diff --git a/.github/actions/setup-podman/action.yml b/.github/actions/setup-podman/action.yml new file mode 100644 index 0000000000..ee30ca3603 --- /dev/null +++ b/.github/actions/setup-podman/action.yml @@ -0,0 +1,23 @@ +name: Setup Podman +description: Install rootless Podman and prepare its per-user runtime directory + +runs: + using: composite + steps: + # Rootless Podman needs subordinate id ranges and a per-user runtime directory; the user + # service is started explicitly so /run/user/ exists before the first podman call. + - name: Prepare rootless Podman + shell: bash + run: | + if ! command -v podman >/dev/null; then + sudo apt-get update + sudo apt-get install -y podman + fi + user="$(id -un)" + grep -q "^${user}:" /etc/subuid || sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 "${user}" + sudo loginctl enable-linger "${user}" + sudo systemctl start "user@$(id -u).service" + echo "XDG_RUNTIME_DIR=/run/user/$(id -u)" >> "$GITHUB_ENV" + XDG_RUNTIME_DIR="/run/user/$(id -u)" podman system migrate + podman version + podman info diff --git a/.github/workflows/stack-stress.yml b/.github/workflows/stack-stress.yml new file mode 100644 index 0000000000..2d10715a3f --- /dev/null +++ b/.github/workflows/stack-stress.yml @@ -0,0 +1,59 @@ +name: Stack stress + +# Repeats the idle/wake and parallel-stack scenarios back to back on each +# backend to surface timing-dependent races. Manual only: a failing iteration +# fails the job outright and is never retried. +on: + workflow_dispatch: + inputs: + iterations: + description: Runs per backend + type: number + default: 10 + +permissions: + contents: read + +concurrency: + group: stack-stress-${{ github.ref }} + cancel-in-progress: true + +jobs: + stress: + name: Run stack stress scenarios (${{ matrix.runtime }}) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 90 + env: + SUPABASE_STACK_TEST_ENGINE: ${{ matrix.runtime == 'podman' && 'podman' || 'docker' }} + strategy: + fail-fast: false + matrix: + runtime: [native, docker, podman] + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + - name: Setup Podman + if: matrix.runtime == 'podman' + uses: ./.github/actions/setup-podman + + # vitest's --repeats counts the runs after the first. + - name: Run idle and parallel-stack scenarios repeatedly + working-directory: packages/stack + env: + ITERATIONS: ${{ inputs.iterations }} + run: | + if ! [ "$ITERATIONS" -ge 1 ] 2>/dev/null; then + echo "iterations must be a whole number of at least 1, got '$ITERATIONS'" >&2 + exit 1 + fi + pnpm run test:e2e:run \ + whole-stack.${{ matrix.runtime }}.idle-parallel \ + --repeats=$((ITERATIONS - 1)) \ + --passWithNoTests=false diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6f1162d342..981705f990 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -128,6 +128,16 @@ jobs: - name: Run integration tests run: pnpm run test:integration + # Not covered by test:integration above (a different vitest project); runs here so the + # registry cross-process and publication-crash scenarios run on Linux too. + - name: Run stack namespace e2e tests + working-directory: packages/stack + run: >- + pnpm test:e2e:run + src/namespace/Namespace.e2e.test.ts + src/port-reservation.e2e.test.ts + --passWithNoTests=false + test-stack-ports: if: | !startsWith(github.head_ref, 'release-notes/') && @@ -160,16 +170,25 @@ jobs: pnpm test:integration src/Ports.integration.test.ts src/storage/DirectoryCopy.integration.test.ts - src/State.integration.test.ts - src/State.windows.integration.test.ts - src/State.process.integration.test.ts + src/namespace/Namespace.integration.test.ts + src/namespace/Paths.integration.test.ts src/HostProcess.integration.test.ts src/Network.integration.test.ts src/Proxy.integration.test.ts src/HttpProxy.integration.test.ts --passWithNoTests=false - # Windows-only CLI subset: paths, PATH lookup, and permission semantics diverge there. + - name: Run stack namespace e2e tests + working-directory: packages/stack + run: pnpm test:e2e:run src/namespace/Namespace.e2e.test.ts --passWithNoTests=false + + # Windows-only: sharing-violation retries and CLI paths, PATH lookup, and permission + # semantics that only diverge there. + - name: Run stack namespace Windows e2e tests + if: runner.os == 'Windows' + working-directory: packages/stack + run: pnpm test:e2e:run src/namespace/Namespace.windows.e2e.test.ts --passWithNoTests=false + - name: Run CLI integration tests if: runner.os == 'Windows' working-directory: apps/cli @@ -179,6 +198,37 @@ jobs: src/commands/experimental/stack/start/start.integration.test.ts --passWithNoTests=false + test-stack-native-scenarios: + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) + name: Run stack native scenarios (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + # Native artifacts exist for macOS and Linux only, and macOS runners have + # no Docker; Linux coverage for both backends comes from test-stack-e2e. + os: [macos-latest] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + - name: Run stack native whole-stack scenarios + working-directory: packages/stack + run: pnpm test:e2e:native-scenarios --passWithNoTests=false + test-summary: if: | always() && @@ -285,8 +335,10 @@ jobs: strategy: fail-fast: false matrix: - runtime: [native, docker] + runtime: [native, docker, podman] suite: [lifecycle, idle-parallel] + env: + SUPABASE_STACK_TEST_ENGINE: ${{ matrix.runtime == 'podman' && 'podman' || 'docker' }} steps: - name: Checkout uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta @@ -297,6 +349,10 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} go-cache: "false" + - name: Setup Podman + if: matrix.runtime == 'podman' + uses: ./.github/actions/setup-podman + - name: Run stack end-to-end tests run: >- pnpm --filter @supabase/stack test:e2e:run @@ -310,6 +366,53 @@ jobs: src/public.e2e.test.ts --passWithNoTests=false + test-stack-podman-integration: + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) + name: Run stack Podman integration tests + runs-on: blacksmith-8vcpu-ubuntu-2404 + env: + SUPABASE_STACK_TEST_ENGINE: podman + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + go-cache: "false" + + - name: Setup Podman + uses: ./.github/actions/setup-podman + + # The suites that drive a real container engine; the rest of the integration project is + # engine-independent and runs in test-integration. + - name: Run stack Podman integration tests + working-directory: packages/stack + run: >- + pnpm test:integration:run + src/Owner.integration.test.ts + src/Sweep.integration.test.ts + src/Commands.integration.test.ts + src/StackHost.container-shutdown.integration.test.ts + src/runtime/Container.integration.test.ts + src/storage/DockerDatabaseStorage.integration.test.ts + src/services/Database.integration.test.ts + src/services/Functions.integration.test.ts + src/services/Realtime.integration.test.ts + src/services/Rest.integration.test.ts + src/services/AuthStorage.integration.test.ts + src/services/StorageGateway.integration.test.ts + src/services/AnalyticsImgproxy.integration.test.ts + src/services/Mail.integration.test.ts + src/services/Pooler.integration.test.ts + tests/docker-volume-run.integration.test.ts + --passWithNoTests=false + # Summary job that gates branch protection. It preserves the original check # name and succeeds iff every CLI shard and stack runtime succeeded. test-e2e-summary: @@ -320,13 +423,16 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run end-to-end tests - needs: [test-e2e, test-stack-e2e] + needs: [test-e2e, test-stack-e2e, test-stack-native-scenarios, test-stack-podman-integration] runs-on: ubuntu-latest steps: - name: Verify all shards succeeded run: | - if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ]; then - echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}" + if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ] || \ + [ "${{ needs.test-stack-podman-integration.result }}" = "failure" ] || [ "${{ needs.test-stack-podman-integration.result }}" = "cancelled" ]; then + echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-podman-integration=${{ needs.test-stack-podman-integration.result }}" exit 1 fi - echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}" + echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-podman-integration=${{ needs.test-stack-podman-integration.result }}" diff --git a/.oxlintrc.json b/.oxlintrc.json index 8c10b88c04..9f4d34677e 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -43,6 +43,386 @@ "rules": { "typescript/await-thenable": "off" } + }, + { + // The namespace owns paths, leases, artifact pins, and port reservations; everywhere + // in the repo, outside `src/namespace/**` itself, reaches that state through + // `StackNamespace.ts`, not by importing namespace internals directly. Repo-wide (apps/cli + // included) so a deep or relative import of a namespace module fails from any package; the + // stack package's own raw-primitive restrictions are layered on top, further down. + "files": ["**/*"], + "excludeFiles": ["packages/stack/src/namespace/**", "packages/stack/src/StackNamespace.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ] + } + ] + } + }, + { + // Default for the whole stack package (`src/**` and `tests/**`): on top of the repo-wide + // namespace restriction above, raw `node:fs`/`node:sqlite` are confined to namespace drivers + // and a short enumerated list of files. The excluded files are direct namespace consumers + // the facade doesn't cover or raw-primitive users; they get narrower overrides further down. + // oxlint JSON has no shared lists, so the direct-consumer files are listed here and again in + // the override that restores the raw-primitive rules for them; keep the two in sync. + "files": ["packages/stack/src/**", "packages/stack/tests/**"], + "excludeFiles": [ + "packages/stack/src/namespace/**", + "packages/stack/src/StackNamespace.ts", + "packages/stack/src/Network.ts", + "packages/stack/src/Network.integration.test.ts", + "packages/stack/src/Owner.ts", + "packages/stack/src/Ports.ts", + "packages/stack/src/Ports.integration.test.ts", + "packages/stack/src/Owner.integration.test.ts", + "packages/stack/src/port-reservation.e2e.test.ts", + "packages/stack/src/host/CommandRunner.ts", + "packages/stack/src/preparation/ArtifactStore.ts", + "packages/stack/src/preparation/artifacts.integration.test.ts", + "packages/stack/src/runtime/Container.ts", + "packages/stack/src/runtime/NativeProcess.ts", + "packages/stack/src/runtime/native-process.integration.test.ts", + "packages/stack/src/runtime/native-launcher.ts", + "packages/stack/src/runtime/postgres-user.ts", + "packages/stack/src/services/Catalog.ts", + "packages/stack/src/services/Database.ts", + "packages/stack/src/services/DatabaseSnapshot.ts", + "packages/stack/src/services/ProcessRecipe.ts", + "packages/stack/src/storage/DockerDatabaseStorage.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ], + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + } + ] + } + ] + } + }, + { + // `StackNamespace.ts` is the facade itself: it needs namespace imports, but is still an + // ordinary consumer of raw primitives, so those restrictions stay on. + "files": ["packages/stack/src/StackNamespace.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + } + ] + } + ] + } + }, + { + // Inside the namespace, drivers are the only place for raw `node:fs`/`node:sqlite`; sibling + // namespace modules import each other freely, so only the raw-primitive restriction repeats + // here (the facade exception above excludes this whole directory from that one). + "files": ["packages/stack/src/namespace/**"], + "excludeFiles": [ + "packages/stack/src/namespace/drivers/**", + "packages/stack/src/namespace/Namespace.integration.test.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { "name": "node:fs", "message": "Raw node:fs stays in namespace/drivers." }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises stays in namespace/drivers." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // Sqlite is the driver directory's whole purpose, so it stays allowed everywhere in it; raw + // `node:fs` is narrower, limited to `FileSystem.ts` (the no-follow lstat/rmdir driver) by + // exact filename, excluded below. + "files": ["packages/stack/src/namespace/drivers/**"], + "excludeFiles": ["packages/stack/src/namespace/drivers/FileSystem.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { "name": "node:fs", "message": "Raw node:fs stays in drivers/FileSystem.ts." }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises stays in drivers/FileSystem.ts." + } + ] + } + ] + } + }, + { + // Synchronous existence check between manual scheduler steps; keeps the disable comment's + // reason, only sqlite stays off-limits. + "files": ["packages/stack/src/namespace/Namespace.integration.test.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // Direct namespace consumers the facade doesn't cover; none of them touch raw + // node:fs/node:sqlite themselves, so that restriction stays on. + // - Network.ts, Ports.ts (+its integration test), port-reservation.e2e.test.ts: read/reserve + // ports through PortReservations, not yet on the facade. + // - Owner.ts: needs PortReservations and owned/borrowed Paths. + // - ArtifactStore.ts, artifacts.integration.test.ts: use the sqlite-pin driver and Pin; + // artifact pins are not on the facade yet. + // - Container.ts, Database.ts, ProcessRecipe.ts, Catalog.ts: need owned/borrowed Paths + // and/or Environment. + // - NativeProcess.ts, native-process.integration.test.ts: need Environment. + // - postgres-user.ts: needs the no-follow lstatPath. + // - DockerDatabaseStorage.ts: needs Publication. + "files": [ + "packages/stack/src/Network.ts", + "packages/stack/src/Network.integration.test.ts", + "packages/stack/src/Owner.ts", + "packages/stack/src/Ports.ts", + "packages/stack/src/Ports.integration.test.ts", + "packages/stack/src/Owner.integration.test.ts", + "packages/stack/src/port-reservation.e2e.test.ts", + "packages/stack/src/preparation/ArtifactStore.ts", + "packages/stack/src/preparation/artifacts.integration.test.ts", + "packages/stack/src/runtime/Container.ts", + "packages/stack/src/runtime/NativeProcess.ts", + "packages/stack/src/runtime/native-process.integration.test.ts", + "packages/stack/src/runtime/postgres-user.ts", + "packages/stack/src/services/Catalog.ts", + "packages/stack/src/services/Database.ts", + "packages/stack/src/services/ProcessRecipe.ts", + "packages/stack/src/storage/DockerDatabaseStorage.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers (plus runtime/native-launcher.ts's standalone boundary); use a namespace driver." + } + ] + } + ] + } + }, + { + // `host/CommandRunner.ts` needs Environment and borrow from the namespace, plus the + // raw `rmdir` Effect's FileSystem has no non-recursive equivalent for; sqlite stays + // off-limits. + "files": ["packages/stack/src/host/CommandRunner.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // `services/DatabaseSnapshot.ts` opens its own sqlite lock file for the snapshot store, a + // concern independent of the namespace's own locking; it also needs the no-follow + // lstatPath. Raw node:fs stays off-limits. + "files": ["packages/stack/src/services/DatabaseSnapshot.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "paths": [ + { + "name": "node:fs", + "message": "Raw node:fs is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + }, + { + "name": "node:fs/promises", + "message": "Raw node:fs/promises is confined to namespace drivers and a short enumerated list of existing files; use Effect's FileSystem or a namespace driver." + } + ] + } + ] + } + }, + { + // Standalone process outside the Effect runtime, launched before any service is wired up: it + // calls the sqlite-pin driver and raw node:fs/node:sqlite directly by design, so none of + // the namespace restrictions apply here. + "files": ["packages/stack/src/runtime/native-launcher.ts"], + "rules": { + "no-restricted-imports": "off" + } + }, + { + // Raw node:fs where Effect's FileSystem can't do the job (each import's disable comment + // gives the reason); none of these import namespace internals or raw sqlite. + "files": [ + "packages/stack/src/internal/host-process.ts", + "packages/stack/src/Sweep.integration.test.ts", + "packages/stack/src/storage/DirectoryCopy.ts", + "packages/stack/src/storage/DirectoryCopy.integration.test.ts", + "packages/stack/src/effect-timeout.integration.test.ts", + "packages/stack/src/HostProcess.ts", + "packages/stack/src/functions/serve-main-resolver.integration.test.ts", + "packages/stack/tests/effect-timeout.fixture.ts", + "packages/stack/tests/host-process-fixture.ts", + "packages/stack/tests/failing-owner-fixture.ts", + "packages/stack/tests/short-registration-poll-fixture.ts", + "packages/stack/tests/gated-docker-stop-fixture.ts", + "packages/stack/tests/release-fifo.ts", + "packages/stack/tests/watch-entry.ts" + ], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ], + "paths": [ + { + "name": "node:sqlite", + "allowTypeImports": true, + "message": "Raw node:sqlite stays in namespace/drivers." + }, + { + "name": "bun:sqlite", + "allowTypeImports": true, + "message": "Raw bun:sqlite stays in namespace/drivers." + } + ] + } + ] + } + }, + { + // Standalone fixture process the lease waits on, outside any Effect runtime: it takes the + // same raw sqlite lock as the lease it waits for and reports over a raw fd. + // It doesn't import namespace internals, so that restriction stays on. + "files": ["packages/stack/tests/lease-wait-fixture.ts"], + "rules": { + "no-restricted-imports": [ + "error", + { + "patterns": [ + { + "group": ["**/namespace/**"], + "message": "Namespace internals are private; consume them through StackNamespace.ts." + } + ] + } + ] + } } ] } diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index e6e0d4a063..9d8e4be993 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -32,8 +32,8 @@ when its engine answers, and otherwise native on Linux x64/arm64 and macOS arm64 platforms without a reachable engine, the command fails and asks you to start Docker or Podman. The selected runtime is saved with the stack and reused without probing; when auto selection skips Docker, the command prints a notice saying so. To switch, destroy the stack or choose a different -`--stack` name. When an explicit `--runtime docker` or a saved Docker stack cannot reach Docker, -the failure asks you to install or start it, and also suggests `--runtime native` for a new stack +`--stack` name. When an explicit `--runtime docker` or `--runtime podman`, or a saved container +stack, cannot reach its engine, the failure asks you to install or start it, and also suggests `--runtime native` for a new stack on platforms that support native. Project stacks created by database commands, and shadow stacks created without a project stack, use the same selection. @@ -313,7 +313,7 @@ retained in stack state, so starting without `--exclude` restores the project's cannot be read or decoded are skipped with a warning on stderr; only a failure to read the stacks directory itself fails discovery, before any stop is attempted. Individual stop failures make the command fail and identify the affected stack IDs with their error details; no -success or unavailable summary is emitted when a stop fails. +success or not-running summary is emitted when a stop fails. `supabase stack destroy --stack feature-a` permanently removes exactly that stack and its data after confirmation. Use `--yes` for unattended execution. There is no bulk destroy option. diff --git a/apps/cli/src/command-internal/bundled-postgres-client.ts b/apps/cli/src/command-internal/bundled-postgres-client.ts index 8495a41661..b57ef39779 100644 --- a/apps/cli/src/command-internal/bundled-postgres-client.ts +++ b/apps/cli/src/command-internal/bundled-postgres-client.ts @@ -15,8 +15,8 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { ArtifactError, postgresVersion, - prepareNativeArtifact, resolveArtifact, + useNativeArtifact, } from "@supabase/stack/internal/artifacts"; import { DockerRun, type DockerRunOpts } from "./docker-run.service.ts"; @@ -249,7 +249,9 @@ export const bundledPostgresClientLayer: Layer.Layer< ); return yield* dockerRun(options, resolved.image, docker); } - const prepared = yield* prepareNativeArtifact( + // Pins the generation for the life of this scope, before the spawn inside `nativeRun`: + // the client child always exits before this scope closes, so the pin covers it fully. + const prepared = yield* useNativeArtifact( { service: "database", version }, path.join(settings.supabaseHome, "cache", "stack"), { os: runtimeInfo.platform, arch: runtimeInfo.arch }, diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 028e39ce19..a556e887e8 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -457,13 +457,9 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { lifecycle: state === "running" ? "running" : "stopped", health: state === "running" ? "healthy" : undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, wakeEnabled: state === "running", - registered: true, }), followStatus: Stream.empty, readLogs: () => Stream.empty, @@ -508,7 +504,6 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }) diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts index 41954d2c35..0aa1c8d538 100644 --- a/apps/cli/src/command-internal/stack-api.ts +++ b/apps/cli/src/command-internal/stack-api.ts @@ -9,7 +9,6 @@ import { open, type CreateOptions, type DeletedStack, - type DestroyResult, type FindOptions, type FoundStack, type OpenOptions, @@ -76,16 +75,4 @@ export const stackApiLayer = Layer.effect( }), ).pipe(Layer.provide(FetchHttpClient.layer)); -/** Describes the engine resources a destroy left behind and the commands that remove them. */ -export const skippedRuntimeCleanupWarning = ( - subject: string, - result: Extract, -): string => { - const engineName = result.engine === "docker" ? "Docker" : "Podman"; - return [ - `${engineName} was unavailable, so ${engineName} resources for ${subject} were not removed. Once it is running, remove them with:`, - ...result.cleanupCommands.map((command) => ` ${command}`), - ].join("\n"); -}; - export type { Stack }; diff --git a/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts b/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts index 791e54aef9..84b21e2788 100644 --- a/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts +++ b/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts @@ -15,7 +15,6 @@ import { Stream, } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; -import { tmpdir } from "node:os"; import { create, StackError, type Stack } from "@supabase/stack/effect"; import { postgres } from "@supabase/stack/commands"; import { mockOutput } from "../../tests/helpers/mocks.ts"; @@ -26,8 +25,9 @@ import { DbConnection, type DbSession } from "./db-connection.service.ts"; import { dbConnectionLayer } from "./db-connection.sql-pg.layer.ts"; import { stackCatalogSetupLayer, StackCatalogSetup } from "./stack-catalog-setup.ts"; import { destroyTestStack } from "../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../tests/helpers/stack-artifacts.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = stackArtifactCacheRoot; const jwtSecret = "stack-catalog-setup-integration-secret"; describe("stack catalog setup", { timeout: 180_000 }, () => { diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index 3009d58552..b8094c3d99 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -127,135 +127,72 @@ const section = (document: Readonly> | undefined, name: return isRecord(value) ? value : undefined; }; -const explicitPort = ( - document: Readonly> | undefined, - sectionName: string, - key: string, -): number | undefined => { - const value = section(document, sectionName)?.[key]; - return typeof value === "number" ? value : undefined; -}; - -const envPortOrConfigured = ( - name: string, - document: Readonly> | undefined, - sectionName: string, - key: string, - configured: number, - env: Readonly>, -): number | undefined => { - if (envOverride(name, undefined, env) !== undefined) return configured; - return explicitPort(document, sectionName, key) === undefined ? undefined : configured; -}; - -const envNestedPortOrConfigured = ( - name: string, - document: Readonly> | undefined, - sectionName: string, - nestedSection: string, - key: string, - configured: number, - env: Readonly>, -): number | undefined => { - if (envOverride(name, undefined, env) !== undefined) return configured; - const nested = section(section(document, sectionName), nestedSection); - return typeof nested?.[key] === "number" ? configured : undefined; -}; - /** A setting's `config.toml` key and the `SUPABASE_*` env var that overrides it. */ export interface StackEndpointSetting { readonly configPath: string; readonly envVar: string; } -/** - * A port setting's `config.toml` key and env var, shared by `createCreations` and - * `stackEndpointSetting` so both report the same names; a new endpoint still needs an entry in - * `endpointSettingsByServiceEndpoint`. - */ -interface PortSetting extends StackEndpointSetting { - readonly section: string; - readonly nestedSection?: string; - readonly key: string; -} - -const DB_PORT: PortSetting = { +// Shared by `createCreations` and `stackEndpointSetting`; a new endpoint also needs an entry in +// `endpointSettingsByServiceEndpoint`. +const DB_PORT: StackEndpointSetting = { envVar: "SUPABASE_DB_PORT", - section: "db", - key: "port", configPath: "db.port", }; -const API_PORT: PortSetting = { +const API_PORT: StackEndpointSetting = { envVar: "SUPABASE_API_PORT", - section: "api", - key: "port", configPath: "api.port", }; -const STUDIO_PORT: PortSetting = { +const STUDIO_PORT: StackEndpointSetting = { envVar: "SUPABASE_STUDIO_PORT", - section: "studio", - key: "port", configPath: "studio.port", }; -const DB_POOLER_PORT: PortSetting = { +const DB_POOLER_PORT: StackEndpointSetting = { envVar: "SUPABASE_DB_POOLER_PORT", - section: "db", - nestedSection: "pooler", - key: "port", configPath: "db.pooler.port", }; -const LOCAL_SMTP_PORT: PortSetting = { +const LOCAL_SMTP_PORT: StackEndpointSetting = { envVar: "SUPABASE_LOCAL_SMTP_PORT", - section: "local_smtp", - key: "port", configPath: "local_smtp.port", }; -const LOCAL_SMTP_SMTP_PORT: PortSetting = { +const LOCAL_SMTP_SMTP_PORT: StackEndpointSetting = { envVar: "SUPABASE_LOCAL_SMTP_SMTP_PORT", - section: "local_smtp", - key: "smtp_port", configPath: "local_smtp.smtp_port", }; -const LOCAL_SMTP_POP3_PORT: PortSetting = { +const LOCAL_SMTP_POP3_PORT: StackEndpointSetting = { envVar: "SUPABASE_LOCAL_SMTP_POP3_PORT", - section: "local_smtp", - key: "pop3_port", configPath: "local_smtp.pop3_port", }; -const ANALYTICS_PORT: PortSetting = { +const ANALYTICS_PORT: StackEndpointSetting = { envVar: "SUPABASE_ANALYTICS_PORT", - section: "analytics", - key: "port", configPath: "analytics.port", }; -const EDGE_RUNTIME_INSPECTOR_PORT: PortSetting = { +const EDGE_RUNTIME_INSPECTOR_PORT: StackEndpointSetting = { envVar: "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", - section: "edge_runtime", - key: "inspector_port", configPath: "edge_runtime.inspector_port", }; -/** Resolves one `PortSetting` against the loaded document and env, picking the nested variant when needed. */ +/** The configured port when the setting is present in `config.toml` or its env var, else undefined. */ const resolvePort = ( - setting: PortSetting, + setting: StackEndpointSetting, document: Readonly> | undefined, configured: number, env: Readonly>, -): number | undefined => - setting.nestedSection === undefined - ? envPortOrConfigured(setting.envVar, document, setting.section, setting.key, configured, env) - : envNestedPortOrConfigured( - setting.envVar, - document, - setting.section, - setting.nestedSection, - setting.key, - configured, - env, - ); +): number | undefined => { + if (envOverride(setting.envVar, undefined, env) !== undefined) return configured; + const path = setting.configPath.split("."); + const key = path[path.length - 1] ?? ""; + const parent = path + .slice(0, -1) + .reduce> | undefined>( + (node, name) => section(node, name), + document, + ); + return typeof parent?.[key] === "number" ? configured : undefined; +}; /** - * Maps a saved stack endpoint (service + endpoint name) to the `PortSetting` that controls it. + * Maps a saved stack endpoint (service + endpoint name) to the `StackEndpointSetting` that controls it. * An endpoint missing here (e.g. `pooler.http`, `realtime.rpc`) is always automatic. */ const endpointSettingsByServiceEndpoint: Readonly> = { diff --git a/apps/cli/src/command-internal/stack-local-database.ts b/apps/cli/src/command-internal/stack-local-database.ts index 613bb47d10..9af7a359dd 100644 --- a/apps/cli/src/command-internal/stack-local-database.ts +++ b/apps/cli/src/command-internal/stack-local-database.ts @@ -24,6 +24,7 @@ import { automaticRuntimeNotice, recordStackRuntimeTelemetry, selectStackRuntime, + type StackRuntime, } from "./stack-runtime.ts"; import { Output } from "../shared/output/output.service.ts"; import type { PgConnInput } from "./db-connection.service.ts"; @@ -127,7 +128,7 @@ const RUNTIME_UNAVAILABLE = new StackRuntimeUnavailableError({ }); export const stackRequireProjectRuntime: Effect.Effect< - "native" | "docker" | "podman", + StackRuntime, StackRuntimeUnavailableError, CommandSettings | StackApi | Path.Path > = stackProjectRuntime.pipe( diff --git a/apps/cli/src/command-internal/stack-runtime.ts b/apps/cli/src/command-internal/stack-runtime.ts index aa7fb4dcf3..4d1144e1fc 100644 --- a/apps/cli/src/command-internal/stack-runtime.ts +++ b/apps/cli/src/command-internal/stack-runtime.ts @@ -1,6 +1,6 @@ import { defaultRuntime } from "@supabase/stack/internal/artifacts"; +import { resolveEngineTarget, type ContainerEngine } from "@supabase/stack/internal/engine"; import { Data, Effect } from "effect"; -import * as ChildProcess from "effect/unstable/process/ChildProcess"; import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; import { @@ -9,7 +9,6 @@ import { ErrorActionabilityId, } from "../shared/telemetry/error-actionability.ts"; import { PropStackRuntime } from "../shared/telemetry/event-catalog.ts"; -import { withChildTraceEnv, withProcessSpan } from "../shared/telemetry/spans.ts"; import { recordCommandTelemetry } from "../telemetry/command-telemetry-attributes.ts"; /** Runtime that executes a local stack's services. */ @@ -35,40 +34,27 @@ export class StackRuntimeSelectionError extends Data.TaggedError("StackRuntimeSe /** A cold Docker Desktop or Podman machine can take several seconds to answer its first request. */ const PROBE_TIMEOUT = "10 seconds"; -/** Each probe exits non-zero unless the engine's daemon or service answers. */ -const engineProbes = [ - { runtime: "docker", args: ["version", "--format", "{{.Server.Version}}"] }, - { runtime: "podman", args: ["info", "--format", "{{.Version.Version}}"] }, -] as const; - -const engineReachable = ( +/** + * Resolves the engine target the owner would pin, so selection and startup agree on which endpoint + * answers; resolution reads the engine's identity, so success means the engine is reachable. + */ +const engineReachable = Effect.fn("StackRuntime.probeEngine")(function* ( spawner: ChildProcessSpawner["Service"], - probe: (typeof engineProbes)[number], -) => - withProcessSpan( - "StackRuntime.probeEngine", - { - executable: probe.runtime, - argCount: probe.args.length, - subcommand: probe.args[0], - hasFallback: true, - }, - (traceEnv) => - spawner.exitCode( - ChildProcess.make( - probe.runtime, - probe.args, - withChildTraceEnv( - { stdin: "ignore", stdout: "ignore", stderr: "ignore", forceKillAfter: "1 second" }, - traceEnv, - ), - ), - ), - ).pipe( + engine: ContainerEngine, +) { + yield* Effect.annotateCurrentSpan({ "container.engine": engine }); + const reachable = yield* resolveEngineTarget(spawner, engine).pipe( + Effect.as(true), Effect.timeout(PROBE_TIMEOUT), - Effect.map((exitCode) => exitCode === 0), Effect.orElseSucceed(() => false), ); + yield* Effect.annotateCurrentSpan({ "container.reachable": reachable }); + return reachable; +}); + +/** Display name of a container runtime. */ +export const containerEngineName = (runtime: ContainerEngine) => + runtime === "docker" ? "Docker" : "Podman"; /** * Notice for a new stack whose automatic selection skipped Docker, since that runtime is saved with @@ -82,10 +68,7 @@ export const automaticRuntimeNotice = ( ? undefined : `Docker didn't answer, so this new stack uses the ${selected === "podman" ? "Podman" : "native"} runtime, which is saved with the stack. To use Docker, start it, then run \`supabase stack destroy\` and start again, or choose a different --stack name with --runtime docker.`; -/** - * Returns the requested or saved runtime unchanged; otherwise the first of Docker, Podman, or - * native that is usable on this host. - */ +/** Returns the requested or saved runtime unchanged; otherwise the first of Docker, Podman, or native that is usable on this host. */ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( requested: StackRuntime | undefined, ) { @@ -104,8 +87,8 @@ export const selectStackRuntime = Effect.fn("StackRuntime.select")(function* ( return requested; } const spawner = yield* ChildProcessSpawner; - for (const probe of engineProbes) { - if (yield* engineReachable(spawner, probe)) return probe.runtime; + for (const engine of ["docker", "podman"] as const) { + if (yield* engineReachable(spawner, engine)) return engine; } const { platform, arch } = yield* RuntimeInfo; if (defaultRuntime({ os: platform, arch }) === "native") return "native"; diff --git a/apps/cli/src/command-internal/stack-runtime.unit.test.ts b/apps/cli/src/command-internal/stack-runtime.unit.test.ts deleted file mode 100644 index 79794d8306..0000000000 --- a/apps/cli/src/command-internal/stack-runtime.unit.test.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { BunServices } from "@effect/platform-bun"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer, Ref } from "effect"; -import { runtimeInfoLayer } from "../shared/runtime/runtime-info.layer.ts"; -import { - CommandTelemetryAttributes, - type CommandTelemetryAttributeValues, -} from "../telemetry/command-telemetry-attributes.ts"; -import { selectStackRuntime } from "./stack-runtime.ts"; - -describe("selectStackRuntime", () => { - it.effect("records the selected runtime on the command event", () => - Effect.gen(function* () { - const recorded = yield* Ref.make({}); - - const runtime = yield* selectStackRuntime("podman").pipe( - Effect.provideService(CommandTelemetryAttributes, { - record: (values) => Ref.update(recorded, (current) => ({ ...current, ...values })), - }), - ); - - expect(runtime).toBe("podman"); - expect((yield* Ref.get(recorded)).stack_runtime).toBe("podman"); - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))), - ); -}); diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 7b12b4f540..b3e4ba76b1 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -458,15 +458,14 @@ describe("stack shadow databases", () => { ); it.live( - "prints the cleanup commands when a shadow's destroy skips its engine", + "reports a shadow whose destroy fails because its engine is unreachable", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-skipped-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-unreachable-" }); const output = mockOutput(); - const cleanupCommand = "docker rm --force $(docker ps --all --quiet)"; - // A real registration whose database creation fails fast and whose destroy reports skipped cleanup. + // A real registration whose database creation fails fast and whose destroy fails with the engine unreachable. const api = Layer.effect( StackApi, Effect.gen(function* () { @@ -482,11 +481,13 @@ describe("stack shadow databases", () => { create: () => Effect.fail(new StackError({ operation: "create", message: "injected" })), }, - destroy: Effect.succeed({ - runtimeCleanup: "skipped", - engine: "docker", - cleanupCommands: [cleanupCommand], - } as const), + destroy: Effect.fail( + new StackError({ + operation: "destroy", + message: "Docker CLI or daemon isn't reachable", + reason: "runtime-unavailable", + }), + ), })), ), }); @@ -510,7 +511,7 @@ describe("stack shadow databases", () => { ); expect(output.stderrText).toMatch( - /Warning: Docker was unavailable, so Docker resources for shadow stack [0-9a-f]{64} were not removed\. Once it is running, remove them with:\n {2}docker rm --force \$\(docker ps --all --quiet\)\n/u, + /Failed to destroy shadow stack [0-9a-f]{64}: Docker CLI or daemon isn't reachable\.\n/u, ); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), 120_000, diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 86c2ee6276..764bc4eb93 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -3,10 +3,10 @@ import type { DatabaseInstance, Stack } from "@supabase/stack/effect"; import { CommandSettings } from "../config/command-settings.service.ts"; import { Output } from "../shared/output/output.service.ts"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; -import { skippedRuntimeCleanupWarning, StackApi } from "./stack-api.ts"; +import { StackApi } from "./stack-api.ts"; import { initializeStackDatabase } from "./stack-bootstrap.ts"; import { stackProjectRuntime } from "./stack-local-database.ts"; -import { selectStackRuntime } from "./stack-runtime.ts"; +import { selectStackRuntime, type StackRuntime } from "./stack-runtime.ts"; import { parseConnectionString } from "./db-config.parse.ts"; import { toPostgresURL } from "./postgres-url.ts"; import { @@ -19,15 +19,13 @@ import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; import { stackShadowCacheEntry, stackShadowCacheRoles } from "./stack-shadow-cache.ts"; -type Runtime = "native" | "docker" | "podman"; - export interface StackShadowAcquiredHandle { readonly stack: Stack; readonly database: DatabaseInstance; readonly url: string; readonly host: string; readonly port: number; - readonly runtime: Runtime; + readonly runtime: StackRuntime; readonly version: string; /** Cache lineage when a baseline was restored or atomically published by this handle. */ readonly snapshotKey?: string; @@ -36,7 +34,7 @@ export interface StackShadowAcquiredHandle { interface ShadowOptions { readonly port?: number; - readonly runtime?: Runtime; + readonly runtime?: StackRuntime; readonly webhooks?: SetupDatabaseOptions["webhooks"]; readonly bypassCache?: boolean; } @@ -81,7 +79,7 @@ const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (op const initialize = Effect.fn("StackShadow.initialize")(function* ( stack: Stack, - runtime: Runtime, + runtime: StackRuntime, input: ShadowSetupInput, opts: ShadowOptions, ) { @@ -235,14 +233,6 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct const output = yield* Output; const namespace = yield* Effect.acquireRelease(acquireNamespace(opts), ({ stack }) => stack.destroy.pipe( - Effect.flatMap((result) => - result.runtimeCleanup === "skipped" - ? output.raw( - `Warning: ${skippedRuntimeCleanupWarning(`shadow stack ${stack.id}`, result)}\n`, - "stderr", - ) - : Effect.void, - ), Effect.catch((cause) => output.raw(`Failed to destroy shadow stack ${stack.id}: ${cause.message}.\n`, "stderr"), ), diff --git a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts index 37603ac7da..168035f975 100644 --- a/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts +++ b/apps/cli/src/commands/db/diff/diff.stack-cache.e2e.test.ts @@ -2,10 +2,10 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path, Redacted } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; -import { create as createStack } from "@supabase/stack/effect"; -import { tmpdir } from "node:os"; +import { create as createStack, discover as discoverStacks } from "@supabase/stack/effect"; import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 8 * 60_000; const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS * 4 + 2 * 60_000; @@ -91,10 +91,9 @@ describe("supabase db diff (stack shadow baseline cache)", () => { const home = yield* fs.makeTempDirectoryScoped({ prefix: `db-diff-cache-home-${runtime}-`, }); - const sharedArtifacts = path.join(tmpdir(), "supabase-stack-artifacts"); - yield* fs.makeDirectory(sharedArtifacts, { recursive: true }); + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true }); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - yield* fs.symlink(sharedArtifacts, path.join(home, "cache", "stack")); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); const supabase = path.join(project, "supabase"); const migrations = path.join(supabase, "migrations"); @@ -130,10 +129,8 @@ describe("supabase db diff (stack shadow baseline cache)", () => { expect(second.stdout).toMatch(/stack_cache_second/iu); expect(second.stdout).not.toMatch(/stack_cache_first/iu); - const stackEntries = (yield* fs.readDirectory(path.join(home, "stacks"))).filter( - (entry) => /^[0-9a-f]{64}$/u.test(entry), - ); - expect(stackEntries).toEqual([stack.id]); + const registered = yield* discoverStacks({ stateRoot: path.join(home, "stacks") }); + expect(registered.map(({ definition }) => definition.id)).toEqual([stack.id]); }).pipe(Effect.scoped, Effect.provide([BunServices.layer, FetchHttpClient.layer])), { timeout: TEST_TIMEOUT_MS }, ); diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index bf6fb0002f..c83eceab39 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -116,13 +116,9 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { lifecycle: "running", health: "healthy", error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, wakeEnabled: true, - registered: true, }), followStatus: Stream.empty, readLogs: () => Stream.empty, @@ -153,7 +149,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { restart: Effect.succeed([]), }, stop: Effect.void, - destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + destroy: Effect.void, gateway: unusedGateway, commands: { run: runCommand }, } satisfies Stack; @@ -171,7 +167,6 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { instances: [], composition: { members: [{ id, activation: "eager" as const }], dependencies: [] }, lifetime: "detached" as const, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 3bbd4b5d6f..cb19373724 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -615,13 +615,9 @@ function makeStackObservation( lifecycle: opts.lifecycle ?? "stopped", health: opts.health, error: opts.error, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, wakeEnabled: opts.wakeEnabled ?? false, - registered: true, }; } @@ -829,7 +825,6 @@ function mockResetStackApi(opts: { })), lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts b/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts index f86b7e44f8..0c395b0ab1 100644 --- a/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts +++ b/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts @@ -4,10 +4,9 @@ import { Context, Data, Effect, FileSystem, Layer, Path, Redacted } from "effect import { FetchHttpClient, HttpClient } from "effect/unstable/http"; import { PgClient } from "@effect/sql-pg"; import { create as createStack } from "@supabase/stack/effect"; -import { tmpdir } from "node:os"; - import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 8 * 60_000; const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS + 2 * 60_000; @@ -166,13 +165,10 @@ describe("supabase db reset (stack e2e)", () => { const home = yield* fs.makeTempDirectoryScoped({ prefix: `db-reset-home-${runtime}-` }); yield* writeFixture(root, fs, path); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - yield* fs.makeDirectory(path.join(tmpdir(), "supabase-stack-artifacts"), { + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true, }); - yield* fs.symlink( - path.join(tmpdir(), "supabase-stack-artifacts"), - path.join(home, "cache", "stack"), - ); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); const stack = yield* composeStack(root, home, runtime); diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index bba2d19e01..7d3cd0fef5 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -129,14 +129,10 @@ function generateStackApi(workdir: string) { endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: STACK_GENERATE_PORT }], lifecycle: "running", health: "healthy", - registered: true, wakeEnabled: true, - intentRevision: 0, currentOperation: undefined, - launchId: undefined, exit: undefined, error: undefined, - cleanupError: undefined, }), }; const composition = { @@ -183,7 +179,6 @@ function generateStackApi(workdir: string) { instances: [], composition, lifetime: "detached" as const, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index 4153e6cd1b..f8e0c62e90 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -131,14 +131,10 @@ function syncStackApi(workdir: string, port: number) { endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port }], lifecycle: "running", health: "healthy", - registered: true, wakeEnabled: true, - intentRevision: 0, currentOperation: undefined, - launchId: undefined, exit: undefined, error: undefined, - cleanupError: undefined, }), }; const composition = { @@ -185,7 +181,6 @@ function syncStackApi(workdir: string, port: number) { instances: [], composition, lifetime: "detached" as const, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index b1fb838403..6fe4dd56c0 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1626,13 +1626,9 @@ describe("db start stack backend", () => { lifecycle: state.running ? ("running" as const) : ("stopped" as const), health: state.running ? ("healthy" as const) : undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, wakeEnabled: state.running, - registered: true, })), followStatus: Stream.empty, readLogs: () => Stream.empty, @@ -1734,7 +1730,7 @@ describe("db start stack backend", () => { restart: Effect.succeed([]), }, stop: Effect.void, - destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + destroy: Effect.void, gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, }; @@ -1757,7 +1753,6 @@ describe("db start stack backend", () => { instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }) diff --git a/apps/cli/src/commands/db/start/start.roles.e2e.test.ts b/apps/cli/src/commands/db/start/start.roles.e2e.test.ts index 51a83da8e8..11cd8ec67c 100644 --- a/apps/cli/src/commands/db/start/start.roles.e2e.test.ts +++ b/apps/cli/src/commands/db/start/start.roles.e2e.test.ts @@ -1,9 +1,8 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path, Schema } from "effect"; -import { tmpdir } from "node:os"; - import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 8 * 60_000; const AUX_TIMEOUT_MS = 3 * 60_000; @@ -27,9 +26,8 @@ describe("supabase db start (e2e, role alignment)", () => { const migrations = path.join(root, "supabase", "migrations"); yield* fs.makeDirectory(migrations, { recursive: true }); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); - yield* fs.makeDirectory(artifacts, { recursive: true }); - yield* fs.symlink(artifacts, path.join(home, "cache", "stack")); + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true }); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); yield* fs.writeFileString( configPath, `project_id = "db-start-roles-e2e" diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 2fea23eb23..266acaa3fb 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -39,18 +39,24 @@ for a deleted `SUPABASE_HOME`, run destroy with that `SUPABASE_HOME`. After confirmation the command opens the selected handle and destroys its entire namespace. Destruction may start an owner to clean up a stopped namespace. Cleanup failures remain errors; the command does not claim success on failure. -A failed destroy may leave its owner running; retry destruction or use -`stack stop` to shut down that owner. +Shutdown is one-way: a failed destroy leaves the stack registered and its +owner exits; the next command retries. + +As with `stack stop`, services stop in reverse dependency order with every +stack listener still open, and traffic that would wake a stopped service during +shutdown is refused. When no owner is running and the engine reports that its daemon cannot be -reached, destruction removes the local namespace and host data anyway, warns on -stderr that the stack's engine resources were not removed, and exits 0. The -warning lists the commands that remove them once the engine is running: one for -the stack's containers, and one per database whose data is kept in an engine -volume. Any other engine failure, an owner starting during destruction, or -host data the current user cannot delete (such as database files a container -wrote as its own user), fails the command before anything is removed and keeps -the stack registered; the last case asks to start the engine and retry. +reached, destruction removes nothing: the registration, port claims and host +data stay as they are, and the command exits 1 with one error whose suggestion +gives the exact `supabase stack destroy --stack-id --yes` command to run +once the engine is reachable. Any other engine failure, an owner starting during +destruction, an instance whose data (including a native database's socket +directory) cannot be removed, or a labelled container that survives removal, +fails the command and keeps the stack registered; the error lists what +remains. A native runtime root that is not trusted (a symlink, owned by another +user or writable by others) is warned about and its socket directories are left +in place. ## Files and network @@ -61,15 +67,30 @@ uploads are preserved. The independent artifact cache is retained. Communicates with the local owner and selected runtime, with no hosted API calls. Shared routing/settings can read project configuration and profiles. +Destruction releases this stack's rows in the per-user port registry at +`/.supabase/ports.sqlite` (independent of `SUPABASE_HOME`, with no +override) only once cleanup is confirmed complete; a destroy that fails +keeps the stack's automatic reservations, and those ports remain unavailable +to other stacks until a later destroy succeeds; configured ports are released +with their listeners. A reservation whose owning +stack's state was deleted without going through destroy (for example a removed +or unmounted state root) is released lazily by whichever stack next needs that +port. That stack's still-running owner notices the lost registration within its +next check (30 seconds by default), stops its workloads and exits without +deleting data; its configured ports are released with their listeners. + ## Output, exit codes and telemetry Text prints `Stack destroyed.` (`Removed the containers stack left behind.` -for a deleted stack's containers), or, when engine cleanup was skipped, -`Stack was removed locally; its resources remain until the commands above are run.` -JSON and stream-json success data contain `destroyed: true`, `id`, and -`runtimeCleanup` (`complete` or `skipped`); a skipped cleanup also carries `engine` and -`cleanupCommands`. Exit 0 on destruction, 1 on invalid flags, missing -selection, rejected/cancelled confirmation or cleanup failure, and 130 on -interruption. Standard command telemetry is unchanged, with no custom events. +for a deleted stack's containers). JSON and stream-json success data contain +`destroyed` (`true`) and `id`. Exit 0 on destruction, 1 on invalid flags, missing +selection, rejected/cancelled confirmation, an unreachable engine, or cleanup +failure, and 130 on interruption. A saved stack whose state cannot be decoded +(for example one saved by an older CLI with a service kind this CLI no longer +knows) fails with an error naming its directory under the state root; removing +that directory discards the stack. Its leftover containers are then removed +when a container stack next starts under the same state root or by +`destroy --stack-id`, and its port reservations are released when another stack +needs them; its data in the shared database volume is not removed. Standard command telemetry is unchanged, with no custom events. Telemetry flushes on success and failure to `/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index 4e6cb9ebc7..030feafbe7 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -9,30 +9,27 @@ import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts import { isStackId, StackApi, - StackTargetError, rejectStackOutput, - skippedRuntimeCleanupWarning, StackTargetResolver, + mapTargetError, validateStackTarget, } from "../stack.shared.ts"; import type { StackDestroyFlags } from "./destroy.command.ts"; import { StackCommandDestroyError } from "./destroy.errors.ts"; -const mapTargetError = (error: StackTargetError, detail?: string) => - new StackCommandDestroyError({ - reason: error.reason, - message: error.message, - ...(detail === undefined ? {} : { detail }), - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - -const destroyError = (cause: StackError) => - new StackCommandDestroyError({ - reason: "unknown", - message: cause.message, - cause, - }); +const destroyError = (id: string) => (cause: StackError) => + cause.reason === "runtime-unavailable" + ? new StackCommandDestroyError({ + reason: "runtime", + message: cause.message, + suggestion: `Start the container engine, then run "supabase stack destroy --stack-id ${id} --yes" again; nothing was removed.`, + cause, + }) + : new StackCommandDestroyError({ + reason: "unknown", + message: cause.message, + cause, + }); export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( flags: StackDestroyFlags, @@ -43,11 +40,13 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( const settings = yield* CommandSettings; const api = yield* StackApi; const outputFlag = yield* Effect.serviceOption(OutputFlag); - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + yield* rejectStackOutput(outputFlag).pipe( + Effect.mapError(mapTargetError((props) => new StackCommandDestroyError(props))), + ); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); + }).pipe(Effect.mapError(mapTargetError((props) => new StackCommandDestroyError(props)))); const resolver = yield* StackTargetResolver; const path = yield* Path.Path; @@ -60,6 +59,7 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( ? yield* Effect.result(api.findDeleted({ ...locations, id: flags.stackId.value })) : Result.succeedNone; const deleted = Result.getOrElse(lookup, Option.none); + const detail = Result.isFailure(lookup) ? lookup.failure.message : undefined; const target = Option.isSome(deleted) ? undefined : yield* resolver @@ -70,8 +70,14 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( runtime: "auto", }) .pipe( - Effect.mapError((error) => - mapTargetError(error, Result.isFailure(lookup) ? lookup.failure.message : undefined), + Effect.mapError( + mapTargetError( + (props) => + new StackCommandDestroyError({ + ...props, + ...(detail === undefined ? {} : { detail }), + }), + ), ), ); const id = Option.isSome(deleted) ? deleted.value.id : target?.id; @@ -112,9 +118,9 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( } const stack = Option.isSome(deleted) ? deleted.value - : yield* api.open({ ...locations, id }).pipe(Effect.mapError(destroyError)); + : yield* api.open({ ...locations, id }).pipe(Effect.mapError(destroyError(id))); const destroying = yield* output.task(`Destroying stack ${id}...`); - const result = yield* stack.destroy.pipe( + yield* stack.destroy.pipe( Effect.onExit((exit) => Exit.isSuccess(exit) ? destroying.clear @@ -122,25 +128,16 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( ? destroying.cancel() : destroying.fail(Option.getOrUndefined(Exit.findErrorOption(exit))?.message), ), - Effect.mapError(destroyError), + Effect.mapError(destroyError(id)), ); - yield* Effect.annotateCurrentSpan({ - "stack.prompted": !yes, - "stack.runtime_cleanup": result.runtimeCleanup, - }); - if (result.runtimeCleanup === "skipped") - yield* output.warn(skippedRuntimeCleanupWarning(`stack ${id}`, result)); - if (output.format !== "text") yield* output.success("", { destroyed: true, id, ...result }); - else if (result.runtimeCleanup === "complete") + yield* Effect.annotateCurrentSpan({ "stack.prompted": !yes }); + if (output.format !== "text") yield* output.success("", { destroyed: true, id }); + else yield* output.raw( target === undefined ? `Removed the containers stack ${id} left behind.\n` : `Stack ${id} destroyed.\n`, ); - else - yield* output.raw( - `Stack ${id} was removed locally; its ${result.engine === "docker" ? "Docker" : "Podman"} resources remain until the commands above are run.\n`, - ); }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts index e3465db0bb..5613044551 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts @@ -120,6 +120,25 @@ describe("stack destroy", () => { }).pipe(Effect.provide(live)), ); + it.live("refuses a stack with an unknown service kind and names its directory", () => + Effect.gen(function* () { + const f = yield* fixture(true); + const statePath = f.path.join(f.locations.stateRoot, f.stack.id, "state.json"); + const saved = yield* f.fs.readFileString(statePath); + yield* f.fs.writeFileString( + statePath, + saved.replace('"instances":[]', '"instances":[{"id":"logs","service":"vector"}]'), + ); + + const error = yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer), Effect.flip); + + expect(error.message).toContain( + `Remove its directory ${f.path.join(f.locations.stateRoot, f.stack.id)}`, + ); + expect(yield* f.fs.exists(statePath)).toBe(true); + }).pipe(Effect.provide(live)), + ); + it.live("destroys only the stack addressed by the short ID that stack list shows", () => Effect.gen(function* () { const f = yield* fixture(true); @@ -155,7 +174,6 @@ describe("stack destroy", () => { id, destroy: Effect.sync(() => { destroyed += 1; - return { runtimeCleanup: "complete" } as const; }), }) : Option.none(), diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts index 625b9339f4..10dc8d656c 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.runtime-unavailable.integration.test.ts @@ -1,6 +1,7 @@ import { BunServices } from "@effect/platform-bun"; import { expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Option, Path } from "effect"; +import { StackError } from "@supabase/stack/effect"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; import { YesFlag } from "../../../../command-internal/global-flags.ts"; import { @@ -10,45 +11,51 @@ import { import { mockOutput, mockStdin, mockTty } from "../../../../../tests/helpers/mocks.ts"; import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackDestroy } from "./destroy.handler.ts"; +import { StackCommandDestroyError } from "./destroy.errors.ts"; const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +/** + * A real registration whose `destroy` fails the way the stack owner reports an unreachable engine: + * the namespace behaviour behind it (registration and claims untouched) is covered by + * packages/stack's own suite; this fixture is only the CLI's mapping of that failure. + */ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function* ( format: "text" | "json", ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-runtime-" }); - const binDir = path.join(root, "bin"); - yield* fs.makeDirectory(binDir, { recursive: true }); - const dockerShim = path.join(binDir, "docker"); - yield* fs.writeFileString( - dockerShim, - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?' >&2\nexit 1\n", - ); - yield* fs.chmod(dockerShim, 0o755); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. - const originalPath = process.env.PATH; - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. - process.env.PATH = `${binDir}:${originalPath ?? ""}`; - yield* Effect.addFinalizer(() => - Effect.sync(() => { - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. - process.env.PATH = originalPath; - }), - ); - - const api = yield* StackApi; + const realApi = yield* StackApi; const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; - const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "docker" }); + const stack = yield* realApi.create({ ...locations, projectRoot: root, runtime: "docker" }); + const api = StackApi.of({ + ...realApi, + open: (options) => + realApi.open(options).pipe( + Effect.map((opened) => ({ + ...opened, + destroy: Effect.fail( + new StackError({ + operation: "destroy", + message: "Docker CLI or daemon isn't reachable", + reason: "runtime-unavailable", + }), + ), + })), + ), + }); const output = mockOutput({ interactive: false, format }); const telemetry = mockTelemetryStateTracked(); const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); const layer = Layer.mergeAll( + Layer.succeed(StackApi, api), output.layer, telemetry.layer, settings, - stackTargetResolverLayer.pipe(Layer.provide(settings)), + stackTargetResolverLayer.pipe( + Layer.provide(Layer.mergeAll(Layer.succeed(StackApi, api), settings)), + ), mockTty({ stdinIsTty: false }), mockStdin(false), Layer.succeed(YesFlag, true), @@ -64,48 +71,22 @@ const fixture = Effect.fn("StackDestroyRuntimeUnavailableTest.fixture")(function }; }); -const containerCleanup = (id: string) => - `sh -c 'ids=\\$\\(docker ps --all --quiet --no-trunc --filter '\\\\''label=com\\.supabase\\.stack=${id}'\\\\'' --filter '\\\\''label=com\\.supabase\\.stack-root=[^']+/${id}/data'\\\\''\\) && \\{ \\[ -z "\\$ids" \\] \\|\\| docker rm --force \\$ids; \\}'`; +for (const format of ["text", "json"] as const) + it.live( + `fails with the exact --yes retry command and keeps the stack registered when its engine is unreachable (${format})`, + () => + Effect.gen(function* () { + const f = yield* fixture(format); -it.live( - "removes a stack locally and lists its cleanup commands when its engine is unreachable", - () => - Effect.gen(function* () { - const f = yield* fixture("text"); - yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText).toBe( - `Stack ${f.stack.id} was removed locally; its Docker resources remain until the commands above are run.\n`, - ); - expect(f.output.messages).toContainEqual({ - type: "warn", - message: expect.stringMatching( - new RegExp( - `^Docker was unavailable, so Docker resources for stack ${f.stack.id} were not removed\\. Once it is running, remove them with:\\n ${containerCleanup(f.stack.id)}$`, - "u", - ), - ), - }); - expect(yield* f.api.discover(f.locations)).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(live)), -); + const failure = yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer), Effect.flip); -it.live("reports skipped engine cleanup and its commands in the JSON result", () => - Effect.gen(function* () { - const f = yield* fixture("json"); - yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); - expect(f.output.messages).toContainEqual( - expect.objectContaining({ - data: { - destroyed: true, - id: f.stack.id, - runtimeCleanup: "skipped", - engine: "docker", - cleanupCommands: [ - expect.stringMatching(new RegExp(`^${containerCleanup(f.stack.id)}$`, "u")), - ], - }, - }), - ); - expect(yield* f.api.discover(f.locations)).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(live)), -); + expect(failure).toBeInstanceOf(StackCommandDestroyError); + expect(failure.reason).toBe("runtime"); + expect(failure.suggestion).toContain( + `supabase stack destroy --stack-id ${f.stack.id} --yes`, + ); + expect(f.output.stdoutText).not.toContain("destroyed"); + expect(f.output.messages.filter(({ type }) => type === "success")).toEqual([]); + expect(yield* f.api.discover(f.locations)).toHaveLength(1); + }).pipe(Effect.scoped, Effect.provide(live)), + ); diff --git a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md index a006346557..33a892745a 100644 --- a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md @@ -10,7 +10,9 @@ Reads `/stacks//state.json` and probes existin local owner control endpoints. It makes no hosted API calls. Shared command setup may read project configuration for the feature gate and the selected profile for CLI settings. Discovery creates the registry directory if absent and sets its mode to 0700; -no stack state files are written. Telemetry flushes to +no stack state files are written. List does not read the separate per-user public +port registry at `/.supabase/ports.sqlite`; listed entries carry no +port or endpoint information. Telemetry flushes to `/telemetry.json` on success and failure. ## Output diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index ece6299dbf..94d090c096 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -9,8 +9,8 @@ starts/stops a service. Select the current project/branch/name, `--stack `, or `--stack-id `. The selectors are mutually exclusive. By default, composition members are -included, plus `gateway` (the shared API port's request lines) once the stack has -claimed that port. `--service ` is repeatable and can also select +included, plus `gateway` (the shared API port's request lines) when a member is +served on that port. `--service ` is repeatable and can also select standalone instances or `gateway`; a value that matches no saved instance, or `gateway` without the shared API port, fails with status 1. A missing stack fails with status 1. @@ -30,9 +30,8 @@ the selected services, with the `launch` and `lost` markers between them. `--since` accepts a duration before now (`30s`, `10m`, `1h30m`, `2d`), an ISO-8601 time, or `start`, which keeps the records of each instance's current launch and later ones, in history and while following. The current launch is the -launch id saved in the stack definition, which keeps increasing across owner -restarts, or else the highest launch record in history (all records when -retention removed that launch record). Records are +highest launch record in history; launch ids keep increasing across owner +restarts (all records when retention removed that launch record). Records are ordered by timestamp, service, instance, and file position. Timestamps are the owner's clock at each line's first byte. Lines end at `\n`, `\r\n`, or a lone `\r`, so carriage-return progress updates print as separate lines. History is diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts index d1a204c66f..080bf1b05c 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts @@ -85,8 +85,7 @@ const compareRecords = (left: StackLogRecord, right: StackLogRecord) => interface Launch { readonly id: number; - /** Unknown until its launch record is read; a saved launch may not have written one. */ - readonly timestamp: string | undefined; + readonly timestamp: string; } interface InstanceTail { @@ -104,21 +103,18 @@ export const isFromLaunch = (record: LogRecord, launchId: number | undefined) => /** Launch ids increase per instance; a gap marker carries none, so its time decides. */ const fromLaunch = (record: StackLogRecord, launch: Launch) => record.launchId === undefined - ? launch.timestamp === undefined || record.timestamp >= launch.timestamp + ? record.timestamp >= launch.timestamp : isFromLaunch(record, launch.id); /** - * Collects each instance's newest `tail` lines and last position. With `startLaunches`, it keeps - * only the records of each instance's current launch: its saved launch id, or else its highest - * launch record, which `launches` reports for following. + * Collects each instance's newest `tail` lines and last position. With `fromLatestLaunch`, it + * keeps only the records of each instance's highest launch record, which `launches` reports for + * following. */ -export const makeHistoryCollector = ( - tail: number, - startLaunches: ReadonlyMap | undefined, -) => { +export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => { const instances = new Map(); const positions = new Map(); - const launches = new Map(startLaunches); + const launches = new Map(); const push = (record: StackLogRecord) => { const position = record.position; const last = positions.get(record.instanceId); @@ -126,24 +122,15 @@ export const makeHistoryCollector = ( positions.set(record.instanceId, position); let state = instances.get(record.instanceId); if (state === undefined) { - const saved = startLaunches?.get(record.instanceId); - state = { - records: [], - head: 0, - lines: 0, - total: 0, - launch: saved === undefined ? undefined : { id: saved, timestamp: undefined }, - }; + state = { records: [], head: 0, lines: 0, total: 0, launch: undefined }; instances.set(record.instanceId, state); } - if (startLaunches !== undefined) { + if (fromLatestLaunch) { const latest = state.launch; if ( record.kind === "launch" && record.launchId !== undefined && - (latest === undefined || - record.launchId > latest.id || - (record.launchId === latest.id && latest.timestamp === undefined)) + (latest === undefined || record.launchId > latest.id) ) { const launch = { id: record.launchId, timestamp: record.timestamp }; state.records = state.records.slice(state.head).filter((kept) => fromLaunch(kept, launch)); diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts index 2e2bf2a006..559f4180c9 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts @@ -48,7 +48,7 @@ describe("makeHistoryCollector", () => { records.filter(({ kind }) => kind !== "launch").map(({ text }) => text); it("keeps the newest lines across instances, counts every line, and tracks positions", () => { - const collector = makeHistoryCollector(2, undefined); + const collector = makeHistoryCollector(2, false); for (let offset = 0; offset < 1_000; offset += 2) { collector.push(at("a", offset, "stdout", `a ${offset}`)); collector.push(at("b", offset + 1, "stdout", `b ${offset + 1}`)); @@ -62,7 +62,7 @@ describe("makeHistoryCollector", () => { }); it("keeps the newest line by time when an older partial line is flushed after it", () => { - const collector = makeHistoryCollector(1, undefined); + const collector = makeHistoryCollector(1, false); collector.push({ ...at("a", 10, "stdout", "newer"), position: { generation: 1, byteOffset: 0 }, @@ -80,7 +80,7 @@ describe("makeHistoryCollector", () => { }); it("starts each instance at its highest launch and keeps instances without one", () => { - const collector = makeHistoryCollector(10, new Map()); + const collector = makeHistoryCollector(10, true); collector.push(at("a", 0, "launch", "", 1)); collector.push(at("a", 1, "stdout", "a old", 1)); collector.push(at("a", 5, "launch", "", 2)); @@ -95,7 +95,7 @@ describe("makeHistoryCollector", () => { }); it("keeps the highest launch when an older launch's first output arrives after it", () => { - const collector = makeHistoryCollector(10, new Map()); + const collector = makeHistoryCollector(10, true); collector.push(at("a", 10, "launch", "", 2)); collector.push(at("a", 11, "stdout", "second launch", 2)); collector.push(at("a", 20, "launch", "", 1)); @@ -109,7 +109,7 @@ describe("makeHistoryCollector", () => { }); it("starts at the launch of the latest owner run, whose ids continue the earlier run's", () => { - const collector = makeHistoryCollector(10, new Map()); + const collector = makeHistoryCollector(10, true); collector.push({ ...at("a", 0, "launch", "", 3), position: { generation: 1, byteOffset: 0 } }); collector.push({ ...at("a", 1, "stdout", "earlier run", 3), diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts index 1b836ec14e..c6c65ab47e 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts @@ -1,6 +1,7 @@ import { Clock, Effect, Option, Path, Stream } from "effect"; import { gatewayLog, + sharesApiEndpoint, streamStackLogs, type SavedStack, type StackLogRecord, @@ -43,24 +44,24 @@ type SavedInstance = SavedStack["instances"][number]; interface Selected { readonly id: string; readonly service: string; - readonly launchId: number | undefined; } const select = ( definition: SavedStack, requested: ReadonlyArray, ): Effect.Effect, StackCommandLogsError> => { - const subject = ({ id, creation, launchId }: SavedInstance): Selected => ({ + const subject = ({ id, creation }: SavedInstance): Selected => ({ id, service: creation.service, - launchId, }); - // The owner records the shared API listener's requests once the stack has claimed its port. - const gateway: ReadonlyArray = definition.ports.some(({ key }) => key === "api") - ? [{ id: gatewayLog.instanceId, service: gatewayLog.service, launchId: undefined }] + const members = new Set(definition.composition.members.map(({ id }) => id)); + // The owner records the shared API listener's requests when a member is served on it. + const gateway: ReadonlyArray = definition.instances.some( + ({ id, creation }) => members.has(id) && sharesApiEndpoint(creation), + ) + ? [{ id: gatewayLog.instanceId, service: gatewayLog.service }] : []; if (requested.length === 0) { - const members = new Set(definition.composition.members.map(({ id }) => id)); const selected = [ ...definition.instances.filter(({ id }) => members.has(id)).map(subject), ...gateway, @@ -151,16 +152,7 @@ export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), }; const sinceTime = since?.kind === "time" ? { since: since.iso } : {}; - const collector = makeHistoryCollector( - flags.tail, - since?.kind === "start" - ? new Map( - selected.flatMap(({ id, launchId }) => - launchId === undefined ? [] : [[id, launchId] as const], - ), - ) - : undefined, - ); + const collector = makeHistoryCollector(flags.tail, since?.kind === "start"); if (flags.tail > 0) yield* streamStackLogs({ stateRoot: locations.stateRoot, diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index b6e4ec5c24..420c3ddfd0 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -39,6 +39,10 @@ const mail = (id: string): SavedInstance => ({ id, creation: { service: "mail", config: {}, endpoints: {} }, }); +const rest = (id: string): SavedInstance => ({ + id, + creation: { service: "rest", config: {}, endpoints: { http: { port: "auto" } } }, +}); const database = (id: string): SavedInstance => ({ id, creation: { @@ -96,7 +100,6 @@ const fixture = Effect.fn("StackLogsTest.fixture")(function* (options: { readonly stateRoot: string; readonly stackId: string; }) => ReadonlyArray>; - readonly ports?: SavedStack["ports"]; readonly gatewayLogs?: (options?: ReadLogsOptions) => Stream.Stream; }) { const fs = yield* FileSystem.FileSystem; @@ -115,7 +118,6 @@ const fixture = Effect.fn("StackLogsTest.fixture")(function* (options: { const definition: SavedStack = { ...found.value.definition, instances: options.instances, - ports: options.ports ?? [], composition: { members: options.members.map((id) => ({ id, activation: "eager" as const })), dependencies: [], @@ -279,18 +281,15 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); - it.live("starts --since start at each instance's saved launch, even before it logged", () => + it.live("starts --since start at each instance's highest launch record", () => Effect.gen(function* () { const f = yield* fixture({ - instances: [ - { ...mail("mail-a"), launchId: 3 }, - { ...mail("mail-b"), launchId: 2 }, - ], + instances: [mail("mail-a"), mail("mail-b")], members: ["mail-a", "mail-b"], }); yield* f.writeSegment("mail", "mail-a", [ launch(t0, 2), - line(t0 + 1, "stdout", "a previous launch", 2), + line(t0 + 1, "stdout", "a only launch", 2), ]); yield* f.writeSegment("mail", "mail-b", [ launch(t0 + 2, 1), @@ -304,6 +303,8 @@ describe("stack logs", () => { expect( output.events.map(({ instance_id, kind, line: text }) => [instance_id, kind ?? text]), ).toEqual([ + ["mail-a", "launch"], + ["mail-a", "a only launch"], ["mail-b", "launch"], ["mail-b", "b current launch"], ]); @@ -449,16 +450,14 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); - const apiPort = [{ key: "api", host: "127.0.0.1", port: 54321 }]; const request = '127.0.0.1 - - [29/Sep/2026:10:00:00 +0000] "GET /rest/v1/ HTTP/1.1" 200 2 "-" "curl/8.7.1" 3ms'; it.live("includes the gateway's requests by default and selects them with --service", () => Effect.gen(function* () { const f = yield* fixture({ - instances: [mail("mail-a")], - members: ["mail-a"], - ports: apiPort, + instances: [mail("mail-a"), rest("rest-a")], + members: ["mail-a", "rest-a"], }); yield* f.writeSegment("mail", "mail-a", [launch(t0), line(t0 + 1, "stdout", "mail up")]); yield* f.writeSegment("gateway", "gateway", [launch(t0), line(t0 + 2, "stdout", request)]); @@ -501,9 +500,8 @@ describe("stack logs", () => { it.live("follows the gateway's new requests through the owner", () => Effect.gen(function* () { const f = yield* fixture({ - instances: [], - members: [], - ports: apiPort, + instances: [rest("rest-a")], + members: ["rest-a"], running: true, gatewayLogs: () => Stream.make({ @@ -527,9 +525,8 @@ describe("stack logs", () => { it.live("keeps only the current owner run's gateway requests with --since start", () => Effect.gen(function* () { const f = yield* fixture({ - instances: [{ ...mail("mail-a"), launchId: 2 }], - members: ["mail-a"], - ports: apiPort, + instances: [mail("mail-a"), rest("rest-a")], + members: ["mail-a", "rest-a"], }); yield* f.writeSegment("mail", "mail-a", [ launch(t0, 1), @@ -560,19 +557,20 @@ describe("stack logs", () => { position: { generation: 1, byteOffset: offset }, }); const f = yield* fixture({ - instances: [{ ...mail("mail-a"), launchId: 3 }], + instances: [mail("mail-a")], members: ["mail-a"], running: true, handles: () => [ followingMail("mail-a", () => Stream.make( - record(0, 3, "current launch"), - record(1, 2, "delayed earlier launch"), - record(2, 4, "newer launch"), + record(100, 3, "current launch"), + record(101, 2, "delayed earlier launch"), + record(102, 4, "newer launch"), ), ), ], }); + yield* f.writeSegment("mail", "mail-a", [launch(t0 - 1, 3)]); const { exit, output } = yield* f.run( { follow: true, since: Option.some("start") }, diff --git a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md index fa73bfd61c..d6021f9344 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md @@ -22,8 +22,6 @@ modified. A preparation failure still runs cleanup for every acquired temporary - A managed stack descriptor/state record when the target does not exist. Existing records are updated while temporary instances and their instance directories are created and removed. - Starting the owner of a target saved with a Vector instance removes that instance from the - record, as described for `stack start`. - Runtime artifacts through the selected native or container runtime's package-owned cache. - Telemetry state after success or failure once the handler starts. diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index c1c50b9cb0..24db98af06 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -23,6 +23,7 @@ import { mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; import { mockOutput, mockRuntimeInfo } from "../../../../../tests/helpers/mocks.ts"; +import { CommandTelemetryAttributes } from "../../../../telemetry/command-telemetry-attributes.ts"; import { StackApi, StackTargetResolver } from "../stack.shared.ts"; import { stackPrepare } from "./prepare.handler.ts"; import type { StackPrepareFlags } from "./prepare.command.ts"; @@ -63,6 +64,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { options.engines ?? { docker: "missing", podman: "missing" }, ); const createdRuntimes: Array = []; + const recordedRuntimes: Array = []; let openCount = 0; let prepareCount = 0; let startCount = 0; @@ -103,13 +105,9 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { lifecycle: "stopped" as const, health: undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, wakeEnabled: false, - registered: true, }), followStatus: Stream.empty, readLogs: () => Stream.empty, @@ -157,7 +155,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { restart: Effect.succeed([]), }, stop: Effect.void, - destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + destroy: Effect.void, gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, }; @@ -170,6 +168,12 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { mockCommandSettings({ workdir: root, supabaseHome: root }), output.layer, telemetry.layer, + Layer.succeed(CommandTelemetryAttributes, { + record: (values) => + Effect.sync(() => { + if (values.stack_runtime !== undefined) recordedRuntimes.push(values.stack_runtime); + }), + }), Layer.succeed(StackTargetResolver, { resolve: (input) => Effect.succeed({ @@ -203,6 +207,9 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { get createdRuntimes() { return createdRuntimes; }, + get recordedRuntimes() { + return recordedRuntimes; + }, get openCount() { return openCount; }, @@ -214,7 +221,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { .map(({ message }) => message); }, get probes() { - return engines.spawned.map(({ command }) => command); + return [...new Set(engines.spawned.map(({ command }) => command))]; }, get prepareCount() { return prepareCount; @@ -445,7 +452,7 @@ describe("stack prepare automatic runtime selection", () => { ), ); - it.live("honors an explicit runtime without probing any engine", () => + it.live("honors an explicit runtime without probing any engine and records it on telemetry", () => makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { const fixture = makeFixture(root, { engines: { docker: "running", podman: "running" } }); @@ -454,6 +461,7 @@ describe("stack prepare automatic runtime selection", () => { Effect.tap(() => Effect.sync(() => { expect(fixture.createdRuntimes).toEqual(["podman"]); + expect(fixture.recordedRuntimes).toEqual(["podman"]); expect(fixture.probes).toEqual([]); expect(fixture.runtimeNotices).toEqual([]); }), diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts index ab24e7622a..02c535afb7 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts @@ -1,8 +1,8 @@ -import { tmpdir } from "node:os"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { StackError } from "@supabase/stack/effect"; -import { Effect, FileSystem, Layer, Option, Path } from "effect"; +import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Schema, Stream } from "effect"; +import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, @@ -11,8 +11,38 @@ import { import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackRestart } from "./restart.handler.ts"; import { destroyTestStack } from "../../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../../tests/helpers/stack-artifacts.ts"; -const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +const live = Layer.provideMerge( + stackApiLayer, + Layer.merge(BunServices.layer, FetchHttpClient.layer), +); +const MailpitInfo = Schema.Struct({ RuntimeStats: Schema.Struct({ SMTPAccepted: Schema.Finite }) }); + +// Mailpit counts accepted messages per process, so the count restarts from zero after a relaunch. +const mailpit = Effect.fn("StackRestartTest.mailpit")(function* (port: number) { + const http = yield* HttpClient.HttpClient; + const origin = `http://127.0.0.1:${port}`; + return { + send: http + .execute( + HttpClientRequest.post(`${origin}/api/v1/send`).pipe( + HttpClientRequest.bodyJsonUnsafe({ + From: { Email: "from@example.com" }, + To: [{ Email: "to@example.com" }], + Subject: "restart", + Text: "restart", + }), + ), + ) + .pipe(Effect.flatMap((response) => response.text)), + accepted: http.get(`${origin}/api/v1/info`).pipe( + Effect.flatMap((response) => response.json), + Effect.flatMap(Schema.decodeUnknownEffect(MailpitInfo)), + Effect.map((info) => info.RuntimeStats.SMTPAccepted), + ), + }; +}); const fixture = Effect.fn("StackRestartTest.fixture")(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -20,7 +50,7 @@ const fixture = Effect.fn("StackRestartTest.fixture")(function* () { const api = yield* StackApi; const locations = { stateRoot: path.join(root, "stacks"), - cacheRoot: path.join(tmpdir(), "supabase-stack-artifacts"), + cacheRoot: stackArtifactCacheRoot, }; const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "native" }); const output = mockOutput(); @@ -71,17 +101,38 @@ describe("stack restart", () => { yield* standalone.start; yield* standalone.ready; const before = yield* member.status; - const standaloneBefore = yield* standalone.status; + const standaloneHttpPort = (yield* standalone.status).endpoints.find( + ({ name }) => name === "http", + )?.port; + if (standaloneHttpPort === undefined) return yield* Effect.die("Mail port missing"); + const standaloneMail = yield* mailpit(standaloneHttpPort); + yield* standaloneMail.send; + expect(yield* standaloneMail.accepted).toBe(1); + + // Both follows are subscribed once their first status arrives, before the restart. + const memberFollowing = yield* Deferred.make(); + const relaunched = yield* member.followStatus.pipe( + Stream.tap(() => Deferred.succeed(memberFollowing, undefined)), + Stream.dropWhile((value) => value.lifecycle === "running"), + Stream.filter( + (value) => value.lifecycle === "running" && value.health === "healthy", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(memberFollowing); + const result = yield* stackRestart(f.flags).pipe(Effect.provide(f.layer)); + yield* Fiber.join(relaunched); const after = yield* member.status; const standaloneAfter = yield* standalone.status; expect(result.map(({ id }) => id)).toEqual([member.id]); expect(after.lifecycle).toBe("running"); expect(after.health).toBe("healthy"); - expect(after.launchId).not.toBe(before.launchId); expect(after.endpoints).toEqual(before.endpoints); expect(standaloneAfter.lifecycle).toBe("running"); - expect(standaloneAfter.launchId).toBe(standaloneBefore.launchId); + expect(yield* standaloneMail.accepted).toBe(1); expect(f.output.stdoutText).toContain("using its saved configuration"); expect(f.telemetry.flushed).toBe(true); yield* stack.stop; diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts index dcdb889a29..b307ec32c9 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts @@ -186,14 +186,19 @@ describe("stack target resolver", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("carries an explicit Podman runtime onto a new stack target", () => + it.live("carries an explicit Podman runtime onto a new stack target and records it", () => Effect.gen(function* () { const { root, resolve } = yield* workspace; + const recorded = yield* Ref.make({}); - const target = yield* resolve({ projectRoot: root, runtime: "podman" }); + const target = yield* resolve({ projectRoot: root, runtime: "podman" }).pipe( + Effect.provideService(CommandTelemetryAttributes, { + record: (values) => Ref.update(recorded, (current) => ({ ...current, ...values })), + }), + ); - expect(target).toMatchObject({ runtime: "podman", hostRunning: false }); - expect(target.id).toBeUndefined(); + expect(target).toEqual({ projectRoot: root, runtime: "podman", hostRunning: false }); + expect((yield* Ref.get(recorded)).stack_runtime).toBe("podman"); }).pipe(Effect.provide(BunServices.layer)), ); @@ -228,6 +233,38 @@ describe("stack target resolver", () => { }).pipe(Effect.provide(BunServices.layer)), ); + it.live( + "tells the user to remove the directory of a saved stack with an unknown service kind", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { root, home, register, resolve } = yield* workspace; + const id = yield* register({ projectRoot: root }); + const statePath = path.join(home, "stacks", id, "state.json"); + const saved = yield* fs.readFileString(statePath); + yield* fs.writeFileString( + statePath, + saved.replace('"instances":[]', '"instances":[{"id":"logs","service":"vector"}]'), + ); + + const byProject = yield* resolve({ projectRoot: root, runtime: "auto" }).pipe(Effect.flip); + const byPrefix = yield* resolve({ + projectRoot: root, + id: id.slice(0, 8), + runtime: "auto", + }).pipe(Effect.flip); + + for (const failure of [byProject, byPrefix]) { + expect(failure.reason).toBe("invalid-config"); + expect(failure.message).toContain(`Stack ${id} could not be read`); + expect(failure.message).toContain( + `Remove its directory ${path.join(home, "stacks", id)}`, + ); + } + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("surfaces an unreadable saved stack that an id prefix selects", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.ts b/apps/cli/src/commands/experimental/stack/stack.shared.ts index 6adc4fd71c..d924692707 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.ts @@ -11,17 +11,13 @@ import { type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../../../shared/telemetry/error-actionability.ts"; -import { - skippedRuntimeCleanupWarning, - StackApi, - stackApiLayer, -} from "../../../command-internal/stack-api.ts"; +import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; import { recordStackRuntimeTelemetry, type StackRuntime, } from "../../../command-internal/stack-runtime.ts"; -export { skippedRuntimeCleanupWarning, StackApi, stackApiLayer }; +export { StackApi, stackApiLayer }; /** The target selected by the CLI adapter for one stack command. */ export interface StackTarget { @@ -44,6 +40,24 @@ export class StackTargetError extends Data.TaggedError("ExperimentalStackTargetE } } +/** Rewraps a `StackTargetError` into a command's own error type, preserving reason, message, and suggestion. */ +export const mapTargetError = + ( + make: (props: { + readonly reason: StackTargetError["reason"]; + readonly message: string; + readonly suggestion?: string; + readonly cause: StackTargetError; + }) => E, + ) => + (error: StackTargetError): E => + make({ + reason: error.reason, + message: error.message, + ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), + cause: error, + }); + interface StackTargetResolverShape { readonly resolve: (input: { readonly projectRoot: string; @@ -144,13 +158,7 @@ export const stackTargetResolverLayer = Layer.effect( }); const [invalid] = unreadable; if (invalid !== undefined) - return yield* new StackTargetError({ - message: `Stack ${invalid.id} could not be read: ${invalid.error.message}`, - reason: "invalid-config", - suggestion: - "Inspect the stack registry under $SUPABASE_HOME/stacks or ~/.supabase/stacks.", - cause: invalid.error, - }); + yield* stackApi.find({ stateRoot, id: invalid.id }).pipe(Effect.mapError(stateError)); return matches[0]; }); const resolve = Effect.fn("StackTargetResolver.resolve")(function* (input: { diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index ee77a0126d..24aedc3736 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -35,8 +35,9 @@ require template serving and shared external JWKS verification respectively. Secrets needed by enabled services are passed to the runtime. State and service data live under `$SUPABASE_HOME/stacks//` (`~/.supabase/stacks//` by default); native artifacts -use `$SUPABASE_HOME/cache/stack`. Storage files use the caller-owned project directory -`supabase/.temp/stack-uploads//`. Functions preparation may build the project's source. +use `$SUPABASE_HOME/cache/stack`, keyed by content digest and retired automatically once unused for +30 days. Storage files use the caller-owned project directory `supabase/.temp/stack-uploads//`. +Functions preparation may build the project's source. The owner persists each service's output under `$SUPABASE_HOME/stacks//logs/`, keeping at most about 10 MiB (plus the segment being written) per service instance. Destroying an instance or the stack deletes those logs; stopping the stack and resetting database data keep them. @@ -55,27 +56,40 @@ refresh, ID, and provider tokens, and the `X-Amz-Signature`, `X-Amz-Credential`, as `redacted`, whatever their parameter is used for, as are values under any name that hold a secret key (`sb_secret_…`) or a JWT, values that themselves carry such a pair (a `redirect_to` URL with a token), and URL userinfo. -When an owner starts a stack saved with a Vector instance, it removes that instance, its composition -members, dependencies and port claims from `state.json`, and its stack-owned Vector config files -under `data//runtime/vector/`; its containers go with the stack's container sweep. A -migration that fails is logged as a warning and retried by the next owner start. - -For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then -Podman when `podman info` reaches its engine, then native on Linux x64/arm64 and macOS arm64. Each -probe is bounded by 10 seconds. Without a reachable engine on other platforms, the command fails and -asks the user to start Docker or Podman. When auto selection skips Docker, an info line names the -saved Podman or native runtime and how to switch to Docker. An existing stack keeps its saved -runtime and runs no probe. Explicit `--runtime docker`, `podman`, or `native` has no fallback. -When an explicit or saved Docker runtime is unreachable, the reported failure suggests starting -Docker, and `--runtime native` for a new stack on platforms that support native. Explicit + +Public ports are reserved in one SQLite registry per OS user at `/.supabase/ports.sqlite`, +which `$SUPABASE_HOME` does not affect and which has no override. A stopped stack's automatic +ports stay reserved across every other stack's starts, including in a different state root, and +starting it again reuses the same ports. A configured port is reserved only while its listener is +open, so a stopped stack never blocks another project's configured port; a configured port takes +over a reservation whose stack has no running owner (after a crash or reboot), and that stack gets a +new automatic port on its next start. A saved or configured port held by another running stack, or +occupied by a process outside the registry, fails start naming the port and, for a stack, its +project, instead of picking a different port; only automatic allocation tries another candidate. On macOS a process can still win +a narrow race against the pre-bind probe before a brand-new listener exists. +Native backend ports (not publicly exposed) are reserved from 10000–19999, disjoint from the public +auto range and below the OS ephemeral range on every supported platform; this range is not +configurable, and a public port pinned inside it is rejected. A host whose ephemeral range has been widened to overlap it reintroduces the +ephemeral-port race this reservation exists to avoid. + +For a new stack, `--runtime auto` selects Docker when its engine answers, then Podman when its +engine answers, then native on Linux x64/arm64 and macOS arm64. Each probe resolves the engine +target the owner later pins (so a local or remote Podman is judged by the same endpoint) and is +bounded by 10 seconds. Without a reachable engine on other platforms, the command fails and asks +the user to start Docker or Podman. When auto selection skips Docker, an info line names the saved +Podman or native runtime and how to switch to Docker. An existing stack keeps its saved runtime and +runs no probe. Explicit `--runtime docker`, `podman`, or `native` has no fallback. +When an explicit or saved container runtime is unreachable, the reported failure suggests starting +that engine, and `--runtime native` for a new stack on platforms that support native. Explicit `--runtime native` on a platform with no native artifacts fails before creating a stack. Native startup refuses root because PostgreSQL `initdb` cannot run as root, unless a Claude Code or Modal Sandbox is detected or `SUPABASE_NATIVE_POSTGRES_USER` names a non-root user. PostgreSQL then runs as that user: the CLI chowns the instance data, root key, socket directory, and the cached bundle's -`pgsodium_getkey.sh` to it, and adds traverse-only `o+x` to their parent directories, including -root's home directory. Later commands that restrict the artifact cache and stack state roots to -their owner keep that grant. +`pgsodium_getkey.sh` to it, and adds traverse-only `o+x` to the parent directories outside the +cached artifact (the artifact cache and stack state roots, including root's home directory), which +stay owner-restricted between launches otherwise. A directory inside the cached artifact itself is +never chmodded; it keeps the archive's own mode, and startup fails if one is not already traversable. Native PostgreSQL receives `SSL_CERT_FILE` and `SSL_CERT_DIR` when they are non-empty. When `SSL_CERT_FILE` is unset or empty, PostgreSQL receives the first of @@ -90,9 +104,9 @@ stack owner, and a running owner keeps them through `stack restart` and repeated export them before starting and stop and start the stack after changing them. Database is eager by default. Other services are lazy; traffic wakes them through their listeners. -Lazy services with idle policies stop after 60 seconds without traffic, Studio after 5 minutes. A +Lazy services stop after 60 seconds without traffic, Studio after 5 minutes. A service that a running service depends on, such as pg-meta for Studio, stays up until that -dependent stops. Functions has no automatic idle stop. `--eager` makes all selected services eager. +dependent stops. `--eager` makes all selected services eager. Changes to activation policy take effect after stopping and starting the stack, including when a later invocation omits an earlier `--eager` flag. `--preparation` selects on-demand or background artifact preparation. diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index f12ddc6c43..0bad4b7785 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -66,13 +66,9 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac lifecycle: "running", health: "healthy", error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 1, wakeEnabled: false, - registered: true, }; const databaseInstance = { id: "database-id", @@ -125,7 +121,6 @@ const definitionFor = (id: string, projectRoot: string, creation: ServiceCreatio runtime: "native", instances: [{ id: "database-id", creation }], composition: { members: [{ id: "database-id", activation: "eager" }], dependencies: [] }, - ports: [], }); interface TargetSpec { diff --git a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts index d73c028be2..9daad40ec4 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts @@ -92,7 +92,6 @@ const FollowEventSchema = Schema.Struct({ const VariablesSchema = Schema.Record(Schema.String, Schema.String); const StartResultSchema = Schema.Struct({ id: Schema.String }); -const StackListSchema = Schema.Struct({ stacks: Schema.Array(Schema.Unknown) }); const minimalConfig = `project_id = "compiled-stack-start-e2e" @@ -186,7 +185,8 @@ describe("stack start (compiled e2e)", () => { const cleanup = Effect.gen(function* () { let cleanupComplete = stackDestroyed; if (!cleanupComplete && home !== undefined) { - const candidates = yield* readDirectory(join(home.dir, "stacks")).pipe( + const stacksDir = join(home.dir, "stacks"); + const candidates = yield* readDirectory(stacksDir).pipe( Effect.catchIf( (cause) => Predicate.isTagged(cause, "PlatformError") && @@ -194,7 +194,10 @@ describe("stack start (compiled e2e)", () => { () => Effect.succeed([]), ), ); - const discovered = candidates.filter((entry) => /^[0-9a-f]{64}$/u.test(entry)); + const discovered = yield* Effect.filter( + candidates.filter((entry) => /^[0-9a-f]{64}$/u.test(entry)), + (entry) => access(join(stacksDir, entry, "state.json")).pipe(Effect.isSuccess), + ); const ownedId = stackId ?? (discovered.length === 1 ? discovered[0] : undefined); if (ownedId !== undefined) { yield* destroyStack(home.dir, ownedId); @@ -577,54 +580,10 @@ describe("stack start (compiled e2e)", () => { yield* destroyStack(homeDir.dir, idText); stackDestroyed = true; - const destroyed = yield* Effect.exit(access(join(homeDir.dir, "stacks", idText))); - expect(Exit.isFailure(destroyed)).toBe(true); - }), - ), - ); - - test( - "removes a stack registration that fails to start because Docker is unreachable", - { timeout: CLEANUP_TIMEOUT_MS }, - () => - runNode( - Effect.gen(function* () { - home = makeTempHome(); - projectDir = yield* makeTempDirectory("/tmp/supabase-stack-start-docker-down-e2e-"); - yield* makeDirectory(join(projectDir, "supabase"), { recursive: true }); - yield* writeText(join(projectDir, "supabase", "config.toml"), minimalConfig); - // A `docker` first on PATH that reports an unreachable daemon, as the real CLI does. - const binDir = join(projectDir, "bin"); - yield* makeDirectory(binDir); - yield* writeText( - join(binDir, "docker"), - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///shim/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", - ); - yield* withFs((fs) => fs.chmod(join(binDir, "docker"), 0o755)); - - const result = yield* runSupabaseEffect(["stack", "start", "--runtime", "docker"], { - cwd: projectDir, - home: home.dir, - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the CLI subprocess resolves `docker` from PATH. - env: { PATH: `${binDir}:${process.env.PATH ?? ""}` }, - exitTimeoutMs: CLEANUP_TIMEOUT_MS, - }); - expect(result.exitCode, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`).not.toBe( - 0, + const destroyed = yield* Effect.exit( + access(join(homeDir.dir, "stacks", idText, "state.json")), ); - expect(result.stderr).toContain("Cannot connect to the Docker daemon"); - expect(result.stderr).not.toContain("Failed to stop stack host"); - - const list = yield* runSupabaseEffect(["stack", "list", "--output-format", "json"], { - home: home.dir, - env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, - exitTimeoutMs: CLEANUP_TIMEOUT_MS, - }); - expect(list.exitCode, `stdout:\n${list.stdout}\nstderr:\n${list.stderr}`).toBe(0); - const listed = yield* Schema.decodeEffect(Schema.fromJsonString(StackListSchema))( - list.stdout.trim(), - ); - expect(listed.stacks).toEqual([]); + expect(Exit.isFailure(destroyed)).toBe(true); }), ), ); diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 83df18a4f5..45a3dfd5e4 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -13,7 +13,9 @@ import { withProjectFunctionsEnv } from "../../../../command-internal/stack-func import { statusEnvPointer } from "./start-summary.format.ts"; import { automaticRuntimeNotice, + containerEngineName, selectStackRuntime, + type StackRuntime, } from "../../../../command-internal/stack-runtime.ts"; import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; import { Effect, FileSystem, Fiber, Option, Path, Redacted, Ref } from "effect"; @@ -58,9 +60,9 @@ import { envOverride } from "../../../../command-internal/local-config-values.ts import { StackApi, stackCapabilityForService, - StackTargetError, StackTargetResolver, failedOutcomesDetail, + mapTargetError, rejectStackOutput, validateStackTarget, } from "../stack.shared.ts"; @@ -92,55 +94,86 @@ const validateExclusions = (exclusions: ReadonlyArray) => { ); }; -const mapTargetError = (error: StackTargetError) => - new StackCommandStartError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null; + +/** + * Names the config setting behind a contested public port: the one configured endpoint pinned to + * that port number. An automatic port, or a port shared by several settings, gets no suggestion. + */ +const portConflictSuggestion = ( + conflict: StackError["conflict"], + requested: ReadonlyArray<{ readonly service: string; readonly endpoints?: unknown }>, +): string | undefined => { + if (conflict === undefined) return undefined; + const settings = new Map(); + for (const { service, endpoints } of requested) { + if (!isRecord(endpoints)) continue; + for (const [name, intent] of Object.entries(endpoints)) { + const setting = stackEndpointSetting(service, name); + if (setting !== undefined && isRecord(intent) && intent.port === conflict.port) + settings.set(setting.envVar, setting); + } + } + const [setting, ...rest] = settings.values(); + return setting === undefined || rest.length > 0 + ? undefined + : `Set \`${setting.configPath}\` in supabase/config.toml (or ${setting.envVar}) to a free port.`; +}; const stackError = ( - cause: { readonly message: string } & Partial>, - members: ReadonlyArray<{ readonly id: string; readonly service: string }> = [], + cause: { readonly message: string } & Partial>, + members: ReadonlyArray<{ readonly id?: string; readonly service: string }> = [], + requested: ReadonlyArray<{ readonly service: string; readonly endpoints?: unknown }> = [], ) => { const detail = failedOutcomesDetail(cause, (id) => { const service = members.find((member) => member.id === id)?.service; return service === undefined ? id : `${service} (${id})`; }); + const suggestion = portConflictSuggestion(cause.conflict, requested); return new StackCommandStartError({ reason: "unknown", message: cause.message, ...(detail === undefined ? {} : { detail }), + ...(suggestion === undefined ? {} : { suggestion }), cause, }); }; // A saved stack keeps its runtime, so only a new stack can switch to native. -const dockerUnavailableSuggestion = ( +const engineUnavailableSuggestion = ( + engine: Exclude, runtimeInfo: { readonly platform: string; readonly arch: string }, creating: boolean, -) => - creating && defaultRuntime({ os: runtimeInfo.platform, arch: runtimeInfo.arch }) === "native" - ? "Docker CLI or daemon isn't reachable. Install or start Docker, or run with --runtime native." - : "Docker CLI or daemon isn't reachable. Install or start Docker."; +) => { + const name = containerEngineName(engine); + const base = `${name} CLI or daemon isn't reachable. Install or start ${name}`; + return creating && + defaultRuntime({ os: runtimeInfo.platform, arch: runtimeInfo.arch }) === "native" + ? `${base}, or run with --runtime native.` + : `${base}.`; +}; const stackAcquireError = ( cause: StackError, runtimeContext: { - readonly selectedRuntime: "native" | "docker" | "podman"; + readonly selectedRuntime: StackRuntime; readonly runtime: { readonly platform: string; readonly arch: string }; readonly creating: boolean; }, ) => { const base = stackError(cause); - if (cause.reason !== "runtime-unavailable" || runtimeContext.selectedRuntime !== "docker") - return base; + const { selectedRuntime } = runtimeContext; + if (cause.reason !== "runtime-unavailable" || selectedRuntime === "native") return base; return new StackCommandStartError({ reason: "runtime", message: base.message, ...(base.detail === undefined ? {} : { detail: base.detail }), - suggestion: dockerUnavailableSuggestion(runtimeContext.runtime, runtimeContext.creating), + suggestion: engineUnavailableSuggestion( + selectedRuntime, + runtimeContext.runtime, + runtimeContext.creating, + ), cause: base.cause, }); }; @@ -171,8 +204,14 @@ const sameKinds = ( return leftKinds.size === rightKinds.size && [...leftKinds].every((kind) => rightKinds.has(kind)); }; -const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null; +/** + * Whether this set of requested services would run Studio without the REST API it depends on. + * The single source of truth for that dependency: the real `--exclude` guard validates against + * it to reject a request that would otherwise leave Studio stranded. + */ +const studioNeedsRest = (requestedServices: ReadonlyArray<{ readonly service: string }>): boolean => + requestedServices.some(({ service }) => service === "studio") && + !requestedServices.some(({ service }) => service === "rest"); const endpointPortLabel = (endpoints: unknown, name: string): string => { const intent = isRecord(endpoints) ? endpoints[name] : undefined; @@ -418,12 +457,14 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags const outputFlag = yield* Effect.serviceOption(OutputFlag); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + yield* rejectStackOutput(outputFlag).pipe( + Effect.mapError(mapTargetError((props) => new StackCommandStartError(props))), + ); const exclusions = yield* validateExclusions(flags.exclude); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); + }).pipe(Effect.mapError(mapTargetError((props) => new StackCommandStartError(props)))); const target = yield* resolver .resolve({ projectRoot: settings.workdir, @@ -431,7 +472,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), runtime: flags.runtime, }) - .pipe(Effect.mapError(mapTargetError)); + .pipe(Effect.mapError(mapTargetError((props) => new StackCommandStartError(props)))); const runtime = yield* RuntimeInfo; const selectedRuntime = yield* selectStackRuntime(target.runtime).pipe( Effect.mapError( @@ -640,10 +681,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags new StackCommandStartError({ reason: "invalid-config", message: cause.message, cause }), ), ); - if ( - requested.some(({ service }) => service === "studio") && - !requested.some(({ service }) => service === "rest") - ) + if (studioNeedsRest(requested)) return yield* new StackCommandStartError({ reason: "flags", message: "Studio cannot be started without the REST API capability", @@ -774,7 +812,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }) .pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(stackError), + Effect.mapError((error) => stackError(error, requested, requested)), ); if (initialComposition) { const existingIds = new Set(existingServices.map(({ id }) => id)); @@ -898,7 +936,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags } yield* stack.composition.start.pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError((error) => stackError(error, members)), + Effect.mapError((error) => stackError(error, members, requested)), ); yield* Effect.forEach(preparation, (fiber) => Fiber.join(fiber)); yield* Ref.set(startupComplete, true); diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index c20b5e0046..2ebd823401 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -29,12 +29,13 @@ import { type StackCredentials, type Stack, } from "@supabase/stack/effect"; -import { planSupabaseComposition } from "@supabase/stack/testing"; +import { planSupabaseComposition } from "@supabase/stack/internal/composition"; import { mockCommandSettings, mockTelemetryStateTracked, withEnvVar, } from "../../../../../tests/helpers/command-mocks.ts"; +import { containerEngineSpawner } from "../../../../../tests/helpers/child-process-spawner.ts"; import { mockOutput, mockProcessControl, @@ -42,7 +43,6 @@ import { mockTty, } from "../../../../../tests/helpers/mocks.ts"; import { unusedGateway } from "../../../../../tests/helpers/unused-stack.ts"; -import { containerEngineSpawner } from "../../../../../tests/helpers/child-process-spawner.ts"; import { DbConnection, type DbSession, @@ -119,11 +119,7 @@ const webhookSchemaDatabase = () => { const instance = ( creation: ServiceCreation, id: string, - lifecycle: () => "stopped" | "starting" | "running", - wakeEnabled: () => boolean, - health: () => "starting" | "healthy" | "unhealthy" | undefined = () => - lifecycle() === "running" ? "healthy" : undefined, - ready: () => Effect.Effect = () => Effect.void, + view: () => MemberView, ): ServiceInstances[ServiceCreation["service"]] => { const status = (config: ServiceCreation) => ({ id, @@ -141,21 +137,17 @@ const instance = ( ] : [], config, - lifecycle: lifecycle(), - health: health(), + lifecycle: view().lifecycle, + health: view().health, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, - wakeEnabled: wakeEnabled(), - registered: true, + wakeEnabled: view().wakeEnabled, }); const base = { id, start: Effect.void, - ready: Effect.suspend(ready), + ready: Effect.suspend(() => view().ready), stop: Effect.void, restart: () => Effect.void, destroy: Effect.void, @@ -173,11 +165,13 @@ const instance = ( service: "database", restart: (input?: Parameters[0]) => Effect.sync(() => { - if (input?.config.version !== undefined) - current = { - ...current, - config: { ...current.config, version: input.config.version }, - }; + if (input === undefined) return; + current = { + ...current, + ...("version" in input ? { version: input.version } : {}), + ...("endpoints" in input ? { endpoints: input.endpoints } : {}), + config: { ...current.config, ...input.config }, + }; }), status: Effect.sync(() => status(current)), credentials: () => @@ -223,8 +217,18 @@ const instance = ( return { ...base, service: "studio" }; case "pgmeta": return { ...base, service: "pgmeta" }; - case "mail": - return { ...base, service: "mail" }; + case "mail": { + let current = creation; + return { + ...base, + service: "mail", + restart: (input?: Parameters[0]) => + Effect.sync(() => { + if (input !== undefined) current = { ...current, ...input }; + }), + status: Effect.sync(() => status(current)), + }; + } case "analytics": return { ...base, service: "analytics" }; case "pooler": @@ -246,12 +250,45 @@ const requireConcreteCreation = (creation: ServiceCreationInput): ServiceCreatio }; }; +interface MemberView { + readonly lifecycle: "stopped" | "starting" | "running"; + readonly wakeEnabled: boolean; + readonly health: "starting" | "healthy" | "unhealthy" | undefined; + readonly ready: Effect.Effect; +} + interface MemberStatus { readonly lifecycle?: "stopped" | "starting" | "running"; readonly wakeEnabled?: boolean; readonly health?: "starting" | "healthy" | "unhealthy"; } +const credentialsFor = ( + database: ServiceCreationInput | undefined, + options: Parameters[1], +): StackCredentials => { + const config = database?.service === "database" ? database.config : undefined; + return { + jwtSecret: + config?.jwtSecret === undefined ? DEFAULT_LOCAL_JWT_SECRET : Redacted.value(config.jwtSecret), + postgresRootKey: + config?.rootKey === undefined ? DEFAULT_POSTGRES_ROOT_KEY : Redacted.value(config.rootKey), + databasePassword: + config?.databasePassword === undefined + ? DEFAULT_LOCAL_DATABASE_PASSWORD + : Redacted.value(config.databasePassword), + publishableKey: options?.keys?.publishableKey ?? "sb_publishable_test", + secretKey: options?.keys?.secretKey ?? "sb_secret_test", + anonKey: options?.keys?.anonKey ?? "anon-token", + serviceRoleKey: options?.keys?.serviceRoleKey ?? "service-token", + jwks: '{"keys":[]}', + gotrueJwtKeys: options?.keys?.gotrueJwtKeys ?? "[]", + remoteJwks: options?.keys?.remoteJwks ?? "[]", + anonKeyIsOverride: options?.keys?.anonKeyIsOverride ?? false, + serviceRoleKeyIsOverride: options?.keys?.serviceRoleKeyIsOverride ?? false, + }; +}; + const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { let members: Array = []; let stopped = 0; @@ -264,19 +301,17 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { let activations = new Map(); const memberStatuses = new Map(); const memberReadiness = new Map>(); - let savedCredentials: StackCredentials = { - jwtSecret: DEFAULT_LOCAL_JWT_SECRET, - postgresRootKey: DEFAULT_POSTGRES_ROOT_KEY, - databasePassword: DEFAULT_LOCAL_DATABASE_PASSWORD, - publishableKey: "sb_publishable_test", - secretKey: "sb_secret_test", - anonKey: "anon-token", - serviceRoleKey: "service-token", - jwks: '{"keys":[]}', - gotrueJwtKeys: "[]", - remoteJwks: "[]", - anonKeyIsOverride: false, - serviceRoleKeyIsOverride: false, + let savedCredentials = credentialsFor(undefined, undefined); + const memberView = (id: string): MemberView => { + const status = memberStatuses.get(id); + const memberLifecycle = status?.lifecycle ?? lifecycle; + return { + lifecycle: memberLifecycle, + wakeEnabled: + status?.wakeEnabled ?? (lifecycle === "running" && activations.get(id) === "lazy"), + health: status?.health ?? (memberLifecycle === "running" ? "healthy" : undefined), + ready: memberReadiness.get(id) ?? Effect.void, + }; }; const stack: Stack = { id: "a".repeat(64), @@ -301,31 +336,10 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { supabase: (creations: ReadonlyArray, options) => Effect.sync(() => { composed += 1; - const database = creations.find((creation) => creation.service === "database"); - const jwtSecret = - database?.service === "database" && database.config.jwtSecret !== undefined - ? Redacted.value(database.config.jwtSecret) - : DEFAULT_LOCAL_JWT_SECRET; - savedCredentials = { - jwtSecret, - postgresRootKey: - database?.service === "database" && database.config.rootKey !== undefined - ? Redacted.value(database.config.rootKey) - : DEFAULT_POSTGRES_ROOT_KEY, - databasePassword: - database?.service === "database" && database.config.databasePassword !== undefined - ? Redacted.value(database.config.databasePassword) - : DEFAULT_LOCAL_DATABASE_PASSWORD, - publishableKey: options?.keys?.publishableKey ?? "sb_publishable_test", - secretKey: options?.keys?.secretKey ?? "sb_secret_test", - anonKey: options?.keys?.anonKey ?? "anon-token", - serviceRoleKey: options?.keys?.serviceRoleKey ?? "service-token", - jwks: '{"keys":[]}', - gotrueJwtKeys: options?.keys?.gotrueJwtKeys ?? "[]", - remoteJwks: options?.keys?.remoteJwks ?? "[]", - anonKeyIsOverride: options?.keys?.anonKeyIsOverride ?? false, - serviceRoleKeyIsOverride: options?.keys?.serviceRoleKeyIsOverride ?? false, - }; + savedCredentials = credentialsFor( + creations.find((creation) => creation.service === "database"), + options, + ); const previousMembers = members; members = creations.map((creation) => { const previous = previousMembers.find(({ service }) => service === creation.service); @@ -333,20 +347,7 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { previous !== undefined && options?.reuseIds?.includes(previous.id) ? previous.id : `${creation.service}-member-${composed}`; - return instance( - requireConcreteCreation(creation), - id, - () => memberStatuses.get(id)?.lifecycle ?? lifecycle, - () => - memberStatuses.get(id)?.wakeEnabled ?? - (lifecycle === "running" && activations.get(id) === "lazy"), - () => { - const status = memberStatuses.get(id); - if (status?.health !== undefined) return status.health; - return (status?.lifecycle ?? lifecycle) === "running" ? "healthy" : undefined; - }, - () => memberReadiness.get(id) ?? Effect.void, - ); + return instance(requireConcreteCreation(creation), id, () => memberView(id)); }); activations = new Map( members.map(({ id, service }) => [ @@ -402,7 +403,6 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { }), destroy: Effect.sync(() => { hostDestroyed += 1; - return { runtimeCleanup: "complete" as const }; }), gateway: unusedGateway, commands: { run: () => Effect.die("command not used") }, @@ -644,6 +644,62 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); + it.live( + "creates and reports a Podman stack when automatic selection skips a stopped Docker", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-auto-podman-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "auto-podman"\n[edge_runtime]\nenabled = false\n', + ); + const output = mockOutput(); + const engines = containerEngineSpawner({ docker: "stopped", podman: "running" }); + const fixture = fakeStack(); + const createdRuntimes: Array = []; + const overrides = Layer.mergeAll( + Layer.succeed(StackTargetResolver, { + resolve: () => Effect.succeed({ projectRoot: root, hostRunning: false }), + }), + Layer.succeed(StackApi, { + create: (options) => + Effect.sync(() => { + createdRuntimes.push(options.runtime); + return fixture.stack; + }), + open: () => Effect.succeed(fixture.stack), + discover: () => Effect.succeed([]), + find: () => Effect.die("identity not used"), + findDeleted: () => Effect.die("deleted stacks not used"), + }), + engines.layer, + ); + yield* stackStart({ + ...flags([ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "mail", + "analytics", + "pooler", + ]), + runtime: "auto", + }).pipe(Effect.provide(Layer.merge(layers(root, fixture, output, false), overrides))); + expect(createdRuntimes).toEqual(["podman"]); + expect(output.messages).toContainEqual({ + type: "info", + message: expect.stringContaining( + "Docker didn't answer, so this new stack uses the Podman runtime", + ), + }); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("applies capability selection across repeated starts", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1296,36 +1352,6 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("rejects a changed endpoint after the stack is stopped", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-changed-port-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "changed-port"\n'); - const fixture = fakeStack(); - yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); - - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "changed-port"\n[api]\nport = 54999\n', - ); - yield* fixture.stack.composition.stop; - const error = yield* stackStart(flags()).pipe( - Effect.provide(layers(root, fixture)), - Effect.flip, - ); - - expect(error).toMatchObject({ - reason: "invalid-config", - message: expect.stringContaining("[api] port: saved automatic, requested 54999"), - suggestion: expect.stringContaining( - `supabase stack destroy --stack-id ${fixture.stack.id}`, - ), - }); - expect(fixture.composed).toBe(1); - }).pipe(Effect.provide(BunServices.layer)), - ); - it.live("names the config key and both values when a saved port changes", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1352,6 +1378,8 @@ describe("experimental stack start", () => { reason: "invalid-config", message: expect.stringContaining("[api] port: saved 54321, requested 54999"), }); + // The rejection leaves the stopped composition untouched: no recompose was attempted. + expect(fixture.composed).toBe(1); }).pipe(Effect.provide(BunServices.layer)), ); @@ -1792,150 +1820,97 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("reports the saved runtime when automatic selection creates a Podman stack", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-auto-podman-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "auto-podman"\n[edge_runtime]\nenabled = false\n', - ); - const output = mockOutput(); - const engines = containerEngineSpawner({ docker: "stopped", podman: "running" }); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => Effect.succeed({ projectRoot: root, hostRunning: false }), - }); - yield* stackStart({ - ...flags([ - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", - ]), - runtime: "auto", - }).pipe( - Effect.provide( - Layer.mergeAll(layers(root, fakeStack(), output, false), target, engines.layer), - ), - ); - expect(engines.spawned.map(({ command }) => command)).toEqual(["docker", "podman"]); - expect(output.messages).toContainEqual({ - type: "info", - message: expect.stringContaining( - "Docker didn't answer, so this new stack uses the Podman runtime", - ), - }); - }).pipe(Effect.provide(BunServices.layer)), - ); - - // The `#!/bin/sh` shim is never picked up on Windows, which only resolves `docker.exe` on PATH. - it.live.skipIf(process.platform === "win32")( - "leaves no stack registered when a new stack's owner cannot reach the Docker daemon", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-create-failure-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString( - `${root}/supabase/config.toml`, - 'project_id = "create-failure"\n', - ); - yield* fs.makeDirectory(`${root}/bin`); - yield* fs.writeFileString( - `${root}/bin/docker`, - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///shim/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", - ); - yield* fs.chmod(`${root}/bin/docker`, 0o755); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. - const originalPath = process.env.PATH; - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. - process.env.PATH = `${root}/bin:${originalPath ?? ""}`; - yield* Effect.addFinalizer(() => - Effect.sync(() => { - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. - process.env.PATH = originalPath; - }), - ); - const output = mockOutput(); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed({ projectRoot: root, runtime: "docker" as const, hostRunning: false }), - }); - const api = stackApiLayer.pipe(Layer.provide(BunServices.layer)); - - const error = yield* stackStart(flags()).pipe( - Effect.flip, - Effect.provide(Layer.mergeAll(layers(root, fakeStack(), output, false), target, api)), - ); - expect(error.message).toContain("Cannot connect to the Docker daemon"); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.reason).toBe("runtime"); - expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable"); - } - expect(output.stderrText).not.toContain("Failed to stop"); - - const stacks = yield* StackApi.pipe( - Effect.flatMap((stackApi) => - stackApi.discover({ stateRoot: `${root}/.supabase/stacks` }), - ), - Effect.provide(api), - ); - expect(stacks).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), - ); - - it.live("leaves no stack registered when a new stack's owner finds no Docker CLI", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-create-failure-" }); - yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); - yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "create-failure"\n'); + const unreachableDocker = [ + { + name: "cannot reach the Docker daemon", + // The `#!/bin/sh` shim is never picked up on Windows, which only resolves `docker.exe` on PATH. + posixOnly: true, + message: "Cannot connect to the Docker daemon", + // A `docker` first on PATH that reports an unreachable daemon, as the real CLI does. + path: (root: string, inherited: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${root}/bin`); + yield* fs.writeFileString( + `${root}/bin/docker`, + "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///shim/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", + ); + yield* fs.chmod(`${root}/bin/docker`, 0o755); + return `${root}/bin:${inherited}`; + }), + }, + { + name: "finds no Docker CLI", + posixOnly: false, + message: "docker context show", // An empty PATH hides any installed Docker CLI on every platform. - yield* fs.makeDirectory(`${root}/empty-bin`); - // oxlint-disable-next-line effecttsgo/process-env-in-effect -- Windows names the variable `Path`; the detached owner inherits it. - const envKeys = Object.keys(process.env); - const pathKey = envKeys.find((key) => key.toUpperCase() === "PATH") ?? "PATH"; - yield* withEnvVar( - pathKey, - `${root}/empty-bin`, + path: (root: string) => Effect.gen(function* () { - const output = mockOutput(); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed({ projectRoot: root, runtime: "docker" as const, hostRunning: false }), - }); - const api = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${root}/empty-bin`); + return `${root}/empty-bin`; + }), + }, + ]; - const error = yield* stackStart(flags()).pipe( - Effect.flip, - Effect.provide(Layer.mergeAll(layers(root, fakeStack(), output, false), target, api)), + for (const row of unreachableDocker) { + it.live.skipIf(row.posixOnly && process.platform === "win32")( + `leaves no stack registered when a new stack's owner ${row.name}`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-create-failure-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "create-failure"\n', ); - expect(error.message).toContain("docker ps"); - expect(error).toBeInstanceOf(StackCommandStartError); - if (error instanceof StackCommandStartError) { - expect(error.reason).toBe("runtime"); - expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable"); - } - expect(output.stderrText).not.toContain("Failed to stop"); - - const stacks = yield* StackApi.pipe( - Effect.flatMap((stackApi) => - stackApi.discover({ stateRoot: `${root}/.supabase/stacks` }), - ), - Effect.provide(api), + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- Windows names the variable `Path`; the detached owner inherits it. + const envKeys = Object.keys(process.env); + const pathKey = envKeys.find((key) => key.toUpperCase() === "PATH") ?? "PATH"; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. + const path = yield* row.path(root, process.env[pathKey] ?? ""); + yield* withEnvVar( + pathKey, + path, + Effect.gen(function* () { + const output = mockOutput(); + const target = Layer.succeed(StackTargetResolver, { + resolve: () => + Effect.succeed({ + projectRoot: root, + runtime: "docker" as const, + hostRunning: false, + }), + }); + const api = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + + const error = yield* stackStart(flags()).pipe( + Effect.flip, + Effect.provide( + Layer.mergeAll(layers(root, fakeStack(), output, false), target, api), + ), + ); + expect(error.message).toContain(row.message); + expect(error).toBeInstanceOf(StackCommandStartError); + if (error instanceof StackCommandStartError) { + expect(error.reason).toBe("runtime"); + expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable"); + } + expect(output.stderrText).not.toContain("Failed to stop"); + + const stacks = yield* StackApi.pipe( + Effect.flatMap((stackApi) => + stackApi.discover({ stateRoot: `${root}/.supabase/stacks` }), + ), + Effect.provide(api), + ); + expect(stacks).toEqual([]); + }), ); - expect(stacks).toEqual([]); - }), - ); - }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), - ); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + } it.live( "stops, without destroying, the owner when startup fails after the stack is acquired", diff --git a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts index 85363964ec..f540a2edee 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts @@ -325,6 +325,65 @@ describe("experimental stack start native lifecycle", () => { { timeout: 180_000 }, ); + it.live( + "names the pinned Studio port setting when another process holds that port", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-native-studio-" }); + yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); + const server = net.createServer(); + const ownedServer = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const onError = (cause: Error) => resume(Effect.fail(cause)); + server.once("error", onError); + server.listen(0, "127.0.0.1", () => { + server.removeListener("error", onError); + resume(Effect.succeed(server)); + }); + }), + (listeningServer) => + Effect.callback((resume) => { + listeningServer.close(() => resume(Effect.void)); + }), + ); + const address = ownedServer.address(); + if (address === null || typeof address === "string") + return yield* Effect.die("Unable to reserve a native test port"); + yield* fs.writeFileString( + path.join(root, "supabase", "config.toml"), + projectConfig.replace( + "[studio]\nenabled = false", + `[studio]\nenabled = true\nport = ${address.port}`, + ), + ); + const fixture = makeLayers(root); + yield* Effect.scoped( + Effect.gen(function* () { + const api = yield* StackApi; + yield* Effect.addFinalizer(() => + destroyTestStacks(api, path.join(root, "stacks"), path.join(root, "cache")), + ); + const failedStart = yield* Effect.scoped( + Effect.exit( + stackStart(flags(excluded.filter((name) => name !== "studio" && name !== "rest"))), + ), + ); + expect(Exit.isFailure(failedStart)).toBe(true); + if (!Exit.isFailure(failedStart)) return; + expect(Cause.findErrorOption(failedStart.cause)).toMatchObject( + Option.some({ + suggestion: + "Set `studio.port` in supabase/config.toml (or SUPABASE_STUDIO_PORT) to a free port.", + }), + ); + }), + ).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(BunServices.layer)), + { timeout: 180_000 }, + ); + it.live( "stops owners after bind and pre-compose config failures across retries", () => @@ -381,9 +440,15 @@ describe("experimental stack start native lifecycle", () => { // Linux rejects the bind; platforms that allow overlapping binds reject the probe. expect(Cause.pretty(failedStart.cause)).toMatch( new RegExp( - `127\\.0\\.0\\.1:${occupiedPort}(: Cannot bind TCP listener| is already in use)`, + `127\\.0\\.0\\.1:${occupiedPort}(: Cannot bind TCP listener| is in use by another process)`, ), ); + expect(Cause.findErrorOption(failedStart.cause)).toMatchObject( + Option.some({ + suggestion: + "Set `db.port` in supabase/config.toml (or SUPABASE_DB_PORT) to a free port.", + }), + ); const ownerPid = ownerPids[attempt]; expect(ownerPid).toBeDefined(); if (ownerPid === undefined) return yield* Effect.die("stack owner PID missing"); diff --git a/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts index c268717965..5c6c6431ca 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.storage.e2e.test.ts @@ -2,13 +2,12 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Data, Effect, Exit, FileSystem, Path, Schema } from "effect"; import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { tmpdir } from "node:os"; - import { makeTempCliProject, makeTempHome, runSupabaseEffect, } from "../../../../../tests/helpers/cli.ts"; +import { stackArtifactCacheRoot } from "../../../../../tests/helpers/stack-artifacts.ts"; const COMMAND_TIMEOUT_MS = 10 * 60_000; const CLEANUP_TIMEOUT_MS = 120_000; @@ -92,7 +91,7 @@ describe("stack start Storage behind the API gateway (compiled e2e)", () => { const project = yield* Effect.promise(() => makeTempCliProject(`stack-storage-${runtime}-e2e-`), ); - const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); + const artifacts = stackArtifactCacheRoot; yield* fs.makeDirectory(path.join(home.dir, "cache"), { recursive: true }); yield* fs.makeDirectory(artifacts, { recursive: true }); yield* fs.symlink(artifacts, path.join(home.dir, "cache", "stack")); diff --git a/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md index 488c519d36..1c2b7673c6 100644 --- a/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md @@ -31,8 +31,7 @@ available while the owner is unavailable. It reports `config_drift.status` as `u loading failure or unreadable saved state keeps the saved stack report available and is shown as `config_drift.status: "unavailable"` with a message in JSON. Status does not apply current configuration. The `services` list may include saved standalone instances; composition members identify the services used for -primary database, environment export, and drift comparisons. Vector instances saved by earlier -releases are omitted without rewriting saved state. +primary database, environment export, and drift comparisons. Text output starts with one line naming the stack, its readiness, runtime, and project directory, noting when the owner is unavailable. It then prints the diff --git a/apps/cli/src/commands/experimental/stack/status/status.handler.ts b/apps/cli/src/commands/experimental/stack/status/status.handler.ts index c0450d6659..cdfaa5c9f1 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.handler.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.handler.ts @@ -6,6 +6,7 @@ import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; +import type { StackRuntime } from "../../../../command-internal/stack-runtime.ts"; import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import { withProjectFunctionsEnv } from "../../../../command-internal/stack-functions-env.ts"; import { bold, gray, green, red, yellow } from "../../../../command-internal/colors.ts"; @@ -20,8 +21,8 @@ import { } from "../stack-summary.ts"; import { StackApi, - StackTargetError, StackTargetResolver, + mapTargetError, rejectStackOutput, validateStackTarget, } from "../stack.shared.ts"; @@ -57,7 +58,7 @@ type StackReport = { readonly project_root: string; readonly branch_context: string; }; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: StackRuntime; readonly owner: "reachable" | "unavailable"; readonly lifecycle: Observation["lifecycle"] | null; readonly readiness: "unavailable" | "starting" | "sleeping" | "stopped" | "ready" | "unhealthy"; @@ -82,14 +83,6 @@ type StackReport = { readonly env: Readonly>; }; -const mapTargetError = (error: StackTargetError) => - new StackCommandStatusError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - const mapStackError = (error: StackError) => new StackCommandStatusError({ reason: @@ -156,7 +149,7 @@ const reportFor = ( readonly branchContext: string; readonly stackName: string; }; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: StackRuntime; }, owner: StackReport["owner"], observed: ReadonlyArray, @@ -263,7 +256,7 @@ const findTarget = Effect.fn("experimental.stack.status.findTarget")(function* ( ...(id === undefined ? {} : { id }), runtime: "auto", }) - .pipe(Effect.mapError(mapTargetError)); + .pipe(Effect.mapError(mapTargetError((props) => new StackCommandStatusError(props)))); if (target.id === undefined || target.definition === undefined) return yield* new StackCommandStatusError({ reason: "not-found", @@ -373,7 +366,7 @@ export const stackStatus = Effect.fn("experimental.stack.status")(function* ( yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); + }).pipe(Effect.mapError(mapTargetError((props) => new StackCommandStatusError(props)))); if (!flags.env && flags.overrideName.length > 0) return yield* new StackCommandStatusError({ reason: "flags", diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 15ba68efff..52eb6aa131 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -103,13 +103,9 @@ const makeObservation = ( lifecycle: "stopped", health: undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 1, wakeEnabled: true, - registered: true, ...input, }); @@ -241,7 +237,6 @@ const runStatus = (input: { })), composition: { members: input.members ?? [], dependencies: [] }, lifetime: "detached" as const, - ports: [], }; const api = Layer.succeed(StackApi, { create: () => Effect.die("create must not run"), diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index 7aadf9738e..36eb5d7241 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -12,16 +12,24 @@ with either selector. Explicit legacy `-o/--output` is rejected; use `--output-format` instead. Discovery probes saved owners before shutdown. If no owner is reachable, the -command reports `No owner is reachable; workload state is unavailable.` It -neither starts an owner nor claims workloads have stopped. If an owner disappears -after that preflight, the shutdown error remains a failure. - -Text confirms each successful shutdown and identifies each unavailable owner. -JSON and stream-json success data contain `stopped` and `unavailable` ID arrays. +command does not start one; it reclaims the stack's leftover workloads under the +stack lease and reports `Stack was not running; leftover resources were reclaimed.` +It fails when another process holds the lease or the cleanup fails, so a stack +whose owner exited after a failed stop can be stopped again. If an owner +disappears after discovery, the shutdown error remains a failure. + +Services stop in reverse dependency order with every stack listener still open, +so a dependent's graceful stop can still reach its prerequisite; each listener +closes once its own service has stopped. Traffic that would wake a stopped +service during shutdown is refused. Shutdown is one-way: a failed stop leaves +the stack registered and the owner exits, and the next stop retries. + +Text confirms each successful shutdown and identifies each stack that had no owner. +JSON and stream-json success data contain `stopped` and `not_running` ID arrays. A missing default selection reports `found: false`; an unknown explicit name or ID fails. A single selection reads only the selected stack's state document; an unreadable document fails the selection instead of being reported as missing. -`--all` attempts every selected reachable owner and reports failures with their +`--all` attempts every selected stack and reports failures with their IDs. It skips state entries that cannot be read or decoded with a warning on stderr identifying each stack; only a failure to read the stacks directory itself fails discovery. @@ -32,13 +40,16 @@ Reads saved state under `/stacks//` and the current project/Git identity when no explicit ID is supplied. Shared routing and settings may read project configuration and the selected profile. Communicates only with local owner endpoints; no hosted API requests. Shutdown updates owned -runtime resources but preserves persistent instance definitions, data and port -claims. No caller-owned upload files are removed. +runtime resources but preserves persistent instance definitions, data and automatic +port reservations in the per-user registry at `/.supabase/ports.sqlite` +(unaffected by `SUPABASE_HOME`); configured ports are released with their listeners. +Killing the owner process preserves every reservation, but another stack may take over +a configured port while no owner runs. No caller-owned upload files are removed. ## Exit codes and telemetry -Exit 0 on successful shutdown, absent default selection, or unavailable owner; -1 for invalid flags, selection/registry failure, or shutdown failure; 130 on +Exit 0 on successful shutdown or reclaim, or absent default selection; +1 for invalid flags, selection/registry failure, or shutdown or reclaim failure; 130 on interruption. The command wrapper retains standard command telemetry, with no custom events. Telemetry flushes on success and failure to `/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts index 6dd789617f..38573871d8 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts @@ -119,17 +119,15 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: hostRunning, }): Effect.Effect<{ readonly id: string; - readonly result: Result.Result<"stopped" | "unavailable", StackError>; + readonly result: Result.Result<"stopped" | "not-running", StackError>; }> => - !hostRunning - ? Effect.succeed({ id, result: Result.succeed("unavailable" as const) }) - : api.open({ ...locations, id }).pipe( - Effect.flatMap((stack) => stack.stop), - Effect.scoped, - Effect.as("stopped" as const), - Effect.result, - Effect.map((result) => ({ id, result })), - ), + api.open({ ...locations, id }).pipe( + Effect.flatMap((stack) => stack.stop), + Effect.scoped, + Effect.as(hostRunning ? ("stopped" as const) : ("not-running" as const)), + Effect.result, + Effect.map((result) => ({ id, result })), + ), ); const failures = results.flatMap(({ id, result }) => Result.isFailure(result) ? [{ id, error: result.failure }] : [], @@ -144,8 +142,8 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: const stopped = results.flatMap(({ id, result }) => Result.isSuccess(result) && result.success === "stopped" ? [id] : [], ); - const unavailable = results.flatMap(({ id, result }) => - Result.isSuccess(result) && result.success === "unavailable" ? [id] : [], + const notRunning = results.flatMap(({ id, result }) => + Result.isSuccess(result) && result.success === "not-running" ? [id] : [], ); yield* Effect.annotateCurrentSpan({ "stack.count": results.length, @@ -153,9 +151,9 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: }); if (output.format === "text") { for (const id of stopped) yield* output.raw(`Stack ${id} stopped.\n`); - for (const id of unavailable) - yield* output.raw(`Stack ${id}: No owner is reachable; workload state is unavailable.\n`); + for (const id of notRunning) + yield* output.raw(`Stack ${id} was not running; leftover resources were reclaimed.\n`); if (results.length === 0) yield* output.raw("No managed stacks found.\n"); - } else yield* output.success("", { stopped, unavailable }); + } else yield* output.success("", { stopped, not_running: notRunning }); }).pipe(Effect.ensuring(telemetry.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts index 6648036cf2..7fd6437ec8 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts @@ -1,12 +1,13 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Layer, Option, Path, Ref } from "effect"; +import { Effect, Exit, FileSystem, Layer, Option, Path, Ref, Scope } from "effect"; import { StackError } from "@supabase/stack/effect"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; +import * as StackNamespace from "../../../../../../../packages/stack/src/StackNamespace.ts"; import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackStop } from "./stop.handler.ts"; @@ -56,24 +57,28 @@ describe("stack stop", () => { }).pipe(Effect.provide(live)), ); - it.live("stop all preserves an offline registry without claiming stopped workloads", () => - Effect.gen(function* () { - const f = yield* fixture(); - yield* f.api.create({ - ...f.locations, - projectRoot: f.root, - name: "second", - runtime: "native", - }); - yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); - yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText.match(/workload state is unavailable/g)).toHaveLength(4); - expect(f.output.stdoutText).not.toContain("stopped."); - expect((yield* f.api.discover(f.locations)).every(({ host }) => host === undefined)).toBe( - true, - ); - expect(f.telemetry.flushed).toBe(true); - }).pipe(Effect.provide(live)), + it.live( + "stop all reclaims ownerless stacks and keeps their definitions without claiming a stop", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.api.create({ + ...f.locations, + projectRoot: f.root, + name: "second", + runtime: "native", + }); + yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); + yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); + expect( + f.output.stdoutText.match(/was not running; leftover resources were reclaimed/g), + ).toHaveLength(4); + expect(f.output.stdoutText).not.toContain("stopped."); + expect((yield* f.api.discover(f.locations)).every(({ host }) => host === undefined)).toBe( + true, + ); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), ); it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( @@ -92,7 +97,7 @@ describe("stack stop", () => { fs.chmod(locked, 0o700).pipe(Effect.orDie), ); yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); - expect(f.output.stdoutText.match(/workload state is unavailable/g)).toHaveLength(1); + expect(f.output.stdoutText.match(/was not running/g)).toHaveLength(1); expect(f.output.stderrText).toContain("Warning: skipping invalid stack broken"); expect(f.output.stderrText).toContain("Warning: skipping invalid stack locked"); expect(f.output.stdoutText).not.toContain("Warning"); @@ -142,4 +147,32 @@ describe("stack stop", () => { expect(f.telemetry.flushed).toBe(true); }).pipe(Effect.provide(live)), ); + + it.live("fails when an ownerless stack's lease is held by another process", () => + Effect.gen(function* () { + const f = yield* fixture(); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: f.locations.stateRoot })), + ); + const holder = yield* Scope.make(); + const lease = yield* state.acquireLease(f.stack.id).pipe(Scope.provide(holder)); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + const error = yield* stackStop(flags(f.stack.id)).pipe( + Effect.provide(f.layer), + Effect.flip, + Effect.ensuring(Scope.close(holder, Exit.void)), + ); + expect(error.reason).toBe("unknown"); + expect(error.detail).toContain(`${f.stack.id}: Another process holds the stack lease`); + expect(f.output.stdoutText).toBe(""); + expect((yield* f.api.discover(f.locations)).map(({ definition }) => definition.id)).toEqual([ + f.stack.id, + ]); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); }); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts index c5230a5c66..3b7c941e3a 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts @@ -1,13 +1,31 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { create as createStack } from "@supabase/stack/effect"; -import { Data, Effect, FileSystem, Layer, Path, Redacted, Schema } from "effect"; +import { + create as createStack, + postgres, + type Observation, + type Stack, + type StackError, +} from "@supabase/stack/effect"; +import { + Data, + Deferred, + Effect, + Fiber, + FileSystem, + Layer, + Path, + Redacted, + Schema, + Stream, +} from "effect"; import { FetchHttpClient, HttpClient } from "effect/unstable/http"; -import { homedir, tmpdir } from "node:os"; +import { homedir } from "node:os"; import { spawnSupabase } from "../../../../tests/helpers/cli.ts"; import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; import { destroyTestStack } from "../../../../tests/helpers/stack-cleanup.ts"; +import { stackArtifactCacheRoot } from "../../../../tests/helpers/stack-artifacts.ts"; const jwtSecret = "functions-serve-stack-e2e-secret-at-least-32-characters"; const nativeSupported = @@ -55,9 +73,8 @@ const fixture = Effect.fn("FunctionsServeE2e.fixture")(function* ( ); yield* fs.writeFileString(path.join(root, "override.env"), "CUSTOM_VALUE=overridden\n"); yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); - const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); - yield* fs.makeDirectory(artifacts, { recursive: true }); - yield* fs.symlink(artifacts, path.join(home, "cache", "stack")); + yield* fs.makeDirectory(stackArtifactCacheRoot, { recursive: true }); + yield* fs.symlink(stackArtifactCacheRoot, path.join(home, "cache", "stack")); const stack = yield* createStack({ projectRoot: root, stateRoot: path.join(home, "stacks"), @@ -182,6 +199,42 @@ const payload = Effect.fn("FunctionsServeE2e.payload")(function* ( const layer = Layer.mergeAll(BunServices.layer, FetchHttpClient.layer); +/** + * Follows a service's statuses from the first one it delivers and returns a fiber that resolves + * with every status delivered before `barrier` matches. Statuses arrive in order, so a later + * transition the test causes proves the recorded prefix is complete. + */ +const statusesBefore = Effect.fn("FunctionsServeE2e.statusesBefore")(function* ( + instance: { readonly followStatus: Stream.Stream }, + barrier: (status: Observation) => boolean, +) { + const subscribed = yield* Deferred.make(); + const before = yield* instance.followStatus.pipe( + Stream.tap(() => Deferred.succeed(subscribed, undefined)), + Stream.takeWhile((status) => !barrier(status)), + Stream.runCollect, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + return before; +}); + +/** The database process start time, which changes whenever Postgres is relaunched. */ +const postmasterStartTime = Effect.fn("FunctionsServeE2e.postmasterStartTime")(function* ( + stack: Stack, + databaseUrl: string, +) { + const decoder = new TextDecoder(); + let stdout = ""; + const result = yield* stack.commands.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "-tA", "--command", "SELECT pg_postmaster_start_time()"], + stdout: (bytes) => Effect.sync(() => (stdout += decoder.decode(bytes))), + stderr: () => Effect.void, + }); + expect(result.exitCode).toBe(0); + return stdout.trim(); +}); + describe("functions serve (stack e2e)", () => { for (const runtime of ["native", "docker"] as const) { if (runtime === "native" && !nativeSupported) continue; @@ -205,7 +258,13 @@ describe("functions serve (stack e2e)", () => { expect(savedStudio.wakeEnabled).toBe(true); const composition = yield* stack.composition.describe; const plain = yield* serve(root, home); - const active = yield* functions.status; + const beforeOverride = yield* statusesBefore( + functions, + (status) => + status.lifecycle === "running" && + status.config.service === "functions" && + status.config.config.env?.CUSTOM_VALUE === "overridden", + ); expect(yield* payload(apiUrl, true)).toEqual({ value: "original", anon: true, @@ -222,7 +281,6 @@ describe("functions serve (stack e2e)", () => { expect((yield* payload(apiUrl, true)).value).toBeNull(); yield* fs.writeFileString(sourcePath, source); yield* interrupt(plain); - expect((yield* functions.status).launchId).toBe(active.launchId); expect(yield* studio.status).toMatchObject({ lifecycle: savedStudio.lifecycle, wakeEnabled: savedStudio.wakeEnabled, @@ -241,6 +299,12 @@ describe("functions serve (stack e2e)", () => { databaseUrl, }); yield* interrupt(overridden); + // The override's own restart is the only launch before its configuration is running. + const launches = (yield* Fiber.join(beforeOverride)).filter( + ({ lifecycle }, index, all) => + lifecycle === "starting" && all[index - 1]?.lifecycle !== "starting", + ); + expect(launches).toHaveLength(1); expect((yield* functions.status).config).toEqual(saved.config); expect(yield* studio.status).toMatchObject({ lifecycle: savedStudio.lifecycle, @@ -262,9 +326,7 @@ describe("functions serve (stack e2e)", () => { false, ); const composition = yield* stack.composition.describe; - const database = services.find((instance) => instance.service === "database"); - if (database === undefined) return yield* Effect.die("Database missing"); - const databaseLaunch = (yield* database.status).launchId; + const startedAt = yield* postmasterStartTime(stack, databaseUrl); const child = yield* serve(root, home); expect(yield* payload(apiUrl, true)).toEqual({ value: null, @@ -279,10 +341,10 @@ describe("functions serve (stack e2e)", () => { services.map(({ id }) => id).sort(), ); expect(yield* stack.composition.describe).toEqual(composition); - expect((yield* database.status).launchId).toBe(databaseLaunch); expect((yield* invoke(apiUrl, false)).status).toBe(404); const http = yield* HttpClient.HttpClient; expect((yield* http.get(`${apiUrl}/rest/v1/`)).status).toBe(200); + expect(yield* postmasterStartTime(stack, databaseUrl)).toBe(startedAt); }).pipe(Effect.provide(layer)), { timeout: 360_000 }, ); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index 82d68fcca0..c3fe1f031f 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -92,12 +92,8 @@ const observation = ( health: "healthy", currentOperation: undefined, error: undefined, - cleanupError: undefined, exit: undefined, - launchId: 1, - intentRevision: 1, wakeEnabled: false, - registered: true, ...overrides, }); @@ -306,7 +302,6 @@ const fixture = ( instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }, host: undefined, }), diff --git a/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts b/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts index 5b4a4e3620..a3e24013e6 100644 --- a/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts +++ b/apps/cli/src/shared/telemetry/error-tag-stability.unit.test.ts @@ -5,11 +5,11 @@ import { fileURLToPath, pathToFileURL } from "node:url"; /** * Guards the telemetry identity of every CLI error: the string passed to - * `Data.TaggedError("...")` becomes `error_fingerprint` on the `cli_command_executed` event, so + * `Data.TaggedError("...")` or `Schema.TaggedError` becomes `error_fingerprint` on the `cli_command_executed` event, so * changing it (not renaming the class) silently splits one error's history into two * fingerprints, undetected until dashboards look wrong. * - * Enumerates every production `Data.TaggedError` tag under `apps/cli/src` and + * Enumerates every production tagged-error tag under `apps/cli/src` and * `packages/config/src`, resolving tags built via template interpolation * (`mintConfigTargetErrors`) at runtime by constructing each class — see * {@link collectComputedTagDeclarations}. Compares the set against the committed @@ -26,10 +26,10 @@ const externalConfigSrcDir = fileURLToPath( ); const fixturePath = fileURLToPath(new URL("./__fixtures__/error-tags.txt", import.meta.url)); -// Matches both the inline declaration form and the formatter-wrapped -// multi-line form, where the string literal lands on its own line before the -// closing paren. -const TAGGED_ERROR_PATTERN = /class\s+(\w+)\s+extends\s+Data\.TaggedError\(\s*["']([^"']+)["']/g; +// Matches `Data.TaggedError("…")` and `Schema.TaggedError()("…", …)`, inline or +// formatter-wrapped with the string literal on its own line. +const TAGGED_ERROR_PATTERN = + /class\s+(\w+)\s+extends\s+(?:Data\.TaggedError|Schema\.TaggedError<\w+>\(\))\(\s*["']([^"']+)["']/g; // Every test tier this repo's vitest config defines (`apps/cli/vitest.config.ts`). // Shared by name so a fifth tier can't quietly slip past `isProductionSourceFile` diff --git a/apps/cli/tests/helpers/child-process-spawner.ts b/apps/cli/tests/helpers/child-process-spawner.ts index 7dc2ad5a0e..366d65b93a 100644 --- a/apps/cli/tests/helpers/child-process-spawner.ts +++ b/apps/cli/tests/helpers/child-process-spawner.ts @@ -107,8 +107,9 @@ export function mockChildProcessSpawner( export type ContainerEngineState = "missing" | "stopped" | "running"; /** - * Spawner for a host whose `docker` and `podman` commands behave as `engines` describe. Other - * commands fail with `NotFound`, or run on the real spawner through `hidingLayer`. + * Spawner for a host whose `docker` and `podman` commands behave as `engines` describe, answering + * the identity queries engine target resolution makes. Other commands fail with `NotFound`, or run + * on the real spawner through `hidingLayer`. */ export function containerEngineSpawner(engines: { readonly docker: ContainerEngineState; @@ -122,6 +123,12 @@ export function containerEngineSpawner(engines: { method: "spawn", pathOrDescriptor: command, }); + const answer = (cmd: "docker" | "podman", args: ReadonlyArray) => { + if (cmd === "docker") return args.includes("context") ? "default" : "docker-daemon-id"; + return args.some((arg) => arg.includes("ServiceIsRemote")) + ? "false|true|host|/graph-root" + : "host"; + }; const spawner = (delegate?: ChildProcessSpawner.ChildProcessSpawner["Service"]) => ChildProcessSpawner.make((command) => { const cmd = Predicate.isTagged(command, "StandardCommand") ? command.command : ""; @@ -131,13 +138,16 @@ export function containerEngineSpawner(engines: { spawned.push({ command: cmd, args }); const state = engines[cmd]; if (state === "missing") return Effect.fail(notFound(cmd)); + const running = state === "running"; return Effect.succeed( ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(2000 + spawned.length), - stdout: Stream.empty, - stderr: Stream.empty, + stdout: running ? Stream.make(encoder.encode(answer(cmd, args))) : Stream.empty, + stderr: running + ? Stream.empty + : Stream.make(encoder.encode("Cannot connect to the engine")), all: Stream.empty, - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(state === "running" ? 0 : 1)), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(running ? 0 : 1)), isRunning: Effect.succeed(false), stdin: Sink.drain, kill: () => Effect.void, diff --git a/apps/cli/tests/helpers/stack-artifacts.ts b/apps/cli/tests/helpers/stack-artifacts.ts new file mode 100644 index 0000000000..70e16863aa --- /dev/null +++ b/apps/cli/tests/helpers/stack-artifacts.ts @@ -0,0 +1,9 @@ +import { homedir } from "node:os"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. +import { join } from "node:path"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. + +/** Shared native artifact cache for stack tests; OS temp directories age out files inside published generations. */ +export const stackArtifactCacheRoot = join( + // oxlint-disable-next-line effecttsgo/process-env -- plain constant shared by test files. + process.env.XDG_CACHE_HOME || join(homedir(), ".cache"), + "supabase-stack-artifacts", +); diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index cb086be56b..8620435576 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -157,13 +157,9 @@ const observation = ( health, error: error === undefined ? undefined : { _tag: "ServiceError", operation: "status", message: error }, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 1, wakeEnabled, - registered: true, }); const instance = ( @@ -277,7 +273,6 @@ export function buildStorageStackApi( ], composition: { members, dependencies: [] }, lifetime: "detached" as const, - ports: [], }; const findStackCalls: Array<{ readonly projectRoot: string }> = []; const layer = diff --git a/docs/adr/0017-simplified-managed-stack-architecture.md b/docs/adr/0017-simplified-managed-stack-architecture.md index f9ee11209f..a4cb733990 100644 --- a/docs/adr/0017-simplified-managed-stack-architecture.md +++ b/docs/adr/0017-simplified-managed-stack-architecture.md @@ -68,7 +68,7 @@ Disabling a capability releases its automatic port assignment; re-enabling it ma select a new port. Stack handles are lightweight identity-scoped clients. Creating or opening one -does not launch a Supervisor. Successful stop drains ingress, removes every +does not launch a Supervisor. Successful stop stops services in dependency order, removes every ephemeral runtime resource, persists stopped state, delivers its response, then closes control and releases ownership. The caller waits for both response and lease release. A stopped stack therefore consumes no live process, container, @@ -132,16 +132,16 @@ Linux, and Windows in CI. Every managed document records one concrete runtime selection. Native and container runtimes never mix. For a new stack, an omitted runtime checks that -the Docker client is installed and then probes the daemon with a short timeout; -it selects Docker when the daemon is reachable and native otherwise. An -installed client with an unreachable daemon selects native, and the created -Effect handle carries a notice that the selection is persisted for that stack, -so a later switch to Docker requires destroying the stack or choosing a new -stack name; callers such as `stack start` decide whether to print it. Native is -refused as uid 0. Existing state is reused without probing. Callers may -explicitly select native, Docker, or Podman without fallback, and an omitted -engine for an explicit container runtime defaults to Docker. Podman is -supported only on local Linux hosts. See +each container engine in turn, Docker then Podman, resolves the same engine +target the stack owner pins and probes it with a 10 s timeout; it selects the +first reachable engine and native otherwise, where native artifacts are +supported. When Docker is skipped, the created Effect handle carries a notice +that the selected runtime (Podman or native) is persisted for that stack, so a +later switch to Docker requires destroying the stack or choosing a new stack +name; callers such as `stack start` decide whether to print it. Native as +root runs only PostgreSQL as an unprivileged user. Existing state is reused +without probing. Callers may explicitly select native, Docker, or Podman without +fallback. See [ADR-0025](0025-ephemeral-postgres-for-schema-tooling.md) for the daemon probe and its interaction with ephemeral Postgres. Persisted state records the resolved exact engine. Capability releases and @@ -300,7 +300,7 @@ Tests follow consumed boundaries: ownership, stale-owner recovery, and interrupted cleanup; - supervisor integration covers detached ownership, RPC, stop, destroy, retirement, and wake-up; and -- one shared stack-package E2E journey runs in native and Docker modes, starts +- one shared stack-package E2E journey runs in native, Docker, and Podman modes, starts with PostgreSQL alone, activates every other service through realistic traffic, verifies cross-service behavior, then exercises stop/start and retained offline observability. diff --git a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md index 0b55385e27..e56fe2faa1 100644 --- a/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md +++ b/docs/adr/0025-ephemeral-postgres-for-schema-tooling.md @@ -1,6 +1,6 @@ # 0025. Ephemeral Postgres for schema tooling -**Status**: superseded by throwaway stack shadow stacks and [database snapshots](../../packages/stack/ARCHITECTURE.md#snapshots-belong-to-the-database-instance) — `EphemeralPostgres` was never built; see [`stack-shadow.ts`](../../apps/cli/src/command-internal/stack-shadow.ts) +**Status**: superseded by throwaway stack shadow stacks and [database snapshots](../../packages/stack/ARCHITECTURE.md#snapshots-belong-to-the-database-instance) — `EphemeralPostgres` was never built; see [`stack-shadow.ts`](../../apps/cli/src/command-internal/stack-shadow.ts); runtime auto-selection and native-as-root handling amended by [ADR 0029](0029-native-postgres-privilege-step-down.md) **Date**: 2026-09-09 ## Problem Statement diff --git a/docs/adr/0029-native-postgres-privilege-step-down.md b/docs/adr/0029-native-postgres-privilege-step-down.md new file mode 100644 index 0000000000..e9c751d569 --- /dev/null +++ b/docs/adr/0029-native-postgres-privilege-step-down.md @@ -0,0 +1,57 @@ +# 0029. Native Postgres privilege step-down + +**Status**: proposed +**Date**: 2026-10-07 + +## Problem Statement + +[ADR 0025](0025-ephemeral-postgres-for-schema-tooling.md) refuses native Postgres when the process +uid is 0, because `initdb` refuses to run as root, and tells users to pick `--runtime docker`. +Containers and agent sandboxes commonly run the CLI as root without a container engine, so that +rule leaves no working runtime there. Automatic runtime selection also only chose between Docker +and native, ignoring Podman. + +## Decision + +- When the stack runs as root, only the PostgreSQL process steps down to an unprivileged user. The + user is named by `SUPABASE_NATIVE_POSTGRES_USER`, or is the account of a detected agent sandbox. + The CLI itself keeps running as root and hands the instance data, root key, and socket directory + to that user. Without either, native startup still refuses root. +- Auto-selecting the runtime of a new stack probes Docker, then Podman, then falls back to native + where native artifacts exist. An explicit `--runtime` has no fallback, and a saved runtime never + flips. + +## Rationale + +Stepping down only the process that requires it keeps the rest of the stack unchanged and avoids a +second privilege model. Probing Podman before native means a machine with either engine gets +containers, as it does when started explicitly. + +## Consequences + +### Positive + +- Native stacks start in root-only environments when a user is named or a sandbox is detected. +- Podman-only machines get a container runtime by default. + +### Negative + +- Root-owned native stacks need ownership changes on stack state and traverse permission on parent + directories, described in the `stack start` side effects. +- A runtime chosen automatically after skipping Docker is sticky until the stack is destroyed or a + new stack name is used. + +## Alternatives Considered + +1. **Keep refusing root**: leaves root-only environments without a runtime. +2. **Run the whole stack as the unprivileged user**: requires re-executing the CLI and splits + state ownership across users. + +## Related Decisions + +- [ADR 0025](0025-ephemeral-postgres-for-schema-tooling.md): the native-as-root and default + runtime rules this decision amends. + +## See Also + +- [`stack start` side effects](../../apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md) diff --git a/docs/adr/README.md b/docs/adr/README.md index 8b74b63173..05b840861b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -70,6 +70,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0026 | [Slim Image and Native Artifact Mirrors](0026-slim-artifact-mirrors.md) | proposed | | 0027 | [CLI Tracing Conventions](0027-cli-tracing-conventions.md) | proposed | | 0028 | [Release Branches and Maintenance Lines](0028-release-branches-and-maintenance-lines.md) | accepted | +| 0029 | [Native Postgres Privilege Step-Down](0029-native-postgres-privilege-step-down.md) | proposed | ## Template diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 9cd7c2606f..e8720ebe54 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -4,26 +4,27 @@ This document defines the stack package architecture. The package implements the ## Core model -Use **a graph of service instances, with a small serialized lifecycle for each instance**. Keep application readiness separate from that lifecycle. Add a proxy that starts services on public traffic and sleeps them on public inactivity. Run finite commands with the same identity, artifacts and execution primitives, without treating them as services. +Use **a graph of service instances whose lifecycle one reducer per owner decides**. Keep application readiness separate from that lifecycle. Add a proxy that starts services on public traffic and sleeps them on public inactivity. Run finite commands with the same identity, artifacts and execution primitives, without treating them as services. The critical simplification is: > Starting a process and waiting for that process to become healthy are different operations. -A service finishes its current lifecycle transition before accepting another lifecycle signal. There is no cancellation of one transition by another, ownership handoff, special pending-stop state, or general desired-state reconciliation engine. +Lifecycle decisions are pure transitions applied under one gate per owner; executing them is separate. An execution reports generation-tagged outcomes, so a late outcome never affects a newer generation. There is no second lifecycle state outside the reducer. Preserve the useful identity and instance use cases while replacing the underlying machinery. The contract intentionally stays small and behavioral. ## The mental model: a service runs itself; the orchestrator connects services -Establish the individual service lifecycle first, then compose it through a dependency-aware orchestrator. Identity and resource ownership are foundations for both. The orchestrator uses the same service operations for one service or a whole stack; it does not implement a second lifecycle. +Each service execution runs one generation's launch, readiness check and cleanup. The orchestrator's reducer decides when each happens, for one service or a whole stack. Identity and resource ownership are foundations for both. ```mermaid flowchart TB Handle["Public service or stack handle"] --> Orchestrator Proxy["Proxy: traffic and idle signals"] --> Orchestrator Composition["Stack composition
dependency edges and input wiring"] --> Orchestrator - Orchestrator["Orchestrator
validate graph, order operations, await readiness"] --> Executor["Individual service executor
serialize its own lifecycle transitions"] + Orchestrator["Orchestrator
owner gate and lifecycle reducer"] -->|launch, stop, reprobe| Executor["Service execution
one generation's runtime and cleanup"] + Executor -.->|generation-tagged outcomes| Orchestrator Executor --> Backend["Native or container backend
launch, observe, stop, remove"] Context["Shared stack context
identity, owned data, sticky ports"] -.-> Orchestrator Context -.-> Executor @@ -37,10 +38,10 @@ A URL implies a connection requirement, not ownership of another service. Do not | ----------------- | ------------------------------------------------------------------------ | ------------------------------------------------------- | | Recipe/definition | Typed configuration inputs, such as a database URL; own runtime behavior | Where inputs originate or which stack services must run | | Stack composition | Concrete dependency edges and how values populate service inputs | Runtime lifecycle implementation | -| Orchestrator | Graph, input resolution, readiness requirements, selected operations | How PostgreSQL launches or a container stops | -| Service executor | Own state, processes, readiness, data and resolved inputs | Dependency ordering or other services' lifecycle | +| Orchestrator | Graph, lifecycle state, demand, admission, timers and input resolution | How PostgreSQL launches or a container stops | +| Service execution | One generation's processes, readiness check, cleanup and data | When to launch, stop or sleep, or other services' state | -Calling public `service.start()` or `service.stop()` still goes through the orchestrator. The executor is internal, so callers cannot accidentally bypass graph checks. Sleep is an internal proxy policy over the same stop/start mechanics; it has no public service method or RPC endpoint. +Calling public `service.start()` or `service.stop()` goes through the orchestrator. The execution boundary is internal, so callers cannot bypass the reducer. Sleep is an internal idle policy over the same stop/launch mechanics; it has no public service method or RPC endpoint. Each long-running service is an individually identified instance with its own executor, lifecycle and health. There is no capability layer, hidden service group or aggregate service handle. Composition connects instances; it does not hide their identities. @@ -50,7 +51,7 @@ Keep the implementation Effect V4 from the domain inward. Promise is the outer f Organize by cohesive responsibilities. The package shape is: -- `src/` modules: the instance executor, orchestrator, owner, detached host, networking, persistence, and RPC boundary. +- `src/` modules: the lifecycle reducer, orchestrator, service execution, owner, detached host, networking, persistence, and RPC boundary. - `services/`: one definition per service, owning its configuration, endpoints, launch settings, and readiness. `Catalog.ts` validates and dispatches creation; `Recipe.ts` defines their contract and `ProcessRecipe.ts` shares process mechanics. - `composition/Supabase.ts`: default Supabase membership, dependency edges, and input wiring. - `host/`: endpoint projections, command execution, and command attachment transport. @@ -61,7 +62,7 @@ Organize by cohesive responsibilities. The package shape is: - `testing.ts`: disposable, composed session stacks for tests, with database checkpoints, in Promise and Effect forms. - `Defaults.ts`: shared local-development credentials, exported once as `./defaults`. -This is navigational guidance, not a required file scaffold. Split modules when a responsibility needs it; avoid one folder or interface per operation. Keep service definitions narrow, with graph edges and input wiring in composition. Do not introduce capabilities, projections, recovery journals, reservations, public sleep APIs, or extra lifecycle states to force this shape. +This is navigational guidance, not a required file scaffold. Split modules when a responsibility needs it; avoid one folder or interface per operation. Keep service definitions narrow, with graph edges and input wiring in composition. Do not introduce capabilities, projections, recovery journals, reservation mechanisms beyond the machine-wide per-OS-user port registry, public sleep APIs, or extra lifecycle states to force this shape. That registry is the one deliberate reservation and the only saved public-port assignment. Application services use Effect `Context.Service` and `Layer.effect`; consumers obtain their dependencies from the Effect context. Each owner receives an isolated orchestrator graph. Commands share the host lifetime alongside the owner. Individual executors, recipes, and process handles remain scoped resources because a stack owns multiple independently identified instances. @@ -75,20 +76,20 @@ Use that separation for both backends: ```mermaid stateDiagram-v2 - [*] --> Stopped: register - Stopped --> Starting: start or authorized wake - Starting --> Running: process launched - Starting --> Stopped: launch failed and cleanup confirmed - Running --> Stopping: stop, proxy idle sleep, destroy, or unexpected exit - Stopping --> Stopped: cleanup confirmed + [*] --> Stopped: register (unarmed) + Stopped --> Starting: demand or explicit start + Starting --> Running: session available + Starting --> Failed: launch failed, cleanup confirmed + Running --> Stopping: unexpected exit, stop, restart or idle sleep + Starting --> Stopping: unexpected exit, stop or restart + Stopping --> Failed: unexpected exit, cleanup confirmed + Stopping --> Stopped: stop, restart or idle sleep, cleanup confirmed + Stopping --> Stopping: cleanup failed, retried on request + Failed --> Starting: demand, behind the breaker + Failed --> Stopped: explicit stop Stopped --> [*]: destroy owned data and registration - note right of Starting - The next lifecycle signal waits. - Preparation already completed. - end note note right of Running - Start transition is complete. Health is observed separately. end note note left of Stopped @@ -97,7 +98,7 @@ stateDiagram-v2 end note ``` -The final marker means the registration no longer exists. `destroy` of a running instance follows stop, then removal, within the admitted operation. If runtime cleanup fails, remain stopping with the current process/exit observation and cleanup error; the next stop/restart retries the remaining cleanup. Do not report a dead process as running or unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it does not add a new stable state. +Each phase after `Stopped` belongs to one generation. The final marker means the registration no longer exists. `destroy` of a running instance stops it, then removes its data under a storage reservation, which it takes once any running storage operation releases it. A generation whose cleanup fails stays stopping, holding its resources; the next stop, restart, start or waiter retries the remaining cleanup. A workload that exits on its own enters `Stopping` at once, so no traffic routes to its dead endpoint while cleanup runs; it counts toward the breaker then, and ends `Failed` once cleanup is confirmed. A dead process is never reported as running, nor unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it adds no stable state. ### Chart 2: health does not hold the lifecycle gate @@ -112,13 +113,13 @@ flowchart LR Stopping -->|cleanup confirmed| Stopped["Lifecycle: stopped"] ``` -Only running + healthy satisfies a readiness wait. A health failure records an error and leaves the process running; it neither locks the lifecycle nor automatically stops the process. +Only running + healthy satisfies a readiness wait. A health failure records an error and leaves the process running; it fails the explicit waits that needed it and is re-checked while traffic still waits on it. **For one service instance, running means its process has launched. It does not mean the application is ready.** Every instance exposes its own observation. Separately observe health: starting, healthy, or unhealthy. A failed launch records its error; unsuccessful cleanup must not be reported as a successful stop. Destroy removes the registration only after stopped resources and owned data are safely removed. -Example: PostgreSQL launches, so its start transition finishes in running. Its initialization and health are still pending. A stop request can now run the ordinary stopping transition; there is no start transition to interrupt. A request arriving during actual launch waits for launch to settle first. +Example: PostgreSQL launches, so its generation is running. Its initialization and health are still pending. A stop now interrupts only the readiness check and cleans the session. A stop arriving while the runtime is being acquired waits for that acquisition, then cleans what it produced; a stop arriving during preparation abandons it. -Dependency waits and health waits never hold a lifecycle transition open. Artifact preparation completes before the start transition is admitted, as a shared cache operation whose callers may cancel their own wait. Downloads and image pulls cannot run inside the serialized launch transition. Bounded native pre-launch steps such as initdb remain part of launch. Actual launch and stop work have bounded success/failure paths. A timeout is the current transition settling through its own failure handling, not another command pre-empting it. +Dependency waits and health waits never hold the owner gate or the execution lock. Preparation (artifact downloads and image pulls) runs at the start of each launch, outside the execution lock, and is idempotent. Bounded native pre-launch steps such as initdb remain part of runtime acquisition. Actual launch and stop work have bounded success and failure paths. ## 2. Ownership, instances and composition @@ -227,88 +228,94 @@ flowchart LR The `service: "database"` discriminator selects a service definition and its configuration type; it does not name a group or identify a unique instance. Two databases have the same definition kind and different instance IDs. Operations and dependency edges address instance IDs. Short-lived initialization helpers remain implementation details of an operation; they do not justify a hidden hierarchy of long-running services. -## 3. One lifecycle implementation, one graph planner +## 3. One lifecycle authority per owner -An instance has one serial transition executor. Requests wait until the current transition settles, then revalidate against the resulting state. An already-satisfied request does no runtime work; an explicit stop still updates intent and disables wake as described below. A waiting caller can cancel its wait; an admitted transition belongs to the StackHost and finishes independently of that caller. +Each owner holds one pure reducer, `Lifecycle.ts`: `reduce(state, event, now)` returns the next state and the commands to execute. It owns every service's intent (eager, lazy or stopped), phase (stopped, starting, running, stopping or failed, each tagged with its generation), work leases, waiters, idle epoch, crash-loop breaker, storage reservation and pending destroy. Every lifecycle writer is an event: explicit start, arm, stop, restart and destroy; traffic admission and release; storage reservation; graph updates; and execution outcomes. No other component holds lifecycle state, so status, admission, storage eligibility and retries all read the same record. -### Interruption and command concurrency +### The owner gate -An executing instance operation cannot be interrupted or replaced by another command. Start, stop, restart, destroy, export and restore use the same serial execution gate. Internal proxy-triggered sleep also uses that gate. Incoming commands wait, revalidate against the resulting state, then execute. An instance's restart keeps that gate across stop and launch; readiness observation happens afterward. There is no priority-stop path or ownership handoff. +`Orchestrator.ts` applies events under one gate per owner. One step reduces a batch of events, starts the resulting executions and timers, publishes the new state, then resolves the waiters the step admitted or failed. Executions and timers run outside the gate and report back as events. An event tagged with a generation, idle epoch or breaker cooldown that is no longer current changes nothing, so a late outcome cannot affect a replacement. -```text -start executing → stop requested → stop waits -start settles → stop executes → stopped -``` +| Command | Executed as | +| ----------------------------------- | -------------------------------------------------------------------- | +| `Launch`, `Stop`, `Reprobe` | A fiber in the owner scope driving that service's execution | +| `ArmIdleTimer`, `ArmCooldownTimer` | A timer that reports `IdleElapsed` or `CooldownElapsed` | +| `AdmitConnection`, `FailConnection` | Completing that waiter, after the state it was decided on is visible | +| `RequestRejected` | The requesting caller's failure | -Cancellation has a limited meaning at each boundary: +Explicit stop and restart wait on the reducer state itself: a stop completes when its generation leaves `stopping`, and fails while that generation's cleanup is pending. -| Caller activity | Effect of cancellation | -| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| Preparing artifacts or waiting for execution admission | Abandon this request before its lifecycle operation begins; shared preparation may continue for other callers | -| Waiting for an executing instance operation | Stop waiting; the StackHost finishes the operation and its cleanup | -| Waiting for readiness or following logs | End the observation without changing lifecycle | -| Running an attached command invocation | Terminate and clean up that invocation | +### The execution boundary -A service is stopped by an explicit stop command, not by a disconnected caller. Launch completes at running, so stop during health-starting uses the ordinary stop operation. If the launch ends while a caller awaits readiness, that observation fails rather than attaching to a later launch. +`Service.ts` executes one generation at a time and owns its resources from before preparation until cleanup is confirmed. A launch prepares, acquires the runtime under the service's execution lock, reports the live session, then runs the readiness check. A stop fences its generation under the same lock, so a launch of that generation that has not acquired its runtime never will; it then interrupts the attempt while it prepares or checks readiness, and cleans only that generation's session, never a newer one. Every outcome is reported: `SessionAvailable`, `LaunchSucceeded`, `ReadinessLost`/`ReadinessRecovered`, `LaunchFailed`, `SessionLost` as soon as a runtime exits on its own, and `Exited` once cleanup is confirmed, or `StopFailed` while resources are still held. -A pending orchestration plan is different from an executing transition: stopping REST while its start request waits for database readiness must prevent the old request from later launching REST. The intent revision described below handles this one stale-request check. It does not interrupt an admitted operation. +A generation whose cleanup failed stays `stopping` with its cleanup failure recorded. Nothing launches and no storage operation is admitted while it holds resources, and its waiters fail with the cleanup error. The next explicit stop, restart or start, or the next waiter, retries only the remaining cleanup. -Independent instances execute concurrently. Admission and dependency validation use short coordination sections; downloads, readiness probes, process operations and snapshots do not hold a global execution lock. Each operation settles through success or its own timeout/failure handling and cleanup. Process crashes and owner loss remain possible; report unavailable observations or cleanup errors rather than claiming success. Crash recovery is outside this design. This ownership rule does not require one blanket uninterruptible region around all Effect code. +### Waits and cancellation -### Chart 3: dependency awareness lives in the orchestrator +| Caller activity | Effect of cancellation | +| --------------------------------------- | ---------------------------------------------------------------------- | +| Waiting for the owner gate | Leave without dispatching anything | +| Waiting for traffic admission | Withdraw the waiter; a lease already granted is released | +| Waiting for an explicit operation | Withdraw the wait; the admitted execution continues in the owner scope | +| Waiting for readiness or following logs | End the observation without changing lifecycle | +| Running an attached command invocation | Terminate and clean up that invocation | -Here stack composition has wired the primary database’s connection URL into REST configuration and declared a readiness dependency. The caller composes `start` and `ready`, as a CLI command that needs a usable service would. The service executors never call each other. With an external database URL, the managed database branch is absent. +Each waiter has a kind. A `traffic` waiter carries a lease and holds through a breaker cooldown. An `explicit` waiter (start, ready, restart, composition start) opens no lease and fails fast with its target's or a prerequisite's outcome when that fails, loses readiness or fails its cleanup; it waits without a budget or cap. The reducer holds no clocks for waiters. The orchestrator enforces the wake budget, 120 seconds by default, with one timeout around the whole traffic wait, from the client's arrival through the gate wait and the wait for readiness. On expiry it withdraws the waiter, which releases any lease already granted, and fails only that caller with the stage it was blocked on; the shared launch continues. At most 256 traffic waiters queue per service; beyond that, admission fails at once. -```mermaid -sequenceDiagram - participant Caller - participant O as Orchestrator - participant DB as Database executor - participant REST as REST executor - - Caller->>O: Start REST and wait for readiness - O->>O: Read composition edge and input mapping - O->>DB: Start - DB-->>O: Running; health still starting - Note over DB: Launch transition has finished - O->>DB: Await readiness of this launch - Note over O,DB: Observation only; lifecycle remains available - DB-->>O: Healthy - O->>O: Revalidate target intent and graph - O->>REST: Start with ordinary database URL input - REST-->>O: Running; health still starting - O->>REST: Await readiness of this launch - REST-->>O: Healthy - O-->>Caller: REST ready -``` +A service is stopped by an explicit stop, never by a disconnected caller. + +### Demand, admission and readiness + +A service has demand while it holds leases or waiters, after an explicit start, while it is eager and has not failed, or while any transitive dependent has demand or is starting, running or stopping. Demand launches a stopped or failed service whose intent is not stopped and whose prerequisites are ready. A failed eager service relaunches only on new demand, so a broken launch is not retried in a loop. + +Admission requires the target and its whole prerequisite closure to be ready. An inspector acquisition needs only a live target session. A readiness failure is not terminal: the session keeps running, and the waiters that awaited that readiness itself, or explicitly through a dependent, fail with its error. While any other waiter is still blocked on it, the reducer keeps re-checking that generation, one check at a time and one second apart after a failed re-check, until it recovers, its generation ends or no waiter needs it; the waiters' budgets bound how long that lasts. Losing a prerequisite never replaces a healthy dependent; a crashed prerequisite with running dependents relaunches because they keep demand on it. -If the prerequisite stops or readiness fails, the wait fails and REST is not launched. The wait does not reserve a long-running start transition. Deliberate stop still goes through the orchestrator's graph checks. +A destroy that holds its storage reservation owns the service: removing it from the graph succeeds even while work leases are outstanding, and a lease released afterwards is ignored. -The orchestrator performs a few graph operations: +### Crash-loop breaker -| Operation | Rule | -| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Start | Start declared prerequisite instances; await their health; launch the selected instance | -| Stop | Admit only when every dependent is stopped with wake disabled, regardless of batch selection; composition stops dependents first | -| Restart | Individual restart follows the same stop check, then launches; composition restart stops in reverse dependency order, then starts dependency-gated members concurrently | -| Sleep | Require proxy inactivity and no running/starting dependent | -| Destroy | Reject while any registered dependent still references the instance; composition/namespace destruction removes dependents first, then performs ordinary stop and owned-data removal | +Each launch failure or unrequested exit counts once per generation. Three failures, each within five minutes of the previous one, open the breaker for a fixed 30 seconds; a failure after a longer quiet spell starts a new count. While open, admission fails at once naming the last cause; the cooldown's end re-admits one shared recovery attempt, and a further failure reopens it for the same 30 seconds. An explicit start, restart or composition start resets the breaker. -For stop/restart, a running, starting, stopping or wake-armed dependent blocks prerequisite shutdown until that dependent has stopped and settled. Batch membership never bypasses this check. For sleep, a dependent that is already sleeping does not prevent prerequisite sleep. These are admission checks under the same short graph coordination boundary, not long-lived reservations. +### Chart 3: dependency awareness lives in the reducer -Validate composition references, input/output types and cycles when registering or changing the graph. Wiring uses only a source instance ID and named output mapped to a target instance ID and named configuration input. The service catalog declares those input/output schemas. There is no expression language, arbitrary callback or general JSON-path evaluator. Resolve bindings into ordinary configuration before handing it to the service. Validate each service’s configuration separately from that graph. Bind references to immutable instance IDs, not names. Derive ordering from this one graph; do not maintain separate capability and instance lifecycle graphs. +Here stack composition has wired the primary database's connection URL into REST configuration and declared a readiness dependency. The service executions never call each other. -**Readiness waiting is observation, not a transition.** If PostgreSQL stops while REST is waiting for its health, the wait fails and REST does not launch. It does not restart PostgreSQL or continue using a stale ready result. Readiness is associated with the exact runtime launch. +```mermaid +sequenceDiagram + participant Caller + participant O as Owner gate and reducer + participant DB as Database execution + participant REST as REST execution + + Caller->>O: StartRequested(REST) and a readiness wait + O->>DB: Launch(generation 1) + Note over O: REST has demand but its prerequisite is not ready + DB-->>O: SessionAvailable, then LaunchSucceeded + O->>REST: Launch(generation 1) with the resolved database URL + REST-->>O: SessionAvailable, then LaunchSucceeded + O-->>Caller: Waiter admitted +``` + +If the database fails, loses readiness or fails its cleanup, the explicit wait fails and REST is not launched. -Before launching a dependent, atomically check its prerequisites, verify its own intent revision is unchanged since the request began, and mark its transition starting. Explicit stop advances that revision and disables wake even if no process exists yet; an older request still preparing artifacts or waiting for dependencies cannot launch afterward. This revision fences unadmitted work; it never interrupts an executing transition. This closes the race between dependency validation and another command stopping the prerequisite. Do not reserve an entire composition or acquire a set of batch locks. Each instance operation waits for its own gate and rechecks the current graph and observations on admission. If another caller starts a dependent between two batch steps, stopping the prerequisite is rejected and the batch reports partial results. A stale start plan cannot resume after another operation stops or replaces its prerequisites. +The reducer applies a few graph rules: -Independent instances still execute concurrently. A shadow startup or snapshot must not block Functions restart. Serialize only short in-memory admission decisions and durable writes; never hold the graph decision boundary while downloading, starting processes, probing health, archiving data or writing files. Gateway traffic on unrelated ready instances must not wait behind a state write. +| Operation | Rule | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Start | Arm the target and its prerequisites; demand launches each prerequisite, then the target once they are ready | +| Stop | Reject while any transitive dependent has demand, is up or winding down, or is still armed; composition stops dependents first | +| Restart | The same dependent check, then stop and relaunch; composition restart stops in reverse dependency order, then performs composition start | +| Sleep | An armed lazy service sleeps after its idle timeout once it is ready and has no demand, including demand from its dependents | +| Destroy | Reject while a dependency edge still references the instance; then the ordinary stop, a storage reservation for data removal taken once any storage operation releases it, and the graph update removing it. A pending destroy refuses new starts, restarts and storage operations; any other exit gives it up | -Composition and namespace methods select instances and call these same operations. Default composition start first assigns/reuses and binds every configured public listener needed by its selected members, including lazy members, and registers their routes. It then starts one dependency-gated worker per selected instance, including prerequisites outside the configured members; independent branches overlap, and outcomes remain in topological order. Eager members launch only after all immediate prerequisites are healthy. Lazy routes arm after their prerequisites settle, including when a prerequisite fails; an eager descendant with a failed prerequisite is reported as blocked and is not launched. It does not select unrelated standalone instances. Successful start returns member observations with their public endpoints; credentials reads render usable connection strings from these assignments without starting lazy backends. It awaits readiness for every launched eager member, including leaves. A failed launch or readiness check makes the operation return an error with per-instance outcomes; a running-but-unhealthy instance remains running, and already completed steps are not rolled back. Armed lazy members need not launch unless required by an eager member. +Validate composition references, input/output types and cycles when registering or changing the graph. Wiring uses only a source instance ID and named output mapped to a target instance ID and named configuration input. The service catalog declares those input/output schemas. There is no expression language, arbitrary callback or general JSON-path evaluator. Resolve bindings into ordinary configuration at launch. Bind references to immutable instance IDs, not names. Derive ordering from this one graph. A graph change requires every affected instance to be stopped and unarmed, and the reducer's verdict is checked before the change is persisted. -After binding the shared API listener, default composition setup uses the endpoint renderer to populate ordinary configuration values: Studio receives `apiUrl` and `publicApiUrl`, Auth receives `externalApiUrl`, and Functions receives `apiUrl`. When the database has a configured SQL endpoint, Functions also receives its rendered `databaseUrl`; this saved value is refreshed when callers recompose the composition, and does not create a database readiness dependency for Function execution. These values are saved in each instance's configuration alongside the saved listener assignment; individual starts and normal host reopening consume that configuration. This is concrete default setup code, not declarative input wiring or a public listener-binding API. The endpoint renderer owns the host/runtime reachability rules; custom configurations supply plain URLs through the same inputs. URL values do not imply graph edges. +Composition start first assigns or reuses and binds every configured public listener of its selected members, including lazy members, and registers their routes. One worker per member then waits for its prerequisites' outcomes: a lazy member is armed (also after a prerequisite failed), and an eager member is started and awaited for readiness, or reported as blocked when a prerequisite failed. Prerequisites outside the configured members run as eager standalone instances. It does not select unrelated standalone instances. Successful start returns member observations with their public endpoints; credentials reads render usable connection strings from these assignments without starting lazy backends. A failed launch or readiness check makes the operation return an error with per-instance outcomes; a running but unhealthy instance remains running, and completed steps are not rolled back. -Composition restart is two passes: stop the selected instances in reverse dependency order, then perform dependency-gated composition start according to eager/lazy policy. It is not a loop of individual `restart()` calls. If the stop pass fails, report partial results without beginning the start pass. Cancelling composition interrupts its waits and work not yet admitted; already admitted instance operations continue to settle in their owner scope, and launched instances remain until an explicit stop or owner close. Readiness does not hold an instance gate or a composition-wide reservation. Explicit empty selection is a no-op; there is no generic rollback engine. +After binding the shared API listener, default composition setup uses the endpoint renderer to populate ordinary configuration values: Studio receives `apiUrl` and `publicApiUrl`, Auth receives `externalApiUrl`, and Functions receives `apiUrl`. When the database has a configured SQL endpoint, Functions also receives its rendered `databaseUrl`; this saved value is refreshed when callers recompose the composition, and does not create a database readiness dependency for Function execution. These values are saved in each instance's configuration; individual starts and normal host reopening consume that configuration. This is concrete default setup code, not declarative input wiring or a public listener-binding API. The endpoint renderer owns the host/runtime reachability rules; custom configurations supply plain URLs through the same inputs. URL values do not imply graph edges. + +Composition restart is two passes: stop the selected instances in reverse dependency order, then perform composition start. If the stop pass fails, report partial results without beginning the start pass. Cancelling composition withdraws its waits; admitted executions continue in the owner scope, and launched instances remain until an explicit stop or owner close. Explicit empty selection is a no-op; there is no generic rollback engine. ## 4. Recipes own runtime details @@ -320,9 +327,9 @@ A successful launch returns an owned runtime handle with readiness observation a The database keeps only the initialization information needed to reuse its data on a normal stop/start. Mark initialization complete only after it succeeds. If an ordinary stop interrupts application initialization, the recipe must either support a safe retry or report that initialization is incomplete; do not add a generic persisted operation journal or recovery workflow. -Unexpected process exit is observed by the living host. Immediately disable forwarding and automatic wake for that instance and record its exit result. The executor uses the ordinary serialized stop/remove cleanup for the exact owned runtime handle, reaching stopped only after cleanup succeeds. Stop cannot be a no-op while runtime resources remain; start/restart must finish that cleanup before launching a replacement. If exit occurs during an executing transition, that transition settles and the same gate orders any remaining cleanup; no transition is pre-empted. Old-handle events cannot affect a replacement. This adds no Exited lifecycle state and no cleanup scanner: the host already owns the handle and its exit observation. +Unexpected process exit is observed by the living host. The execution records the exit result, cleans the exact owned session under its execution lock, and reports the exit once cleanup is confirmed, or a cleanup failure while resources remain. Nothing launches a replacement while resources are held. Old-handle events cannot affect a replacement. This adds no cleanup scanner: the host already owns the handle and its exit observation. -There is **no automatic restart, continuous dependency shutdown cascade, or self-healing controller**. A crashed service stays stopped until explicit start/restart. A wake operation may use a running prerequisite or launch a stopped prerequisite whose wake route is armed; an explicitly stopped or unexpectedly exited prerequisite makes wake fail. Only an explicit start may launch those prerequisites. Explicit composition startup can arm initially lazy instances, and sleep retains that permission; unexpected exit removes it. Existing dependents may observe connection failures, as they can with Compose. Deliberate lifecycle changes still obey graph checks. +A crashed service stays armed. Demand relaunches it behind the crash-loop breaker; a crashed prerequisite with running dependents relaunches because they keep demand on it. There is no continuous dependency shutdown cascade: existing dependents may observe connection failures until it is back, as they can with Compose. An explicitly stopped prerequisite makes wake fail. Deliberate lifecycle changes still obey graph checks. ## 5. Sleep stays simple @@ -332,9 +339,9 @@ There is **no automatic restart, continuous dependency shutdown cascade, or self - The idle timer and wake route belong to one instance. Sleep stops that instance after the ordinary dependency checks, retains its data and keeps its wake route armed. It does not implicitly stop companion instances. Instances without a supported public wake route remain running until explicitly stopped; no group sleep mechanism is introduced. - The next request starts the instance through the orchestrator, ensuring its declared prerequisites are ready, then waits for its own health before forwarding. - Explicit stop disables wake. Traffic never reverses it. -- If a prerequisite's sleep is blocked by a running dependent, recheck its existing idle eligibility when that dependent sleeps/stops. No new activity heuristic is needed. +- A prerequisite stays awake while any dependent has demand or is starting, running or stopping. Once the dependent's stop is confirmed, the prerequisite idles on its own timeout. -Sleep is therefore ordinary stop mechanics plus an armed wake route. There is no separate sleep runtime. Activity acquisition and sleep admission must make one atomic in-memory decision so sleep cannot race a request. A request that arrives after sleep was admitted waits for it to finish, then wakes the instance. +Sleep is therefore ordinary stop mechanics plus an armed wake route. There is no separate sleep runtime. The idle timer carries an epoch that any new admission advances, so a request admitted before the reducer commits the sleep keeps the instance running. A request that arrives after the sleep was committed waits for the confirmed stop, then is served by the next generation. ### Chart 4: sleep, stop and destroy have different retained state @@ -351,9 +358,9 @@ flowchart TB Stopped -->|explicit start| Start["Orchestrator starts selected services
Reuses saved public ports"] ``` -Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving saved assignments; the next start rebinds the same ports. Destruction releases only the removed instance's owned assignments, not unrelated services' shared listeners or claims. +Sleep retains the public listener needed to wake. Whole-stack stop closes listeners and releases pinned reservations while preserving automatic ones; the next start rebinds the same ports. Destroying an individual instance deletes only the registry rows it owns, not unrelated services' shared listeners or rows; a stack-wide destroy deletes its rows only after the registration is removed, so a failed destroy keeps the automatic ones (a configured port's row is released when its listener closes) and a failed release leaves them for lazy reclamation. -Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for instances with a supported public wake route and an enabled idle policy. Database and Functions need no automatic idle timer by default. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. +Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for lazy instances with a supported public wake route and an enabled idle policy. The default composition gives every lazy service one, Functions included; the database is eager. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. ## 6. Commands share execution, not service lifecycle @@ -470,8 +477,7 @@ stateDiagram-v2 Serving --> Serving: clients connect or disconnect Serving --> Serving: instances start, stop or sleep Serving --> Draining: host.stop or host.destroy - Draining --> Exited: operations settled and cleanup complete - Draining --> Serving: cleanup failed; report error and retain ownership + Draining --> Exited: cleanup complete, or failed and reported Exited --> [*] ``` @@ -483,9 +489,13 @@ During Starting, acquire the exclusive stack lease, load the instance definition **Session lifetime.** A session stack is registered by its owner under the lease, so it never exists without a live owner except after that owner dies. Its spawner keeps the owner's stdin pipe open for the life of the creating handle. End of input means the creator is gone, whether it closed the handle or its process died: the owner destroys the stack and exits. Only the creating handle starts a session stack's owner; other handles attach. `create({ startOwner: true })` registers a detached stack through its owner the same way; an owner whose startup fails removes the registration it made, so a failed or interrupted first launch leaves no stack behind. -**Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. A stack whose directory, and with it its lease, was deleted is visited the same way when containers labelled with its data root in this state root remain on an engine the root's stacks use. Only those containers are removed, even while an owner of that stack still runs; its data in the root's shared database volume stays, and starting the stack again waits for the visit to end. `findDeleted` selects any stack that is no longer registered and still has such containers, such as one whose directory was deleted, by its full id, so a client can destroy those containers the same way. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. +**Orphan sweep.** Stack-labelled containers and session stacks exist only while their lease is held. After readiness, each owner visits every other stack in its state root in the background, with a bounded time per stack. It skips stacks whose lease is held. For a free lease it takes that lease for the duration of the visit, publishing a sweeper record in `owner.json` so clients wait for the visit instead of mistaking it for a starting owner, removes containers labelled with the stack and its data root, and destroys the stack through the owner's own destroy path when its lifetime is `session`. A container-runtime owner also visits stacks that have no registration but left containers labelled with this state root on its pinned engine, such as when their directory was deleted, under the same lease and unregistered check, and removes only those containers; the stack's data in the shared database volume stays. `findDeleted` selects such a stack by its full id, so a client can destroy those containers on demand. Filtering on the data-root label keeps other state roots untouched. Creating a stack whose identity belongs to a dead session stack reclaims that stack the same way first. The sweep also removes the empty `/` and `data/` directories a destroy leaves behind, once they are over ten minutes old and hold nothing else. + +**Lost ownership.** The orphan sweep above covers a dead owner whose registration survives; the opposite case is a live owner whose registration disappears out from under it (the stack directory or state root is deleted, moved or unmounted, possibly followed by a new start of the same stack). Every owner polls on an interval (30 seconds by default) and treats its ownership as ended when `//state.json` is confirmed missing (ENOENT) or `owner.lock` at its path is no longer the file its lease locked. Any other read error keeps the owner running and logs a warning. When ownership ends, the owner stops what it launched once any stop already running settles, closes its listeners and exits. It does not destroy instance data, sweep labels or write the registry, and it retracts `owner.json` only while the lock file at its path is still its own, so a successor owner's lease is never disturbed. Closing the listeners releases the reservations of configured ports; automatic reservations are kept. A stop already running when the directory is deleted and the same stack restarted can still sweep the successor's containers by label. Leftovers of a stack whose directory is gone are reclaimed by the existing paths: the next container owner's sweep of unregistered labelled stacks, lazy reclamation of reservations whose registration is gone, and `destroy --stack-id`. Docker database data in the shared volume of such a stack is not reclaimed. + +**Engine targets.** A container stack's owner pins the engine target it resolves at startup, so every command it runs reaches the same engine. The target is not saved with the stack: an owner started after the Docker context or `DOCKER_HOST` changed resolves the new target, and containers left on the previous engine are not reached from there; switch back to clean them up. Docker pins `DOCKER_HOST` as `--host`, else the current context. Podman pins `CONTAINER_HOST` as `--url`, else `CONTAINER_CONNECTION` as `--connection`; with neither set, a bare `podman info` decides whether the engine is local or remote. A remote engine (macOS `podman machine`) pins the default system connection by name, and a local one adds no flags. On rootless Podman, containers that run as the caller's uid with writable borrowed bind mounts get `--userns=keep-id` so files stay owned by the caller. Containers reach host listeners through the engine's host alias, `host.docker.internal` or `host.containers.internal`. -**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then proceeds without an owner: it takes the free lease, publishing a sweeper record like the orphan sweep, refuses when any stack data directory cannot be deleted by the current user, removes the host data and the registration with its port claims, and returns the shell commands that remove the stack's containers and engine-volume data once the engine runs. `stop` without a live owner already succeeds without contacting the engine. +**Unreachable engine.** An owner of a container stack fails startup with a `runtime-unavailable` reason when its first container sweep finds the engine CLI missing or its daemon or service not listening; permission, TLS, authentication and timeout failures are ordinary startup errors. `destroy` then fails with that reason, leaving the registration, port reservations and host data untouched; run it again once the engine is reachable. `stop` without a live owner takes the same lease and reclaims the registered stack's leftovers (the orphan-sweep path: containers, plus destruction of a session stack) before succeeding; it fails if another process holds the lease or the cleanup fails, and so needs the engine for a container stack. During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. @@ -493,20 +503,36 @@ Where the platform delivers SIGTERM or SIGINT to the host, treat it as the same During Draining: -1. Close admission to new mutations and proxy wake requests. +1. Close admission to new mutations and to proxy traffic that would wake a service at rest. + Traffic to starting or running services stays admitted. 2. Reject queued work that has not begun. 3. Let executing instance operations settle. -4. Cancel and settle attached commands and stop owned services through their existing operations. +4. Cancel and settle attached commands and stop owned services in reverse dependency order, + with every listener still open (ordered stop, below). 5. For destruction, remove proven-owned data and metadata after shutdown. 6. Send the outcome, close the control endpoint and release ownership. -**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the live host retains ownership for inspection and retry; the stack is not reported stopped. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and saved port assignments. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. +**Ordered stop.** Stop and destroy close no listener up front. Public and internal +dependency traffic share the same listeners, so a dependent's graceful stop still reaches its +prerequisite through the stack proxy while it is running. Services stop in reverse dependency order, and each listener closes once its own +service has stopped, through the ordinary scope teardown. Traffic that would wake a service at rest +is refused for the whole shutdown. Accepted, documented tradeoff: the proxy offers no completion +window, so an in-flight request survives only as far as its service's own graceful stop carries it, +and a slow client can lose a response. + +**Draining is one-way.** The host holds the single serving/draining fact; the owner reads it for +admission. Once a stop or destroy begins, the +owner never serves again. A failed stop or destroy reports its error, the stack stays registered +with uncertain state, and the owner exits. The next stop or destroy finds no owner and reclaims +the stack under its lease. + +**Successful whole-stack shutdown.** `stack.stop()` reports success only after admitted work settles, every owned live service and command workload has stopped (including native processes, descendants and containers labeled with this stack and data root), stack listeners close with their services, the shutdown request is acknowledged, and the detached host's exit is confirmed. If workload cleanup cannot be confirmed, stop fails and the host exits without resuming service; the stack is not reported stopped and remains registered for the next stop or destroy to reclaim. If cleanup succeeds but host exit cannot be confirmed, stop also fails, though the host may already have exited. A delivered SIGTERM or SIGINT follows this cleanup path. Stop preserves stack definitions, service data, caches and port reservations. `destroy` follows the same live-workload cleanup, then removes only proven-owned persistent data. The client captures the live owner PID from the validated identity endpoint or readiness handshake, completes the shutdown request, then performs bounded process-existence checks. An absent PID confirms exit; a permission-denied probe remains inconclusive until the deadline. Failure to confirm exit is a `shutdown-exit` error carrying the PID in its message, even when workload cleanup has already succeeded. This does not require persisted PID records or forceful termination. Caller cancellation ends its wait without cancelling admitted owner cleanup. Callers must not start or restart the same stack concurrently with whole-stack shutdown. In particular, replacing an owner between identity lookup and the shutdown request is outside this guarantee. Parallel stacks with separate identities remain independent. Client disposal and Effect scope closure do not implicitly stop a detached stack; disposable fixtures register explicit destruction or use a session stack, which closing its creating handle destroys. -Returning to Serving after cleanup failure does not undo completed cleanup. Unexpected host death does not resume interrupted operations or restore live service state. Native launchers stop their process groups when the dead host's pipe closes. The next host startup for that stack sweeps its containers without removing volumes or saved definitions, and any host start in the same state root sweeps them in the background, also destroying the stack if it is a session stack. A forced host termination does not guarantee immediate container cleanup. A lost control response is reported as uncertain; do not blindly retry a mutation. +A cleanup failure does not undo completed cleanup. Unexpected host death does not resume interrupted operations or restore live service state. Native launchers stop their process groups when the dead host's pipe closes. The next host startup for that stack sweeps its containers without removing volumes or saved definitions, and any host start in the same state root sweeps them in the background, also destroying the stack if it is a session stack. A forced host termination does not guarantee immediate container cleanup. A lost control response is reported as uncertain; do not blindly retry a mutation. ### Request lifetime is separate from execution lifetime @@ -516,16 +542,16 @@ Once an instance operation starts executing, the host owns it until settlement. sequenceDiagram participant Client participant RPC as RPC handler - participant Executor as Host-owned executor + participant Executor as Owner gate and executions Client->>RPC: service.start(id) - RPC->>Executor: Submit start - Executor->>Executor: Admit and execute launch + RPC->>Executor: StartRequested and a readiness wait + Executor->>Executor: Reduce and launch in the owner scope Client--xRPC: Client disconnects - Note over RPC: Caller stops waiting - Note over Executor: Executing operation continues - Executor->>Executor: Record running or failure + Note over RPC: Caller's wait is withdrawn + Note over Executor: Launch continues + Executor->>Executor: Reduce the reported outcome Client->>RPC: Reconnect; service.status(id) - RPC->>Executor: Read observation + RPC->>Executor: Read status Executor-->>RPC: Current state RPC-->>Client: Current state ``` @@ -536,16 +562,17 @@ Attached commands have a separate contract. `runCommand` streams `started(jobId) ## 8. Keep safety infrastructure at its boundary -### Instance observations are the runtime source of truth +### The reducer state is the lifecycle source of truth -Each instance owns one observable runtime state. Its executor updates lifecycle and current operation; runtime observations update health, exits and errors. `service.status`, `service.followStatus`, dependency readiness checks and CLI presentation consume those same observations. Endpoint availability comes from the component that owns the listener. The StackHost transports these observations without maintaining another lifecycle copy. +Lifecycle and health come from the owner's reducer state; the service execution adds only execution facts: configuration, the last error, a pending cleanup error, the exit result and the current operation. `service.status` and `service.followStatus` join the two, and admission, dependency readiness and CLI presentation read the same state. Endpoint availability comes from the component that owns the listener. The StackHost transports these observations without maintaining another lifecycle copy. ```mermaid flowchart LR - Executor["Executor transitions"] --> Instance["Instance observation"] - Runtime["Health and process events"] --> Instance - Instance --> RPC["status and followStatus"] - Instance --> Dependencies["Orchestrator readiness checks"] + Outcomes["Execution outcomes and requests"] --> Reducer["Reducer state"] + Execution["Execution facts"] --> Status["Instance status"] + Reducer --> Status + Reducer --> Admission["Admission and readiness"] + Status --> RPC["status and followStatus"] RPC --> CLI["CLI display"] ``` @@ -553,19 +580,19 @@ A composition returns the observations of its members. Remove the separate capab ### Persist only what normal stop/start needs -Retain stack/instance identity, service configuration and selected versions, composition membership/wiring, credentials, public port assignments and the service-specific information needed to reopen the data. Derive stack and instance data locations from the state root and saved identities rather than persisting duplicate paths. Keep the data itself across stop. Shared port claims remain necessary for stickiness across parallel stacks. The active host's exclusive lease prevents competing owners; it is not a crash-recovery subsystem. +Retain stack/instance identity, service configuration and selected versions, composition membership/wiring, credentials and the service-specific information needed to reopen the data. Derive stack and instance data locations from the state root and saved identities rather than persisting duplicate paths. Keep the data itself across stop. Public port assignments are not part of the stack document: the machine-wide port registry (`ports.sqlite`) holds the only saved assignment and keeps ports sticky across parallel stacks. The active host's exclusive lease prevents competing owners; it is not a crash-recovery subsystem. Do not persist health, runtime lifecycle projections, transition progress, command queues, operation results or recovery checkpoints. A disconnected caller can reconnect to the same living host; resuming after host death is not part of the contract. Incomplete operations and leftover resources after a crash may require manual cleanup, without automatic deletion of valuable data. Some foundations are not optional, but they do not need to dominate the service model: -| Foundation | Keep | -| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| Identity | Stack and instance IDs; deterministic names and paths for their resources | -| Ownership | One detached StackHost and exclusive lease; client handles do not own service lifetime | -| Ports | Persist public port assignments across stop/start; reuse the same ports, including initially automatic assignments; never silently relocate | -| Data | Stop retains data; destroy removes only proven owned data | -| Snapshots | DB-specific keyed save, compatible empty-target restore, complete entry publication and credential reconciliation | +| Foundation | Keep | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------- | +| Identity | Stack and instance IDs; deterministic names and paths for their resources | +| Ownership | One detached StackHost and exclusive lease; client handles do not own service lifetime | +| Ports | Persist automatic port assignments across stop/start; a pinned port is reserved only while listening; never silently relocate | +| Data | Stop retains data; destroy removes only proven owned data | +| Snapshots | DB-specific keyed save, compatible empty-target restore, complete entry publication and credential reconciliation | ### Port allocation and endpoint ownership @@ -587,7 +614,7 @@ A shared listener is a networking resource, not a capability or service group. C A lazy database already has a bound SQL proxy listener and a concrete connection URL; lazy REST already has its path on the bound shared API listener. Their backend processes may still be absent. The first request launches the needed backends, waits for readiness and forwards through the already published endpoint. Backend address allocation can wait until that launch; public port assignment cannot. If a required listener cannot bind, startup returns an error with partial outcomes rather than reporting successful startup with unusable URLs. -A host-wide allocation registry coordinates claims across parallel stacks, including stopped instances and stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Persist the assignment before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. Fixed requests obey the same ownership and binding checks. +A host-wide allocation registry coordinates claims across parallel stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Reserve the assignment in the registry before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. An automatic assignment stays reserved while its stack is stopped. A configured (pinned) port is reserved only while its listener is open, so a stopped stack blocks no other project; the endpoint's address always comes from the configured number, never from the registry. A pinned port inside the native backend range is rejected. A conflict names the port and, for a registry holder, its project and the commands that free it. ```mermaid flowchart LR @@ -598,18 +625,18 @@ flowchart LR Runtime["Native or container runtime"] -.-> Backend ``` -| Operation | Dedicated instance listener | Route on a shared listener | -| ---------------------- | --------------------------------------------------------------- | ---------------------------------------------------------- | -| Start | Bind saved port and connect backend | Enable this route and bind the saved shared port if needed | -| Restart | Retain assignment and update target | Retain shared assignment and update this route's target | -| Sleep | Keep listener for wake | Keep route armed for wake | -| Stop | Close listener; retain claim | Disable this route; leave other routes alone | -| Instance destroy | Remove endpoint and release its claim | Remove this route; retain the stack-owned listener claim | -| Namespace stop/destroy | Close all listeners; stop retains claims, destroy releases them | Same rule | +| Operation | Dedicated instance listener | Route on a shared listener | +| ---------------------- | ------------------------------------------------------------------------- | ---------------------------------------------------------- | +| Start | Bind saved port and connect backend | Enable this route and bind the saved shared port if needed | +| Restart | Retain assignment and update target | Retain shared assignment and update this route's target | +| Sleep | Keep listener for wake | Keep route armed for wake | +| Stop | Close listener; retain an automatic claim | Disable this route; leave other routes alone | +| Instance destroy | Remove endpoint and release its claim | Remove this route; retain the stack-owned listener claim | +| Namespace stop/destroy | Close all listeners; stop retains automatic claims, destroy releases them | Same rule | -A shared listener stays bound while any route is running or armed. When no route needs it, its socket may close while its stack-owned claim remains. Removing the final route does not release that claim; namespace destruction or explicit listener reconfiguration does. Sleeping routes still need the listener. +A shared listener stays bound while any route is running or armed. When no route needs it, its socket closes; an automatic claim remains, a pinned one is released with the socket. Removing the final route does not release an automatic claim; namespace destruction or explicit listener reconfiguration does. Sleeping routes still need the listener. -**Public port stickiness is a retained contract.** Sleep/wake, stop/start, restart and host reopening preserve every established assignment, including the shared API port. Other managed stacks cannot take retained claims. An unrelated process may occupy a saved port while its socket is closed; the next activation reports a conflict and preserves the assignment, never silently relocating it. A destroyed and recreated instance has a new identity and receives a new dedicated allocation. +**Public port stickiness is a retained contract for automatic assignments.** Sleep/wake, stop/start, restart and host reopening preserve every established automatic assignment, including the shared API port. Automatic allocation never takes another stack's retained claim. A pinned port is the configuration's own number, so it needs no retention; its registry row exists only while the listener is open, and a pinned request reclaims a row whose stack has no live owner (a free owner lease), such as after a crash or reboot. If that row was a stopped stack's automatic assignment, that stack receives a new automatic port on its next start. An unrelated process may occupy a saved port while its socket is closed; the next activation reports a conflict and preserves the assignment, never silently relocating it. A destroyed and recreated instance has a new identity and receives a new dedicated allocation. Standalone and composed instances use the same allocator. Two shadow databases have separate IDs, data and dedicated listeners. Composition membership does not change listener ownership. The runtime reports the backend address; the proxy updates its target without altering the public assignment. @@ -654,8 +681,8 @@ The owner is the only subscriber of each instance's output and persists it as re - **Retention:** the oldest closed segments except the newest are deleted while an instance holds more than 10 MiB or 64 segments. A failed deletion is retried at the next rotation, with one warning until a pass succeeds. -- **Launch ids:** keep increasing across owner starts, since each saved instance records its latest - one; an instance saved without one continues after the launch ids at the end of its newest segment. +- **Launch ids:** keep increasing across owner starts: an instance continues after the launch id at + the end of its newest segment. - **Lines:** state is kept per launch, process and stream. A late partial line of an ended launch waits for its newline for two seconds of quiet, or until the store closes. - **Loss:** chunk sequence numbers span a launch's streams and processes, so any dropped chunk @@ -700,7 +727,7 @@ launch changes. launch its latest post went to, and when that post ends at the latest). A missing or unreadable cursor starts at the oldest retained segment, and a cursor in a deleted segment resumes at the next retained one. A failed log read restarts the instance from its cursor after a capped - backoff, and attaching an instance removes temporary cursor writes a dead owner left behind. + backoff. - **Why delivery is confirmed:** Logflare answers a post once it queued the events and stores them later in per-source batches; a batch holding an id already stored is dropped whole. - **Stored-id check:** the owner reads which ids `_analytics.log_events_` holds, resolving the @@ -746,6 +773,9 @@ process death also releases ownership. The file stays in place and is accessed o SQLite. No tables, state records, or WAL are created there. Saved stack data remains in JSON; the lock does not make multi-file operations transactional or recover interrupted operations. This uses the built-in SQLite API available in the pinned Bun runtime and modern Node.js. +Every lock connection in a process goes through that one SQLite library, which tracks locks per +file across connections, so a second connection in the same process is excluded exactly as another +process would be, and closing one connection never releases a lock another still holds. #### Stop and destroy @@ -781,8 +811,8 @@ Keep the contract narrow: - Save requires confirmed stopped, initialized data. The caller explicitly stops first; snapshotting does not secretly stop dependents or restart services. A save publishes only a complete entry and replaces the previous entry for the same key. - Restore requires a confirmed stopped instance with empty data. Validate format, artifact/runtime compatibility and initialization profile before installing restored data. A missing key returns `false`; a compatible published entry returns `true`; reject a nonempty target rather than overwriting it. -- Both operations occupy the instance's existing serial operation gate and leave lifecycle stopped. Queued start, destroy or another storage operation waits for settlement and revalidates. No new lifecycle states are necessary; the observable pending operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. -- Native snapshots copy or clone the host data; container snapshots copy database data through a managed volume and helper. Docker data normally lives in a managed volume, while existing host data can be retained through the host-backed fallback. The host storage marker detects a missing or mismatched Docker volume; deleting that volume loses its database data. +- Both operations hold the instance's storage reservation in the lifecycle reducer and leave it stopped. While the reservation is held, a start keeps the instance stopped until it is released and another storage operation is rejected at once. A destroy waits for the storage operation to settle, then takes the reservation; once a destroy is pending, new storage operations are refused, so it cannot be starved. No new lifecycle states are necessary; the observable current operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. +- Native snapshots copy or clone the host data; container snapshots copy database data through a managed volume and helper. Docker data normally lives in a managed volume, while existing host data can be retained through the host-backed fallback; Podman always uses host-backed data. The host storage marker detects a missing or mismatched Docker volume; deleting that volume loses its database data. - Restore transfers compatible database contents, not the source instance's identity, public port claims or composition membership. The target retains its own data location and configuration, with database-specific credentials reconciled before readiness. Cache snapshots survive destruction of the source instance because their managed storage is separate; instance snapshots restore only into their own instance. These are physical database snapshots for the cache use case. A `pg_dump` invocation remains an ordinary client command for logical exports. CLI code owns cache keys, migrations and the decision to fall back to rebuilding a baseline; managed storage owns publication and retention. The snapshot API does not acquire CLI cache policy. @@ -791,7 +821,7 @@ These are physical database snapshots for the cache use case. A `pg_dump` invoca `DatabaseInstance.resetData` removes the selected database instance's owned data and initialization metadata while retaining its registration, configuration, composition bindings, and public port assignments. It is database-specific, alongside snapshot save and restore; other service types do not expose a reset operation. -The caller must stop the database with wake disabled first. Reset runs through the same serialized storage-operation gate as snapshots and leaves the database stopped. Deletion uses the database's ownership checks and runtime-specific filesystem handling. The next normal start initializes a fresh PostgreSQL cluster using the retained configuration. Reset does not apply project migrations or seeds, stop other services, or resume the composition; those decisions belong to the CLI. It adds no lifecycle state or persisted recovery phase. +The caller must stop the database with wake disabled first. Reset holds the same storage reservation as snapshots and leaves the database stopped. Deletion uses the database's ownership checks and runtime-specific filesystem handling. The next normal start initializes a fresh PostgreSQL cluster using the retained configuration. Reset does not apply project migrations or seeds, stop other services, or resume the composition; those decisions belong to the CLI. It adds no lifecycle state or persisted recovery phase. ## 9. What changes for the CLI @@ -809,7 +839,7 @@ Make lifecycle and health explicit in status instead of forcing them into the ol | Process exited | Exit result is visible while ordinary cleanup settles, then stopped; report unexpected exit as failure | | Stop cleanup failed | Runtime cleanup remains stopping with current process/exit observation and error; stop/restart retries cleanup | -Functions serve observes lifecycle and health separately and reports an unhealthy or stopped runtime. DB/storage readiness checks require running and healthy. The current operation remains observable in memory, but shadow finalizers no longer poll it: `destroy()` queues behind an executing snapshot and runs after settlement. Do not persist it for crash recovery. Health observations are attached to the current launch; they cannot make a replacement healthy or failed. +Functions serve observes lifecycle and health separately and reports an unhealthy or stopped runtime. DB/storage readiness checks require running and healthy. The current operation remains observable in memory. `destroy()` issued during a snapshot waits for it to settle, then removes the data. Do not persist it for crash recovery. Health observations are attached to the current launch; they cannot make a replacement healthy or failed. The public surface stays recognizable: create/open/discover, services create/get/list, lifecycle, credentials, status/logs and preparation. Promise stays an outer adapter over Effect. The important API clarification is **launch versus readiness**. diff --git a/packages/stack/README.md b/packages/stack/README.md index 4efc77320a..7b448e1584 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -14,7 +14,7 @@ import { postgres } from "@supabase/stack/commands"; const stack = await create({ projectRoot: process.cwd(), - runtime: "native", // or "docker" / "podman" + runtime: "native", // or "docker" or "podman" stateRoot: "/tmp/example-stack/state", cacheRoot: "/tmp/example-stack/cache", }); @@ -57,37 +57,73 @@ await stack.close(); // disconnects this client `start` and `ready` are separate operations. `restart({ config })` replaces recipe configuration while retaining the instance identity and endpoint intentions. A health failure leaves a launched process running and observable; it does not prevent `stop`. Database snapshots require a stopped instance with wake disabled. `saveSnapshot(key)` publishes complete data to managed backend storage and replaces the previous entry for that key; `restoreSnapshot(key)` returns `false` on a miss and `true` after restoring a compatible entry. By default snapshots live in the shared cache, whose retention may evict older keys, and survive destruction of the source stack. `{ scope: "instance" }` keeps a snapshot with the instance instead: it is never evicted, restores only into that instance, survives `resetData`, and is removed when the instance is destroyed. Other service handles have no snapshot methods. -Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Automatic ports avoid numbers saved by any stack under the same `stateRoot`; a fixed port is decided by binding it, so another stack's saved port blocks only while something listens on it; that conflict names the stack that saved the port. Omitted public endpoints are not exposed. +Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Automatic ports avoid every port reserved in the per-user port registry (see Public ports); a fixed port another stack holds is a conflict that names that stack. Omitted public endpoints are not exposed. -Native public listeners bind to loopback. Docker and Podman public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. +Native public listeners bind to loopback. Container-runtime (Docker and Podman) public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. + +On macOS, Podman runs containers in a `podman machine` VM. A full stack needs about 8 GiB of VM memory (`podman machine set --memory 8192`); the 2 GiB default runs out of memory and the kernel kills services. Functions use a package-provided, self-contained Edge Runtime main service unless the configuration supplies `bootstrap` source; only an explicit `bootstrap` is saved with the service definition. Database versions belong in `config.version`; other recipes accept an optional top-level artifact `version`. -On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker and Podman mount project files at a separate runtime path. +On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker mounts project files at a separate runtime path. -`open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions and port assignments separately from live-owner availability. It skips each entry that cannot be read or decoded and reports it to `onInvalidState(id, error)`; only a failure to read `stateRoot` itself fails discovery. Port allocation skips the same entries. Offline definitions are not live lifecycle observations. +`open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions separately from live-owner availability. It skips each entry that cannot be read or decoded and reports it to `onInvalidState(id, error)`; only a failure to read `stateRoot` itself fails discovery. Port allocation skips the same entries. Offline definitions are not live lifecycle observations. -`find({ stateRoot, projectRoot, name })` derives the stack ID with the same identity rules as `create` and reads only that stack; `find({ stateRoot, id })` reads a known ID. It returns the saved definition with the live owner's endpoint, if any, or nothing when no such stack is saved. Unlike `discover`, an unreadable state document fails the call instead of being skipped. The Effect entrypoint's `StackId` schema validates an ID before lookup. +`find({ stateRoot, projectRoot, name })` derives the stack ID with the same identity rules as `create` and reads only that stack; `find({ stateRoot, id })` reads a known ID. It returns the saved definition with the live owner's endpoint, if any, or nothing when no such stack is saved. Unlike `discover`, an unreadable state document fails the call, naming the stack's directory, instead of being skipped. The Effect entrypoint's `StackId` schema validates an ID before lookup. Pass `startOwner: true` to `open` when live status and other owner-backed operations are needed; this starts only the detached owner and does not start services. The owner writes its output to `//owner.log`, which each owner start truncates; owner start and connection failures name that file. A client drives only an owner of its own release; other operations fail and ask you to stop or destroy the stack, which work across releases. ### Lifetimes -`create` defaults to `lifetime: "detached"`: the stack and its owner outlive the creating client. `create({ ..., lifetime: "session" })` starts the owner immediately and ties the stack to the creating client. Closing that client, or its process exiting for any reason, destroys the stack: instances stop, data and registrations are removed, and port claims are released. Other clients may attach to a running session stack but cannot start its owner. A session stack whose owner has stopped is disposable: the next owner start in the state root removes it, and creating a stack with the same identity replaces it. +`create` defaults to `lifetime: "detached"`: the stack and its owner outlive the creating client. `create({ ..., lifetime: "session" })` starts the owner immediately and ties the stack to the creating client. Closing that client, or its process exiting for any reason, destroys the stack: instances stop, data and registrations are removed, and port reservations are released. Other clients may attach to a running session stack but cannot start its owner. A session stack whose owner has stopped is disposable: the next owner start in the state root removes it, and creating a stack with the same identity replaces it. `create({ ..., startOwner: true })` starts a detached stack's owner immediately and lets it register the stack under its lease, as a session stack always does. If the owner fails to start or the launch is interrupted before it reports ready, the owner removes that registration and `create` fails with the launch error, so a failed launch leaves no stack behind. -`destroy` normally returns `{ runtimeCleanup: "complete" }`. When no owner is running and a new owner cannot start because the stack's container engine reports that its daemon cannot be reached, `destroy` takes the stack's lease, so no owner can start meanwhile, and removes the local registration, port claims and host data anyway and returns `{ runtimeCleanup: "skipped", engine, cleanupCommands }`; its containers and any database data in engine volumes remain, and `cleanupCommands` are the shell commands that remove them once the engine is running. If some host data cannot be deleted by the current user, `destroy` fails before removing anything so it can be retried with the engine running. +`destroy` succeeds once the stack and everything it owns are removed, except containers left on a container engine other than the one currently selected, which are cleaned only after switching back (see Engine targets in `ARCHITECTURE.md`). When no owner is running and a new owner cannot start because the stack's container engine (Docker or Podman) cannot be reached, `destroy` fails with reason `runtime-unavailable` and leaves the registration, port reservations and host data untouched. When an owner does run but a labelled container cannot be removed (for example one on a different daemon) or an instance's data cannot be removed, `destroy` likewise keeps the stack registered and fails, listing what remains. Either way, retry `destroy` once the failure is resolved; there is no `cleanupCommands` field. -The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. +The stack owns database, Functions bootstrap, and command-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. Host metadata remains under `stateRoot`; native database data uses host files. Docker database data normally uses a managed volume, while existing host data is retained through the host-backed fallback; Podman always keeps database data in a host-backed directory. A host marker records the selected Docker storage and detects a missing or mismatched volume; deleting that volume loses the associated database data. Native snapshot entries live below `cacheRoot`. Docker snapshots share the managed data volume in a separate namespace derived from `cacheRoot`, so they survive source destruction and can use filesystem cloning. A Docker cache hit requires the same daemon, `stateRoot`, and `cacheRoot`. There is no portable tar snapshot API. Omitted database `jwtSecret` and `rootKey` inputs use the shared local-development values exported as `DEFAULT_LOCAL_JWT_SECRET` and `DEFAULT_POSTGRES_ROOT_KEY`. Explicit values override these defaults. The effective root key is supplied through a stack-owned file for both native and container runtimes. -Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory, the PostgreSQL bundle directory, and their ancestors receive traverse-only (`o+x`) permission, which the artifact cache and stack state keep when they restrict their roots to the owner. Running as root elsewhere fails before PostgreSQL launches. - -Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. pg_cron runs jobs in background workers instead of loopback TCP sessions, so a job body cannot contain its own `BEGIN` or `COMMIT`. `max_worker_processes` defaults to 17 for those workers; `PGDATA/postgresql.conf`, its includes, `ALTER SYSTEM`, and caller settings override it. Because workers skip session preload libraries, `supautils` is appended to the effective `shared_preload_libraries`; disposable shadow databases (`stopGraceSeconds: 0`) skip all of this and keep pg_cron's default mode. +The native artifact cache holds one immutable, content-addressed generation per published archive at `cacheRoot///`, published by renaming a lock-guarded `cacheRoot//.staging//content` into place. A generation is never modified after publish. `Artifacts.use` is the one scoped launch-time operation: it takes a SHARED lock on the generation's `cacheRoot//.lock` file before resolving or preparing it, and every consumer (service launch, startup commands, the PostgreSQL handover, one-shot commands, and the native launcher itself) uses the returned paths only inside that scope. Each pin holds its own connection, and the SQLite lock a pin takes excludes retirement whether the pin belongs to this process or another. After a cache miss publishes a generation, a sweep over the whole cache root retires a generation only once it is both unlocked and untouched for 30 days; cache hits never sweep, and digest lock files are never deleted. A stale, lock-guarded staging directory left by a killed preparer is reaped by the next cache miss, never by age. + +Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory and its ancestors outside the generation (the artifact cache and stack state roots, which otherwise stay restricted to their owner) receive traverse-only (`o+x`) permission. A directory inside the generation itself is never chmodded; it must already carry that bit from the archive, or the handover fails naming the path. The `pgsodium_getkey.sh` chown is the one accepted exception to generation immutability: the packaged init script hard-codes that cached path and `chmod +x`s it, which only its owner may do. Running as root elsewhere fails before PostgreSQL launches. + +Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. The socket directory is `pg-` under the validated per-user runtime root, where the digest derives from the data root and instance id, so nothing about it is persisted: launch replaces a leftover and stop removes it. Destroying the instance removes it before the instance is unregistered, so a failed removal keeps the instance registered and a retry removes it; orphan recovery of a session stack runs the same destroy. A runtime root that is a symlink, belongs to another user or is writable by others is never trusted: removal warns and leaves its socket directories in place. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. pg_cron runs jobs in background workers instead of loopback TCP sessions, so a job body cannot contain its own `BEGIN` or `COMMIT`. `max_worker_processes` defaults to 17 for those workers; `PGDATA/postgresql.conf`, its includes, `ALTER SYSTEM`, and caller settings override it. Because workers skip session preload libraries, `supautils` is appended to the effective `shared_preload_libraries`; disposable shadow databases (`stopGraceSeconds: 0`) skip all of this and keep pg_cron's default mode. + +### Public ports + +Every public port is reserved, before any listener binds it, in one SQLite registry per OS user at +`/.supabase/ports.sqlite`. The home is resolved once per process straight from the OS +user database (`getent passwd ` on Linux, `dscacheutil -q user -a uid ` on macOS), never +`$HOME` or `SUPABASE_HOME`, and with no override; a process cannot steer the registry's location by +setting its own environment. The registry is the only saved port assignment (the stack's `state.json` holds none) and the only authority on which stack owns a port across +every state root on the machine, so a stopped stack keeps its automatic ports across other +stacks' automatic allocation, and a restart reuses the same ports or fails with a `StackError` whose `conflict` +field names the port, the endpoint, and either the live holder (`{ stackId, stateRoot }`) or +`"foreign"` for a process outside the registry. A fixed or previously saved port is never silently +reassigned; only automatic allocation tries another candidate. A reservation is released when +deleting an individual service releases the endpoints only it owned, or when a stack-wide `destroy` +has removed the registration. A failed stack-wide `destroy` keeps automatic reservations (a configured port's reservation is released when its listener closes); a failed +individual-service destroy may already have released some of that service's endpoints, and the +service can stay unavailable until the destroy is retried. Stopping a stack or killing its +owner leaves automatic reservations in place, but a configured port claimed by another stack takes +over a reservation whose stack has no running owner; that stack's next start allocates a new +automatic port. A reservation whose owning stack's `state.json` is confirmed gone (`ENOENT`, for example +after deleting its state root) is reclaimed lazily by the next stack that needs its port. + +Known limitations: on macOS, BSD, and Windows, where the kernel allows overlapping binds, a loopback +probe before the real bind narrows but cannot close the race with a foreign process binding in the +same window; two of this user's stacks can never collide, because the registry excludes them. An +unmounted or otherwise temporarily unreachable state root looks identical to a deleted one and can +be reclaimed the same way. The registry does not reserve across OS users; isolation between users' +stacks still relies only on the kernel refusing a second bind. A native backend's own (never public) +ports are reserved directly from 10000–19999, disjoint from the public auto range and below the OS +ephemeral range on every supported platform; this is not configurable, and a public port pinned +inside it is rejected. A host whose ephemeral range has been widened to overlap 10000–19999 +reintroduces the ephemeral-port race this reservation exists to avoid. Native PostgreSQL receives the owner's `SSL_CERT_FILE` and `SSL_CERT_DIR` when they are non-empty. When `SSL_CERT_FILE` is unset or empty, it receives the first of `/etc/ssl/certs/ca-certificates.crt`, `/etc/pki/tls/certs/ca-bundle.crt`, `/etc/ssl/ca-bundle.pem`, and `/etc/ssl/cert.pem` that exists, if any. The `http` and `pg_net` extensions verify HTTPS certificates against these. On macOS, `/etc/ssl/cert.pem` does not include roots added to the Keychain; to trust those, set `SSL_CERT_FILE` to a bundle that also holds the public roots, such as a copy of `/etc/ssl/cert.pem` with those roots appended. When PostgreSQL runs as a separate user, that user must be able to read the owner's `SSL_CERT_FILE` and `SSL_CERT_DIR` paths, which are not chowned to it. The detached owner inherits the environment of the client that starts it and keeps it until it exits, through restarts and repeated starts, so set these before the owner starts, and stop the stack before starting it again after changing them. @@ -116,13 +152,13 @@ const services = await stack.composition.supabase([ await stack.composition.start(); ``` -The factory accepts one instance of each selected recipe, binds its configured public endpoints, and wires managed inputs such as REST's database URL. When the database SQL endpoint is configured, Functions receives the saved database URL as an ordinary input too; recompose the composition after rotating database credentials to refresh that value. This binding does not make Functions wait for database readiness. Database is eager; Functions are lazy without an idle timeout; Studio is lazy with a 5-minute idle timeout; other public services are lazy with a 60-second idle timeout. Services without public endpoints are eager. Pass `{ eager: true }` to start every member eagerly. A managed URL binding requires its producer's endpoint to be configured. The factory also supplies ordinary host/runtime API URLs to Auth, Studio, and Functions without adding dependencies from those URLs. It rejects an already configured composition. +The factory accepts one instance of each selected recipe, binds its configured public endpoints, and wires managed inputs such as REST's database URL. When the database SQL endpoint is configured, Functions receives the saved database URL as an ordinary input too; recompose the composition after rotating database credentials to refresh that value. This binding does not make Functions wait for database readiness. Database is eager; Studio is lazy with a 5-minute idle timeout; other public services, Functions included, are lazy with a 60-second idle timeout. Services without public endpoints are eager. Pass `{ eager: true }` to start every member eagerly. A managed URL binding requires its producer's endpoint to be configured. The factory also supplies ordinary host/runtime API URLs to Auth, Studio, and Functions without adding dependencies from those URLs. It rejects an already configured composition. Inputs that the composition binds or the owner fills from the stack credentials, such as REST's `databaseUrl` or a JWT secret, are optional in creation configuration. Starting or restarting an instance without a required input that was neither bound nor provided fails before any lifecycle change with a `ServiceError` whose `operation` is `"input"` and whose message names the input; a running instance keeps running. To recompose a stopped stack, pass the IDs to keep in `reuseIds`. `composition.plan(services)` compares requested creations with every saved instance of the same kinds without contacting the owner or changing state. Each entry reports its instance `id`, `service`, whether it is a composition `member`, and a `change`: `unchanged`, `changed` with the differing config `paths`, or `incompatible` with the `paths` of an endpoint, artifact version, or PostgreSQL major-version change that the saved instance cannot adopt. Inputs that the composition or the stack credentials supply are ignored, an automatic API port is compared as the shared fixed port the composition would assign, and a PostgreSQL major alias matches its pinned version. Recomposing with `reuseIds` replaces a `changed` instance's configuration while keeping its identity, data, and ports. -The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native and Docker runtimes, split into a `lifecycle` and an `idle-parallel` file per runtime. Run one runtime with a filter such as `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native` (matches both native files) or target one file, for example `src/whole-stack.docker.idle-parallel.e2e.test.ts`. +The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native, Docker, and Podman runtimes, split into a `lifecycle` and an `idle-parallel` file per runtime. The Podman files need `SUPABASE_STACK_TEST_ENGINE=podman`; CI runs them, and the container integration suite, on rootless Podman 4.9 on Linux. macOS `podman machine` is not covered by CI. Run one runtime with a filter such as `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native` (matches both native files) or target one file, for example `src/whole-stack.docker.idle-parallel.e2e.test.ts`. For multiple instances or custom dependencies, configure members and bindings explicitly instead: @@ -149,11 +185,11 @@ await stack.composition.configure({ await stack.composition.start(); ``` -Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health; inspector connections can attach before health succeeds. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. +Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health, up to a 120-second wake budget; inspector connections can attach before health succeeds. A prerequisite stays awake while any dependent runs. A crashed instance stays armed and the next request wakes it again; after repeated failures it fails fast for a 30-second cooldown, until the cooldown elapses or an explicit start. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. - Instance methods affect that instance, subject to dependency checks. - Composition methods affect selected members; startup also includes declared prerequisites. -- `stack.stop()` stops every owned instance and attached command and returns after confirming owner exit. Without a live owner nothing runs, so it returns without starting one; an instance's `stop()` behaves the same way. +- `stack.stop()` stops every owned instance and attached command and returns after confirming owner exit. Without a live owner it does not start one: it reclaims the stack's leftover workloads under the stack lease, and fails if another process holds the lease or the cleanup fails. An instance's `stop()` without a live owner has nothing to stop and returns without starting one. - `stack.destroy()` additionally removes owned data and registrations, and also waits for owner exit. - `stack.close()` disposes the client and invalidates its active observation iterators. Closing the creating client of a session stack destroys the stack. Stopping the last instance leaves the owner available. @@ -178,7 +214,7 @@ await test.reset("seeded"); // discard writes made since the checkpoint The stack is a session stack composed with `composition.supabase` and `{ eager: true }`; each selected service is configured for local development with every endpoint on an automatic port, and `test.services` holds a typed handle per selected kind. Select a kind by name, or pass `{ service, config, endpoints }` to override part of its configuration. Disposal destroys the stack, then closes the client and removes the temporary project root. A session stack is also destroyed when the test process exits. -Every test stack for the current OS user shares one state root and the package's artifact cache root under the OS temp directory, so their ports are coordinated and Docker uses one data volume; each gets a unique temporary project root and name. Pass `stateRoot`, `cacheRoot`, or `projectRoot` to override them. The runtime comes from `runtime`, then `SUPABASE_STACK_TEST_RUNTIME`, then the platform default: native where the catalog publishes native artifacts and Docker elsewhere. A startup failure names the state of each registered service and the owner log. +Every test stack for the current OS user shares one state root and the package's artifact cache root under the OS temp directory, so their ports are coordinated and Docker uses one data volume (Podman uses host directories); each gets a unique temporary project root and name. Pass `stateRoot`, `cacheRoot`, or `projectRoot` to override them. The runtime comes from `runtime`, then `SUPABASE_STACK_TEST_RUNTIME`, then the platform default: native where the catalog publishes native artifacts and Docker elsewhere. A startup failure names the state of each registered service and the owner log. `checkpoint(name)` stops the composition, saves the database data as an instance-scoped snapshot, and starts the composition again. `reset(name)` stops it, clears the database data, restores that snapshot, and waits until every service is ready. Checkpoints are never evicted by other stacks' snapshots and are removed when the test stack is destroyed. diff --git a/packages/stack/package.json b/packages/stack/package.json index e71911609f..a8ba06429f 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -11,6 +11,8 @@ "./commands": "./src/Commands.ts", "./internal/dispatch": "./src/internal/dispatch.ts", "./internal/artifacts": "./src/internal/artifacts.ts", + "./internal/composition": "./src/internal/composition.ts", + "./internal/engine": "./src/internal/engine.ts", "./internal/release": "./src/internal/release.ts", "./internal/test-run-label": "./src/internal/test-run-label.ts" }, @@ -23,6 +25,7 @@ "test:integration:run": "bun --bun vitest run --project integration", "test:e2e": "pnpm run test:e2e:run", "test:e2e:run": "bun --bun vitest run --project e2e", + "test:e2e:native-scenarios": "pnpm run test:e2e:run whole-stack.native", "types:check": "tsc --noEmit" }, "dependencies": { diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index c9895c01b9..f3f00a6f02 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -79,6 +79,8 @@ export interface PreparedNativeArtifact { readonly version: string; readonly root: string; readonly executable: string; + /** The generation's digest lock file; a native workload spawned from `root` pins this too. */ + readonly lockPath: string; } interface ArtifactDefinition { @@ -606,6 +608,49 @@ export const catalogPins = (): ReadonlyArray<{ const artifactKey = (artifact: SlimServicesArtifact): string => `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; +/** Builds the native store request shared by ahead-of-time preparation and launch-time use. */ +const nativeStoreRequest = Effect.fn("Artifacts.nativeStoreRequest")(function* ( + request: { readonly service: ArtifactKind; readonly version?: string }, + platform: { readonly os: string; readonly arch: string }, +) { + const resolved = yield* resolveArtifact(request); + const target = targetForPlatform(platform); + if (target === undefined) + return yield* new ArtifactError({ + message: `Native artifacts are unsupported on ${platformText(platform)}`, + service: request.service, + version: resolved.version, + platform: platformText(platform), + }); + const sourceArtifact = artifactFor(request.service, resolved, target); + const key = artifactKey(sourceArtifact); + const source = makeSlimServicesSource((candidate) => + candidate.key === key ? sourceArtifact : undefined, + ); + return { + resolved, + storeRequest: { + key, + requiredRuntimePaths: resolved.requiredRuntimePaths, + executablePath: resolved.executablePath, + }, + source, + }; +}); + +const toNativeArtifact = ( + resolved: ArtifactResolution, + prepared: { readonly path: string; readonly lockPath: string }, + path: Path.Path, +): PreparedNativeArtifact => ({ + service: resolved.service, + version: resolved.version, + root: prepared.path, + executable: path.join(prepared.path, resolved.executablePath), + lockPath: prepared.lockPath, +}); + +/** Ahead-of-time: downloads and publishes the generation, but pins nothing. */ export const prepareNativeArtifact = Effect.fn("Artifacts.prepareNativeArtifact")(function* ( request: { readonly service: ArtifactKind; readonly version?: string }, cacheRoot: string, @@ -616,32 +661,44 @@ export const prepareNativeArtifact = Effect.fn("Artifacts.prepareNativeArtifact" ) { const resolved = yield* resolveArtifact(request); return yield* Effect.gen(function* () { - const target = targetForPlatform(platform); - if (target === undefined) - return yield* new ArtifactError({ - message: `Native artifacts are unsupported on ${platformText(platform)}`, - service: request.service, - version: resolved.version, - platform: platformText(platform), - }); - const sourceArtifact = artifactFor(request.service, resolved, target); - const key = artifactKey(sourceArtifact); - const source = makeSlimServicesSource((candidate) => - candidate.key === key ? sourceArtifact : undefined, - ); - const store = yield* makeArtifactStore({ cacheRoot, source }); - const prepared = yield* store.prepare({ - key, - requiredRuntimePaths: resolved.requiredRuntimePaths, - executablePath: resolved.executablePath, - }); + const built = yield* nativeStoreRequest(request, platform); + const store = yield* makeArtifactStore({ cacheRoot, source: built.source }); + const prepared = yield* store.prepare(built.storeRequest); const path = yield* Path.Path; - return { - service: resolved.service, - version: resolved.version, - root: prepared.path, - executable: path.join(prepared.path, resolved.executablePath), - }; + return toNativeArtifact(built.resolved, prepared, path); + }).pipe( + Effect.mapError((cause) => + cause instanceof ArtifactError + ? cause + : new ArtifactError({ + message: `Unable to prepare ${request.service} artifact: ${errorMessage(cause)}`, + service: request.service, + version: resolved.version, + cause, + }), + ), + ); +}); + +/** + * The one scoped launch-time operation: pins the generation, resolves or prepares it, and returns + * its paths. Every consumer must use the returned paths only inside this scope. + */ +export const useNativeArtifact = Effect.fn("Artifacts.useNativeArtifact")(function* ( + request: { readonly service: ServiceKind; readonly version?: string }, + cacheRoot: string, + platform: { readonly os: string; readonly arch: string } = { + os: process.platform, + arch: process.arch, + }, +) { + const resolved = yield* resolveArtifact(request); + return yield* Effect.gen(function* () { + const built = yield* nativeStoreRequest(request, platform); + const store = yield* makeArtifactStore({ cacheRoot, source: built.source }); + const prepared = yield* store.use(built.storeRequest); + const path = yield* Path.Path; + return toNativeArtifact(built.resolved, prepared, path); }).pipe( Effect.mapError((cause) => cause instanceof ArtifactError diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts index 50c55fd78c..0e0ecf510d 100644 --- a/packages/stack/src/Artifacts.unit.test.ts +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -1,14 +1,5 @@ import { expect, it } from "@effect/vitest"; -import { catalogPins, slimImageMirrors } from "./Artifacts.ts"; - -it("carries a normalized upstreamImage for every catalog pin", () => { - for (const { pin } of catalogPins()) { - expect(pin.upstreamImage).not.toBe(""); - expect(pin.upstreamImage).not.toContain("docker.io/"); - expect(pin.upstreamImage).not.toContain("@sha256:"); - expect(pin.upstreamImage.split(":").at(-1)).not.toBe(""); - } -}); +import { slimImageMirrors } from "./Artifacts.ts"; it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { expect( diff --git a/packages/stack/src/Commands.integration.test.ts b/packages/stack/src/Commands.integration.test.ts index 21d81d129a..ef44dfb3ac 100644 --- a/packages/stack/src/Commands.integration.test.ts +++ b/packages/stack/src/Commands.integration.test.ts @@ -15,18 +15,20 @@ import { Schedule, Stream, } from "effect"; -import { tmpdir } from "node:os"; import { randomUUID } from "node:crypto"; import { postgres } from "./Commands.ts"; +import type { ContainerEngine } from "./runtime/Container.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; import { CommandError } from "./host/CommandRunner.ts"; import type { PgProveOptions, PostgresCommand } from "./Commands.ts"; import { makeDatabase } from "./services/Database.ts"; -import { makeService } from "./Service.ts"; +import { makeStandaloneService } from "../tests/standalone-service.ts"; import { bindTcp, serveTcp } from "./Proxy.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; class PgProveTestError extends Data.TaggedError("PgProveTestError")<{ readonly message: string; @@ -36,11 +38,14 @@ const makeTestCommandRunner = (options: { readonly stackId: string; readonly root: string; readonly cacheRoot: string; - readonly runtime: "native" | "docker" | "podman"; + readonly runtime: "native" | ContainerEngine; }) => - Layer.build(CommandRunner.layer(options)).pipe( - Effect.map((context) => Context.get(context, CommandRunner.Service)), - ); + Layer.build( + CommandRunner.layer({ + ...options, + ...(options.runtime !== "native" ? { engineTarget } : {}), + }), + ).pipe(Effect.map((context) => Context.get(context, CommandRunner.Service))); const runPostgres = ( runner: CommandRunner.Interface, @@ -73,14 +78,14 @@ const runPostgres = ( }); describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { - for (const runtime of ["native", "docker"] as const) { + for (const runtime of ["native", testEngine] as const) { it.live(`${runtime} runs SQL from stdin, streams a dump, and returns client failure`, () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const stackId = `tools-integration-${randomUUID()}`; const root = - runtime === "docker" + runtime !== "native" ? yield* makeDockerDatabaseRoot("stack-tools-", stackId) : yield* fs.makeTempDirectoryScoped({ prefix: "stack-tools-" }); const database = yield* makeDatabase({ @@ -89,8 +94,9 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { stackId, instanceId: "database", runtime, + ...(runtime !== "native" ? { engineTarget } : {}), }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database", config: { version: "17", @@ -223,14 +229,14 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { } for (const major of [15, 17] as const) { - for (const runtime of ["native", "docker"] as const) { + for (const runtime of ["native", testEngine] as const) { it.live(`${runtime} runs bundled pg_prove for PostgreSQL ${major}`, () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const stackId = `tools-pgprove-${runtime}-${major}-${randomUUID()}`; const root = - runtime === "docker" + runtime !== "native" ? yield* makeDockerDatabaseRoot(`stack-pgprove-${major}-`, stackId) : yield* fs.makeTempDirectoryScoped({ prefix: `stack-pgprove-${major}-` }); const database = yield* makeDatabase({ @@ -239,8 +245,9 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { stackId, instanceId: "database", runtime, + ...(runtime !== "native" ? { engineTarget } : {}), }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database", config: { version: String(major), @@ -270,8 +277,12 @@ describe("finite PostgreSQL commands", { timeout: 180_000 }, () => { PGPASSWORD: `pgprove-${runtime}-${major}-password`, PGDATABASE: "postgres", }; - const tests = `${root}/tests`; - yield* fs.makeDirectory(tests, { recursive: true }); + // Ownership is by location: pg_prove's mount source is a caller path and must + // live outside the stack's data root, not merely outside the database's own + // instance root. + const tests = yield* fs.makeTempDirectoryScoped({ + prefix: `stack-pgprove-${major}-tests-`, + }); yield* fs.writeFileString(`${tests}/main.sql`, "\\ir included.sql\n"); yield* fs.writeFileString(`${tests}/included.sql`, "\\i nested.sql\n"); yield* fs.writeFileString( diff --git a/packages/stack/src/HostProcess.integration.test.ts b/packages/stack/src/HostProcess.integration.test.ts index f924707b68..985f8e4f74 100644 --- a/packages/stack/src/HostProcess.integration.test.ts +++ b/packages/stack/src/HostProcess.integration.test.ts @@ -12,6 +12,7 @@ import { Layer, Option, Path, + Ref, Schema, Scope, Stream, @@ -34,7 +35,8 @@ import { } from "./HostProcess.ts"; import { discover } from "./effect.ts"; import { watchLeaseRelease } from "../tests/owner.ts"; -import * as State from "./State.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; +import * as StackNamespace from "./StackNamespace.ts"; class ProcessTestError extends Data.TaggedError("ProcessTestError")<{ readonly message: string }> {} @@ -42,14 +44,13 @@ const fixtureEntrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), ); -const savedStack = (root: string, stackName: string): State.SavedStack => ({ +const savedStack = (root: string, stackName: string): StackNamespace.SavedStack => ({ id: "stack", runtime: "native", identity: { projectRoot: root, branchContext: "main", stackName }, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); /** A loopback listener that accepts connections and never answers, counting each one. */ @@ -80,10 +81,21 @@ const silentListener = Effect.acquireRelease( ); const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); +/** Writes a holder record directly, simulating a process that published then crashed before retracting. */ +const writeStaleHolder = (root: string, id: string, record: StackNamespace.LeaseHolder) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(StackNamespace.LeaseHolder))( + record, + ); + yield* fs.writeFileString(path.join(root, id, "owner.json"), encoded); + }); + type ChildHandle = { readonly child: ChildProcessSpawner.ChildProcessHandle; closed: boolean; @@ -190,7 +202,11 @@ const bestEffortShutdown = (stateRoot: string) => (access: HostAccess) => }), ).pipe(Effect.exit, Effect.asVoid); -const bestEffortShutdownByState = (stateRoot: string, state: State.Interface, stackId: string) => +const bestEffortShutdownByState = ( + stateRoot: string, + state: StackNamespace.Interface, + stackId: string, +) => Effect.exit(connectHost(state, stackId).pipe(Effect.flatMap(bestEffortShutdown(stateRoot)))).pipe( Effect.asVoid, ); @@ -215,30 +231,29 @@ const closeServer = (server: Net.Server) => server.close(() => resume(Effect.void)); }); +/** Attaches now; the returned effect awaits the marker and reads its value. Subscribe before + * triggering the fixture that writes the marker, then await the returned effect after. */ const waitForMarker = (marker: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - yield* fs.watch(path.dirname(marker)).pipe( - Stream.filter((event) => event.path === path.basename(marker)), - Stream.runHead, - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(new ProcessTestError({ message: "marker watcher ended" })), - onSome: Effect.succeed, + const ready = yield* watchEntry(path.dirname(marker), path.basename(marker), true); + return yield* Effect.succeed( + ready.pipe( + Effect.flatMap(() => + fs.readFileString(marker).pipe( + Effect.map(Number), + Effect.mapError(() => new ProcessTestError({ message: "marker read failed" })), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.fail(new ProcessTestError({ message: "slow fixture did not start" })), }), ), ); - return yield* fs.readFileString(marker).pipe( - Effect.map(Number), - Effect.mapError(() => new ProcessTestError({ message: "marker read failed" })), - ); - }).pipe( - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => Effect.fail(new ProcessTestError({ message: "slow fixture did not start" })), - }), - ); + }); it.live("starts exactly one owner for concurrent launchers and attaches the others", () => Effect.scoped( @@ -253,7 +268,6 @@ it.live("starts exactly one owner for concurrent launchers and attaches the othe instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const entrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), @@ -333,7 +347,7 @@ it.live("ignores a stale endpoint record once no process holds the lease", () => const state = yield* makeTestState(root); yield* state.save(savedStack(root, "local")); const unresponsive = yield* silentListener; - yield* state.publishHolder("stack", { + yield* writeStaleHolder(root, "stack", { role: "owner", secret: "stale", port: unresponsive.port, @@ -371,8 +385,8 @@ it.live("discovers dead stacks from their free leases without contacting recorde const ids = ["dead-a", "dead-b", "dead-c"]; for (const id of ids) { yield* state.save({ ...savedStack(root, id), id }); - yield* Effect.scoped(state.lease(id)); - yield* state.publishHolder(id, { + yield* Effect.scoped(state.acquireLease(id)); + yield* writeStaleHolder(root, id, { role: "owner", secret: "stale", port: unresponsive.port, @@ -407,7 +421,6 @@ it.live( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const entrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), @@ -442,22 +455,19 @@ it.live("terminates a detached child when readiness is interrupted", () => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const entrypoint = fileURLToPath( new URL("../tests/host-process-fixture.ts", import.meta.url), ); const marker = path.join(root, "slow-handshake.pid"); - const markerReady = yield* waitForMarker(marker).pipe( - Effect.forkChild({ startImmediately: true }), - ); + const markerReady = yield* waitForMarker(marker); const launch = yield* launchHost(state, { stateRoot: root, cacheRoot: root, stackId: "stack", entrypoint, }).pipe(Effect.forkChild); - const pid = yield* Fiber.join(markerReady); + const pid = yield* markerReady; expect(Number.isInteger(pid) && pid > 0).toBe(true); expect(() => process.kill(pid, 0)).not.toThrow(); yield* Fiber.interrupt(launch); @@ -466,7 +476,7 @@ it.live("terminates a detached child when readiness is interrupted", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("keeps the owner log tail in a start failure after the owner removed its log", () => +it.live("reports a structured owner startup error without the owner log tail", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -480,7 +490,8 @@ it.live("keeps the owner log tail in a start failure after the owner removed its register: savedStack(root, "failing"), }).pipe(Effect.flip); expect(failure.message).toContain("owner startup failed"); - expect(failure.message).toContain("failing-owner-diagnostic"); + expect(failure.message).toContain("owner log:"); + expect(failure.message).not.toContain("failing-owner-diagnostic"); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); @@ -529,23 +540,29 @@ it.live("waits out a sweeper's hold before spawning the owner of a stack it regi const state = yield* makeTestState(root); const hold = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(hold, Exit.void)); - expect(yield* state.lease("stack").pipe(Scope.provide(hold))).toBe(true); - yield* state.publishHolder("stack", { + const lease = yield* state.acquireLease("stack").pipe(Scope.provide(hold)); + yield* lease.publishHolder({ role: "sweeper", pid: process.pid, startedAt: "2026-01-01T00:00:00.000Z", }); const sweeperSeen = yield* Deferred.make(); - const watched: State.Interface = { + const observations = yield* Ref.make(0); + const watched: StackNamespace.Interface = { ...state, readHolder: (id) => - state - .readHolder(id) - .pipe( - Effect.tap((holder) => - holder?.role === "sweeper" ? Deferred.succeed(sweeperSeen, undefined) : Effect.void, - ), + state.readHolder(id).pipe( + Effect.tap((holder) => + holder?.role !== "sweeper" + ? Effect.void + : Ref.updateAndGet(observations, (count) => count + 1).pipe( + Effect.flatMap((count) => + // The first attempt has ended once the retry observes the hold again. + count === 2 ? Deferred.succeed(sweeperSeen, undefined) : Effect.void, + ), + ), ), + ), }; const launching = yield* launchHost(watched, { stateRoot: root, @@ -558,7 +575,7 @@ it.live("waits out a sweeper's hold before spawning the owner of a stack it regi expect(yield* fs.exists(state.ownerLog("stack")), "no owner spawned during the hold").toBe( false, ); - yield* state.retractHolder("stack"); + yield* lease.retractHolder; yield* Scope.close(hold, Exit.void); yield* Effect.acquireRelease(Fiber.join(launching), bestEffortShutdown(root)); expect(yield* state.read("stack")).toBeDefined(); diff --git a/packages/stack/src/HostProcess.ts b/packages/stack/src/HostProcess.ts index 50aea1b49b..bf27544438 100644 --- a/packages/stack/src/HostProcess.ts +++ b/packages/stack/src/HostProcess.ts @@ -31,7 +31,11 @@ import { HOST_PROCESS_DISPATCH_SENTINEL, isBunVirtualPath } from "./internal/dis import { failureMessage } from "./internal/failure-message.ts"; import { stackSourceDigest } from "./internal/release.ts"; import { StackRpc } from "./Rpc.ts"; -import { SavedStack, type Interface as StateInterface, type StateError } from "./State.ts"; +import { + SavedStack, + type Interface as StateInterface, + type NamespaceError, +} from "./StackNamespace.ts"; declare const SUPABASE_STACK_BUILD_ID: string | undefined; @@ -259,7 +263,7 @@ export const connectHost = Effect.fn("HostProcess.connectHost")(function* ( options: ConnectOptions = {}, ): Effect.fn.Return< HostAccess, - HostProcessError | StateError, + HostProcessError | NamespaceError, HttpClient.HttpClient | FileSystem.FileSystem | Path.Path | Crypto.Crypto > { const stack = yield* registered(state, stackId); @@ -342,6 +346,8 @@ export interface LaunchOptions { readonly cacheRoot: string; readonly stackId: string; readonly entrypoint?: string; + /** Extra positional arguments a non-default `entrypoint` reads, ahead of `register`. */ + readonly entrypointArgs?: ReadonlyArray; /** Registers this definition once the spawned owner holds the lease. */ readonly register?: SavedStack; /** Ties a spawned owner to the enclosing scope, whose closure destroys the stack. */ @@ -459,7 +465,14 @@ const spawnOwner = Effect.fn("HostProcess.spawnOwner")(function* ( try: () => { const started = spawn( process.execPath, - [entrypoint, options.stateRoot, options.cacheRoot, options.stackId, ...register], + [ + entrypoint, + options.stateRoot, + options.cacheRoot, + options.stackId, + ...(options.entrypointArgs ?? []), + ...register, + ], { cwd: process.cwd(), detached: true, @@ -527,7 +540,11 @@ const spawnOwner = Effect.fn("HostProcess.spawnOwner")(function* ( } return yield* line.reason === "lease-held" || line.reason === "exists" ? error("startup", "Stack already exists; use open") - : failure(line.message, line.reason); + : error( + "startup", + `Stack owner failed to start: ${line.message} (owner log: ${log})`, + line.reason, + ); } if (line.endpoint.stackId !== options.stackId) return yield* failure( @@ -564,7 +581,7 @@ export const launchHost = Effect.fn("HostProcess.launchHost")(function* ( options: LaunchOptions, ): Effect.fn.Return< HostAccess, - HostProcessError | StateError, + HostProcessError | NamespaceError, Scope.Scope | HttpClient.HttpClient | FileSystem.FileSystem | Path.Path | Crypto.Crypto > { const entrypoint = options.entrypoint ?? hostEntrypointFor(import.meta.url); diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 57858f28eb..1bd5626981 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -869,6 +869,71 @@ it.live("gives up after a single retry when the upstream keeps dropping connecti ); }); +it.live( + "re-resolves the backend on retry instead of reaching a listener that reused the invalidated address", + () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const recovered = createServer((_request, response) => response.end("recovered")); + const recoveredAddress = yield* listen(recovered); + + let foreignConnections = 0; + let resolveInvalidated: (() => void) | undefined; + // One address plays the real backend for the connection that is retried, then the + // address's next owner for any later one, without an OS-level close/rebind race. + let role: "backend" | "foreign" = "backend"; + const shared = createTcpServer((socket) => { + if (role === "backend") { + role = "foreign"; + socket.destroy(); + resolveInvalidated?.(); + } else { + foreignConnections += 1; + socket.destroy(); + } + }); + const sharedAddress = yield* listen(shared); + + const invalidated = yield* Deferred.make(); + yield* Effect.callback((resume) => { + resolveInvalidated = () => resume(Effect.void); + return Effect.void; + }).pipe(Effect.andThen(Deferred.succeed(invalidated, undefined)), Effect.forkScoped); + + let attempts = 0; + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { + id: "retry-target", + prefix: "/", + target: Effect.gen(function* () { + attempts += 1; + if (attempts === 1) return sharedAddress; + yield* Deferred.await(invalidated); + return recoveredAddress; + }), + }, + ]); + + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://127.0.0.1:${proxy.port}/hello`); + expect(response.status).toBe(200); + expect(yield* response.text).toBe("recovered"); + expect(foreignConnections).toBe(0); + expect(logs).toHaveLength(1); + expect(logs[0]).toContain( + "Route retry-target GET upstream failed before responding, retrying", + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll(NodeHttpClient.layerNodeHttp, NodeServices.layer, captureWarnings(logs)), + ), + ); + }, +); + it.live("closes an upgrade naming the route and cause when a target cannot wake", () => { const logs: Array = []; return Effect.scoped( diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index f72a4ec39c..a73b4dafdf 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -413,7 +413,9 @@ const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( ) => Effect.gen(function* () { yield* Effect.annotateCurrentSpan({ route_id: route.id }); - const backend = yield* Effect.raceFirst(route.target, disconnected(request, response)); + // Re-resolved on retry: a backend invalidated between attempts may have its address reused. + const resolveBackend = () => Effect.raceFirst(route.target, disconnected(request, response)); + const backend = yield* resolveBackend(); yield* forward( request, response, @@ -426,7 +428,12 @@ const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( Effect.logWarning( `Route ${route.id} ${request.method ?? "GET"} upstream failed before responding, retrying`, error, - ).pipe(Effect.andThen(forward(request, response, route, backend, false, sent))), + ).pipe( + Effect.andThen(resolveBackend()), + Effect.andThen((retryBackend) => + forward(request, response, route, retryBackend, false, sent), + ), + ), ), ); }).pipe( @@ -635,7 +642,9 @@ export const makeHttpProxy = (options: { }); server.on("connection", (socket) => { sockets.add(socket); - socket.once("close", () => sockets.delete(socket)); + socket.once("close", () => { + sockets.delete(socket); + }); }); server.on("upgrade", (request, socket, head) => { socket.on("error", () => socket.destroy()); diff --git a/packages/stack/src/Lifecycle.ts b/packages/stack/src/Lifecycle.ts new file mode 100644 index 0000000000..f4716c31c7 --- /dev/null +++ b/packages/stack/src/Lifecycle.ts @@ -0,0 +1,1232 @@ +import { Data } from "effect"; + +const waiterCap = 256; +const breakerThreshold = 3; +const breakerCooldownMillis = 30_000; +/** Failures further apart than this are unrelated; it exceeds the cooldown so a reopened breaker keeps counting. */ +const breakerWindowMillis = 300_000; +/** The pause between a failed re-check of a blocking session and the next one. */ +const reprobeSpacingMillis = 1_000; + +/** One member of the owner's composition graph, as the reducer needs to see it. */ +export interface ServiceSpec { + readonly id: string; + readonly activation: "eager" | "lazy"; + readonly prerequisites: ReadonlyArray; + /** Undefined means the service never sleeps on idle. */ + readonly idleMillis?: number; +} + +/** + * The owner's composition graph, with its topological order precomputed, plus each service's + * direct dependents (for the stop/restart guard), full transitive-dependent closure (every + * service that needs it, directly or through another prerequisite, so a wake deep in a stopped + * chain is visible to every ancestor in one pass) and full transitive prerequisite closure (so + * readiness checks see a lost ancestor, not just a direct prerequisite). + */ +export interface LifecycleGraph { + readonly services: ReadonlyMap; + readonly directDependents: ReadonlyMap>; + readonly transitiveDependents: ReadonlyMap>; + readonly prerequisiteClosure: ReadonlyMap>; + readonly order: ReadonlyArray; +} + +export const makeGraph = (specs: ReadonlyArray): LifecycleGraph => { + const services = new Map(specs.map((spec): [string, ServiceSpec] => [spec.id, spec])); + const order: Array = []; + const seen = new Set(); + const prerequisiteClosure = new Map>(); + const closureOf = (id: string): ReadonlySet => { + const cached = prerequisiteClosure.get(id); + if (cached !== undefined) return cached; + const closure = new Set(); + for (const prerequisite of services.get(id)?.prerequisites ?? []) { + closure.add(prerequisite); + for (const transitive of closureOf(prerequisite)) closure.add(transitive); + } + prerequisiteClosure.set(id, closure); + return closure; + }; + const visit = (id: string): void => { + if (seen.has(id)) return; + seen.add(id); + for (const prerequisite of services.get(id)?.prerequisites ?? []) visit(prerequisite); + order.push(id); + }; + const directDependents = new Map>(); + const transitiveDependents = new Map>(); + for (const id of services.keys()) { + visit(id); + for (const prerequisite of services.get(id)?.prerequisites ?? []) { + const list = directDependents.get(prerequisite) ?? []; + list.push(id); + directDependents.set(prerequisite, list); + } + for (const prerequisite of closureOf(id)) { + const list = transitiveDependents.get(prerequisite) ?? new Set(); + list.add(id); + transitiveDependents.set(prerequisite, list); + } + } + return { services, directDependents, transitiveDependents, prerequisiteClosure, order }; +}; + +type Intent = "eager" | "lazy" | "stopped"; + +/** A launch's progress, named so a waiter's budget error can report where it's still blocked. */ +type Stage = "preparing" | "launching"; + +/** + * A service's point in its launch/stop cycle. `Running.ready` tracks the live health signal + * separately from the phase itself: a readiness failure is not terminal, only a launch + * failure or an exit is. + */ +type Phase = Data.TaggedEnum<{ + Stopped: {}; + Starting: { readonly generation: number; readonly stage: Stage }; + Running: { readonly generation: number; readonly ready: boolean }; + /** `crash` marks a generation whose runtime exited on its own: its cleanup ends in `Failed`. */ + Stopping: { readonly generation: number; readonly crash?: { readonly cause: unknown } }; + Failed: { readonly generation: number; readonly cause: unknown }; +}>; +const Phase = Data.taggedEnum(); + +interface BreakerState { + readonly consecutiveFailures: number; + readonly lastFailureAt: number | undefined; + readonly openUntil: number | undefined; + readonly lastCause: unknown; +} + +const initialBreaker: BreakerState = { + consecutiveFailures: 0, + lastFailureAt: undefined, + openUntil: undefined, + lastCause: undefined, +}; + +interface Waiter { + readonly id: number; + /** + * A traffic wait holds a lease and counts toward the cap; an explicit operation's wait only + * observes readiness and fails with its prerequisite's outcome. + */ + readonly kind: "traffic" | "explicit"; + /** False for an acquisition (such as the Functions inspector) that bypasses target readiness. */ + readonly requireReady: boolean; +} + +/** Whether admission opens a work lease; an explicit operation's wait only observes readiness. */ +const isLeaseWaiter = (waiter: Waiter): boolean => waiter.kind === "traffic"; + +export interface ServiceState { + readonly intent: Intent; + readonly phase: Phase; + readonly leases: number; + readonly idleEpoch: number; + /** The epoch an outstanding idle timer was armed with, or undefined while none is armed. */ + readonly idleArmedEpoch: number | undefined; + readonly breaker: BreakerState; + readonly waiters: ReadonlyMap; + /** Set by an explicit start or restart so the next launch attempt proceeds without demand. */ + readonly relaunchForced: boolean; + /** Blocks every launch while a storage operation or destroy owns the stopped service. */ + readonly storageReserved: boolean; + /** + * A destroy waiting for the service to stop and any storage operation to release + * (`requested`), or holding the storage reservation to remove its data (`reserved`). + */ + readonly destroy: "requested" | "reserved" | undefined; + /** + * The current generation's last failed readiness check, cleared once it recovers or ends. + * `reprobed` marks a failure that a re-check already confirmed, so the next one is spaced out. + */ + readonly readinessFailure: { readonly cause: unknown; readonly reprobed: boolean } | undefined; + /** Whether a `Reprobe` of the current generation is in flight; only one runs at a time. */ + readonly reprobing: boolean; + /** + * Set while `Stopping` when the last cleanup of that generation failed: its resources are still + * held and no stop is in flight. A stop, restart, start or new waiter retries the cleanup. + */ + readonly cleanupFailure: { readonly cause: unknown } | undefined; + /** The newest restart candidate, consumed by the next `Launch` command. */ + readonly restartCandidate: unknown; +} + +const initialService = (spec: ServiceSpec): ServiceState => ({ + intent: spec.activation, + phase: Phase.Stopped(), + leases: 0, + idleEpoch: 0, + idleArmedEpoch: undefined, + breaker: initialBreaker, + waiters: new Map(), + relaunchForced: false, + storageReserved: false, + destroy: undefined, + readinessFailure: undefined, + reprobing: false, + cleanupFailure: undefined, + restartCandidate: undefined, +}); + +export interface LifecycleState { + readonly graph: LifecycleGraph; + readonly services: ReadonlyMap; + /** + * The next generation to assign per service, kept outside `services` and never pruned: a + * service removed and re-added by a `GraphUpdated` still can't reuse a generation number. + */ + readonly generationCounters: ReadonlyMap; +} + +export const initialState = (graph: LifecycleGraph): LifecycleState => ({ + graph, + services: new Map( + [...graph.services.values()].map((spec): [string, ServiceState] => [ + spec.id, + initialService(spec), + ]), + ), + generationCounters: new Map([...graph.services.keys()].map((id) => [id, 1])), +}); + +export type LifecycleEvent = Data.TaggedEnum<{ + /** `requireReady: false` lets an acquisition (the Functions inspector) bypass target readiness. */ + ConnectionOpened: { + readonly id: string; + readonly waiterId: number; + readonly requireReady: boolean; + }; + ConnectionClosed: { readonly id: string }; + /** An explicit operation waiting for a session (or readiness) with no lease. */ + ReadinessAwaited: { + readonly id: string; + readonly waiterId: number; + readonly requireReady: boolean; + }; + /** A live session exists but hasn't passed its first health check; admits `requireReady: false`. */ + SessionAvailable: { readonly id: string; readonly generation: number }; + LaunchSucceeded: { readonly id: string; readonly generation: number }; + LaunchFailed: { readonly id: string; readonly generation: number; readonly cause: unknown }; + /** + * A generation's runtime exited on its own and its cleanup has begun. Reported before the + * cleanup runs, so nothing routes to the dead session while its resources are still held. + */ + SessionLost: { readonly id: string; readonly generation: number; readonly cause: unknown }; + Exited: { readonly id: string; readonly generation: number; readonly requested: boolean }; + /** + * A stop or a failed launch's cleanup of `generation` failed; its resources are still held. + * `failure` carries the launch failure or crash that ended the generation, if one did, so it + * still counts toward the breaker once. + */ + StopFailed: { + readonly id: string; + readonly generation: number; + readonly cause: unknown; + readonly failure?: unknown; + }; + IdleElapsed: { readonly id: string; readonly generation: number; readonly epoch: number }; + ReadinessLost: { readonly id: string; readonly generation: number; readonly cause: unknown }; + ReadinessRecovered: { readonly id: string; readonly generation: number }; + StageChanged: { readonly id: string; readonly generation: number; readonly stage: Stage }; + /** Fired by the runtime once a breaker's cooldown elapses; stale against the current breaker is a no-op. */ + CooldownElapsed: { readonly id: string; readonly openUntil: number }; + StartRequested: { readonly id: string }; + /** Arms a service for demand-driven launches without forcing one. */ + ArmRequested: { readonly id: string }; + StopRequested: { readonly id: string }; + /** `candidate`, when given, is the launch input the next launch uses in place of the saved one. */ + RestartRequested: { readonly id: string; readonly candidate?: unknown }; + StorageReserved: { readonly id: string }; + StorageReleased: { readonly id: string }; + /** Stops the service and reserves its storage for data removal once both are free. */ + DestroyRequested: { readonly id: string }; + /** A destroy gives up its request or reservation, whether or not it removed the data. */ + DestroyReleased: { readonly id: string }; + WaiterCancelled: { readonly id: string; readonly waiterId: number }; + /** + * Atomically swaps the graph while preserving every continuing service's generation and epoch. + * A service the update adds starts unarmed, so configuring a graph never launches anything. + */ + GraphUpdated: { readonly graph: LifecycleGraph }; +}>; +export const LifecycleEvent = Data.taggedEnum(); + +export type LifecycleCommand = Data.TaggedEnum<{ + /** `candidate` is the restart's launch input, or undefined when the saved one applies. */ + Launch: { readonly id: string; readonly generation: number; readonly candidate: unknown }; + Stop: { readonly id: string; readonly generation: number }; + ArmIdleTimer: { + readonly id: string; + readonly generation: number; + readonly epoch: number; + readonly delayMillis: number; + }; + /** Schedules the `CooldownElapsed` event once the breaker's cooldown elapses. */ + ArmCooldownTimer: { + readonly id: string; + readonly openUntil: number; + readonly delayMillis: number; + }; + /** Re-runs a live session's failed readiness check, after `delayMillis`, because a waiter needs it. */ + Reprobe: { readonly id: string; readonly generation: number; readonly delayMillis: number }; + AdmitConnection: { readonly id: string; readonly waiterId: number }; + FailConnection: { + readonly id: string; + readonly waiterId: number; + readonly message: string; + readonly cause: unknown; + }; + /** An explicit start, stop, restart, storage reservation or graph update the reducer refused. */ + RequestRejected: { + readonly id: string; + readonly operation: "start" | "stop" | "restart" | "storage" | "graph"; + readonly message: string; + }; +}>; +export const LifecycleCommand = Data.taggedEnum(); + +const generationOf = (phase: Phase): number | undefined => + Phase.$match(phase, { + Stopped: () => undefined, + Starting: (p) => p.generation, + Running: (p) => p.generation, + Stopping: (p) => p.generation, + Failed: (p) => p.generation, + }); + +const isUpOrWindingDown = (phase: Phase): boolean => + phase._tag === "Starting" || phase._tag === "Running" || phase._tag === "Stopping"; + +const isReady = (phase: Phase): boolean => phase._tag === "Running" && phase.ready; +const isSessionAvailable = (phase: Phase): boolean => phase._tag === "Running"; + +/** + * An eager service's standing demand lapses once it fails, so a broken launch isn't retried in a + * loop; traffic, a running dependent or an explicit start still relaunch it. + */ +const ownsDemand = (service: ServiceState): boolean => + (service.intent === "eager" && service.phase._tag !== "Failed") || + service.leases > 0 || + service.waiters.size > 0 || + service.relaunchForced; + +/** + * A dependent, anywhere in the transitive closure, that an explicit stop or restart of its + * prerequisite would strand: one with live or retained demand, one still winding down, or one + * merely wake-armed (lazy or eager, not explicitly stopped) and so able to need it again at any + * moment. Internal idle sleep uses its own, narrower demand rule and is unaffected by this. + */ +const dependentIsActive = (service: ServiceState): boolean => + service.intent !== "stopped" || ownsDemand(service) || isUpOrWindingDown(service.phase); + +/** A service active enough that a `GraphUpdated` removing or changing it would drop live or retained work. */ +const isActiveForGraphUpdate = (service: ServiceState): boolean => + isUpOrWindingDown(service.phase) || + service.leases > 0 || + service.waiters.size > 0 || + service.storageReserved || + service.relaunchForced; + +/** Whether two specs for the same id describe the same service, ignoring prerequisite order. */ +const specsEqual = (a: ServiceSpec, b: ServiceSpec): boolean => + a.activation === b.activation && + a.idleMillis === b.idleMillis && + a.prerequisites.length === b.prerequisites.length && + a.prerequisites.every((prerequisite) => b.prerequisites.includes(prerequisite)); + +/** + * A service has demand when it owns work itself (including an explicit start/restart that hasn't + * launched yet), or a transitive dependent does, or a transitive dependent is admitted, starting + * or running (held until its exit is confirmed). The dependent closure is checked directly (not + * just the immediate one) so a wake deep in a fully stopped chain is visible to every prerequisite + * in the same pass. + */ +const hasDemand = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + if (service === undefined) return false; + if (ownsDemand(service)) return true; + for (const dependent of state.graph.transitiveDependents.get(id) ?? []) { + const dependentState = state.services.get(dependent); + if ( + dependentState !== undefined && + (ownsDemand(dependentState) || isUpOrWindingDown(dependentState.phase)) + ) + return true; + } + return false; +}; + +/** Every prerequisite in the full transitive closure must itself be ready, not just direct ones. */ +const prerequisitesSatisfied = (state: LifecycleState, id: string): boolean => { + for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { + const prerequisiteState = state.services.get(prerequisite); + if (prerequisiteState === undefined || !isReady(prerequisiteState.phase)) return false; + } + return true; +}; + +/** Effective readiness for admission: the target's own health and its prerequisite closure. */ +const admissionReady = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + return service !== undefined && isReady(service.phase) && prerequisitesSatisfied(state, id); +}; + +/** A live session, bypassing the target's own health check, still behind its prerequisite closure. */ +const sessionReady = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + return ( + service !== undefined && isSessionAvailable(service.phase) && prerequisitesSatisfied(state, id) + ); +}; + +/** + * Whether the breaker is currently blocking admission. This is a plain flag, not a `now` + * comparison: only the dedicated `CooldownElapsed` event clears it, so a stale or unrelated event + * processed after the cooldown has elapsed on the wall clock can never itself reopen admission. + */ +const isBreakerOpen = (breaker: BreakerState): boolean => breaker.openUntil !== undefined; + +/** Names what a pending waiter is still blocked on, for a budget-expiry error. */ +export const blockingStage = (state: LifecycleState, id: string): string => { + const service = state.services.get(id); + if (service === undefined) return `${id} is unknown`; + for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { + const prerequisiteState = state.services.get(prerequisite); + if (prerequisiteState === undefined || !isReady(prerequisiteState.phase)) + return `prerequisite ${prerequisite}`; + } + if (service.phase._tag === "Starting") return `${id} to finish ${service.phase.stage}`; + if (service.phase._tag !== "Running") return `${id} to start`; + if (!service.phase.ready) return `${id} to become healthy`; + return `${id}`; +}; + +const setService = ( + state: LifecycleState, + id: string, + update: (service: ServiceState) => ServiceState, +): LifecycleState => { + const service = state.services.get(id); + if (service === undefined) return state; + return { ...state, services: new Map(state.services).set(id, update(service)) }; +}; + +/** Fails every pending waiter of a service with one cause and clears its waiter queue. */ +const failAllWaiters = ( + service: ServiceState, + id: string, + message: string, + cause: unknown, + commands: Array, +): ServiceState => { + for (const waiter of service.waiters.values()) + commands.push(LifecycleCommand.FailConnection({ id, waiterId: waiter.id, message, cause })); + return { ...service, waiters: new Map() }; +}; + +/** + * Fails every explicit operation's wait on a transitive dependent of a prerequisite that just + * failed or lost readiness: such a wait has no budget, so it must end with the prerequisite's + * outcome. Traffic waiters keep their demand and stay bounded by the orchestrator's budget. + */ +const failExplicitDependentWaiters = ( + state: LifecycleState, + id: string, + message: string, + cause: unknown, + commands: Array, +): LifecycleState => { + let next = state; + for (const dependent of state.graph.transitiveDependents.get(id) ?? []) { + const service = next.services.get(dependent); + if (service === undefined) continue; + const explicit = [...service.waiters.values()].filter((waiter) => !isLeaseWaiter(waiter)); + if (explicit.length === 0) continue; + const waiters = new Map(service.waiters); + for (const waiter of explicit) { + waiters.delete(waiter.id); + commands.push( + LifecycleCommand.FailConnection({ id: dependent, waiterId: waiter.id, message, cause }), + ); + } + next = setService(next, dependent, (s) => ({ ...s, waiters })); + } + return next; +}; + +/** Re-issues the stop of a generation whose last cleanup failed, if one is pending. */ +const retryCleanup = ( + state: LifecycleState, + id: string, + commands: Array, +): LifecycleState => { + const service = state.services.get(id); + if (service?.phase._tag !== "Stopping" || service.cleanupFailure === undefined) return state; + commands.push(LifecycleCommand.Stop({ id, generation: service.phase.generation })); + return setService(state, id, (s) => ({ ...s, cleanupFailure: undefined })); +}; + +/** + * Retries a failed cleanup that a newly queued waiter is blocked on, in its prerequisite closure + * or the target itself. Only a new arrival retries, so a cleanup that keeps failing never loops. + */ +const retryBlockedCleanup = ( + state: LifecycleState, + id: string, + commands: Array, +): LifecycleState => { + let next = state; + for (const blocker of [...(state.graph.prerequisiteClosure.get(id) ?? []), id]) + next = retryCleanup(next, blocker, commands); + return next; +}; + +/** + * Whether a pending waiter needs this service ready: one of its own that requires readiness, or + * any waiter of a transitive dependent, since admission requires the whole prerequisite closure. + */ +const blocksWaiters = (state: LifecycleState, id: string): boolean => { + const service = state.services.get(id); + if (service === undefined) return false; + for (const waiter of service.waiters.values()) if (waiter.requireReady) return true; + for (const dependent of state.graph.transitiveDependents.get(id) ?? []) + if ((state.services.get(dependent)?.waiters.size ?? 0) > 0) return true; + return false; +}; + +/** Queues or immediately resolves one traffic acquisition or explicit readiness wait. */ +const openWaiter = ( + state: LifecycleState, + id: string, + waiterId: number, + requireReady: boolean, + traffic: boolean, +): readonly [LifecycleState, ReadonlyArray] => { + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) return [state, []]; + const fail = (message: string, cause: unknown) => + [state, [LifecycleCommand.FailConnection({ id, waiterId, message, cause })]] as const; + if (service.intent === "stopped") return fail(`${id} is explicitly stopped`, undefined); + const eligibleNow = requireReady ? admissionReady(state, id) : sessionReady(state, id); + if (eligibleNow) + return [ + traffic ? setService(state, id, (s) => ({ ...s, leases: s.leases + 1 })) : state, + [LifecycleCommand.AdmitConnection({ id, waiterId })], + ]; + if (isBreakerOpen(service.breaker)) + return fail(`${id} circuit breaker is open`, service.breaker.lastCause); + // The cap bounds sockets held by traffic; explicit operations are few and already bounded. + const trafficWaiters = [...service.waiters.values()].filter(isLeaseWaiter).length; + if (traffic && trafficWaiters >= waiterCap) return fail(`${id} has too many waiters`, undefined); + const commands: Array = []; + const queued = setService(state, id, (s) => ({ + ...s, + waiters: new Map(s.waiters).set(waiterId, { + id: waiterId, + kind: traffic ? "traffic" : "explicit", + requireReady, + }), + })); + return [retryBlockedCleanup(queued, id, commands), commands]; +}; + +/** The transitive dependents currently active enough to block a stop or restart of a prerequisite. */ +const activeDependents = (state: LifecycleState, id: string): ReadonlyArray => + [...(state.graph.transitiveDependents.get(id) ?? [])].filter((dependentId) => { + const dependentState = state.services.get(dependentId); + return dependentState !== undefined && dependentIsActive(dependentState); + }); + +/** Applies an explicit stop, or returns undefined after rejecting it for active dependents. */ +const requestStop = ( + state: LifecycleState, + id: string, + commands: Array, +): LifecycleState | undefined => { + const service = state.services.get(id); + if (service === undefined) return undefined; + // Checked independently of whether this service itself needs a `Stop` command: a `Failed` + // target with an open breaker must still protect a dependent's retained demand. + const blockers = activeDependents(state, id); + if (blockers.length > 0) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "stop", + message: `${id} has active dependents: ${blockers.join(", ")}`, + }), + ); + return undefined; + } + const winding = isUpOrWindingDown(service.phase) && service.phase._tag !== "Stopping"; + const failed = failAllWaiters(service, id, `${id} was explicitly stopped`, undefined, commands); + const generation = generationOf(failed.phase); + if (winding && generation !== undefined) commands.push(LifecycleCommand.Stop({ id, generation })); + const phase = + winding && generation !== undefined + ? Phase.Stopping({ generation }) + : failed.phase._tag === "Failed" + ? Phase.Stopped() + : failed.phase; + const stopped = setService(state, id, () => ({ + ...failed, + intent: "stopped", + relaunchForced: false, + phase, + })); + return retryCleanup(stopped, id, commands); +}; + +/** Rejects an explicit start or restart of a service a destroy has claimed. */ +const rejectWhileDestroying = ( + service: ServiceState, + id: string, + operation: "start" | "restart", + commands: Array, +): boolean => { + if (service.destroy === undefined) return false; + commands.push( + LifecycleCommand.RequestRejected({ id, operation, message: `${id} is being destroyed` }), + ); + return true; +}; + +/** Re-arms every prerequisite an explicit start or restart depends on, clearing a stopped intent. */ +const rearmPrerequisites = (state: LifecycleState, id: string): LifecycleState => { + let next = state; + for (const prerequisite of state.graph.prerequisiteClosure.get(id) ?? []) { + const prerequisiteSpec = state.graph.services.get(prerequisite); + if (prerequisiteSpec === undefined) continue; + next = setService(next, prerequisite, (s) => + s.intent === "stopped" ? { ...s, intent: prerequisiteSpec.activation } : s, + ); + } + return next; +}; + +/** Counts one generation's launch failure or crash, opening the breaker at the threshold. */ +const countFailure = ( + service: ServiceState, + id: string, + cause: unknown, + now: number, + commands: Array, +): BreakerState => { + const { lastFailureAt } = service.breaker; + const consecutiveFailures = + lastFailureAt !== undefined && now - lastFailureAt > breakerWindowMillis + ? 1 + : service.breaker.consecutiveFailures + 1; + if (consecutiveFailures < breakerThreshold) + return { consecutiveFailures, lastFailureAt: now, openUntil: undefined, lastCause: cause }; + const openUntil = now + breakerCooldownMillis; + commands.push( + LifecycleCommand.ArmCooldownTimer({ id, openUntil, delayMillis: breakerCooldownMillis }), + ); + return { consecutiveFailures, lastFailureAt: now, openUntil, lastCause: cause }; +}; + +const recordFailure = ( + state: LifecycleState, + id: string, + generation: number, + cause: unknown, + now: number, + commands: Array, +): LifecycleState => + setService( + failExplicitDependentWaiters(state, id, `prerequisite ${id} failed`, cause, commands), + id, + (service) => { + const failed = failAllWaiters(service, id, `${id} failed`, cause, commands); + return { + ...failed, + phase: Phase.Failed({ generation, cause }), + breaker: countFailure(service, id, cause, now, commands), + idleArmedEpoch: undefined, + relaunchForced: false, + readinessFailure: undefined, + reprobing: false, + }; + }, + ); + +const applyEvent = ( + state: LifecycleState, + event: LifecycleEvent, + now: number, +): readonly [LifecycleState, ReadonlyArray] => { + const commands: Array = []; + + switch (event._tag) { + case "ConnectionOpened": + return openWaiter(state, event.id, event.waiterId, event.requireReady, true); + case "ReadinessAwaited": + return openWaiter(state, event.id, event.waiterId, event.requireReady, false); + case "ConnectionClosed": { + return [ + setService(state, event.id, (s) => ({ ...s, leases: Math.max(0, s.leases - 1) })), + commands, + ]; + } + case "SessionAvailable": { + const { id, generation } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Starting" || + service.phase.generation !== generation + ) + break; + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: false }), + readinessFailure: undefined, + reprobing: false, + })), + commands, + ]; + } + case "LaunchSucceeded": { + const { id, generation } = event; + const service = state.services.get(id); + if (service === undefined || generationOf(service.phase) !== generation) break; + // Accepted either straight from `Starting` or after a `SessionAvailable` left it unready. + const fromStarting = service.phase._tag === "Starting"; + const fromUnreadySession = service.phase._tag === "Running" && !service.phase.ready; + if (!fromStarting && !fromUnreadySession) break; + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: true }), + readinessFailure: undefined, + reprobing: false, + })), + commands, + ]; + } + case "LaunchFailed": { + const { id, generation, cause } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Starting" || + service.phase.generation !== generation + ) + break; + return [recordFailure(state, id, generation, cause, now, commands), commands]; + } + case "SessionLost": { + const { id, generation, cause } = event; + const service = state.services.get(id); + if ( + service === undefined || + (service.phase._tag !== "Starting" && service.phase._tag !== "Running") || + service.phase.generation !== generation + ) + break; + // The crash counts and fails its waiters now; the generation then winds down like a stop. + const failed = recordFailure(state, id, generation, cause, now, commands); + return [ + setService(failed, id, (s) => ({ + ...s, + phase: Phase.Stopping({ generation, crash: { cause } }), + })), + commands, + ]; + } + case "Exited": { + const { id, generation, requested } = event; + const service = state.services.get(id); + if (service === undefined || generationOf(service.phase) !== generation) break; + if (service.phase._tag === "Stopping" || requested) { + // An explicit stop that arrived during a crash's cleanup wins: it ends `Stopped`. + const crash = + service.phase._tag === "Stopping" && service.intent !== "stopped" + ? service.phase.crash + : undefined; + return [ + setService(state, id, (s) => ({ + ...s, + phase: + crash === undefined + ? Phase.Stopped() + : Phase.Failed({ generation, cause: crash.cause }), + idleArmedEpoch: undefined, + readinessFailure: undefined, + reprobing: false, + cleanupFailure: undefined, + })), + commands, + ]; + } + break; + } + case "StopFailed": { + const { id, generation, cause, failure } = event; + const service = state.services.get(id); + if (service === undefined || generationOf(service.phase) !== generation) break; + // Nothing may launch or reserve storage while resources are held, and waiters can't be + // served by a generation that is going away, so they get the cleanup failure now. + const message = `${id} cleanup failed`; + const dependentsFailed = failExplicitDependentWaiters( + state, + id, + `prerequisite ${message}`, + cause, + commands, + ); + const crash = + service.phase._tag === "Stopping" + ? service.phase.crash + : failure === undefined + ? undefined + : { cause: failure }; + return [ + setService(dependentsFailed, id, (s) => ({ + ...failAllWaiters(s, id, message, cause, commands), + phase: Phase.Stopping({ generation, ...(crash === undefined ? {} : { crash }) }), + breaker: + failure === undefined || + service.phase._tag === "Stopping" || + service.phase._tag === "Failed" + ? s.breaker + : countFailure(s, id, failure, now, commands), + idleArmedEpoch: undefined, + relaunchForced: false, + readinessFailure: undefined, + reprobing: false, + cleanupFailure: { cause }, + })), + commands, + ]; + } + case "IdleElapsed": { + const { id, generation, epoch } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Running" || + service.phase.generation !== generation || + service.idleArmedEpoch !== epoch || + hasDemand(state, id) + ) + break; + commands.push(LifecycleCommand.Stop({ id, generation })); + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Stopping({ generation }), + idleArmedEpoch: undefined, + })), + commands, + ]; + } + case "ReadinessLost": { + const { id, generation, cause } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Running" || + service.phase.generation !== generation + ) + break; + // Not terminal: the live session is kept, and the next waiter that needs it triggers a + // `Reprobe`. Waiters already awaiting this readiness get the check's failure. + const message = `${id} is not ready`; + const dependentsFailed = failExplicitDependentWaiters( + state, + id, + `prerequisite ${message}`, + cause, + commands, + ); + return [ + setService(dependentsFailed, id, (s) => { + const waiters = new Map(s.waiters); + for (const waiter of s.waiters.values()) { + if (!waiter.requireReady) continue; + waiters.delete(waiter.id); + commands.push( + LifecycleCommand.FailConnection({ id, waiterId: waiter.id, message, cause }), + ); + } + return { + ...s, + phase: Phase.Running({ generation, ready: false }), + waiters, + readinessFailure: { cause, reprobed: s.reprobing }, + reprobing: false, + }; + }), + commands, + ]; + } + case "ReadinessRecovered": { + const { id, generation } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Running" || + service.phase.generation !== generation + ) + break; + return [ + setService(state, id, (s) => ({ + ...s, + phase: Phase.Running({ generation, ready: true }), + readinessFailure: undefined, + reprobing: false, + })), + commands, + ]; + } + case "StageChanged": { + const { id, generation, stage } = event; + const service = state.services.get(id); + if ( + service === undefined || + service.phase._tag !== "Starting" || + service.phase.generation !== generation + ) + break; + return [ + setService(state, id, (s) => ({ ...s, phase: Phase.Starting({ generation, stage }) })), + commands, + ]; + } + case "CooldownElapsed": { + const { id, openUntil } = event; + const service = state.services.get(id); + if (service === undefined || service.breaker.openUntil !== openUntil) break; + return [ + setService(state, id, (s) => ({ ...s, breaker: { ...s.breaker, openUntil: undefined } })), + commands, + ]; + } + case "StartRequested": { + const { id } = event; + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) break; + if (rejectWhileDestroying(service, id, "start", commands)) break; + // A service already `Running` has nothing left to force, and forcing it here would wrongly + // survive into a later, unrelated idle cycle. One already `Stopping` is different: that + // generation's end is imminent and this request's demand must carry through to the next one + // (its own exit confirmation is the only thing that will ever consume this flag). + const forced = + service.phase._tag === "Stopped" || + service.phase._tag === "Failed" || + service.phase._tag === "Stopping"; + const updated = setService(state, id, (s) => ({ + ...s, + intent: spec.activation, + breaker: initialBreaker, + relaunchForced: s.relaunchForced || forced, + })); + return [rearmPrerequisites(retryCleanup(updated, id, commands), id), commands]; + } + case "ArmRequested": { + const { id } = event; + const spec = state.graph.services.get(id); + const service = state.services.get(id); + if (spec === undefined || service === undefined || service.destroy !== undefined) break; + return [ + setService(state, id, (s) => ({ ...s, intent: spec.activation, breaker: initialBreaker })), + commands, + ]; + } + case "StopRequested": { + const stopped = requestStop(state, event.id, commands); + return [stopped ?? state, commands]; + } + case "DestroyRequested": { + const stopped = requestStop(state, event.id, commands); + if (stopped === undefined) break; + return [ + setService(stopped, event.id, (s) => ({ ...s, destroy: s.destroy ?? "requested" })), + commands, + ]; + } + case "DestroyReleased": { + return [ + setService(state, event.id, (s) => + s.destroy === undefined + ? s + : { + ...s, + destroy: undefined, + storageReserved: s.destroy === "reserved" ? false : s.storageReserved, + }, + ), + commands, + ]; + } + case "RestartRequested": { + const { id } = event; + const service = state.services.get(id); + const spec = state.graph.services.get(id); + if (service === undefined || spec === undefined) break; + if (rejectWhileDestroying(service, id, "restart", commands)) break; + const blockers = activeDependents(state, id); + if (blockers.length > 0) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "restart", + message: `${id} has active dependents: ${blockers.join(", ")}`, + }), + ); + break; + } + const winding = isUpOrWindingDown(service.phase) && service.phase._tag !== "Stopping"; + const generation = generationOf(service.phase); + if (winding && generation !== undefined) + commands.push(LifecycleCommand.Stop({ id, generation })); + const updated = setService(state, id, (s) => ({ + ...s, + intent: spec.activation, + breaker: initialBreaker, + relaunchForced: true, + restartCandidate: event.candidate === undefined ? s.restartCandidate : event.candidate, + phase: winding && generation !== undefined ? Phase.Stopping({ generation }) : s.phase, + })); + return [rearmPrerequisites(retryCleanup(updated, id, commands), id), commands]; + } + case "StorageReserved": { + const { id } = event; + const service = state.services.get(id); + if (service === undefined) break; + if (service.destroy !== undefined) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "storage", + message: `${id} is being destroyed`, + }), + ); + break; + } + if ( + service.phase._tag !== "Stopped" || + service.intent !== "stopped" || + service.storageReserved + ) { + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "storage", + message: `${id} must be stopped with wake disabled before modifying data`, + }), + ); + break; + } + return [setService(state, id, (s) => ({ ...s, storageReserved: true })), commands]; + } + case "StorageReleased": { + // Destroy's own reservation ends only with `DestroyReleased`. + return [ + setService(state, event.id, (s) => + s.destroy === "reserved" ? s : { ...s, storageReserved: false }, + ), + commands, + ]; + } + case "WaiterCancelled": { + const { id, waiterId } = event; + return [ + setService(state, id, (s) => { + if (!s.waiters.has(waiterId)) return s; + const waiters = new Map(s.waiters); + waiters.delete(waiterId); + return { ...s, waiters }; + }), + commands, + ]; + } + case "GraphUpdated": { + const { graph } = event; + const removedIds = [...state.graph.services.keys()].filter((id) => !graph.services.has(id)); + const changedIds = [...state.graph.services.keys()].filter((id) => { + const previousSpec = state.graph.services.get(id); + const nextSpec = graph.services.get(id); + return ( + previousSpec !== undefined && + nextSpec !== undefined && + !specsEqual(previousSpec, nextSpec) + ); + }); + // A destroy holding its reservation owns the service, so its removal can't be blocked. + const destroyed = (id: string) => state.services.get(id)?.destroy === "reserved"; + const blockedIds = [...removedIds.filter((id) => !destroyed(id)), ...changedIds].filter( + (id) => { + const service = state.services.get(id); + return service !== undefined && isActiveForGraphUpdate(service); + }, + ); + if (blockedIds.length > 0) { + for (const id of blockedIds) + commands.push( + LifecycleCommand.RequestRejected({ + id, + operation: "graph", + message: `${id} is active and can't be removed or changed by a graph update`, + }), + ); + break; + } + const services = new Map(); + const generationCounters = new Map(state.generationCounters); + for (const spec of graph.services.values()) { + const existing = state.services.get(spec.id); + services.set(spec.id, existing ?? { ...initialService(spec), intent: "stopped" }); + if (!generationCounters.has(spec.id)) generationCounters.set(spec.id, 1); + } + return [{ graph, services, generationCounters }, commands]; + } + } + + return [state, commands]; +}; + +/** + * Cascades launches, idle arming and waiter resolution across the graph after one event. A + * single topological pass suffices: a launch started here only reaches `Running` on a later + * event, so a dependent several hops away naturally waits for its own future turn. Every + * condition here is a function of state alone, so replaying this pass for a stale or unrelated + * event can never surface a new launch or admission on its own; only the dedicated time-driven + * events (`CooldownElapsed`, `IdleElapsed`) do that. + */ +const settle = ( + state: LifecycleState, + commands: ReadonlyArray, +): readonly [LifecycleState, ReadonlyArray] => { + let next = state; + const emitted: Array = [...commands]; + + // A pending destroy takes the storage reservation as soon as the service is stopped and free. + for (const [id, service] of next.services) + if ( + service.destroy === "requested" && + service.phase._tag === "Stopped" && + !service.storageReserved + ) + next = setService(next, id, (s) => ({ ...s, destroy: "reserved", storageReserved: true })); + + for (const id of next.graph.order) { + const spec = next.graph.services.get(id); + const service = next.services.get(id); + if (spec === undefined || service === undefined) continue; + + if (service.phase._tag === "Stopped" || service.phase._tag === "Failed") { + if (service.intent === "stopped") continue; + if (service.storageReserved) continue; + if (isBreakerOpen(service.breaker)) continue; + if (!prerequisitesSatisfied(next, id)) continue; + if (!hasDemand(next, id) && !service.relaunchForced) continue; + const generation = next.generationCounters.get(id) ?? 1; + emitted.push( + LifecycleCommand.Launch({ id, generation, candidate: service.restartCandidate }), + ); + next = { + ...setService(next, id, (s) => ({ + ...s, + phase: Phase.Starting({ generation, stage: "preparing" }), + relaunchForced: false, + restartCandidate: undefined, + readinessFailure: undefined, + reprobing: false, + })), + generationCounters: new Map(next.generationCounters).set(id, generation + 1), + }; + continue; + } + + if (service.phase._tag === "Running") { + const generation = service.phase.generation; + // A blocking unhealthy session is re-checked until it recovers, its generation ends or no + // waiter needs it; their budgets bound how long that lasts. + if ( + !service.phase.ready && + service.readinessFailure !== undefined && + !service.reprobing && + blocksWaiters(next, id) + ) { + emitted.push( + LifecycleCommand.Reprobe({ + id, + generation, + delayMillis: service.readinessFailure.reprobed ? reprobeSpacingMillis : 0, + }), + ); + next = setService(next, id, (s) => ({ ...s, reprobing: true })); + } + if (prerequisitesSatisfied(next, id) && service.waiters.size > 0) { + const ready = service.phase.ready; + const admitted: Array = []; + for (const waiter of service.waiters.values()) + if (!waiter.requireReady || ready) admitted.push(waiter.id); + if (admitted.length > 0) { + const leased = admitted.filter((waiterId) => { + const waiter = service.waiters.get(waiterId); + return waiter !== undefined && isLeaseWaiter(waiter); + }).length; + for (const waiterId of admitted) + emitted.push(LifecycleCommand.AdmitConnection({ id, waiterId })); + next = setService(next, id, (s) => { + const waiters = new Map(s.waiters); + for (const waiterId of admitted) waiters.delete(waiterId); + return { ...s, leases: s.leases + leased, waiters }; + }); + } + } + const settled = next.services.get(id); + if (settled === undefined) continue; + const idles = + settled.intent === "lazy" && + spec.idleMillis !== undefined && + settled.phase._tag === "Running"; + if (idles && settled.phase._tag === "Running") { + const demand = hasDemand(next, id); + if (demand && settled.idleArmedEpoch !== undefined) { + // Demand returning invalidates any outstanding timer regardless of readiness: an idle + // arm/clear decision must never be stranded by a readiness loss that follows it, or the + // service could never become idle-eligible again once readiness recovers. + next = setService(next, id, (s) => ({ ...s, idleArmedEpoch: undefined })); + } else if ( + !demand && + (settled.phase.ready || settled.readinessFailure !== undefined) && + settled.idleArmedEpoch === undefined + ) { + const epoch = settled.idleEpoch + 1; + emitted.push( + LifecycleCommand.ArmIdleTimer({ + id, + generation, + epoch, + delayMillis: spec.idleMillis ?? 0, + }), + ); + next = setService(next, id, (s) => ({ ...s, idleEpoch: epoch, idleArmedEpoch: epoch })); + } + } + } + } + + return [next, emitted]; +}; + +/** The one pure `(state, event, now) -> (state, commands)` step for an owner's whole graph. */ +export const reduce = ( + state: LifecycleState, + event: LifecycleEvent, + now: number, +): readonly [LifecycleState, ReadonlyArray] => { + const [applied, directCommands] = applyEvent(state, event, now); + return settle(applied, directCommands); +}; diff --git a/packages/stack/src/Lifecycle.unit.test.ts b/packages/stack/src/Lifecycle.unit.test.ts new file mode 100644 index 0000000000..1ac8054522 --- /dev/null +++ b/packages/stack/src/Lifecycle.unit.test.ts @@ -0,0 +1,1674 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + blockingStage, + initialState, + LifecycleEvent, + makeGraph, + reduce, + type LifecycleCommand as Command, + type LifecycleState, + type ServiceSpec, +} from "./Lifecycle.ts"; + +const lazy = (id: string, overrides: Partial = {}): ServiceSpec => ({ + id, + activation: "lazy", + prerequisites: [], + idleMillis: 1_000, + ...overrides, +}); + +const open = (id: string, waiterId: number, requireReady = true) => + LifecycleEvent.ConnectionOpened({ id, waiterId, requireReady }); + +/** Drives a sequence of events through one lifecycle state, returning each step's commands. */ +const run = ( + initial: LifecycleState, + steps: ReadonlyArray<{ event: LifecycleEvent; now: number }>, +) => { + let state = initial; + const commandsByStep: Array> = []; + for (const { event, now } of steps) { + const [next, commands] = reduce(state, event, now); + state = next; + commandsByStep.push(commands); + } + return { state, commandsByStep }; +}; + +/** A runtime's unexpected exit as the service reports it: lost first, then cleaned up. */ +const crashSteps = (id: string, generation: number, now: number, cause = "crash") => [ + { event: LifecycleEvent.SessionLost({ id, generation, cause }), now }, + { event: LifecycleEvent.Exited({ id, generation, requested: false }), now }, +]; + +const tagsOf = (commands: ReadonlyArray | undefined) => + (commands ?? []).map((command) => command._tag); + +/** Whether a storage reservation requested now would be accepted rather than rejected. */ +const storageAdmitted = (state: LifecycleState, id: string): boolean => + !tagsOf( + run(state, [{ event: LifecycleEvent.StorageReserved({ id }), now: 0 }]).commandsByStep[0], + ).includes("RequestRejected"); + +/** Narrows a service's current phase to `Starting` and returns its generation, or throws. */ +const startingGeneration = (state: LifecycleState, id: string): number => { + const phase = state.services.get(id)?.phase; + if (phase?._tag !== "Starting") throw new Error(`Expected ${id} to be starting`); + return phase.generation; +}; + +/** Reads a service's current generation from any non-`Stopped` phase, or throws. */ +const currentGeneration = (state: LifecycleState, id: string): number => { + const phase = state.services.get(id)?.phase; + if (phase === undefined || phase._tag === "Stopped") + throw new Error(`Expected ${id} to have a generation`); + return phase.generation; +}; + +/** Indexed access with `noUncheckedIndexedAccess`; throws with a clear message if out of bounds. */ +const at = (array: ReadonlyArray, index: number): T => { + const value = array[index]; + if (value === undefined) throw new Error(`Expected an element at index ${index}`); + return value; +}; + +/** Finds one command by tag, or throws; avoids a non-null assertion at call sites. */ +const commandTagged = ( + commands: ReadonlyArray, + tag: Tag, +): Extract => { + const found = commands.find( + (command): command is Extract => command._tag === tag, + ); + if (found === undefined) throw new Error(`Expected a ${tag} command`); + return found; +}; + +describe("Lifecycle reducer", () => { + it("ignores a launch outcome, an idle elapse and an exit tagged with a stale generation", () => { + const graph = makeGraph([lazy("api")]); + const { state, commandsByStep } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 99 }), now: 10 }, + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 99 }), now: 20 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 99, + requested: false, + }), + now: 30, + }, + { event: LifecycleEvent.SessionLost({ id: "api", generation: 99, cause: "boom" }), now: 40 }, + ]); + + expect(commandsByStep[1]).toEqual([]); + expect(commandsByStep[2]).toEqual([]); + expect(commandsByStep[3]).toEqual([]); + expect(commandsByStep[4]).toEqual([]); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Starting", + generation: 1, + stage: "preparing", + }); + }); + + it("ignores an idle elapse tagged with a stale epoch even while a newer timer is armed with zero demand", () => { + const graph = makeGraph([lazy("api")]); + const { state } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, // arms epoch 1 + { event: open("api", 2), now: 30 }, // demand returns: epoch 1's timer goes stale + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 40 }, // arms epoch 2 + ]); + expect(state.services.get("api")?.idleArmedEpoch).toBe(2); + + const stale = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 1_040 }, + ]); + expect(stale.commandsByStep[0]).toEqual([]); + expect(stale.state.services.get("api")?.phase._tag).toBe("Running"); + + const current = run(stale.state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 2 }), now: 1_041 }, + ]); + expect(tagsOf(current.commandsByStep[0])).toEqual(["Stop"]); + }); + + it("keeps a service running when traffic is admitted before the idle commit", () => { + const graph = makeGraph([lazy("api")]); + const { state, commandsByStep } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + // Traffic arrives while the timer is armed but before it fires: this prevents the sleep. + { event: open("api", 2), now: 500 }, + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 1_000 }, + ]); + + expect(tagsOf(commandsByStep[3])).toEqual(["AdmitConnection"]); + expect(commandsByStep[4]).toEqual([]); + expect(state.services.get("api")?.phase._tag).toBe("Running"); + expect(state.services.get("api")?.leases).toBe(1); + }); + + it("holds traffic arriving after a committed sleep for the confirmed stop and the next generation, without failing it", () => { + const graph = makeGraph([lazy("api")]); + const { state, commandsByStep } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + // The idle timer commits to stopping before any new traffic arrives. + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 1_020 }, + { event: open("api", 2), now: 1_030 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: true, + }), + now: 1_040, + }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 2 }), now: 1_050 }, + ]); + + expect(tagsOf(commandsByStep[3])).toEqual(["Stop"]); + expect(tagsOf(commandsByStep[4])).toEqual([]); + expect(tagsOf(commandsByStep[5])).toEqual(["Launch"]); + expect((at(at(commandsByStep, 5), 0) as Extract).generation).toBe( + 2, + ); + expect(tagsOf(commandsByStep[6])).toEqual(["AdmitConnection"]); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 2, + ready: true, + }); + }); + + it("keeps every prerequisite running while a dependent chain runs, and sleeps each one only once its own dependents confirm their stop", () => { + const graph = makeGraph([ + lazy("database"), + lazy("pg-meta", { prerequisites: ["database"] }), + lazy("functions", { prerequisites: ["database"] }), + lazy("studio", { prerequisites: ["pg-meta", "functions"] }), + ]); + let state = initialState(graph); + let commandsByStep: ReadonlyArray>; + + ({ state } = run(state, [{ event: open("studio", 1), now: 0 }])); + expect(state.services.get("database")?.phase).toEqual({ + _tag: "Starting", + generation: 1, + stage: "preparing", + }); + expect(state.services.get("pg-meta")?.phase._tag).toBe("Stopped"); + expect(state.services.get("functions")?.phase._tag).toBe("Stopped"); + + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 10 }, + ])); + expect(state.services.get("pg-meta")?.phase._tag).toBe("Starting"); + expect(state.services.get("functions")?.phase._tag).toBe("Starting"); + + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "pg-meta", generation: 1 }), now: 20 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "functions", generation: 1 }), now: 30 }, + ])); + expect(state.services.get("studio")?.phase._tag).toBe("Starting"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "studio", generation: 1 }), now: 40 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["AdmitConnection"]); + + // Studio idles and its exit is confirmed: only then do its prerequisites become idle-eligible. + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "studio" }), now: 50 }, + { event: LifecycleEvent.IdleElapsed({ id: "studio", generation: 1, epoch: 1 }), now: 1_050 }, + ])); + expect(tagsOf(commandsByStep[1])).toEqual(["Stop"]); + expect(state.services.get("pg-meta")?.idleArmedEpoch).toBeUndefined(); + expect(state.services.get("functions")?.idleArmedEpoch).toBeUndefined(); + // The database is a dependent-of-a-dependent: it still sees pg-meta/functions `Running`. + expect(state.services.get("database")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "studio", + generation: 1, + requested: true, + }), + now: 1_060, + }, + ])); + expect(state.services.get("studio")?.phase).toEqual({ _tag: "Stopped" }); + expect(tagsOf(commandsByStep[0]).sort()).toEqual(["ArmIdleTimer", "ArmIdleTimer"]); + // pg-meta and functions are now merely idle-armed, still `Running`, so the database stays up. + expect(state.services.get("database")?.phase._tag).toBe("Running"); + expect(state.services.get("database")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "pg-meta", generation: 1, epoch: 1 }), now: 2_060 }, + { + event: LifecycleEvent.IdleElapsed({ id: "functions", generation: 1, epoch: 1 }), + now: 2_060, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + expect(tagsOf(commandsByStep[1])).toEqual(["Stop"]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "pg-meta", + generation: 1, + requested: true, + }), + now: 2_070, + }, + ])); + // Functions is still running: the database still has a live dependent. + expect(state.services.get("database")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "functions", + generation: 1, + requested: true, + }), + now: 2_080, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["ArmIdleTimer"]); + expect(state.services.get("database")?.idleArmedEpoch).toBe(1); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.IdleElapsed({ id: "database", generation: 1, epoch: 1 }), + now: 3_080, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "database", + generation: 1, + requested: true, + }), + now: 3_090, + }, + ])); + expect(state.services.get("database")?.phase).toEqual({ _tag: "Stopped" }); + }); + + it("blocks admission across the full prerequisite closure when a transitive prerequisite loses readiness", () => { + const graph = makeGraph([ + lazy("a"), + lazy("b", { prerequisites: ["a"] }), + lazy("c", { prerequisites: ["b"] }), + ]); + let state = initialState(graph); + let commandsByStep: ReadonlyArray>; + + ({ state } = run(state, [{ event: open("c", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "c", generation: 1 }), now: 30 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["AdmitConnection"]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.ReadinessLost({ id: "a", generation: 1, cause: "db down" }), + now: 40, + }, + ])); + // b and c are untouched directly: losing a prerequisite doesn't replace a healthy dependent. + expect(state.services.get("b")?.phase).toEqual({ _tag: "Running", generation: 1, ready: true }); + expect(state.services.get("c")?.phase).toEqual({ _tag: "Running", generation: 1, ready: true }); + + const blocked = run(state, [{ event: open("c", 2), now: 50 }]); + expect(tagsOf(blocked.commandsByStep[0])).toEqual(["Reprobe"]); + expect(commandTagged(at(blocked.commandsByStep, 0), "Reprobe").id).toBe("a"); + expect(blocked.state.services.get("c")?.waiters.size).toBe(1); + expect(blockingStage(blocked.state, "c")).toBe("prerequisite a"); + }); + + describe("explicit start and restart demand", () => { + it("launches an idle lazy service on its own explicit start, with no connection involved", () => { + const graph = makeGraph([lazy("api")]); + const { commandsByStep } = run(initialState(graph), [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + }); + + it("restarts after an explicit stop, even though restart alone would otherwise leave intent stopped", () => { + const graph = makeGraph([lazy("api")]); + let state = initialState(graph); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 0 }])); + + const restarted = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "api" }), now: 1 }, + ]); + expect(tagsOf(restarted.commandsByStep[0])).toEqual(["Launch"]); + expect(restarted.state.services.get("api")?.intent).toBe("lazy"); + }); + + it("propagates an explicit restart's demand to a fully stopped prerequisite", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + const { commandsByStep } = run(initialState(graph), [ + { event: LifecycleEvent.RestartRequested({ id: "b" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + expect(commandTagged(at(commandsByStep, 0), "Launch").id).toBe("a"); + }); + + it("launches the newest restart candidate once, then returns to the saved input", () => { + const graph = makeGraph([lazy("api")]); + const { commandsByStep } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + { event: LifecycleEvent.RestartRequested({ id: "api", candidate: "first" }), now: 2 }, + { event: LifecycleEvent.RestartRequested({ id: "api", candidate: "second" }), now: 3 }, + { event: LifecycleEvent.RestartRequested({ id: "api" }), now: 4 }, + { event: LifecycleEvent.Exited({ id: "api", generation: 1, requested: true }), now: 5 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 2 }), now: 6 }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 7 }, + { event: LifecycleEvent.Exited({ id: "api", generation: 2, requested: true }), now: 8 }, + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 9 }, + ]); + expect(commandTagged(at(commandsByStep, 0), "Launch").candidate).toBeUndefined(); + expect(commandTagged(at(commandsByStep, 5), "Launch").candidate).toBe("second"); + expect(commandTagged(at(commandsByStep, 9), "Launch").candidate).toBeUndefined(); + }); + + it("keeps no candidate for a restart the reducer rejects", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + const { state, commandsByStep } = run(initialState(graph), [ + { event: LifecycleEvent.RestartRequested({ id: "a", candidate: "rejected" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); + expect(state.services.get("a")?.restartCandidate).toBeUndefined(); + }); + }); + + describe("circuit breaker", () => { + it("opens after 3 consecutive failures, fails fast naming the cause, and recovers once after the cooldown for several waiters at once", () => { + const graph = makeGraph([lazy("flaky")]); + let state = initialState(graph); + let commandsByStep: ReadonlyArray>; + + const failAttempt = (waiterId: number, now: number) => { + ({ state, commandsByStep } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ + id: "flaky", + generation, + cause: `boom-${generation}`, + }), + now: now + 1, + }, + ])); + return commandsByStep; + }; + + failAttempt(1, 0); + failAttempt(2, 100); + const thirdFailure = failAttempt(3, 200); + expect(at(thirdFailure, 0).some((command) => command._tag === "FailConnection")).toBe(true); + expect(state.services.get("flaky")?.breaker.consecutiveFailures).toBe(3); + expect(state.services.get("flaky")?.breaker.openUntil).toBe(201 + 30_000); + + // Fails fast while the breaker is open, naming the last cause, without a new launch. + ({ state, commandsByStep } = run(state, [{ event: open("flaky", 4), now: 300 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["FailConnection"]); + const fastFail = commandTagged(at(commandsByStep, 0), "FailConnection"); + expect(fastFail.cause).toBe("boom-3"); + expect(state.services.get("flaky")?.phase._tag).toBe("Failed"); + + // The dedicated cooldown event is what reopens admission, not the mere passage of time. + ({ state } = run(state, [ + { + event: LifecycleEvent.CooldownElapsed({ id: "flaky", openUntil: 201 + 30_000 }), + now: 201 + 30_000, + }, + ])); + + // After the cooldown elapses, several waiters share exactly one recovery attempt. + ({ state, commandsByStep } = run(state, [{ event: open("flaky", 5), now: 201 + 30_000 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + const recovery = commandTagged(at(commandsByStep, 0), "Launch"); + + ({ state, commandsByStep } = run(state, [ + { event: open("flaky", 6), now: 201 + 30_000 + 1 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual([]); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation: recovery.generation }), + now: 201 + 30_000 + 10, + }, + ])); + expect(tagsOf(commandsByStep[0]).toSorted()).toEqual(["AdmitConnection", "AdmitConnection"]); + }); + + it("never opens for failures spaced wider than the window, but a fast start-ready-crash loop still does", () => { + const graph = makeGraph([lazy("flaky")]); + const crashCycle = (initial: LifecycleState, waiterId: number, now: number) => { + let state = initial; + ({ state } = run(state, [{ event: open("flaky", waiterId), now }])); + const generation = startingGeneration(state, "flaky"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "flaky", generation }), now: now + 1 }, + ...crashSteps("flaky", generation, now + 2), + ])); + return state; + }; + + const day = 86_400_000; + let daily = initialState(graph); + for (const waiterId of [1, 2, 3]) daily = crashCycle(daily, waiterId, waiterId * day); + expect(daily.services.get("flaky")?.breaker.openUntil).toBeUndefined(); + expect(daily.services.get("flaky")?.breaker.consecutiveFailures).toBe(1); + + let loop = initialState(graph); + for (const waiterId of [1, 2, 3]) loop = crashCycle(loop, waiterId, waiterId * 1_000); + expect(loop.services.get("flaky")?.breaker.openUntil).toBeDefined(); + }); + + it("schedules a cooldown timer when the breaker opens and only the dedicated event retries, even with demand retained the whole time", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph); + let commandsByStep: ReadonlyArray>; + + // b's own waiter on a never resolves, so it keeps retaining demand on a across every retry. + ({ state, commandsByStep } = run(state, [{ event: open("b", 1), now: 0 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + + const failRetained = (now: number) => { + const generation = startingGeneration(state, "a"); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchFailed({ id: "a", generation, cause: "boom" }), now }, + ])); + return commandsByStep; + }; + + // The first two failures retry on their own, because b's waiter still holds demand on a. + failRetained(1); + expect(state.services.get("a")?.phase._tag).toBe("Starting"); + failRetained(2); + const third = failRetained(3); + expect(state.services.get("a")?.phase._tag).toBe("Failed"); + const armed = commandTagged(at(third, 0), "ArmCooldownTimer"); + + // Demand is still retained, but the breaker stays shut until the dedicated event fires. + const stillClosed = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "a" }), now: armed.openUntil - 1 }, + ]); + expect(stillClosed.state.services.get("a")?.phase._tag).toBe("Failed"); + + const elapsed = run(state, [ + { + event: LifecycleEvent.CooldownElapsed({ id: "a", openUntil: armed.openUntil }), + now: armed.openUntil, + }, + ]); + expect(tagsOf(elapsed.commandsByStep[0])).toEqual(["Launch"]); + }); + + it("leaves a stale launch outcome a complete no-op even with demand retained and the cooldown elapsed on the wall clock", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph); + + // b's own waiter on a is never resolved, so demand on a is retained through every failure: + // under the old time-based breaker check, this is exactly the state a stale event could + // have exploited to sneak a new launch past the still-open breaker. + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + + const fail = (now: number) => { + const generation = startingGeneration(state, "a"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchFailed({ id: "a", generation, cause: "boom" }), now }, + ])); + }; + fail(1); + fail(2); + fail(3); + expect(state.services.get("a")?.breaker.openUntil).toBeDefined(); + expect(state.services.get("a")?.phase._tag).toBe("Failed"); + + // Far past the cooldown on the wall clock, but no `CooldownElapsed` has been processed yet. + const stale = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10_000_000 }, + ]); + expect(stale.commandsByStep[0]).toEqual([]); + expect(stale.state).toEqual(state); + }); + }); + + it("admits a non-ready-requiring acquisition once a live session exists, bypassing target readiness", () => { + const graph = makeGraph([lazy("functions")]); + let state = initialState(graph); + ({ state } = run(state, [{ event: open("functions", 1), now: 0 }])); + const generation = startingGeneration(state, "functions"); + + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "functions", generation }), now: 10 }, + { + event: LifecycleEvent.ReadinessLost({ + id: "functions", + generation, + cause: "still booting", + }), + now: 11, + }, + ])); + expect(state.services.get("functions")?.phase).toEqual({ + _tag: "Running", + generation, + ready: false, + }); + + const readyRequiring = run(state, [{ event: open("functions", 2), now: 20 }]); + expect(tagsOf(readyRequiring.commandsByStep[0])).toEqual(["Reprobe"]); + + const inspector = run(state, [ + { + event: LifecycleEvent.ConnectionOpened({ + id: "functions", + waiterId: 3, + requireReady: false, + }), + now: 20, + }, + ]); + expect(tagsOf(inspector.commandsByStep[0])).toEqual(["AdmitConnection"]); + }); + + it("treats a readiness loss as non-terminal, re-probing once for the next waiter, unlike an exit which counts toward the breaker", () => { + const graph = makeGraph([lazy("api")]); + const { state: ready } = run(initialState(graph), [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ]); + + const { state: afterReadinessLoss, commandsByStep: lostCommands } = run(ready, [ + { + event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "probe failed" }), + now: 20, + }, + ]); + expect(lostCommands[0]).toEqual([]); + expect(afterReadinessLoss.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: false, + }); + expect(afterReadinessLoss.services.get("api")?.breaker.consecutiveFailures).toBe(0); + + const { state: probing, commandsByStep: probeCommands } = run(afterReadinessLoss, [ + { event: open("api", 2), now: 21 }, + { event: open("api", 3), now: 22 }, + ]); + expect(tagsOf(probeCommands[0])).toEqual(["Reprobe"]); + expect(tagsOf(probeCommands[1])).toEqual([]); + + const { state: recovered, commandsByStep: recoveredCommands } = run(probing, [ + { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 30 }, + ]); + expect(tagsOf(recoveredCommands[0])).toEqual(["AdmitConnection", "AdmitConnection"]); + expect(recovered.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: true, + }); + expect(recovered.services.get("api")?.breaker.consecutiveFailures).toBe(0); + + // Closing the lease first removes all demand, so the crash below settles into `Failed` + // instead of an immediate crash re-wake, isolating the phase/breaker assertion that follows. + const { state: idle } = run(ready, [ + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 15 }, + ]); + const { state: afterExit, commandsByStep: exitCommands } = run(idle, crashSteps("api", 1, 20)); + expect(afterExit.services.get("api")?.phase).toMatchObject({ _tag: "Failed", cause: "crash" }); + expect(afterExit.services.get("api")?.breaker.consecutiveFailures).toBe(1); + expect(exitCommands).toEqual([[], []]); + }); + + describe("crash cleanup", () => { + const sessionLost = (id: string, generation: number, now: number) => ({ + event: LifecycleEvent.SessionLost({ id, generation, cause: "crash" }), + now, + }); + + it("stops routing to a crashed session at once and re-wakes it for traffic that arrived during the cleanup", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: open("api", 1), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 2 }, + ])); + + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + sessionLost("api", 1, 10), + { event: open("api", 2), now: 11 }, + { event: LifecycleEvent.SessionAvailable({ id: "api", generation: 1 }), now: 12 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 13 }, + ])); + expect(commandsByStep).toEqual([[], [], [], []]); + expect(state.services.get("api")?.phase).toMatchObject({ _tag: "Stopping", generation: 1 }); + expect(state.services.get("api")?.waiters.size).toBe(1); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: false, + }), + now: 20, + }, + ])); + expect(commandTagged(at(commandsByStep, 0), "Launch").generation).toBe(2); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + }); + + it("fails the waiters of a generation that crashes while starting and ignores its late session", () => { + let state = initialState(makeGraph([lazy("api")])); + let commandsByStep: ReadonlyArray>; + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const generation = startingGeneration(state, "api"); + + ({ state, commandsByStep } = run(state, [ + sessionLost("api", generation, 5), + { event: LifecycleEvent.SessionAvailable({ id: "api", generation }), now: 6 }, + { + event: LifecycleEvent.Exited({ id: "api", generation, requested: false }), + now: 7, + }, + ])); + expect(commandTagged(at(commandsByStep, 0), "FailConnection")).toMatchObject({ + waiterId: 1, + cause: "crash", + }); + expect(commandsByStep[1]).toEqual([]); + expect(commandsByStep[2]).toEqual([]); + expect(state.services.get("api")?.phase).toMatchObject({ _tag: "Failed", cause: "crash" }); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + }); + + it("counts a crash once when its cleanup fails and a retried stop then confirms it", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + sessionLost("api", 1, 2), + { + event: LifecycleEvent.StopFailed({ + id: "api", + generation: 1, + cause: "busy", + failure: "crash", + }), + now: 3, + }, + ])); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 4 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: true, + }), + now: 5, + }, + ])); + expect(state.services.get("api")?.phase._tag).toBe("Stopped"); + expect(state.services.get("api")?.breaker.consecutiveFailures).toBe(1); + }); + + it("ends Failed without relaunching an eager service whose crash cleanup failed and was retried by traffic", () => { + let state = initialState( + makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]), + ); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, + sessionLost("database", 1, 2), + { + event: LifecycleEvent.StopFailed({ id: "database", generation: 1, cause: "busy" }), + now: 3, + }, + ])); + + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: open("database", 1), now: 4 }, + { event: LifecycleEvent.WaiterCancelled({ id: "database", waiterId: 1 }), now: 5 }, + { + event: LifecycleEvent.Exited({ id: "database", generation: 1, requested: true }), + now: 6, + }, + ])); + expect(tagsOf(commandsByStep[0])).toContain("Stop"); + expect(commandsByStep[2]).toEqual([]); + expect(state.services.get("database")?.phase).toMatchObject({ + _tag: "Failed", + cause: "crash", + }); + }); + + it("ends Failed without relaunching an eager service whose launch cleanup was retained and retried by traffic", () => { + let state = initialState( + makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]), + ); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + { + event: LifecycleEvent.LaunchFailed({ id: "database", generation: 1, cause: "boom" }), + now: 1, + }, + { + event: LifecycleEvent.StopFailed({ + id: "database", + generation: 1, + cause: "busy", + failure: "boom", + }), + now: 2, + }, + ])); + + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: open("database", 1), now: 3 }, + { event: LifecycleEvent.WaiterCancelled({ id: "database", waiterId: 1 }), now: 4 }, + { + event: LifecycleEvent.Exited({ id: "database", generation: 1, requested: true }), + now: 5, + }, + ])); + expect(tagsOf(commandsByStep[0])).toContain("Stop"); + expect(commandsByStep[2]).toEqual([]); + expect(state.services.get("database")?.phase).toMatchObject({ + _tag: "Failed", + cause: "boom", + }); + }); + + it("ends Stopped when an explicit stop arrives during the crash's cleanup", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + sessionLost("api", 1, 2), + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 3 }, + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: false, + }), + now: 4, + }, + ])); + expect(state.services.get("api")?.phase._tag).toBe("Stopped"); + expect(storageAdmitted(state, "api")).toBe(true); + }); + }); + + it("rejects stopping or restarting a prerequisite while a dependent is active", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + expect(state.services.get("b")?.leases).toBe(1); + + const stopResult = run(state, [{ event: LifecycleEvent.StopRequested({ id: "a" }), now: 30 }]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state.services.get("a")?.phase._tag).toBe("Running"); + + const restartResult = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "a" }), now: 30 }, + ]); + expect(tagsOf(restartResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(restartResult.state.services.get("a")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: true, + }); + }); + + describe("storage reservation", () => { + it("admits storage operations only once a service is stopped with intent stopped", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph); + expect(storageAdmitted(state, "database")).toBe(false); + + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + ])); + expect(storageAdmitted(state, "database")).toBe(true); + + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 1 }, + ])); + ({ state } = run(state, [{ event: open("database", 1), now: 2 }])); + expect(storageAdmitted(state, "database")).toBe(false); + }); + + it("rejects a storage reservation unless the service is already stopped", () => { + const graph = makeGraph([lazy("database")]); + const { commandsByStep } = run(initialState(graph), [ + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 0 }, + ]); + expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); + }); + + it("blocks a launch while storage is reserved and relaunches once it's released", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph); + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + ])); + + const reserved = run(state, [ + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 1 }, + ]); + expect(reserved.commandsByStep[0]).toEqual([]); + expect(reserved.state.services.get("database")?.storageReserved).toBe(true); + state = reserved.state; + + // An explicit start during the restore can't emit a launch while storage is reserved. + const duringRestore = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 2 }, + ]); + expect(duringRestore.commandsByStep[0]).toEqual([]); + state = duringRestore.state; + + const released = run(state, [ + { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 3 }, + ]); + expect(tagsOf(released.commandsByStep[0])).toEqual(["Launch"]); + }); + + it("hands a running storage operation's reservation to a destroy requested meanwhile", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph); + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 1 }, + { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 2 }, + ])); + expect(state.services.get("database")?.destroy).toBe("requested"); + + const refused = run(state, [ + { event: LifecycleEvent.StorageReserved({ id: "database" }), now: 3 }, + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 3 }, + ]); + expect(refused.commandsByStep.map(tagsOf)).toEqual([ + ["RequestRejected"], + ["RequestRejected"], + ]); + expect(storageAdmitted(refused.state, "database")).toBe(false); + + ({ state } = run(refused.state, [ + { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 4 }, + ])); + expect(state.services.get("database")).toMatchObject({ + destroy: "reserved", + storageReserved: true, + }); + }); + + it("reserves storage for a destroy once its running service has stopped", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + ])); + const generation = startingGeneration(state, "database"); + + const requested = run(state, [ + { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 1 }, + ]); + expect(tagsOf(requested.commandsByStep[0])).toEqual(["Stop"]); + expect(requested.state.services.get("database")?.destroy).toBe("requested"); + + ({ state } = run(requested.state, [ + { + event: LifecycleEvent.Exited({ + id: "database", + generation, + requested: true, + }), + now: 2, + }, + ])); + expect(state.services.get("database")).toMatchObject({ + phase: { _tag: "Stopped" }, + destroy: "reserved", + storageReserved: true, + }); + }); + + it("makes a service usable again once a destroy releases it", () => { + const graph = makeGraph([lazy("database")]); + let state = initialState(graph); + ({ state } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "database" }), now: 0 }, + { event: LifecycleEvent.DestroyRequested({ id: "database" }), now: 1 }, + ])); + expect(state.services.get("database")?.destroy).toBe("reserved"); + // A storage operation's release can't end the destroy's own reservation. + ({ state } = run(state, [ + { event: LifecycleEvent.StorageReleased({ id: "database" }), now: 2 }, + ])); + expect(storageAdmitted(state, "database")).toBe(false); + + ({ state } = run(state, [ + { event: LifecycleEvent.DestroyReleased({ id: "database" }), now: 3 }, + ])); + expect(state.services.get("database")).toMatchObject({ + destroy: undefined, + storageReserved: false, + }); + expect(storageAdmitted(state, "database")).toBe(true); + }); + }); + + it("rejects new admission for an explicitly stopped service without creating a waiter", () => { + const graph = makeGraph([lazy("api")]); + let state = initialState(graph); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 0 }])); + + const result = run(state, [{ event: open("api", 1), now: 1 }]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["FailConnection"]); + expect(result.state.services.get("api")?.waiters.size).toBe(0); + }); + + it("starts an eager service's prerequisite on its own and cascades once that prerequisite is ready", () => { + const graph = makeGraph([ + lazy("database", { idleMillis: undefined }), + { id: "kong", activation: "eager", prerequisites: ["database"] }, + ]); + let state = initialState(graph); + let commandsByStep: ReadonlyArray>; + + // Kong is eager from composition, so the owner's first settle pass (triggered by any event, + // here the composition's own start signal) already starts its prerequisite without a waiter. + ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "kong" }), now: 0 }])); + expect(state.services.get("database")?.phase._tag).toBe("Starting"); + expect(state.services.get("kong")?.phase._tag).toBe("Stopped"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 10 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + expect(state.services.get("kong")?.phase._tag).toBe("Starting"); + }); + + describe("graph updates", () => { + it("preserves a continuing service's state and never reuses a generation after a remove and re-add", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }, + ])); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 30 }])); + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: true, + }), + now: 31, + }, + ])); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopped" }); + + ({ state } = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([]) }), now: 40 }, + ])); + expect(state.services.has("api")).toBe(false); + + ({ state } = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([lazy("api")]) }), now: 50 }, + ])); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopped" }); + + const relaunch = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 60 }, + ]); + expect(commandTagged(at(relaunch.commandsByStep, 0), "Launch").generation).toBe(2); + }); + + it("rejects removing a service that's starting with a pending waiter, atomically", () => { + let state = initialState(makeGraph([lazy("a")])); + ({ state } = run(state, [{ event: open("a", 1), now: 0 }])); + expect(state.services.get("a")?.phase._tag).toBe("Starting"); + + const rejected = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([]) }), now: 10 }, + ]); + expect(tagsOf(rejected.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(rejected.state).toEqual(state); + }); + + it("rejects removing a service with a storage reservation, so a graph update can't launch mid-restore", () => { + let state = initialState(makeGraph([lazy("db")])); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "db" }), now: 0 }])); + ({ state } = run(state, [{ event: LifecycleEvent.StorageReserved({ id: "db" }), now: 1 }])); + expect(state.services.get("db")?.storageReserved).toBe(true); + + const rejected = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: makeGraph([]) }), now: 2 }, + ]); + expect(tagsOf(rejected.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(rejected.state).toEqual(state); + }); + + it("rejects changing an active continuing service's prerequisites, leaving its live generation untouched", () => { + const graphV1 = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graphV1); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + expect(state.services.get("b")?.leases).toBe(1); + + // b keeps its id but drops a from its prerequisites, while b is still actively running on a. + const graphV2 = makeGraph([lazy("a"), lazy("b", { prerequisites: [] })]); + const rejected = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph: graphV2 }), now: 30 }, + ]); + expect(tagsOf(rejected.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(rejected.state).toEqual(state); + + // The update never took effect: a is still b's prerequisite and still can't be stopped. + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 40 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + }); + }); + + describe("stage reporting", () => { + it("updates the starting stage so a waiter's budget error names it", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const generation = startingGeneration(state, "api"); + + ({ state } = run(state, [ + { + event: LifecycleEvent.StageChanged({ id: "api", generation, stage: "launching" }), + now: 1, + }, + ])); + expect(state.services.get("api")?.phase).toEqual({ + _tag: "Starting", + generation, + stage: "launching", + }); + + expect(blockingStage(state, "api")).toBe("api to finish launching"); + }); + + it("ignores a stage change tagged with a stale generation", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + const result = run(state, [ + { + event: LifecycleEvent.StageChanged({ id: "api", generation: 99, stage: "launching" }), + now: 1, + }, + ]); + expect(result.state).toEqual(state); + }); + }); + + describe("the dependent guard", () => { + it("protects stranded downstream demand even when the immediate dependent is itself inactive", () => { + const graph = makeGraph([ + lazy("a"), + lazy("b", { prerequisites: ["a"] }), + lazy("c", { prerequisites: ["b"] }), + ]); + let state = initialState(graph); + ({ state } = run(state, [{ event: open("c", 1), now: 0 }])); + expect(state.services.get("a")?.phase._tag).toBe("Starting"); + expect(state.services.get("b")?.phase._tag).toBe("Stopped"); + expect(state.services.get("c")?.waiters.size).toBe(1); + + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 10 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state).toEqual(state); + }); + + it("still protects a dependent's demand once the prerequisite's own breaker has opened", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + expect(state.services.get("b")?.leases).toBe(1); + + // b's lease retains demand on a, so each crash immediately relaunches the next generation, + // until the third failure opens the breaker and a finally rests in `Failed`. + const crash = (now: number) => { + const generation = currentGeneration(state, "a"); + ({ state } = run(state, crashSteps("a", generation, now))); + }; + crash(30); + crash(31); + crash(32); + expect(state.services.get("a")?.breaker.openUntil).toBeDefined(); + expect(state.services.get("a")?.phase._tag).toBe("Failed"); + + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 40 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state.services.get("a")?.phase._tag).toBe("Failed"); + + const restartResult = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "a" }), now: 40 }, + ]); + expect(tagsOf(restartResult.commandsByStep[0])).toEqual(["RequestRejected"]); + }); + + it("blocks an explicit stop or restart of a prerequisite while a sleeping dependent is still wake-armed", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph); + ({ state } = run(state, [{ event: open("b", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "a", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "b", generation: 1 }), now: 20 }, + ])); + ({ state } = run(state, [{ event: LifecycleEvent.ConnectionClosed({ id: "b" }), now: 30 }])); + expect(state.services.get("b")?.idleArmedEpoch).toBe(1); + + ({ state } = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "b", generation: 1, epoch: 1 }), now: 1_030 }, + ])); + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "b", + generation: 1, + requested: true, + }), + now: 1_040, + }, + ])); + expect(state.services.get("b")?.phase).toEqual({ _tag: "Stopped" }); + expect(state.services.get("b")?.intent).toBe("lazy"); + + // b is asleep, not explicitly stopped, so it can still wake on its own: a can't be torn down. + const stopResult = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 1_050 }, + ]); + expect(tagsOf(stopResult.commandsByStep[0])).toEqual(["RequestRejected"]); + expect(stopResult.state.services.get("a")?.phase._tag).toBe("Running"); + + const restartResult = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "a" }), now: 1_050 }, + ]); + expect(tagsOf(restartResult.commandsByStep[0])).toEqual(["RequestRejected"]); + + // Once b is explicitly stopped too, a's stop is no longer blocked. + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "b" }), now: 1_060 }])); + const stopAfter = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "a" }), now: 1_070 }, + ]); + expect(tagsOf(stopAfter.commandsByStep[0])).not.toContain("RequestRejected"); + }); + }); + + it("re-arms a stopped prerequisite when its dependent is explicitly restarted", () => { + const graph = makeGraph([lazy("a"), lazy("b", { prerequisites: ["a"] })]); + let state = initialState(graph); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "b" }), now: 0 }])); + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "a" }), now: 1 }])); + expect(state.services.get("a")?.intent).toBe("stopped"); + + const restarted = run(state, [{ event: LifecycleEvent.RestartRequested({ id: "b" }), now: 2 }]); + expect(restarted.state.services.get("a")?.intent).toBe("lazy"); + expect(tagsOf(restarted.commandsByStep[0])).toEqual(["Launch"]); + expect(commandTagged(at(restarted.commandsByStep, 0), "Launch").id).toBe("a"); + }); + + it("keeps an explicit start's demand through a committed stop so the next generation launches once the exit is confirmed", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }])); + expect(state.services.get("api")?.phase._tag).toBe("Starting"); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ])); + expect(state.services.get("api")?.leases).toBe(0); + + ({ state } = run(state, [{ event: LifecycleEvent.StopRequested({ id: "api" }), now: 20 }])); + expect(state.services.get("api")?.phase._tag).toBe("Stopping"); + + ({ state } = run(state, [{ event: LifecycleEvent.StartRequested({ id: "api" }), now: 21 }])); + expect(state.services.get("api")?.relaunchForced).toBe(true); + + const result = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: true, + }), + now: 30, + }, + ]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["Launch"]); + expect(commandTagged(at(result.commandsByStep, 0), "Launch").generation).toBe(2); + }); + + it("admits a cold inspector acquisition before the first readiness check while ordinary traffic still waits", () => { + let state = initialState(makeGraph([lazy("functions")])); + ({ state } = run(state, [{ event: open("functions", 1), now: 0 }])); + const generation = startingGeneration(state, "functions"); + expect(state.services.get("functions")?.waiters.size).toBe(1); + + const inspectorQueued = run(state, [ + { + event: LifecycleEvent.ConnectionOpened({ + id: "functions", + waiterId: 2, + requireReady: false, + }), + now: 1, + }, + ]); + expect(tagsOf(inspectorQueued.commandsByStep[0])).toEqual([]); + state = inspectorQueued.state; + + const result = run(state, [ + { event: LifecycleEvent.SessionAvailable({ id: "functions", generation }), now: 2 }, + ]); + expect(tagsOf(result.commandsByStep[0])).toEqual(["AdmitConnection"]); + expect(commandTagged(at(result.commandsByStep, 0), "AdmitConnection").waiterId).toBe(2); + expect(result.state.services.get("functions")?.waiters.size).toBe(1); + + const ready = run(result.state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "functions", generation }), now: 3 }, + ]); + expect(tagsOf(ready.commandsByStep[0])).toEqual(["AdmitConnection"]); + expect(commandTagged(at(ready.commandsByStep, 0), "AdmitConnection").waiterId).toBe(1); + }); + + it("re-arms the idle timer once demand returns and readiness later recovers, instead of staying armed forever", () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 10 }, + ])); + ({ state } = run(state, [{ event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 20 }])); + expect(state.services.get("api")?.idleArmedEpoch).toBe(1); + + ({ state } = run(state, [ + { event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "blip" }), now: 30 }, + ])); + expect(state.services.get("api")?.idleArmedEpoch).toBe(1); + + ({ state } = run(state, [{ event: open("api", 2), now: 40 }])); + expect(state.services.get("api")?.idleArmedEpoch).toBeUndefined(); + + const staleElapse = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch: 1 }), now: 50 }, + ]); + expect(staleElapse.commandsByStep[0]).toEqual([]); + + ({ state } = run(state, [ + { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 2 }), now: 60 }, + ])); + ({ state } = run(state, [ + { event: LifecycleEvent.ReadinessRecovered({ id: "api", generation: 1 }), now: 61 }, + ])); + expect(state.services.get("api")?.idleArmedEpoch).toBe(2); + }); + + it("idles an unhealthy lazy service without demand and releases its prerequisites", () => { + let state = initialState( + makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]), + ); + let commandsByStep: ReadonlyArray>; + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 2 }, + { event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "503" }), now: 3 }, + ])); + expect(state.services.get("api")?.idleArmedEpoch).toBeUndefined(); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.ConnectionClosed({ id: "api" }), now: 4 }, + ])); + expect(tagsOf(commandsByStep[0])).toContain("ArmIdleTimer"); + const epoch = state.services.get("api")?.idleArmedEpoch ?? -1; + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.IdleElapsed({ id: "api", generation: 1, epoch }), now: 1_004 }, + { event: LifecycleEvent.Exited({ id: "api", generation: 1, requested: true }), now: 1_005 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopped" }); + expect(tagsOf(commandsByStep[1])).toEqual(["ArmIdleTimer"]); + expect(state.services.get("database")?.idleArmedEpoch).toBe(1); + }); + + it("fails only the waiters awaiting a failed readiness check, keeping the session and inspector waiters", () => { + const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); + let state = initialState(graph); + ({ state } = run(state, [{ event: open("api", 1), now: 0 }])); + ({ state } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "database", generation: 1 }), now: 1 }, + { event: LifecycleEvent.SessionAvailable({ id: "api", generation: 1 }), now: 2 }, + { event: open("api", 2, false), now: 3 }, + ])); + + const lost = run(state, [ + { event: LifecycleEvent.ReadinessLost({ id: "api", generation: 1, cause: "503" }), now: 4 }, + ]); + const failed = commandTagged(at(lost.commandsByStep, 0), "FailConnection"); + expect(failed).toMatchObject({ waiterId: 1, message: "api is not ready", cause: "503" }); + expect(tagsOf(lost.commandsByStep[0])).toEqual(["FailConnection"]); + expect(lost.state.services.get("api")?.phase).toEqual({ + _tag: "Running", + generation: 1, + ready: false, + }); + expect(lost.state.services.get("api")?.leases).toBe(1); + }); + + it("gives an explicit readiness wait no budget and no lease, while it still wakes the service", () => { + let state = initialState(makeGraph([lazy("api")])); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1, requireReady: true }), + now: 0, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Launch"]); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 500_000 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["AdmitConnection", "ArmIdleTimer"]); + expect(state.services.get("api")?.leases).toBe(0); + }); + + it("adds services unarmed through a graph update until an arm or start request", () => { + let state = initialState(makeGraph([])); + let commandsByStep: ReadonlyArray>; + const graph = makeGraph([ + { id: "database", activation: "eager", prerequisites: [] }, + lazy("api", { prerequisites: ["database"] }), + ]); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.GraphUpdated({ graph }), now: 0 }, + ])); + expect(commandsByStep[0]).toEqual([]); + expect(state.services.get("database")?.intent).toBe("stopped"); + expect(state.services.get("api")?.intent).toBe("stopped"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.ArmRequested({ id: "api" }), now: 1 }, + ])); + expect(commandsByStep[0]).toEqual([]); + expect(state.services.get("api")?.intent).toBe("lazy"); + expect(state.services.get("database")?.intent).toBe("stopped"); + + ({ commandsByStep } = run(state, [ + { event: LifecycleEvent.ArmRequested({ id: "database" }), now: 2 }, + ])); + expect(commandTagged(at(commandsByStep, 0), "Launch").id).toBe("database"); + }); + + it("lets an eager service's standing demand lapse once it fails, until traffic re-wakes it", () => { + const graph = makeGraph([{ id: "database", activation: "eager", prerequisites: [] }]); + let state = initialState(graph); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "database" }), now: 0 }, + { + event: LifecycleEvent.LaunchFailed({ id: "database", generation: 1, cause: "boom" }), + now: 1, + }, + ])); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("database")?.phase._tag).toBe("Failed"); + + ({ commandsByStep } = run(state, [{ event: open("database", 1), now: 2 }])); + expect(commandTagged(at(commandsByStep, 0), "Launch").generation).toBe(2); + }); + + it("ends an explicit wait on a dependent with its prerequisite's failure, while traffic keeps waiting", () => { + const graph = makeGraph([lazy("database"), lazy("api", { prerequisites: ["database"] })]); + let state = initialState(graph); + ({ state } = run(state, [ + { event: open("api", 1), now: 0 }, + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 2, requireReady: false }), + now: 0, + }, + ])); + + const failed = run(state, [ + { + event: LifecycleEvent.LaunchFailed({ id: "database", generation: 1, cause: "boom" }), + now: 1, + }, + ]); + const failure = commandTagged(at(failed.commandsByStep, 0), "FailConnection"); + expect(failure).toMatchObject({ + id: "api", + waiterId: 2, + message: "prerequisite database failed", + }); + expect([...(failed.state.services.get("api")?.waiters.keys() ?? [])]).toEqual([1]); + }); + + describe("re-checking a blocking session", () => { + const blocked = () => { + const graph = makeGraph([ + lazy("analytics"), + lazy("studio", { prerequisites: ["analytics"] }), + ]); + let state = initialState(graph); + ({ state } = run(state, [ + { event: open("studio", 1), now: 0 }, + { event: LifecycleEvent.SessionAvailable({ id: "analytics", generation: 1 }), now: 1 }, + ])); + return state; + }; + const lost = (now: number) => ({ + event: LifecycleEvent.ReadinessLost({ id: "analytics", generation: 1, cause: "503" }), + now, + }); + + it("keeps re-probing one at a time, spaced after a failed reprobe, until the dependent is served", () => { + const { state, commandsByStep } = run(blocked(), [ + lost(2), + { event: open("studio", 2), now: 3 }, + lost(4), + lost(1_005), + { + event: LifecycleEvent.ReadinessRecovered({ id: "analytics", generation: 1 }), + now: 2_006, + }, + ]); + expect(commandTagged(at(commandsByStep, 0), "Reprobe")).toMatchObject({ + id: "analytics", + generation: 1, + delayMillis: 0, + }); + expect(tagsOf(commandsByStep[1])).toEqual([]); + expect(commandTagged(at(commandsByStep, 2), "Reprobe").delayMillis).toBe(1_000); + expect(commandTagged(at(commandsByStep, 3), "Reprobe").delayMillis).toBe(1_000); + expect(commandTagged(at(commandsByStep, 4), "Launch").id).toBe("studio"); + expect(state.services.get("analytics")?.reprobing).toBe(false); + }); + + it("stops re-probing once no waiter is blocked or the generation ends", () => { + let state = blocked(); + let commandsByStep: ReadonlyArray>; + ({ state } = run(state, [lost(2)])); + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.WaiterCancelled({ id: "studio", waiterId: 1 }), now: 3 }, + lost(4), + ])); + expect(tagsOf(commandsByStep[1])).toEqual([]); + + ({ state } = run(state, [{ event: open("studio", 2), now: 5 }])); + ({ commandsByStep } = run(state, [...crashSteps("analytics", 1, 6, "gone"), lost(7)])); + expect(tagsOf(commandsByStep[2]).includes("Reprobe")).toBe(false); + }); + }); + + describe("failed cleanup", () => { + const runningApi = () => { + let state = initialState(makeGraph([lazy("api")])); + ({ state } = run(state, [ + { event: LifecycleEvent.StartRequested({ id: "api" }), now: 0 }, + { event: LifecycleEvent.LaunchSucceeded({ id: "api", generation: 1 }), now: 1 }, + ])); + return state; + }; + + it("holds the generation in cleanup until a retried stop confirms it, refusing storage and launches", () => { + let state = runningApi(); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 2 }, + { + event: LifecycleEvent.StopFailed({ id: "api", generation: 1, cause: "busy" }), + now: 3, + }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("api")?.phase).toEqual({ _tag: "Stopping", generation: 1 }); + expect(storageAdmitted(state, "api")).toBe(false); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.StorageReserved({ id: "api" }), now: 4 }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 5 }, + { event: LifecycleEvent.StopRequested({ id: "api" }), now: 6 }, + ])); + expect(tagsOf(commandsByStep[0])).toEqual(["RequestRejected"]); + expect(commandTagged(at(commandsByStep, 1), "Stop").generation).toBe(1); + expect(commandsByStep[2]).toEqual([]); + + ({ state } = run(state, [ + { + event: LifecycleEvent.Exited({ + id: "api", + generation: 1, + requested: true, + }), + now: 7, + }, + ])); + expect(storageAdmitted(state, "api")).toBe(true); + }); + + it("fails a restart's waiters when the stop fails, and retries the cleanup for the next waiter", () => { + let state = runningApi(); + let commandsByStep: ReadonlyArray>; + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.RestartRequested({ id: "api" }), now: 2 }, + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1, requireReady: false }), + now: 2, + }, + { + event: LifecycleEvent.StopFailed({ id: "api", generation: 1, cause: "busy" }), + now: 3, + }, + ])); + expect(commandTagged(at(commandsByStep, 2), "FailConnection")).toMatchObject({ + waiterId: 1, + message: "api cleanup failed", + cause: "busy", + }); + expect(state.services.get("api")?.relaunchForced).toBe(false); + + ({ commandsByStep } = run(state, [{ event: open("api", 2), now: 4 }])); + expect(tagsOf(commandsByStep[0])).toEqual(["Stop"]); + }); + }); + + it("caps only traffic waiters, keeps explicit waits out of the count and frees a slot on cancellation", () => { + let state = initialState(makeGraph([lazy("api")])); + let commandsByStep: ReadonlyArray>; + ({ state } = run( + state, + Array.from({ length: 256 }, (_, index) => ({ event: open("api", index), now: 0 })), + )); + ({ state, commandsByStep } = run(state, [ + { event: open("api", 1_000), now: 0 }, + { + event: LifecycleEvent.ReadinessAwaited({ id: "api", waiterId: 1_001, requireReady: true }), + now: 0, + }, + ])); + expect(commandTagged(at(commandsByStep, 0), "FailConnection").message).toBe( + "api has too many waiters", + ); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("api")?.waiters.get(1_001)?.kind).toBe("explicit"); + + ({ state, commandsByStep } = run(state, [ + { event: LifecycleEvent.WaiterCancelled({ id: "api", waiterId: 0 }), now: 50 }, + { event: open("api", 1_003), now: 50 }, + ])); + expect(commandsByStep[1]).toEqual([]); + expect(state.services.get("api")?.waiters.get(1_003)?.kind).toBe("traffic"); + }); +}); diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts index 680424c0ad..188c166e3a 100644 --- a/packages/stack/src/Network.integration.test.ts +++ b/packages/stack/src/Network.integration.test.ts @@ -1,38 +1,47 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Data, Deferred, Effect, Fiber, FileSystem, Layer, Path, Ref } from "effect"; +import { Context, Data, Deferred, Effect, Fiber, FileSystem, Hash, Layer, Path, Ref } from "effect"; +import { randomUUID } from "node:crypto"; import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- fixture retains an idle HTTP connection. import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. import { HttpClient } from "effect/unstable/http"; import * as Network from "./Network.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const makeTestNetwork = (options: { readonly stackId: string; readonly runtime: Network.NetworkRuntime; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; }) => Layer.build( Network.layer({ stackId: options.stackId, runtime: options.runtime }).pipe( - Layer.provide(Layer.succeed(State.Service, options.state)), + Layer.provide(Layer.succeed(StackNamespace.Service, options.state)), ), ).pipe(Effect.map((context) => Context.get(context, Network.Service))); -const stack = (id: string, port: number | "auto") => ({ +const stack = (id: string) => ({ id, identity: { projectRoot: "/tmp", branchContext: "test", stackName: id }, runtime: "native" as const, lifetime: "detached" as const, instances: [], composition: { members: [], dependencies: [] }, - ports: port === "auto" ? [] : [{ key: "api", host: "127.0.0.1", port }], }); +/** The port the machine-wide registry holds for this stack's endpoint key, if any. */ +const reservedPort = (root: string, stackId: string, key: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const registry = yield* PortReservations.Service; + return yield* registry.find(yield* fs.realPath(root), stackId, key); + }).pipe(Effect.provide(PortReservations.layer)); + class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} const backend = Effect.acquireRelease( @@ -101,7 +110,7 @@ it.live("retains dedicated assignments across network reopen", () => const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-retain-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const enabled = yield* Ref.make(true); const first = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -112,8 +121,7 @@ it.live("retains dedicated assignments across network reopen", () => yield* firstNamespace.bind; const firstAddress = yield* firstNamespace.address("api", "host"); expect(yield* request(firstAddress.host, firstAddress.port, "/one")).toBe("backend:/one"); - const saved = yield* state.read("stack"); - expect(saved?.ports).toHaveLength(1); + expect(yield* reservedPort(root, "stack", "one:api")).toBe(firstAddress.port); yield* Ref.set(enabled, false); yield* firstNamespace.close; yield* firstNamespace.bind; @@ -134,7 +142,9 @@ it.live("retains dedicated assignments across network reopen", () => "backend:/reopen", ); yield* secondNamespace.release; - expect((yield* state.read("stack"))?.ports).toHaveLength(0); + expect(yield* reservedPort(root, "stack", "one:api")).toBe(firstAddress.port); + yield* secondNamespace.releasePorts; + expect(yield* reservedPort(root, "stack", "one:api")).toBeUndefined(); expect(yield* fs.exists(path.join(root, "stack", "state.json"))).toBe(true); }), ).pipe(Effect.provide(NodeServices.layer)), @@ -146,7 +156,7 @@ it.live("keeps shared routes independent and retains the shared claim", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-shared-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const firstEnabled = yield* Ref.make(true); const secondEnabled = yield* Ref.make(true); @@ -186,7 +196,7 @@ it.live("keeps shared routes independent and retains the shared claim", () => expect(yield* request(address.host, address.port, "/auth/v1")).toBe("backend:/auth/v1"); yield* Ref.set(secondEnabled, false); yield* second.close; - expect((yield* state.read("stack"))?.ports).toHaveLength(1); + expect(yield* reservedPort(root, "stack", "api")).toBe(address.port); yield* first.bind; expect((yield* first.address("api", "host")).port).toBe(address.port); expect(yield* request(address.host, address.port, "/rest/reopen")).toBe( @@ -194,20 +204,22 @@ it.live("keeps shared routes independent and retains the shared claim", () => ); yield* first.close; yield* second.release; - expect((yield* state.read("stack"))?.ports).toHaveLength(1); + yield* second.releasePorts; + expect(yield* reservedPort(root, "stack", "api")).toBe(address.port); expect(yield* fs.exists(root)).toBe(true); }), ).pipe(Effect.provide(NodeServices.layer)), ); -it.live("reports a dedicated port conflict without rewriting saved ownership", () => +it.live("reports a dedicated port conflict without taking over the holder's reservation", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-conflict-" }); const state = yield* makeTestState(root); - yield* state.save(stack("first", "auto")); - yield* state.save(stack("second", 20001)); + yield* state.save(stack("first")); + yield* state.save(stack("second")); + yield* state.acquireLease("first"); const target = yield* backend; const firstEnabled = yield* Ref.make(true); const firstNetwork = yield* makeTestNetwork({ stackId: "first", runtime: "native", state }); @@ -231,57 +243,14 @@ it.live("reports a dedicated port conflict without rewriting saved ownership", ( }); const failure = yield* second.bind.pipe(Effect.flip); expect(failure.operation).toBe("bind"); - expect((yield* state.read("second"))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: 20001 }, - ]); + expect(yield* reservedPort(root, "first", "db:sql")).toBe(firstPort); + expect(yield* reservedPort(root, "second", "db:sql")).toBeUndefined(); yield* Ref.set(firstEnabled, false); yield* first.release; + yield* first.releasePorts; yield* Ref.set(secondEnabled, false); yield* second.release; - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -it.live("can bind a shared listener after saving its first assignment failed", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-save-" }); - const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); - const failing = yield* Ref.make(true); - const network = yield* makeTestNetwork({ - stackId: "stack", - runtime: "native", - state: { - ...state, - save: (value) => - Ref.get(failing).pipe( - Effect.flatMap((fail) => - fail - ? Effect.fail( - new State.StateError({ operation: "save", message: "write failed" }), - ) - : state.save(value), - ), - ), - }, - }); - const target = yield* backend; - const namespace = yield* network.register({ - id: "rest", - endpoints: { - api: { ...endpoint(target, Effect.succeed(false)), shared: [{ prefix: "/rest" }] }, - }, - }); - yield* namespace.bind.pipe(Effect.flip); - yield* Ref.set(failing, false); - yield* namespace.bind; - const address = yield* namespace.address("api", "host"); - expect(yield* request(address.host, address.port, "/rest")).toBe("backend:/rest"); - yield* namespace.release; - yield* network.release; - expect((yield* state.read("stack"))?.ports).toHaveLength(0); + yield* second.releasePorts; }), ).pipe(Effect.provide(NodeServices.layer)), ); @@ -292,7 +261,7 @@ it.live("releases dedicated HTTP activity after the response while keep-alive st const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-http-activity-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const released = yield* Deferred.make(); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -347,7 +316,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-fixed-port-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const studio = yield* network.register({ @@ -357,8 +326,7 @@ it.live( yield* studio.bind; // The join route is queued, not installed: it never claims or asserts the shared "api" // port, so no shared listener or claim exists yet, whatever port a later claimant picks. - const beforeClaimant = yield* state.read("stack"); - expect(beforeClaimant?.ports.some((claim) => claim.key === "api")).toBe(false); + expect(yield* reservedPort(root, "stack", "api")).toBeUndefined(); const rest = yield* network.register(claimant("rest", target, Effect.succeed(true))); yield* rest.bind; const address = yield* rest.address("api", "host"); @@ -376,7 +344,7 @@ it.live("drops a queued join route when its namespace closes before any claimant const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-cancel-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const studio = yield* network.register({ @@ -404,7 +372,7 @@ it.live("stays idempotent across repeated binds of the joining namespace", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-idempotent-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const rest = yield* network.register(claimant("rest", target, Effect.succeed(true))); @@ -430,7 +398,7 @@ it.live("keeps a namespace's own shared route when its join endpoint also binds" prefix: "network-join-shares-namespace-", }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); const combo = yield* network.register({ @@ -457,7 +425,7 @@ it.live("re-adds a joined route after its namespace closes and rebinds", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-rebind-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const enabled = yield* Ref.make(true); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -490,7 +458,7 @@ it.live("never restores a joined route once its namespace is released", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-release-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const enabled = yield* Ref.make(true); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -524,7 +492,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-last-close-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const restEnabled = yield* Ref.make(true); const studioEnabled = yield* Ref.make(true); @@ -579,7 +547,7 @@ it.live("wakes a sleeping backend when a request reaches its joined route", () = const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-join-wake-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const woken = yield* Deferred.make(); const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); @@ -611,7 +579,7 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-docker-alias-" }); const state = yield* makeTestState(root); - yield* state.save(stack("stack", "auto")); + yield* state.save(stack("stack")); const target = yield* backend; const network = yield* makeTestNetwork({ stackId: "stack", runtime: "docker", state }); const namespace = yield* network.register({ @@ -624,6 +592,108 @@ it.live("addresses docker runtime endpoints through the stack host alias", () => expect(host.host).toBe("127.0.0.1"); expect(runtime).toEqual({ ...host, host: DOCKER_HOST_ALIAS }); yield* namespace.release; + yield* namespace.releasePorts; + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +/** A pinned port above the auto range, so it never meets a registry row from an auto allocation. */ +const pinnedPort = () => 33000 + (Math.abs(Hash.string(randomUUID())) % 16000); + +it.live("addresses a pinned endpoint from its configured port without a registry row", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-pinned-address-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack")); + const target = yield* backend; + const port = pinnedPort(); + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const namespace = yield* network.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Effect.succeed(false)), port } }, + }); + + expect((yield* namespace.address("sql", "host")).port).toBe(port); + expect((yield* namespace.address("sql", "runtime")).port).toBe(port); + expect(yield* reservedPort(root, "stack", "db:sql")).toBeUndefined(); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("a pinned port is reserved only while its listener is open", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-pinned-stop-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("first")); + yield* state.save(stack("second")); + const target = yield* backend; + const port = pinnedPort(); + const firstEnabled = yield* Ref.make(true); + const firstNetwork = yield* makeTestNetwork({ stackId: "first", runtime: "native", state }); + const first = yield* firstNetwork.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Ref.get(firstEnabled)), port } }, + }); + yield* first.bind; + expect(yield* reservedPort(root, "first", "db:sql")).toBe(port); + + yield* Ref.set(firstEnabled, false); + yield* first.close; + expect(yield* reservedPort(root, "first", "db:sql")).toBeUndefined(); + + const secondNetwork = yield* makeTestNetwork({ stackId: "second", runtime: "native", state }); + const second = yield* secondNetwork.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Effect.succeed(true)), port } }, + }); + yield* second.bind; + expect((yield* second.address("sql", "host")).port).toBe(port); + expect(yield* request("127.0.0.1", port, "/started")).toBe("backend:/started"); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("a pinned shared listener keeps its reservation until its last route closes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-pinned-shared-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack")); + const target = yield* backend; + const port = pinnedPort(); + const firstEnabled = yield* Ref.make(true); + const secondEnabled = yield* Ref.make(true); + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const first = yield* network.register({ + id: "rest", + endpoints: { + api: { ...endpoint(target, Ref.get(firstEnabled)), port, shared: [{ prefix: "/rest" }] }, + }, + }); + const second = yield* network.register({ + id: "auth", + endpoints: { + api: { ...endpoint(target, Ref.get(secondEnabled)), port, shared: [{ prefix: "/auth" }] }, + }, + }); + yield* first.bind; + yield* second.bind; + expect(yield* reservedPort(root, "stack", "api")).toBe(port); + expect(yield* request("127.0.0.1", port, "/auth/v1")).toBe("backend:/auth/v1"); + + yield* Ref.set(firstEnabled, false); + yield* first.close; + expect(yield* reservedPort(root, "stack", "api")).toBe(port); + expect(yield* request("127.0.0.1", port, "/auth/v1")).toBe("backend:/auth/v1"); + + yield* Ref.set(secondEnabled, false); + yield* second.close; + expect(yield* reservedPort(root, "stack", "api")).toBeUndefined(); }), ).pipe(Effect.provide(NodeServices.layer)), ); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index d5b2ca44ad..6c8f06f8d3 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -1,7 +1,8 @@ import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effect"; import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; -import * as State from "./State.ts"; -import { makePorts, PortError } from "./Ports.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { makePorts, probeVacant } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; import { makeHttpProxy, type HttpAccessSink, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; @@ -42,7 +43,10 @@ export interface NetworkNamespace { readonly bindings: Effect.Effect>; readonly bind: Effect.Effect, NetworkError>; readonly close: Effect.Effect; + /** Closes the endpoints for good once the instance is stopped; saved port assignments stay. */ readonly release: Effect.Effect; + /** Deletes the saved port assignments of this instance's dedicated endpoints. */ + readonly releasePorts: Effect.Effect; readonly address: ( name: string, from: "host" | "runtime", @@ -50,7 +54,8 @@ export interface NetworkNamespace { } export interface Interface { - readonly release: Effect.Effect; + /** Releases every reservation this stack holds, dedicated and shared alike, in one step. */ + readonly releaseStack: Effect.Effect; readonly register: (options: { readonly id: string; readonly endpoints: Readonly>; @@ -66,16 +71,27 @@ const errorFor = (operation: string, cause: unknown) => cause, }); +/** A later claimant reuses the open shared listener, so it never touches the port registry. */ +const joinClaim = ( + existing: { readonly claim: number; readonly proxy: HttpProxy }, + port: number | "auto", +) => + port !== "auto" && port !== existing.claim + ? Effect.fail(errorFor("bind", "Shared API port differs from existing claim")) + : Effect.succeed({ port: existing.claim, listener: { proxy: existing.proxy } }); + const makeNetwork = (options: { readonly stackId: string; readonly runtime: NetworkRuntime; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; + readonly platform?: NodeJS.Platform; readonly onAccess?: HttpAccessSink; }) => Effect.gen(function* () { const ports = yield* makePorts(options.state).pipe( Effect.mapError((cause) => errorFor("ports", cause)), ); + const probe = probeVacant(options.platform ?? process.platform); const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); const shared = yield* Ref.make< @@ -92,12 +108,7 @@ const makeNetwork = (options: { const listenHost = options.runtime === "native" ? "127.0.0.1" : "0.0.0.0"; const hostAddress = "127.0.0.1"; - const runtimeAddress = - options.runtime === "native" - ? "127.0.0.1" - : options.runtime === "docker" - ? DOCKER_HOST_ALIAS - : "host.containers.internal"; + const runtimeAddress = options.runtime === "native" ? "127.0.0.1" : DOCKER_HOST_ALIAS; const routeKey = (route: Pick) => `${route.id}:${route.prefix}`; @@ -159,55 +170,54 @@ const makeNetwork = (options: { Effect.gen(function* () { const endpointScope = yield* Scope.fork(owner, "sequential"); const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; + const claimed = + endpoint.shared === undefined ? undefined : yield* Ref.get(shared); const result = yield* restore( - ports - .acquire<{ readonly proxy?: HttpProxy }, Scope.Scope>( - { stackId: options.stackId, key, host: listenHost, port: endpoint.port }, - (host, port) => - Effect.gen(function* () { - if (endpoint.shared !== undefined) { - const current = yield* Ref.get(shared); - if (current !== undefined) { - if (current.claim !== port) - return yield* new PortError({ - key: "api", - message: "Shared API port differs from existing claim", + claimed !== undefined + ? joinClaim(claimed, endpoint.port) + : ports + .acquire<{ readonly proxy?: HttpProxy }, Scope.Scope>( + { + stackId: options.stackId, + key, + host: listenHost, + port: endpoint.port, + }, + (host, port) => + Effect.gen(function* () { + if (endpoint.shared !== undefined) { + yield* probe(key, host, port); + const proxy = yield* makeHttpProxy({ + host, + port, + onAccess: options.onAccess, }); - return { proxy: current.proxy }; - } - return { - proxy: yield* makeHttpProxy({ - host, - port, - onAccess: options.onAccess, - }), - }; - } - if (endpoint.protocol === "http") { - const proxy = yield* makeHttpProxy({ host, port }); - yield* proxy.setRoutes([ - { id, prefix: "/", target: endpoint.backend }, - ]); - return { proxy }; - } - const listener = yield* bindTcp(host, port); - yield* Effect.forkIn( - serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( - Effect.provideService(Scope.Scope, endpointScope), - Effect.catch((cause) => - Effect.logWarning("Public listener failed", cause), - ), - ), - endpointScope, - { startImmediately: true }, - ); - return {}; - }), - ) - .pipe( - Effect.provideService(Scope.Scope, endpointScope), - Effect.mapError((cause) => errorFor("bind", cause)), - ), + return { proxy }; + } + if (endpoint.protocol === "http") { + yield* probe(key, host, port); + const proxy = yield* makeHttpProxy({ host, port }); + yield* proxy.setRoutes([ + { id, prefix: "/", target: endpoint.backend }, + ]); + return { proxy }; + } + yield* probe(key, host, port); + const listener = yield* bindTcp(host, port); + yield* Effect.forkIn( + serveTcp(listener, endpoint.backend, `${id}:${name}`).pipe( + Effect.provideService(Scope.Scope, endpointScope), + ), + endpointScope, + { startImmediately: true }, + ); + return {}; + }), + ) + .pipe( + Effect.provideService(Scope.Scope, endpointScope), + Effect.mapError((cause) => errorFor("bind", cause)), + ), ).pipe( Effect.onExit((exit) => Exit.isFailure(exit) ? Scope.close(endpointScope, exit) : Effect.void, @@ -299,33 +309,38 @@ const makeNetwork = (options: { "Stop the instance before releasing its endpoints", ); yield* Ref.set(closed, true); - for (const name of Object.keys(endpoints)) { - const endpoint = endpoints[name]; - if (endpoint?.shared === undefined) - yield* ports - .release(options.stackId, `${id}:${name}`) - .pipe(Effect.mapError((cause) => errorFor("release", cause))); - } }), ), ), ); + const releasePorts = Effect.fn("Network.releasePorts")(() => + Effect.forEach( + Object.entries(endpoints).filter(([, endpoint]) => endpoint.shared === undefined), + ([name]) => + ports + .release(options.stackId, `${id}:${name}`) + .pipe(Effect.mapError((cause) => errorFor("release", cause))), + { discard: true }, + ), + ); const address = Effect.fn("Network.address")((name: string, from: "host" | "runtime") => Effect.gen(function* () { const endpoint = endpoints[name]; if (endpoint === undefined) return yield* errorFor("address", `Unknown endpoint ${name}`); - const saved = yield* options.state - .read(options.stackId) - .pipe(Effect.mapError((cause) => errorFor("address", cause))); const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; - const claim = saved?.ports.find((value) => value.key === key); - if (claim === undefined) + const port = + endpoint.port === "auto" + ? yield* ports + .assigned(options.stackId, key) + .pipe(Effect.mapError((cause) => errorFor("address", cause))) + : endpoint.port; + if (port === undefined) return yield* errorFor("address", `Endpoint ${name} is not assigned`); return { name, protocol: endpoint.protocol, host: from === "host" ? hostAddress : runtimeAddress, - port: claim.port, + port, }; }), ); @@ -333,19 +348,21 @@ const makeNetwork = (options: { bind: bind(), close: close(), release: release(), + releasePorts: releasePorts(), address, bindings: Ref.get(bound).pipe(Effect.map((values) => [...values.values()])), } satisfies NetworkNamespace; }); - const release = Effect.fn("Network.releaseNamespace")(() => - gate.withPermits(1)( - ports - .release(options.stackId, "api") - .pipe(Effect.mapError((cause) => errorFor("release", cause))), - ), + const releaseStack = Effect.fn("Network.releaseStack")(() => + ports + .releaseStack(options.stackId) + .pipe(Effect.mapError((cause) => errorFor("release", cause))), ); - return { register, release: release() } satisfies Interface; + return { + register, + releaseStack: releaseStack(), + } satisfies Interface; }); export const layer = (options: { @@ -357,7 +374,7 @@ export const layer = (options: { Layer.effect( Service, Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; return yield* makeNetwork({ ...options, state }); }).pipe(Effect.map(Service.of)), - ); + ).pipe(Layer.provide(PortReservations.layer)); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts index 1fe51757ff..e2dd01fab3 100644 --- a/packages/stack/src/Orchestrator.integration.test.ts +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -1,10 +1,33 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, Fiber, Option, Ref, Schema, Scope, Stream } from "effect"; +import { + Cause, + Context, + Deferred, + Duration, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Queue, + Ref, + Schema, + Scope, + Stream, +} from "effect"; import * as TestClock from "effect/testing/TestClock"; +import { HttpClient } from "effect/unstable/http"; +import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real backend behind the listener. import { captureLogs } from "../tests/logs.ts"; +import type { LifecycleEvent } from "./Lifecycle.ts"; +import * as Network from "./Network.ts"; import * as Orchestrator from "./Orchestrator.ts"; import type { RegisteredInstance } from "./Orchestrator.ts"; +import { ProxyError } from "./Proxy.ts"; import { makeService, ServiceError } from "./Service.ts"; +import * as StackNamespace from "./StackNamespace.ts"; const failure = (message: string) => new ServiceError({ operation: "fixture", message }); const makeTestOrchestrator = () => Orchestrator.make(); @@ -20,8 +43,15 @@ const makeInstance = ( readonly launch?: Effect.Effect; readonly stop?: Effect.Effect; readonly removeData?: Effect.Effect; + /** Replaces the default close, which only tracks the bound state. */ + readonly close?: Effect.Effect; + /** Resolves the first launch's runtime exit; later launches exit only when stopped. */ readonly exit?: Deferred.Deferred>; + /** Every launch's runtime exit, for workloads that crash on demand. */ + readonly crash?: Effect.Effect>; readonly events?: Ref.Ref>; + /** Runs after the lifecycle has applied each event the service reports. */ + readonly onReport?: (event: LifecycleEvent) => Effect.Effect; } = {}, ) => Effect.gen(function* () { @@ -37,13 +67,18 @@ const makeInstance = ( launch: ({ config }) => Effect.gen(function* () { yield* options.launch ?? Effect.void; - yield* Ref.update(starts, (values) => [...values, config]); + const launched = yield* Ref.updateAndGet(starts, (values) => [...values, config]); yield* event("start"); const exited = yield* Deferred.make>(); + const exit = + options.crash ?? + (options.exit !== undefined && launched.length === 1 + ? Deferred.await(options.exit) + : Deferred.await(exited)); return { health: options.health ?? Effect.void, ...(options.probe === undefined ? {} : { probe: options.probe }), - exit: Deferred.await(options.exit ?? exited), + exit, stop: (options.stop ?? Effect.void).pipe( Effect.andThen(event("stop")), Effect.andThen(Deferred.succeed(exited, Exit.void)), @@ -54,41 +89,93 @@ const makeInstance = ( }), removeData: () => event("destroy").pipe(Effect.andThen(options.removeData ?? Effect.void)), }, - { id, config: {}, coordinate: orchestrator.admissionFor(id) }, + { + id, + config: {}, + report: (reported) => + orchestrator + .report(reported) + .pipe(Effect.andThen(options.onReport?.(reported) ?? Effect.void)), + }, ); + const configFor = (inputs: Record, candidate: unknown) => + candidate === undefined + ? Effect.succeed(inputs) + : Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.String))(candidate).pipe( + Effect.mapError(() => failure("Invalid configuration")), + ); const instance: RegisteredInstance = { id, service: id, core, - startAt: (revision, inputs, wake, guard) => core.startAt(revision, inputs, wake, guard), - restart: (revision, inputs, candidate, guard) => - (candidate === undefined - ? Effect.succeed(inputs) - : Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.String))(candidate).pipe( - Effect.mapError(() => failure("Invalid configuration")), - ) - ).pipe(Effect.flatMap((config) => core.restart(config, revision, guard))), + launch: (generation, inputs, candidate) => + configFor(inputs, candidate).pipe( + Effect.flatMap((config) => core.launch(generation, config)), + ), + prepare: (inputs, candidate) => + configFor(inputs, candidate).pipe(Effect.flatMap((config) => core.prepare(config))), bind: options.bind ?? Ref.set(bound, true), - close: core.get.pipe( - Effect.flatMap((state) => - state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, + confirmRemoved: Effect.void, + release: Effect.void, + releasePorts: Effect.void, + close: + options.close ?? + orchestrator.status(id).pipe( + Effect.flatMap((state) => + state.lifecycle === "stopped" && !state.wakeEnabled + ? Ref.set(bound, false) + : Effect.void, + ), + Effect.mapError((error) => failure(error.message)), ), - ), hasEndpoint: options.endpoint ?? true, inputs: ["databaseUrl"], outputs: { url: Effect.succeed(`postgres://${id}`) }, }; yield* orchestrator.register(instance); - return { ...instance, starts, bound }; + return { + ...instance, + starts, + bound, + status: orchestrator.status(id), + ready: orchestrator.ready(id), + observation: orchestrator.changes(id), + }; }); -const stopped = (instance: RegisteredInstance) => - instance.core.observation.pipe( +type Instance = Effect.Success>; + +const stopped = (instance: Instance) => + instance.observation.pipe( Stream.filter((state) => state.lifecycle === "stopped" && state.currentOperation === undefined), Stream.take(1), Stream.runDrain, ); +/** + * Forks a wait for the next lifecycle transition after now. It returns once the subscription has + * seen the current state, so a transition triggered afterwards can't be missed. + */ +const nextTransition = (observation: Stream.Stream) => + Effect.gen(function* () { + const subscribed = yield* Deferred.make(); + const transition = yield* observation.pipe( + Stream.zipWithIndex, + Stream.tap(([, index]) => + index === 0 ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.drop(1), + Stream.take(1), + Stream.runDrain, + Effect.forkChild, + ); + yield* Deferred.await(subscribed); + return transition; + }); + +const errorOf = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; + describe("service composition", () => { it.live("starts independent eager services concurrently", () => Effect.scoped( @@ -193,11 +280,11 @@ describe("service composition", () => { yield* Deferred.await(databaseLaunched); yield* Deferred.await(authLaunched); yield* Deferred.succeed(authHealthy, undefined); - yield* auth.core.ready; + yield* auth.ready; yield* Deferred.succeed(databaseHealthGate, undefined); - yield* database.core.ready; + yield* database.ready; yield* Deferred.await(restLaunched); - yield* rest.core.ready; + yield* rest.ready; return { boundInputs: yield* Ref.get(rest.starts), eventOrder: yield* Ref.get(events), @@ -247,13 +334,7 @@ describe("service composition", () => { dependencies: [{ from: "prerequisite", to: "descendant" }], }); - const result = yield* orchestrator.startComposition.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isSuccess(result)) - return yield* Effect.die( - "composition with an unhealthy prerequisite unexpectedly passed", - ); - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + const error = errorOf(yield* orchestrator.startComposition.pipe(Effect.exit)); expect(error).toBeInstanceOf(Orchestrator.OrchestratorError); if (!(error instanceof Orchestrator.OrchestratorError)) return yield* Effect.die("composition failure did not include ordered outcomes"); @@ -265,12 +346,7 @@ describe("service composition", () => { ["independent", true], ]); const descendantOutcome = error.outcomes?.find(({ id }) => id === "descendant")?.result; - if (descendantOutcome === undefined || Exit.isSuccess(descendantOutcome)) - return yield* Effect.die("unhealthy prerequisite did not block descendant startup"); - const descendantError = Option.getOrUndefined( - Cause.findErrorOption(descendantOutcome.cause), - ); - expect(descendantError).toMatchObject({ + expect(descendantOutcome && errorOf(descendantOutcome)).toMatchObject({ message: "Blocked by prerequisites: prerequisite", }); expect(yield* Ref.get(descendant.starts)).toEqual([]); @@ -306,11 +382,7 @@ describe("service composition", () => { ], }); - const result = yield* orchestrator.startComposition.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isSuccess(result)) - return yield* Effect.die("composition with a prebinding failure unexpectedly passed"); - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + const error = errorOf(yield* orchestrator.startComposition.pipe(Effect.exit)); if (!(error instanceof Orchestrator.OrchestratorError)) return yield* Effect.die("prebinding failure did not return composition outcomes"); expect( @@ -322,17 +394,15 @@ describe("service composition", () => { ["independent", true], ]); const blocked = error.outcomes?.find(({ id }) => id === "descendant")?.result; - if (blocked === undefined || Exit.isSuccess(blocked)) - return yield* Effect.die("eager descendant unexpectedly started"); - expect(Option.getOrUndefined(Cause.findErrorOption(blocked.cause))).toMatchObject({ + expect(blocked && errorOf(blocked)).toMatchObject({ message: "Blocked by prerequisites: prerequisite", }); expect(yield* Ref.get(descendant.starts)).toEqual([]); expect(yield* Ref.get(lazyDescendant.starts)).toEqual([]); - expect((yield* lazyDescendant.core.get).wakeEnabled).toBe(true); + expect((yield* lazyDescendant.status).wakeEnabled).toBe(true); expect(yield* Ref.get(independent.starts)).toHaveLength(1); yield* orchestrator.stopNamespace; - expect((yield* lazyDescendant.core.get).wakeEnabled).toBe(false); + expect((yield* lazyDescendant.status).wakeEnabled).toBe(false); }), ), ); @@ -349,7 +419,7 @@ describe("service composition", () => { }); const dependent = yield* makeInstance(orchestrator, "dependent"); const independentLazy = yield* makeInstance(orchestrator, "independent-lazy"); - const independentArmed = independentLazy.core.observation.pipe( + const independentArmed = independentLazy.observation.pipe( Stream.filter((state) => state.wakeEnabled), Stream.take(1), Stream.runDrain, @@ -372,14 +442,14 @@ describe("service composition", () => { const observedIndependentArm = yield* Fiber.join(armedObserver).pipe( Effect.timeoutOption("5 seconds"), ); - expect((yield* prerequisite.core.get).health).toBe("starting"); - expect((yield* dependent.core.get).wakeEnabled).toBe(false); - expect((yield* independentLazy.core.get).wakeEnabled).toBe(true); + expect((yield* prerequisite.status).health).toBe("starting"); + expect((yield* dependent.status).wakeEnabled).toBe(false); + expect((yield* independentLazy.status).wakeEnabled).toBe(true); yield* Deferred.succeed(healthGate, undefined); expect(Option.isSome(observedIndependentArm)).toBe(true); yield* Fiber.join(composition); - expect((yield* dependent.core.get).wakeEnabled).toBe(true); + expect((yield* dependent.status).wakeEnabled).toBe(true); expect(yield* Ref.get(dependent.starts)).toEqual([]); yield* orchestrator.stopNamespace; }), @@ -399,9 +469,7 @@ describe("service composition", () => { Effect.andThen(Deferred.await(launchGate)), ), }); - const descendant = yield* makeInstance(orchestrator, "descendant", { - events, - }); + const descendant = yield* makeInstance(orchestrator, "descendant", { events }); yield* Effect.addFinalizer(() => Deferred.succeed(launchGate, undefined).pipe(Effect.asVoid), ); @@ -420,12 +488,11 @@ describe("service composition", () => { ); yield* Deferred.succeed(launchGate, undefined); expect(Option.isSome(interruptFinished)).toBe(true); - const interrupted = yield* Fiber.await(composition); - expect(Exit.isFailure(interrupted)).toBe(true); + expect(Exit.isFailure(yield* Fiber.await(composition))).toBe(true); expect(yield* Ref.get(descendant.starts)).toEqual([]); - yield* prerequisite.core.ready; - expect((yield* prerequisite.core.get).lifecycle).toBe("running"); + yield* prerequisite.ready; + expect((yield* prerequisite.status).lifecycle).toBe("running"); expect(yield* Ref.get(descendant.starts)).toEqual([]); yield* orchestrator.stopNamespace; expect( @@ -447,18 +514,18 @@ describe("service composition", () => { health: Deferred.await(health), }); yield* orchestrator.start("functions"); - expect((yield* functions.core.get).health).toBe("starting"); + expect((yield* functions.status).health).toBe("starting"); yield* Effect.scoped(orchestrator.acquire("functions", false)); - expect((yield* functions.core.get).health).toBe("starting"); + expect((yield* functions.status).health).toBe("starting"); yield* Deferred.succeed(health, undefined); yield* Effect.scoped(orchestrator.acquire("functions")); - expect((yield* functions.core.get).health).toBe("healthy"); + expect((yield* functions.status).health).toBe("healthy"); }), ), ); it.live( - "launches an individual service before health and invalidates a dependent readiness wait on stop", + "rejects stopping a prerequisite while a dependent's start waits on it, then starts the dependent", () => Effect.scoped( Effect.gen(function* () { @@ -475,45 +542,19 @@ describe("service composition", () => { ], }); yield* orchestrator.start("database"); - expect((yield* database.core.get).health).toBe("starting"); + expect((yield* database.status).health).toBe("starting"); const dependent = yield* orchestrator .start("rest") .pipe(Effect.forkChild({ startImmediately: true })); - yield* orchestrator.stop("database"); - expect(Exit.isFailure(yield* Fiber.await(dependent))).toBe(true); - expect(yield* Ref.get(rest.starts)).toEqual([]); - expect((yield* database.core.get).lifecycle).toBe("stopped"); - }), - ), - ); - it.live( - "rejects a stale dependency plan after its prerequisite was stopped and replaced during preparation", - () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const preparing = yield* Deferred.make(); - const continuePreparation = yield* Deferred.make(); - const database = yield* makeInstance(orchestrator, "database"); - const rest = yield* makeInstance(orchestrator, "rest", { - prepare: Deferred.succeed(preparing, undefined).pipe( - Effect.andThen(Deferred.await(continuePreparation)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "rest", activation: "eager" }], - dependencies: [{ from: "database", to: "rest" }], - }); - const request = yield* orchestrator.start("rest").pipe(Effect.forkChild); - yield* Deferred.await(preparing); - yield* orchestrator.stop("database"); - yield* orchestrator.start("database"); - yield* database.core.ready; - yield* Deferred.succeed(continuePreparation, undefined); - expect(Exit.isFailure(yield* Fiber.await(request))).toBe(true); + const rejected = yield* orchestrator.stop("database").pipe(Effect.flip); + expect(rejected.message).toBe("database has active dependents: rest"); expect(yield* Ref.get(rest.starts)).toEqual([]); - expect((yield* database.core.get).lifecycle).toBe("running"); + + yield* Deferred.succeed(health, undefined); + yield* Fiber.join(dependent); + expect(yield* Ref.get(rest.starts)).toEqual([{ databaseUrl: "postgres://database" }]); + yield* orchestrator.stopNamespace; }), ), ); @@ -551,8 +592,8 @@ describe("service composition", () => { ]); expect(yield* Ref.get(standalone.starts)).toHaveLength(1); yield* orchestrator.stopNamespace; - expect((yield* standalone.core.get).lifecycle).toBe("stopped"); - expect((yield* database.core.get).lifecycle).toBe("stopped"); + expect((yield* standalone.status).lifecycle).toBe("stopped"); + expect((yield* database.status).lifecycle).toBe("stopped"); }), ), ); @@ -580,18 +621,19 @@ describe("service composition", () => { yield* orchestrator.startComposition; const result = yield* orchestrator.restartComposition.pipe(Effect.exit); expect(Exit.isFailure(result)).toBe(true); - expect((yield* bad.core.get).lifecycle).toBe("stopping"); - expect((yield* independent.core.get).lifecycle).toBe("stopped"); + expect((yield* bad.status).lifecycle).toBe("stopping"); + expect((yield* independent.status).lifecycle).toBe("stopped"); expect(yield* Ref.get(bad.starts)).toHaveLength(1); expect(yield* Ref.get(independent.starts)).toHaveLength(1); yield* Ref.set(failStop, false); yield* orchestrator.stopNamespace; + expect((yield* bad.status).lifecycle).toBe("stopped"); }), ), ); it.live( - "binds lazy endpoints without launching, counts traffic, and sleeps idle prerequisites after dependents", + "binds lazy endpoints without launching, counts traffic, and sleeps an idle prerequisite after its dependent", () => Effect.scoped( Effect.gen(function* () { @@ -611,17 +653,19 @@ describe("service composition", () => { expect(yield* Ref.get(database.starts)).toEqual([]); const requestScope = yield* Scope.make(); yield* orchestrator.acquire("rest").pipe(Scope.provide(requestScope)); - expect((yield* rest.core.get).health).toBe("healthy"); + expect((yield* rest.status).health).toBe("healthy"); yield* TestClock.adjust("2 seconds"); - expect((yield* rest.core.get).lifecycle).toBe("running"); - expect((yield* database.core.get).lifecycle).toBe("running"); + expect((yield* rest.status).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); yield* Scope.close(requestScope, Exit.void); const restStopped = yield* stopped(rest).pipe(Effect.forkChild); const databaseStopped = yield* stopped(database).pipe(Effect.forkChild); yield* TestClock.adjust("1 second"); yield* Fiber.join(restStopped); + expect((yield* database.status).lifecycle).toBe("running"); + yield* TestClock.adjust("100 millis"); yield* Fiber.join(databaseStopped); - expect((yield* database.core.get).wakeEnabled).toBe(true); + expect((yield* database.status).wakeEnabled).toBe(true); expect(yield* Ref.get(rest.bound)).toBe(true); yield* orchestrator.stopComposition; yield* orchestrator.acquire("rest").pipe(Effect.flip); @@ -655,7 +699,7 @@ describe("service composition", () => { yield* orchestrator.acquire("rest").pipe(Scope.provide(requestScope)); yield* Scope.close(requestScope, Exit.void); yield* TestClock.adjust("200 millis"); - expect((yield* database.core.get).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); const restStopped = yield* stopped(rest).pipe(Effect.forkChild); const databaseStopped = yield* stopped(database).pipe(Effect.forkChild); yield* Deferred.succeed( @@ -663,12 +707,57 @@ describe("service composition", () => { Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), ); yield* Deferred.await(cleanupStarted); - expect((yield* database.core.get).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); yield* Deferred.succeed(cleanupGate, undefined); yield* Fiber.join(restStopped); yield* TestClock.adjust("1 second"); yield* Fiber.join(databaseStopped); - expect((yield* database.core.get).lifecycle).toBe("stopped"); + expect((yield* database.status).lifecycle).toBe("stopped"); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("holds traffic through a crashed workload's cleanup, then re-wakes it", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashed = yield* Deferred.make>(); + const cleanupStarted = yield* Deferred.make(); + const cleanupGate = yield* Deferred.make(); + const rest = yield* makeInstance(orchestrator, "rest", { + exit: crashed, + stop: Deferred.succeed(cleanupStarted, undefined).pipe( + Effect.andThen(Deferred.await(cleanupGate)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "rest", activation: "lazy", idleMillis: 10_000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("rest")); + + yield* Deferred.succeed( + crashed, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* Deferred.await(cleanupStarted); + expect((yield* rest.status).lifecycle).toBe("stopping"); + + const admitted = yield* Deferred.make(); + const waking = yield* Effect.scoped(orchestrator.acquire("rest")).pipe( + Effect.andThen(Deferred.succeed(admitted, undefined)), + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("1 second"); + expect(yield* Deferred.isDone(admitted)).toBe(false); + expect(yield* Ref.get(rest.starts)).toHaveLength(1); + + yield* Deferred.succeed(cleanupGate, undefined); + yield* Fiber.join(waking); + expect(yield* Ref.get(rest.starts)).toHaveLength(2); + expect((yield* rest.status).lifecycle).toBe("running"); yield* orchestrator.stopNamespace; }), ).pipe(Effect.provide(TestClock.layer())), @@ -680,8 +769,8 @@ describe("service composition", () => { const orchestrator = yield* makeTestOrchestrator(); const events = yield* Ref.make>([]); const failRemoval = yield* Ref.make(true); - const next = yield* makeInstance(orchestrator, "next", { events }); - const failing = yield* makeInstance(orchestrator, "failing", { + yield* makeInstance(orchestrator, "next", { events }); + yield* makeInstance(orchestrator, "failing", { events, removeData: Ref.get(failRemoval).pipe( Effect.flatMap((fail) => @@ -693,10 +782,7 @@ describe("service composition", () => { yield* orchestrator.start("next"); const result = yield* Effect.exit(orchestrator.destroyNamespace); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isSuccess(result)) - return yield* Effect.die("namespace destroy unexpectedly passed"); - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + const error = errorOf(result); expect(error).toBeInstanceOf(Orchestrator.OrchestratorError); if (!(error instanceof Orchestrator.OrchestratorError)) return yield* Effect.die("namespace destroy did not return a combined error"); @@ -711,231 +797,1222 @@ describe("service composition", () => { "destroy:failing", "destroy:next", ]); - expect((yield* failing.core.get).registered).toBe(true); - expect((yield* next.core.get).registered).toBe(false); + expect((yield* orchestrator.instances).map(({ id }) => id)).toEqual(["failing"]); yield* Ref.set(failRemoval, false); yield* orchestrator.destroyNamespace; - expect((yield* failing.core.get).registered).toBe(false); + expect(yield* orchestrator.instances).toEqual([]); }), ), ); }); -it.live("joins an in-progress wake and wakes again after an admitted idle stop", () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const launching = yield* Deferred.make(); - const launch = yield* Deferred.make(); - const stopping = yield* Deferred.make(); - const stop = yield* Deferred.make(); - const instance = yield* makeInstance(orchestrator, "api", { - launch: Deferred.succeed(launching, undefined).pipe(Effect.andThen(Deferred.await(launch))), - stop: Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Deferred.await(stop))), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); - yield* Deferred.await(launching); - const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(launch, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); - expect(yield* Ref.get(instance.starts)).toHaveLength(1); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopping); - const third = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(stop, undefined); - yield* Fiber.join(third); - expect(yield* Ref.get(instance.starts)).toHaveLength(2); - yield* orchestrator.stopNamespace; - }), - ).pipe(Effect.provide(TestClock.layer())), -); - -it.live("allows later traffic to retry an armed service after a failed wake launch", () => - Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const failing = yield* Ref.make(true); - const instance = yield* makeInstance(orchestrator, "api", { - launch: Ref.get(failing).pipe( - Effect.flatMap((value) => (value ? Effect.fail(failure("launch failed")) : Effect.void)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); - yield* Ref.set(failing, false); - yield* Effect.scoped(orchestrator.acquire("api")); - expect((yield* instance.core.get).health).toBe("healthy"); - yield* orchestrator.stopNamespace; - }), - ), -); - -it.live("logs a named failure for a failed wake and readiness for a successful one", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const failing = yield* Ref.make(true); - yield* makeInstance(orchestrator, "api", { - launch: Ref.get(failing).pipe( - Effect.flatMap((value) => (value ? Effect.fail(failure("launch failed")) : Effect.void)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); - yield* Ref.set(failing, false); - yield* Effect.scoped(orchestrator.acquire("api")); - yield* orchestrator.stopNamespace; - expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(true); - expect(logs.some((line) => line.includes("api api is ready"))).toBe(true); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); +describe("destroying an instance with admitted traffic", () => { + it.live("removes the instance even while its traffic lease is held", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "database"); + yield* makeInstance(orchestrator, "rest"); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy" }, + { id: "rest", activation: "lazy" }, + ], + dependencies: [{ from: "database", to: "rest" }], + }); + yield* orchestrator.startComposition; + const request = yield* Scope.make(); + yield* orchestrator.acquire("rest").pipe(Scope.provide(request)); -it.live("logs exactly one wake for many concurrent acquires of a sleeping member", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const launching = yield* Deferred.make(); - const proceed = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - launch: Deferred.succeed(launching, undefined).pipe( - Effect.andThen(Deferred.await(proceed)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const callers = yield* Effect.forEach(Array.from({ length: 20 }), () => - Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ), - ); - yield* Deferred.await(launching); - yield* Deferred.succeed(proceed, undefined); - yield* Effect.forEach(callers, Fiber.join); - yield* orchestrator.stopNamespace; - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); - expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + yield* orchestrator.destroy("rest"); + expect((yield* orchestrator.instances).map(({ id }) => id)).not.toContain("rest"); + yield* orchestrator.stop("database"); -it.live("logs a distinct failure when a launched member never becomes ready", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - yield* makeInstance(orchestrator, "api", { - health: Effect.fail(failure("still booting")), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); - yield* orchestrator.stopNamespace; - expect(logs.some((line) => line.includes("api api failed to become ready"))).toBe(true); - expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(false); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + yield* Scope.close(request, Exit.void); + expect((yield* orchestrator.status("database")).lifecycle).toBe("stopped"); + }), + ), + ); -it.live("logs no extra wake while the initiator is still waiting for readiness", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const healthEntered = yield* Deferred.make(); - const readyGate = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - health: Deferred.succeed(healthEntered, undefined).pipe( - Effect.andThen(Deferred.await(readyGate)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "lazy" }], - dependencies: [], - }); - yield* orchestrator.startComposition; - const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.await(healthEntered); - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); - const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(readyGate, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); - yield* orchestrator.stopNamespace; - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); - expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); -}); + it.live( + "unregisters an instance whose data removal was confirmed even if its final close fails", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const removed = yield* Ref.make(false); + yield* makeInstance(orchestrator, "database", { + removeData: Ref.set(removed, true), + close: Ref.get(removed).pipe( + Effect.flatMap((gone) => (gone ? Effect.fail(failure("close failed")) : Effect.void)), + ), + }); -it.live("does not log a spurious wake for a member already starting outside acquire", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const orchestrator = yield* makeTestOrchestrator(); - const launching = yield* Deferred.make(); - const proceed = yield* Deferred.make(); - yield* makeInstance(orchestrator, "api", { - launch: Deferred.succeed(launching, undefined).pipe( - Effect.andThen(Deferred.await(proceed)), - ), - }); - yield* orchestrator.configure({ - members: [{ id: "api", activation: "eager" }], - dependencies: [], - }); - const composition = yield* orchestrator.startComposition.pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.await(launching); - const acquired = yield* Effect.scoped(orchestrator.acquire("api")).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.succeed(proceed, undefined); - yield* Fiber.join(composition); - yield* Fiber.join(acquired); - yield* orchestrator.stopNamespace; - expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(0); - }), - ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + const error = yield* orchestrator.destroy("database").pipe(Effect.flip); + + expect(error.message).toBe("close failed"); + expect(yield* orchestrator.instances).toEqual([]); + }), + ), + ); }); -describe("readiness recovery", () => { - const recoverableHealth = (healthy: Ref.Ref) => - Ref.get(healthy).pipe( - Effect.flatMap((ok) => (ok ? Effect.void : Effect.fail(failure("still booting")))), - ); +describe("wake and idle sleep", () => { + it.live( + "does not let a prerequisite idle-sleep while its dependent is still preparing to wake", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + const database = yield* makeInstance(orchestrator, "database"); + const studio = yield* makeInstance(orchestrator, "studio", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy", idleMillis: 100 }, + { id: "studio", activation: "lazy" }, + ], + dependencies: [{ from: "database", to: "studio" }], + }); + yield* orchestrator.startComposition; + const wake = yield* Effect.scoped(orchestrator.acquire("studio")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(preparing); + expect((yield* database.status).lifecycle).toBe("running"); + yield* TestClock.adjust("1 second"); + expect((yield* database.status).lifecycle).toBe("running"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(wake); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "does not idle-sleep a launch before its first healthy observation, then idles normally once healthy", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const health = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + health: Deferred.await(health), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 100 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + expect((yield* api.status).wakeEnabled).toBe(true); + + yield* orchestrator.start("api"); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "starting" }); + yield* TestClock.adjust("1 second"); + expect((yield* api.status).lifecycle).toBe("running"); + + yield* Deferred.succeed(health, undefined); + yield* api.ready; + expect((yield* api.status).health).toBe("healthy"); + yield* TestClock.adjust("1 second"); + yield* stopped(api); + expect((yield* api.status).wakeEnabled).toBe(true); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "holds an eager member's prerequisite active through a slow preparation instead of bypassing the reservation", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + yield* makeInstance(orchestrator, "database"); + const functions = yield* makeInstance(orchestrator, "functions"); + const studio = yield* makeInstance(orchestrator, "studio", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "eager" }, + { id: "functions", activation: "lazy", idleMillis: 100 }, + { id: "studio", activation: "eager" }, + ], + dependencies: [ + { from: "database", to: "studio" }, + { from: "functions", to: "studio" }, + ], + }); + const composition = yield* orchestrator.startComposition.pipe(Effect.forkChild); + yield* Deferred.await(preparing); + + // Independent traffic wakes and releases `functions` while studio is still preparing. + yield* Effect.scoped(orchestrator.acquire("functions")); + expect((yield* functions.status).lifecycle).toBe("running"); + yield* TestClock.adjust("1 second"); + expect((yield* functions.status).lifecycle).toBe("running"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(composition); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("holds a restarted member's prerequisite active through its own slow preparation", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + yield* makeInstance(orchestrator, "database"); + const functions = yield* makeInstance(orchestrator, "functions"); + const studio = yield* makeInstance(orchestrator, "studio", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy" }, + { id: "functions", activation: "lazy", idleMillis: 100 }, + { id: "studio", activation: "lazy" }, + ], + dependencies: [ + { from: "database", to: "studio" }, + { from: "functions", to: "studio" }, + ], + }); + yield* orchestrator.startComposition; + // Functions is independently already running; studio itself is still asleep, so nothing + // but studio's own restart keeps functions awake while studio prepares. + yield* Effect.scoped(orchestrator.acquire("functions")); + expect((yield* functions.status).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("stopped"); + + const restart = yield* orchestrator + .restart("studio") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(preparing); + yield* TestClock.adjust("1 second"); + expect((yield* functions.status).lifecycle).toBe("running"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(restart); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("keeps every prerequisite of a running dependent awake past its own idle timeout", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const pgmeta = yield* makeInstance(orchestrator, "pgmeta"); + const functions = yield* makeInstance(orchestrator, "functions"); + const studio = yield* makeInstance(orchestrator, "studio"); + yield* orchestrator.configure({ + members: [ + { id: "pgmeta", activation: "lazy", idleMillis: 60_000 }, + { id: "functions", activation: "lazy", idleMillis: 60_000 }, + { id: "studio", activation: "lazy", idleMillis: 300_000 }, + ], + dependencies: [ + { from: "pgmeta", to: "studio" }, + { from: "functions", to: "studio" }, + ], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("studio")); + + yield* TestClock.adjust("299 seconds"); + expect((yield* studio.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + expect((yield* functions.status).lifecycle).toBe("running"); + + const studioStopped = yield* stopped(studio).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(studioStopped); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + expect((yield* functions.status).lifecycle).toBe("running"); + + const prerequisitesStopped = yield* Effect.all([stopped(pgmeta), stopped(functions)], { + concurrency: "unbounded", + }).pipe(Effect.forkChild); + yield* TestClock.adjust("60 seconds"); + yield* Fiber.join(prerequisitesStopped); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "fails only a waiter whose wake budget expires, naming the stage, while the shared launch continues", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + + const early = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(preparing); + yield* TestClock.adjust("60 seconds"); + const late = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("60 seconds"); + + const expired = yield* Fiber.join(early); + expect(expired.message).toBe( + "api wake budget exceeded while waiting for api to finish preparing", + ); + expect((yield* api.status).lifecycle).toBe("starting"); + + yield* Deferred.succeed(continuePreparation, undefined); + yield* Fiber.join(late); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + describe("a traffic waiter that leaves before its launch finishes", () => { + // A leaked lease or waiter would keep demand on the service and stop it from ever idling. + const idlesAfterWaiterLeaves = ( + leave: ( + waiter: Fiber.Fiber>, + ) => Effect.Effect, + ) => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const waiter = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.exit, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(launching); + yield* leave(waiter); + + yield* Deferred.succeed(proceed, undefined); + yield* api.ready; + yield* TestClock.adjust("1 second"); + yield* stopped(api); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + }), + ).pipe(Effect.provide(TestClock.layer())); + + it.live( + "releases its claim when its wake budget expires, so the service idles afterwards", + () => + idlesAfterWaiterLeaves((waiter) => + TestClock.adjust("120 seconds").pipe( + Effect.andThen(Fiber.join(waiter)), + Effect.tap((exit) => + Effect.sync(() => expect(errorOf(exit)?.message).toContain("wake budget exceeded")), + ), + Effect.asVoid, + ), + ), + ); + + it.live("releases its claim when the client cancels, so the service idles afterwards", () => + idlesAfterWaiterLeaves((waiter) => Fiber.interrupt(waiter)), + ); + }); + + it.live("releases only a cancelled waiter and keeps the shared launch for the others", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const cancelled = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const kept = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(launching); + yield* Fiber.interrupt(cancelled); + + yield* Deferred.succeed(proceed, undefined); + yield* Fiber.join(kept); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + yield* orchestrator.stopNamespace; + }), + ), + ); + + it.live("joins an in-progress wake and wakes again after an admitted idle stop", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const launch = yield* Deferred.make(); + const stopping = yield* Deferred.make(); + const stop = yield* Deferred.make(); + const instance = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(launch)), + ), + stop: Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Deferred.await(stop))), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); + yield* Deferred.await(launching); + const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(launch, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Ref.get(instance.starts)).toHaveLength(1); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(stopping); + const third = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(stop, undefined); + yield* Fiber.join(third); + expect(yield* Ref.get(instance.starts)).toHaveLength(2); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + describe("while the owner refuses new work", () => { + const makeRefusingOrchestrator = Effect.gen(function* () { + const draining = yield* Ref.make(false); + const orchestrator = yield* Orchestrator.make({ + admit: () => + Ref.get(draining).pipe( + Effect.flatMap((refusing) => + refusing ? Effect.fail(failure("Owner is draining")) : Effect.void, + ), + ), + }); + return { orchestrator, draining }; + }); + + it.live( + "refuses traffic that would wake a stopped lazy service, then wakes once admitted", + () => + Effect.scoped( + Effect.gen(function* () { + const { orchestrator, draining } = yield* makeRefusingOrchestrator; + const api = yield* makeInstance(orchestrator, "api"); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + + yield* Ref.set(draining, true); + const refused = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + expect(refused.message).toBe("Owner is draining"); + expect(yield* Ref.get(api.starts)).toHaveLength(0); + expect((yield* api.status).lifecycle).toBe("stopped"); + + yield* Ref.set(draining, false); + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live( + "refuses traffic that arrives while an idle stop is in progress, without relaunching", + () => + Effect.scoped( + Effect.gen(function* () { + const { orchestrator, draining } = yield* makeRefusingOrchestrator; + const stopping = yield* Deferred.make(); + const finishStop = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + stop: Deferred.succeed(stopping, undefined).pipe( + Effect.andThen(Deferred.await(finishStop)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(stopping); + + yield* Ref.set(draining, true); + const refused = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + expect(refused.message).toBe("Owner is draining"); + + yield* Deferred.succeed(finishStop, undefined); + yield* orchestrator.stopNamespace; + expect(yield* Ref.get(api.starts)).toHaveLength(1); + expect((yield* api.status).lifecycle).toBe("stopped"); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("still admits traffic to a starting or running service", () => + Effect.scoped( + Effect.gen(function* () { + const { orchestrator, draining } = yield* makeRefusingOrchestrator; + const launching = yield* Deferred.make(); + const launch = yield* Deferred.make(); + const api = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(launch)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const waking = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); + yield* Deferred.await(launching); + + yield* Ref.set(draining, true); + const joining = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(launch, undefined); + yield* Fiber.join(waking); + yield* Fiber.join(joining); + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* Ref.get(api.starts)).toHaveLength(1); + expect((yield* api.status).lifecycle).toBe("running"); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + }); +}); + +describe("idle races through a real listener", () => { + interface HeldResponse { + readonly release: () => void; + } + + /** + * A lazy service whose every generation is a real HTTP server answering `generation:N`, reached + * only through its namespace listener. `/hold` keeps the response open until released, and a + * stop can be held open the same way. + */ + const makeListenerStack = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "orchestrator-listener-" }); + const stackId = `listener-${root.split("/").at(-1) ?? "stack"}`; + const state = yield* Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), + ); + yield* state.save({ + id: stackId, + identity: { projectRoot: root, branchContext: "test", stackName: stackId }, + runtime: "native", + lifetime: "detached", + instances: [], + composition: { members: [], dependencies: [] }, + }); + const network = yield* Layer.build( + Network.layer({ stackId, runtime: "native" }).pipe( + Layer.provide(Layer.succeed(StackNamespace.Service, state)), + ), + ).pipe(Effect.map((context) => Context.get(context, Network.Service))); + const orchestrator = yield* makeTestOrchestrator(); + const address = yield* Ref.make<{ readonly host: string; readonly port: number } | undefined>( + undefined, + ); + const generations = yield* Ref.make(0); + const held: Array = []; + const holdEntered = yield* Deferred.make(); + const holdStops = yield* Ref.make< + | { readonly entered: Deferred.Deferred; readonly release: Deferred.Deferred } + | undefined + >(undefined); + + const core = yield* makeService>( + { + launch: () => + Effect.gen(function* () { + const generation = yield* Ref.updateAndGet(generations, (value) => value + 1); + const server = createServer((request, response) => { + const answer = () => response.end(`generation:${generation}`); + if (request.url !== "/hold") return answer(); + held.push({ release: answer }); + Deferred.doneUnsafe(holdEntered, Effect.void); + }); + const port = yield* Effect.callback((resume) => { + server.listen(0, "127.0.0.1", () => { + const bound = server.address(); + resume( + bound === null || typeof bound === "string" + ? Effect.fail(failure("no address")) + : Effect.succeed(bound.port), + ); + }); + }); + yield* Ref.set(address, { host: "127.0.0.1", port }); + const exited = yield* Deferred.make>(); + const close = Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + server.closeAllConnections(); + }); + return { + health: Effect.void, + exit: Deferred.await(exited), + stop: Ref.get(holdStops).pipe( + Effect.flatMap((gate) => + gate === undefined + ? Effect.void + : Deferred.succeed(gate.entered, undefined).pipe( + Effect.andThen(Deferred.await(gate.release)), + ), + ), + Effect.andThen(close), + Effect.andThen(Deferred.succeed(exited, Exit.void)), + Effect.asVoid, + ), + remove: Effect.void, + }; + }), + removeData: () => Effect.void, + }, + { id: "api", config: {}, report: orchestrator.report }, + ); + const namespace = yield* network.register({ + id: "api", + endpoints: { + http: { + protocol: "http", + port: "auto", + backend: orchestrator.acquire("api").pipe( + Effect.andThen(Ref.get(address)), + Effect.flatMap((current) => + current === undefined ? Effect.fail(failure("no session")) : Effect.succeed(current), + ), + Effect.mapError((cause) => new ProxyError({ message: cause.message, cause })), + ), + enabled: Effect.succeed(true), + }, + }, + }); + yield* orchestrator.register({ + id: "api", + service: "api", + core, + launch: (generation, inputs) => core.launch(generation, inputs), + prepare: () => Effect.void, + bind: namespace.bind.pipe( + Effect.asVoid, + Effect.mapError((cause) => failure(cause.message)), + ), + close: Effect.void, + confirmRemoved: Effect.void, + release: Effect.void, + releasePorts: Effect.void, + hasEndpoint: true, + inputs: [], + outputs: {}, + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const [binding] = yield* namespace.bindings; + if (binding === undefined) return yield* Effect.die("listener was not bound"); + const get = (path: string) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://127.0.0.1:${binding.port}${path}`); + return { status: response.status, body: yield* response.text }; + }).pipe(Effect.provide(NodeHttpClient.layerNodeHttp)); + const releaseHeld = Effect.sync(() => { + for (const response of held.splice(0)) response.release(); + }); + return { + orchestrator, + get, + holdEntered, + releaseHeld, + holdStops, + status: orchestrator.status("api"), + observation: orchestrator.changes("api"), + }; + }); + + it.live( + "keeps serving a request admitted before the idle timer fires, from the same generation", + () => + Effect.scoped( + Effect.gen(function* () { + const stack = yield* makeListenerStack; + // A scoped acquisition releases its lease before returning, so the idle timer is armed. + yield* Effect.scoped(stack.orchestrator.acquire("api")); + + const request = yield* stack + .get("/hold") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(stack.holdEntered); + yield* TestClock.adjust("10 seconds"); + expect((yield* stack.status).lifecycle).toBe("running"); + + yield* stack.releaseHeld; + expect(yield* Fiber.join(request)).toEqual({ status: 200, body: "generation:1" }); + yield* stack.orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, TestClock.layer()))), + ); + + it.live( + "holds a request arriving after a committed sleep until the stop is confirmed, then serves it from the next generation", + () => + Effect.scoped( + Effect.gen(function* () { + const stack = yield* makeListenerStack; + // A scoped acquisition releases its lease before returning, so the idle timer is armed. + yield* Effect.scoped(stack.orchestrator.acquire("api")); + const gate = { + entered: yield* Deferred.make(), + release: yield* Deferred.make(), + }; + yield* Ref.set(stack.holdStops, gate); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(gate.entered); + expect((yield* stack.status).lifecycle).toBe("stopping"); + + // The request's own admission is the next lifecycle transition while the stop is held. + const queued = yield* nextTransition(stack.observation); + const request = yield* stack.get("/").pipe(Effect.forkChild({ startImmediately: true })); + yield* Fiber.join(queued); + yield* Ref.set(stack.holdStops, undefined); + yield* Deferred.succeed(gate.release, undefined); + + expect(yield* Fiber.join(request)).toEqual({ status: 200, body: "generation:2" }); + expect((yield* stack.status).lifecycle).toBe("running"); + yield* stack.orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, TestClock.layer()))), + ); +}); + +describe("admission while the lifecycle gate is held", () => { + // A configure whose persistence never finishes holds the gate the way a stalled state write does. + const holdGate = (orchestrator: Orchestrator.Interface) => + Effect.gen(function* () { + const persisting = yield* Deferred.make(); + const release = yield* Deferred.make(); + const configuring = yield* orchestrator + .configure( + { members: [{ id: "api", activation: "lazy" }], dependencies: [] }, + Deferred.succeed(persisting, undefined).pipe(Effect.andThen(Deferred.await(release))), + ) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(persisting); + return { release: Deferred.succeed(release, undefined), configuring }; + }); + + it.live("lets a client cancel an acquisition still waiting for the gate", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "api"); + const held = yield* holdGate(orchestrator); + const waiting = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const cancelled = yield* Fiber.interrupt(waiting).pipe(Effect.timeoutOption("2 seconds")); + expect(Option.isSome(cancelled)).toBe(true); + yield* held.release; + yield* Fiber.join(held.configuring); + }), + ), + ); + + it.live("fails an acquisition whose wake budget runs out while it waits for the gate", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "api"); + const held = yield* holdGate(orchestrator); + const waiting = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + yield* TestClock.adjust("120 seconds"); + yield* held.release; + yield* Fiber.join(held.configuring); + expect((yield* Fiber.join(waiting)).message).toBe( + "api wake budget exceeded while waiting for admission", + ); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); +}); + +describe("failed and crashed services", () => { + it.live("allows later traffic to retry an armed service after a failed wake launch", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const failing = yield* Ref.make(true); + const instance = yield* makeInstance(orchestrator, "api", { + launch: Ref.get(failing).pipe( + Effect.flatMap((value) => + value ? Effect.fail(failure("launch failed")) : Effect.void, + ), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* Ref.set(failing, false); + yield* Effect.scoped(orchestrator.acquire("api")); + expect((yield* instance.status).health).toBe("healthy"); + yield* orchestrator.stopNamespace; + }), + ), + ); + + it.live( + "keeps a crashed armed service wakeable and wakes it exactly once on the next request", + () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crash = yield* Deferred.make>(); + const api = yield* makeInstance(orchestrator, "api", { exit: crash }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + + // The runtime exits on its own, with no stop/restart/destroy ever invalidating the intent. + yield* Deferred.succeed( + crash, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* stopped(api); + expect((yield* api.status).wakeEnabled).toBe(true); + + const recoveryLogStart = logs.length; + const callers = yield* Effect.forEach(Array.from({ length: 5 }), () => + Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ), + ); + yield* Effect.forEach(callers, Fiber.join); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + expect( + logs.slice(recoveryLogStart).filter((line) => line.includes("Waking api api")), + ).toHaveLength(1); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }, + ); + + it.live( + "opens the breaker after three crashes, fails fast naming the cause, reopens for the same fixed cooldown after a further crash, and resets on an explicit start", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashes = yield* Queue.unbounded>(); + const api = yield* makeInstance(orchestrator, "api", { crash: Queue.take(crashes) }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const segfault = Exit.fail( + new ServiceError({ operation: "process", message: "segfault" }), + ); + const serveThenCrash = Effect.gen(function* () { + yield* Effect.scoped(orchestrator.acquire("api")); + const down = yield* stopped(api).pipe(Effect.forkChild({ startImmediately: true })); + yield* Queue.offer(crashes, segfault); + yield* Fiber.join(down); + }); + + yield* serveThenCrash; + yield* serveThenCrash; + yield* serveThenCrash; + const fastFail = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + expect(fastFail.message).toBe("api circuit breaker is open: segfault"); + expect(yield* Ref.get(api.starts)).toHaveLength(3); + + // The cooldown's own state change is the only signal that it elapsed. + const coolDown = (duration: Duration.Input) => + Effect.gen(function* () { + const cooled = yield* nextTransition(api.observation); + yield* TestClock.adjust(duration); + yield* Fiber.join(cooled); + }); + yield* coolDown("30 seconds"); + yield* serveThenCrash; + expect(yield* Ref.get(api.starts)).toHaveLength(4); + + // A further failure reopens the breaker for the same fixed cooldown. + expect( + (yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip)).message, + ).toBe("api circuit breaker is open: segfault"); + yield* coolDown("30 seconds"); + yield* serveThenCrash; + expect(yield* Ref.get(api.starts)).toHaveLength(5); + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + + yield* orchestrator.start("api"); + expect((yield* api.status).lifecycle).toBe("running"); + expect(yield* Ref.get(api.starts)).toHaveLength(6); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("counts a crash toward the breaker even when its cleanup has to be retried", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crashes = yield* Queue.unbounded>(); + const stops = yield* Ref.make(0); + const cleanupFailures = yield* Queue.unbounded(); + const api = yield* makeInstance(orchestrator, "api", { + onReport: (event) => + event._tag === "StopFailed" ? Queue.offer(cleanupFailures, undefined) : Effect.void, + crash: Queue.take(crashes), + // Every generation's first cleanup attempt fails; the retry succeeds. + stop: Ref.updateAndGet(stops, (count) => count + 1).pipe( + Effect.flatMap((count) => + count % 2 === 1 ? Effect.fail(failure("container busy")) : Effect.void, + ), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const serveThenCrash = Effect.gen(function* () { + yield* Effect.scoped(orchestrator.acquire("api")); + yield* Queue.offer(crashes, Exit.fail(failure("segfault"))); + yield* Queue.take(cleanupFailures); + expect((yield* api.status).lifecycle).toBe("stopping"); + }); + + yield* serveThenCrash; + yield* serveThenCrash; + yield* serveThenCrash; + const fourth = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.exit); + expect(errorOf(fourth)?.message).toBe("api circuit breaker is open: segfault"); + expect(yield* Ref.get(api.starts)).toHaveLength(3); + }), + ), + ); + + it.live("relaunches a crashed prerequisite without replacing its healthy dependent", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const crash = yield* Deferred.make>(); + const launched = yield* Queue.unbounded(); + const database = yield* makeInstance(orchestrator, "database", { + exit: crash, + onReport: (event) => + event._tag === "LaunchSucceeded" ? Queue.offer(launched, undefined) : Effect.void, + }); + const rest = yield* makeInstance(orchestrator, "rest"); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy" }, + { id: "rest", activation: "lazy" }, + ], + dependencies: [{ from: "database", to: "rest" }], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("rest")); + yield* Queue.take(launched); + + yield* Deferred.succeed(crash, Exit.fail(failure("segfault"))); + yield* Queue.take(launched); + + expect((yield* rest.status).lifecycle).toBe("running"); + expect(yield* Ref.get(database.starts)).toHaveLength(2); + expect(yield* Ref.get(rest.starts)).toHaveLength(1); + yield* Effect.scoped(orchestrator.acquire("rest")); + yield* orchestrator.stopNamespace; + }), + ), + ); +}); + +describe("wake logging", () => { + it.live("logs a named failure for a failed wake and readiness for a successful one", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const failing = yield* Ref.make(true); + yield* makeInstance(orchestrator, "api", { + launch: Ref.get(failing).pipe( + Effect.flatMap((value) => + value ? Effect.fail(failure("launch failed")) : Effect.void, + ), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* Ref.set(failing, false); + yield* Effect.scoped(orchestrator.acquire("api")); + yield* orchestrator.stopNamespace; + expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(true); + expect(logs.some((line) => line.includes("api api is ready"))).toBe(true); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("logs exactly one wake for many concurrent acquires of a sleeping member", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const callers = yield* Effect.forEach(Array.from({ length: 20 }), () => + Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ), + ); + yield* Deferred.await(launching); + yield* Deferred.succeed(proceed, undefined); + yield* Effect.forEach(callers, Fiber.join); + yield* orchestrator.stopNamespace; + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); + expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("logs a distinct failure when a launched member never becomes ready", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + yield* makeInstance(orchestrator, "api", { + health: Effect.fail(failure("still booting")), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* orchestrator.stopNamespace; + expect(logs.some((line) => line.includes("api api failed to become ready"))).toBe(true); + expect(logs.some((line) => line.includes("api api failed to wake"))).toBe(false); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("logs no extra wake while the initiator is still waiting for readiness", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const healthEntered = yield* Deferred.make(); + const readyGate = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + health: Deferred.succeed(healthEntered, undefined).pipe( + Effect.andThen(Deferred.await(readyGate)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(healthEntered); + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); + const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(readyGate, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + yield* orchestrator.stopNamespace; + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(1); + expect(logs.filter((line) => line.includes("api api is ready"))).toHaveLength(1); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); + + it.live("does not log a spurious wake for a member already starting outside acquire", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const proceed = yield* Deferred.make(); + yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe( + Effect.andThen(Deferred.await(proceed)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "eager" }], + dependencies: [], + }); + const composition = yield* orchestrator.startComposition.pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(launching); + const acquired = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(proceed, undefined); + yield* Fiber.join(composition); + yield* Fiber.join(acquired); + yield* orchestrator.stopNamespace; + expect(logs.filter((line) => line.includes("Waking api api"))).toHaveLength(0); + }), + ).pipe(Effect.provide(captureLogs(["Error", "Info"])(logs))); + }); +}); + +describe("readiness recovery", () => { + const recoverableHealth = (healthy: Ref.Ref) => + Ref.get(healthy).pipe( + Effect.flatMap((ok) => (ok ? Effect.void : Effect.fail(failure("still booting")))), + ); + + it.live( + "keeps re-checking a slow prerequisite after a failed reprobe until a waiting dependent is served", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const probes = yield* Ref.make(0); + const firstProbe = yield* Deferred.make(); + const failFirstProbe = yield* Deferred.make(); + const analytics = yield* makeInstance(orchestrator, "analytics", { + health: Effect.fail(failure("still booting")), + probe: Ref.updateAndGet(probes, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Deferred.succeed(firstProbe, undefined).pipe( + Effect.andThen(Deferred.await(failFirstProbe)), + Effect.andThen(Effect.fail(failure("still booting"))), + ) + : Effect.void, + ), + ), + }); + const studio = yield* makeInstance(orchestrator, "studio"); + yield* orchestrator.configure({ + members: [ + { id: "analytics", activation: "lazy" }, + { id: "studio", activation: "lazy" }, + ], + dependencies: [{ from: "analytics", to: "studio" }], + }); + yield* orchestrator.startComposition; + + const request = yield* Effect.scoped(orchestrator.acquire("studio")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(firstProbe); + const failed = yield* nextTransition(analytics.observation); + yield* Deferred.succeed(failFirstProbe, undefined); + yield* Fiber.join(failed); + yield* TestClock.adjust("1 second"); + + yield* Fiber.join(request); + expect(yield* Ref.get(probes)).toBe(2); + expect(yield* Ref.get(analytics.starts)).toHaveLength(1); + expect(yield* studio.status).toMatchObject({ lifecycle: "running", health: "healthy" }); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); it.live("serves traffic once a running instance recovers without restarting it", () => Effect.scoped( @@ -957,7 +2034,7 @@ describe("readiness recovery", () => { yield* Effect.scoped(orchestrator.acquire("api")); expect(yield* Ref.get(api.starts)).toHaveLength(1); - expect(yield* api.core.get).toMatchObject({ lifecycle: "running", health: "healthy" }); + expect(yield* api.status).toMatchObject({ lifecycle: "running", health: "healthy" }); yield* orchestrator.stopNamespace; }), ), @@ -988,7 +2065,7 @@ describe("readiness recovery", () => { expect(yield* Ref.get(database.starts)).toHaveLength(1); expect(yield* Ref.get(rest.starts)).toHaveLength(1); - expect((yield* rest.core.get).health).toBe("healthy"); + expect((yield* rest.status).health).toBe("healthy"); yield* orchestrator.stopNamespace; }), ), diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts index 2ae7d33c58..dce4e4ad4f 100644 --- a/packages/stack/src/Orchestrator.ts +++ b/packages/stack/src/Orchestrator.ts @@ -6,35 +6,36 @@ import { Effect, Exit, FiberMap, - Match, + Option, Ref, Schema, Scope, Semaphore, Stream, + SubscriptionRef, } from "effect"; import { - ServiceError, - ServiceDestroyed, - ServiceNotRunning, - ServiceStaleLaunch, - type ServiceObservation, -} from "./Service.ts"; -import type { ServiceAdmission } from "./Service.ts"; + blockingStage, + initialState, + LifecycleEvent, + makeGraph, + reduce, + type LifecycleCommand, + type LifecycleGraph, + type LifecycleState, + type ServiceSpec, +} from "./Lifecycle.ts"; +import { ServiceError, type ServiceInstance, type ServiceObservation } from "./Service.ts"; import { errorChainMessage } from "./internal/error-message.ts"; export type OrchestratorOperation = | "start" | "stop" | "restart" - | "sleep" | "destroy" - | "arm" - | "bind" - | "close" | "configure" | "register" - | "proxy"; + | "storage"; export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ readonly operation: OrchestratorOperation; @@ -42,7 +43,7 @@ export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ readonly cause?: unknown; readonly outcomes?: ReadonlyArray<{ readonly id: string; - readonly result: Exit.Exit; + readonly result: Exit.Exit; }>; }> {} @@ -50,51 +51,47 @@ export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ export const causeMessage = (cause: Cause.Cause): string => Cause.prettyErrors(cause).map(errorChainMessage).join("; ") || "interrupted"; -type CoreObservation = ServiceObservation; - -interface RegisteredCore { - readonly get: Effect.Effect; - readonly ready: Effect.Effect; - readonly stop: Effect.Effect; - readonly destroy: Effect.Effect; - readonly armAt: ( - revision: number, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly sleep: Effect.Effect; - readonly observation: Stream.Stream; -} +type ExecutionCore = Pick< + ServiceInstance, + "get" | "observation" | "stop" | "reprobe" | "storage" | "removeData" +>; export interface RegisteredInstance { readonly id: string; /** The service kind this instance runs, for wake observability; the id is the graph identity. */ readonly service: string; - readonly core: RegisteredCore; - readonly startAt: ( - revision: number, + readonly core: ExecutionCore; + /** Resolves the launch configuration from bound inputs and an optional restart candidate, then launches. */ + readonly launch: ( + generation: number, inputs: Record, - wake: boolean, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly restart: ( - revision: number, + candidate: unknown, + ) => Effect.Effect; + /** Resolves the same configuration as `launch` and only prepares it. */ + readonly prepare: ( inputs: Record, - config?: unknown, - guard?: Effect.Effect, - ) => Effect.Effect; + candidate: unknown, + ) => Effect.Effect; readonly bind: Effect.Effect; readonly close: Effect.Effect; + /** + * Publishes this instance's registration removal; run only once `core.removeData`'s resources + * are confirmed removed, as the `confirm` step of that same execution lock. A no-op when the + * registration or its state root is already gone. + */ + readonly confirmRemoved: Effect.Effect; + /** + * Closes this instance's network endpoints for good once its data is confirmed removed. Never + * deletes saved port assignments. + */ + readonly release: Effect.Effect; + /** Deletes this instance's saved port assignments; an individual destroy runs it, a stack-wide destroy leaves them to `Network.releaseStack`. */ + readonly releasePorts: Effect.Effect; readonly hasEndpoint: boolean; readonly inputs: ReadonlyArray; readonly outputs: Readonly>>; } -export type LifecycleError = - | ServiceError - | ServiceDestroyed - | ServiceNotRunning - | ServiceStaleLaunch; - interface CompositionMember { readonly id: string; readonly activation: "eager" | "lazy"; @@ -136,113 +133,290 @@ export const CompositionConfig = Schema.Struct({ ), }); +/** The wake budget a traffic waiter gets, counted from its arrival. */ +const waiterBudgetMillis = 120_000; + +/** One service's lifecycle phase and health from the reducer, joined with its execution facts. */ +export interface Status extends ServiceObservation { + readonly lifecycle: "stopped" | "starting" | "running" | "stopping"; + readonly health: "starting" | "healthy" | "unhealthy" | undefined; + /** Whether demand (traffic, a dependent or eager intent) relaunches the service. */ + readonly wakeEnabled: boolean; +} + +/** + * An operation the owner may refuse, for example while it drains. `wake` is traffic that would + * relaunch a service at rest. + */ +export type AdmittedOperation = "start" | "restart" | "storage" | "wake"; + /** The lifecycle authority for one stack's registered instances and their composition. */ export interface Interface { - readonly admissionFor: ( - id: string, - ) => ( - operation: ServiceAdmission, - transition: Effect.Effect, - ) => Effect.Effect; + /** Delivers a service's execution outcome to the lifecycle reducer. */ + readonly report: (event: LifecycleEvent) => Effect.Effect; readonly register: (entry: Entry) => Effect.Effect; readonly get: (id: string) => Effect.Effect; + /** A snapshot of every currently registered entry, independent of composition membership. */ + readonly instances: Effect.Effect>; + readonly status: (id: string) => Effect.Effect; + readonly changes: (id: string) => Stream.Stream; readonly composition: Effect.Effect; /** * Validates and installs a composition; a failed `persist` keeps the previous one. - * `persist` runs under the graph gate, so callers take the cross-process state lock first. + * `persist` runs under the lifecycle gate, so callers take the cross-process state lock first. */ readonly configure: ( configuration: CompositionConfig, persist?: Effect.Effect, ) => Effect.Effect; - readonly start: (id: string) => Effect.Effect; - readonly stop: (id: string) => Effect.Effect; + readonly start: (id: string) => Effect.Effect; + readonly ready: (id: string) => Effect.Effect; + readonly stop: (id: string) => Effect.Effect; readonly restart: ( id: string, config?: unknown, - ) => Effect.Effect; - readonly destroy: (id: string) => Effect.Effect; - readonly startComposition: Effect.Effect< - ReadonlyArray, - OrchestratorError | LifecycleError - >; - readonly stopComposition: Effect.Effect< - ReadonlyArray, - OrchestratorError | LifecycleError - >; + ) => Effect.Effect; + readonly destroy: (id: string) => Effect.Effect; + /** Runs storage work while the lifecycle keeps the stopped service from launching. */ + readonly storage: ( + id: string, + operation: Effect.Effect, + ) => Effect.Effect; + /** + * Runs `effect` under the lifecycle gate once every listed instance is stopped with wake + * disabled, failing with `refuse` for the first that is not. Callers take the cross-process + * state lock first. + */ + readonly whileStopped: ( + ids: Iterable, + refuse: (id: string) => F, + effect: Effect.Effect, + ) => Effect.Effect; + readonly startComposition: Effect.Effect, OrchestratorError | ServiceError>; + readonly stopComposition: Effect.Effect, OrchestratorError | ServiceError>; readonly restartComposition: Effect.Effect< - ReadonlyArray, - OrchestratorError | LifecycleError + ReadonlyArray, + OrchestratorError | ServiceError >; - readonly stopNamespace: Effect.Effect; - readonly destroyNamespace: Effect.Effect; + readonly stopNamespace: Effect.Effect; + readonly destroyNamespace: Effect.Effect; + /** Admits one unit of traffic, waking the service if needed; the lease lasts for the scope. */ readonly acquire: ( id: string, awaitReady?: boolean, trigger?: string, - ) => Effect.Effect; + ) => Effect.Effect; } -interface ActivityState { - readonly active: number; - readonly lastActivity: number; +interface Waiter { + readonly deferred: Deferred.Deferred; } -interface Graph { - readonly members: ReadonlyMap; - readonly prerequisites: ReadonlyMap>; - readonly dependents: ReadonlyMap>; +interface Applied { + readonly before: LifecycleState; + readonly after: LifecycleState; + readonly commands: ReadonlyArray; } -interface StartPlan { - readonly order: ReadonlyArray; - readonly revisions: ReadonlyMap; - readonly structure: Graph; - readonly configuration: CompositionConfig; -} - -const serviceFailure = (operation: string, message: string): ServiceError => - new ServiceError({ operation, message }); - const graphError = (operation: OrchestratorOperation, message: string, cause?: unknown) => new OrchestratorError({ operation, message, ...(cause === undefined ? {} : { cause }) }); -const isActive = (observation: CoreObservation): boolean => - observation.lifecycle !== "stopped" || observation.wakeEnabled; - -const topo = ( - ids: ReadonlyArray, - prerequisites: ReadonlyMap>, -): ReadonlyArray => { - const result: Array = []; - const seen = new Set(); - const visit = (id: string): void => { - if (seen.has(id)) return; - seen.add(id); - for (const prerequisite of prerequisites.get(id) ?? []) visit(prerequisite); - result.push(id); +/** The composition without `id`, as a member or as either end of a dependency. */ +export const withoutMember = (config: CompositionConfig, id: string): CompositionConfig => ({ + members: config.members.filter((member) => member.id !== id), + dependencies: config.dependencies.filter( + (dependency) => dependency.from !== id && dependency.to !== id, + ), +}); + +const validateComposition = ( + configuration: CompositionConfig, + values: ReadonlyMap, +): OrchestratorError | undefined => { + const memberIds = new Set(); + for (const member of configuration.members) { + if (memberIds.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); + memberIds.add(member.id); + } + const boundInputs = new Set(); + const prerequisites = new Map>(); + for (const dependency of configuration.dependencies) { + const list = prerequisites.get(dependency.to) ?? []; + if (list.includes(dependency.from)) + return graphError("configure", `Duplicate dependency ${dependency.from}->${dependency.to}`); + list.push(dependency.from); + prerequisites.set(dependency.to, list); + const target = values.get(dependency.to); + const source = values.get(dependency.from); + if (source === undefined || target === undefined) + return graphError("configure", `Dependency references an unregistered instance`); + for (const binding of dependency.bindings ?? []) { + if (source.outputs[binding.output] === undefined) + return graphError("configure", `Unknown output ${dependency.from}.${binding.output}`); + if (!target.inputs.includes(binding.input)) + return graphError("configure", `Unknown input ${dependency.to}.${binding.input}`); + const inputKey = `${dependency.to}:${binding.input}`; + if (boundInputs.has(inputKey)) + return graphError("configure", `Duplicate binding for ${dependency.to}.${binding.input}`); + boundInputs.add(inputKey); + } + } + for (const member of configuration.members) { + if (!values.has(member.id)) + return graphError("configure", `Member references an unregistered instance ${member.id}`); + if (member.activation === "lazy" && !values.get(member.id)?.hasEndpoint) + return graphError("configure", `Lazy member ${member.id} has no public endpoint`); + if (member.idleMillis !== undefined && member.idleMillis < 0) + return graphError("configure", `Negative idle timeout for ${member.id}`); + } + const visiting = new Set(); + const visited = new Set(); + const visit = (id: string): OrchestratorError | undefined => { + if (visiting.has(id)) + return graphError("configure", "Composition dependencies contain a cycle"); + if (visited.has(id)) return; + visiting.add(id); + for (const prerequisite of prerequisites.get(id) ?? []) { + const failure = visit(prerequisite); + if (failure !== undefined) return failure; + } + visiting.delete(id); + visited.add(id); + }; + for (const id of values.keys()) { + const failure = visit(id); + if (failure !== undefined) return failure; + } +}; + +/** Every registered instance as the reducer sees it; a non-member runs as an eager standalone. */ +const lifecycleGraph = (ids: Iterable, configuration: CompositionConfig): LifecycleGraph => + makeGraph( + [...ids].map((id): ServiceSpec => { + const member = configuration.members.find((candidate) => candidate.id === id); + const prerequisites = configuration.dependencies + .filter((dependency) => dependency.to === id) + .map((dependency) => dependency.from); + return { + id, + activation: member?.activation ?? "eager", + prerequisites, + ...(member?.idleMillis === undefined ? {} : { idleMillis: member.idleMillis }), + }; + }), + ); + +/** The listed instances, prerequisites first. */ +const inOrder = (graph: LifecycleGraph, ids: Iterable): ReadonlyArray => { + const wanted = new Set(ids); + return graph.order.filter((id) => wanted.has(id)); +}; + +const prerequisitesOf = (graph: LifecycleGraph, id: string): ReadonlyArray => + inOrder(graph, graph.prerequisiteClosure.get(id) ?? []); + +const lifecycleOf = (state: LifecycleState, id: string): Status["lifecycle"] => { + const phase = state.services.get(id)?.phase; + switch (phase?._tag) { + case "Starting": + return "starting"; + case "Running": + return "running"; + case "Stopping": + return "stopping"; + default: + return "stopped"; + } +}; + +const wakeEnabledOf = (state: LifecycleState, id: string): boolean => { + const intent = state.services.get(id)?.intent; + return intent !== undefined && intent !== "stopped"; +}; + +/** Whether a service is stopped and nothing can wake it, which graph, data and credential changes require. */ +export const isStoppedAndWakeDisabled = ( + status: Pick, +): boolean => status.lifecycle === "stopped" && !status.wakeEnabled; + +const statusOf = ( + state: LifecycleState, + id: string, + execution: ServiceObservation, +): Status => { + const service = state.services.get(id); + const phase = service?.phase; + const health = + phase?._tag === "Starting" + ? "starting" + : phase?._tag === "Running" + ? phase.ready + ? "healthy" + : service?.readinessFailure === undefined + ? "starting" + : "unhealthy" + : undefined; + return { + ...execution, + lifecycle: lifecycleOf(state, id), + health, + wakeEnabled: wakeEnabledOf(state, id), }; - for (const id of ids) visit(id); - return result; }; -/** Builds an orchestrator whose watchers and idle timers live in the current scope. */ -export const make = Effect.fn("Orchestrator.make")(function* < - Entry extends RegisteredInstance, ->(): Effect.fn.Return, never, Scope.Scope> { +const rejectionFor = ( + commands: ReadonlyArray, + id: string, +): ServiceError | undefined => { + const rejected = commands.find( + (command) => command._tag === "RequestRejected" && command.id === id, + ); + return rejected?._tag === "RequestRejected" + ? new ServiceError({ operation: rejected.operation, message: rejected.message }) + : undefined; +}; + +const cleanupError = (id: string, cause: unknown) => + new ServiceError({ + operation: "stop", + message: + cause instanceof Error + ? `${id} cleanup failed: ${errorChainMessage(cause)}` + : `${id} cleanup failed`, + cause, + }); + +/** Builds an orchestrator whose lifecycle actor, timers and executions live in the current scope. */ +export const make = Effect.fn("Orchestrator.make")(function* ( + options: { + /** Refuses a start, restart, storage operation or traffic wake, checked under the lifecycle gate. */ + readonly admit?: (operation: AdmittedOperation) => Effect.Effect; + } = {}, +): Effect.fn.Return, never, Scope.Scope> { const owner = yield* Scope.Scope; - const graphGate = yield* Semaphore.make(1); - const idleTimers = yield* FiberMap.make(); + const admit = options.admit ?? (() => Effect.void); + // The one lifecycle gate: every reducer step for this owner's graph is applied under it. + const gate = yield* Semaphore.make(1); + const lifecycle = yield* SubscriptionRef.make(initialState(makeGraph([]))); const registry = yield* Ref.make>(new Map()); const composition = yield* Ref.make({ members: [], dependencies: [] }); - const activity = yield* Ref.make>(new Map()); - /** Tracks members with a wake in progress so only its initiating caller logs the transition. */ - const wakes = yield* Ref.make>(new Set()); - - const withGraph = Effect.fn("Orchestrator.withGraph")((effect: Effect.Effect) => - Effect.uninterruptibleMask((restore) => - restore(graphGate.take(1)).pipe( - Effect.andThen(effect.pipe(Effect.ensuring(graphGate.release(1)))), + const waiters = yield* Ref.make>(new Map()); + const nextWaiterId = yield* Ref.make(1); + const idleTimers = yield* FiberMap.make(); + // Nothing reduces events once the owner closes, so its pending waiters are released here. + yield* Scope.addFinalizer( + owner, + Ref.get(waiters).pipe( + Effect.flatMap((pending) => + Effect.forEach( + pending.values(), + ({ deferred }) => + Deferred.fail( + deferred, + new ServiceError({ operation: "lifecycle", message: "Lifecycle owner stopped" }), + ), + { discard: true }, + ), ), ), ); @@ -250,166 +424,80 @@ export const make = Effect.fn("Orchestrator.make")(function* < const node = Effect.fn("Orchestrator.node")(function* ( id: string, ): Effect.fn.Return { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const nodes = yield* Ref.get(registry); - const value = nodes.get(id); + const value = (yield* Ref.get(registry)).get(id); if (value === undefined) return yield* graphError("register", `Unknown instance ${id}`); return value; }); - const graph = ( - configuration: CompositionConfig, - values: ReadonlyMap, - ): Graph | OrchestratorError => { - const members = new Map(); - for (const member of configuration.members) { - if (members.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); - members.set(member.id, member); - } - const boundInputs = new Set(); - const prerequisites = new Map>(); - const dependents = new Map>(); - for (const dependency of configuration.dependencies) { - const list = prerequisites.get(dependency.to) ?? []; - if (list.includes(dependency.from)) - return graphError("configure", `Duplicate dependency ${dependency.from}->${dependency.to}`); - list.push(dependency.from); - prerequisites.set(dependency.to, list); - const reverse = dependents.get(dependency.from) ?? []; - reverse.push(dependency.to); - dependents.set(dependency.from, reverse); - const target = values.get(dependency.to); - const source = values.get(dependency.from); - if (source === undefined || target === undefined) - return graphError("configure", `Dependency references an unregistered instance`); - for (const binding of dependency.bindings ?? []) { - if (source.outputs[binding.output] === undefined) - return graphError("configure", `Unknown output ${dependency.from}.${binding.output}`); - if (!target.inputs.includes(binding.input)) - return graphError("configure", `Unknown input ${dependency.to}.${binding.input}`); - const inputKey = `${dependency.to}:${binding.input}`; - if (boundInputs.has(inputKey)) - return graphError("configure", `Duplicate binding for ${dependency.to}.${binding.input}`); - boundInputs.add(inputKey); - } - } - for (const member of configuration.members) { - if (!values.has(member.id)) - return graphError("configure", `Member references an unregistered instance ${member.id}`); - if (member.activation === "lazy" && !values.get(member.id)?.hasEndpoint) - return graphError("configure", `Lazy member ${member.id} has no public endpoint`); - if (member.idleMillis !== undefined && member.idleMillis < 0) - return graphError("configure", `Negative idle timeout for ${member.id}`); - } - const visiting = new Set(); - const visited = new Set(); - const visit = (id: string): OrchestratorError | undefined => { - if (visiting.has(id)) - return graphError("configure", "Composition dependencies contain a cycle"); - if (visited.has(id)) return; - visiting.add(id); - for (const prerequisite of prerequisites.get(id) ?? []) { - const failure = visit(prerequisite); - if (failure !== undefined) return failure; - } - visiting.delete(id); - visited.add(id); - }; - for (const id of values.keys()) { - const failure = visit(id); - if (failure !== undefined) return failure; - } - return { members, prerequisites, dependents }; - }; + /** + * Holds the gate for `effect`. The semaphore hands the permit over and installs its release in + * one protected step; only the wait for it keeps the caller's interruptibility. + */ + const withGate = (effect: Effect.Effect) => + gate.withPermit(Effect.uninterruptible(effect)); - const configurationGraph = Effect.fn("Orchestrator.configurationGraph")(function* ( - configuration: CompositionConfig, - ): Effect.fn.Return { - const values = yield* Ref.get(registry); - const result = graph(configuration, values); - if (result instanceof OrchestratorError) return yield* result; - return result; - }); + const fork = (effect: Effect.Effect) => + Effect.forkIn(effect, owner, { uninterruptible: false }).pipe(Effect.asVoid); - const operationBlocksDependents = (operation: ServiceAdmission) => - Match.value(operation).pipe( - Match.when("stop", () => true), - Match.when("restart", () => true), - Match.when("destroy", () => true), - Match.when("sleep", () => true), - Match.when("start", () => false), - Match.when("storage", () => false), - Match.when("arm", () => false), - Match.exhaustive, - ); - const dependentBlocks = (operation: ServiceAdmission, observation: CoreObservation) => - Match.value(operation).pipe( - Match.when("destroy", () => observation.registered), - Match.when("sleep", () => observation.lifecycle !== "stopped"), - Match.when("stop", () => isActive(observation)), - Match.when("restart", () => isActive(observation)), - Match.when("start", () => false), - Match.when("storage", () => false), - Match.when("arm", () => false), - Match.exhaustive, + const settleWaiter = (waiterId: number, result: Effect.Effect) => + Ref.get(waiters).pipe( + Effect.flatMap((current) => { + const waiter = current.get(waiterId); + return waiter === undefined ? Effect.void : Deferred.complete(waiter.deferred, result); + }), ); - const admissionFor = - (id: string) => (operation: ServiceAdmission, transition: Effect.Effect) => - withGraph( - Effect.gen(function* () { - const configured = yield* Ref.get(composition); - const structure = yield* configurationGraph(configured); - const dependents = structure.dependents.get(id) ?? []; - if (operationBlocksDependents(operation)) { - for (const dependentId of dependents) { - const dependent = yield* node(dependentId); - const observation = yield* dependent.core.get; - const blocked = dependentBlocks(operation, observation); - if (blocked) - return yield* serviceFailure( - "graph", - `Dependent ${dependentId} blocks ${operation} of ${id}`, - ); - } - } - if (operation === "sleep") { - const timeout = structure.members.get(id)?.idleMillis; - const current = (yield* Ref.get(activity)).get(id); - const now = yield* Clock.currentTimeMillis; - if ( - timeout === undefined || - current === undefined || - current.active > 0 || - now - current.lastActivity < timeout - ) - return yield* serviceFailure("graph", `Instance ${id} is not idle`); - } - if (operation === "start") { - for (const prerequisiteId of structure.prerequisites.get(id) ?? []) { - const prerequisite = yield* node(prerequisiteId); - const observation = yield* prerequisite.core.get; - if (observation.lifecycle !== "running" || observation.health !== "healthy") - return yield* serviceFailure( - "graph", - `Prerequisite ${prerequisiteId} is not healthy`, - ); - } - } - yield* transition; - }), - ).pipe( - Effect.mapError((error) => - error instanceof OrchestratorError ? serviceFailure("graph", error.message) : error, - ), - ); - const resolveInputs = Effect.fn("Orchestrator.resolveInputs")(function* ( + /** + * Applies events under a gate the caller already holds. Executions and timers are started before + * the new state is published, so whoever observes a transition also observes its armed timers; + * waiters are resolved last, once the state they were admitted against is visible. + */ + const applyLocked = Effect.fnUntraced(function* (events: ReadonlyArray) { + const now = yield* Clock.currentTimeMillis; + const before = yield* SubscriptionRef.get(lifecycle); + let after = before; + const commands: Array = []; + for (const event of events) { + const [next, emitted] = reduce(after, event, now); + after = next; + commands.push(...emitted); + } + const resolvesWaiter = (command: LifecycleCommand) => + command._tag === "AdmitConnection" || command._tag === "FailConnection"; + for (const command of commands) if (!resolvesWaiter(command)) yield* execute(command, after); + yield* SubscriptionRef.set(lifecycle, after); + for (const command of commands) if (resolvesWaiter(command)) yield* execute(command, after); + return { before, after, commands } satisfies Applied; + }); + + const dispatch = (events: ReadonlyArray) => withGate(applyLocked(events)); + + const report = (event: LifecycleEvent) => dispatch([event]).pipe(Effect.asVoid); + + /** Executes one `Stop` command; the service reports `Exited` or `StopFailed` itself. */ + const stopGeneration = Effect.fnUntraced(function* ( id: string, - configuration: CompositionConfig, - ): Effect.fn.Return, OrchestratorError | LifecycleError> { + generation: number, + state: LifecycleState, + ) { + const service = state.services.get(id); + const operation = + service?.intent === "stopped" ? "stop" : service?.relaunchForced ? "restart" : "sleep"; + const discard = service?.destroy !== undefined; + yield* fork( + node(id).pipe( + Effect.flatMap((entry) => entry.core.stop(generation, { operation, discard })), + Effect.withSpan("Lifecycle.stop", { + attributes: { instance_id: id, generation, operation }, + }), + ), + ); + }); + + const resolveInputs = Effect.fn("Orchestrator.resolveInputs")(function* (id: string) { + const configuration = yield* Ref.get(composition); const inputs: Record = {}; - const edges = configuration.dependencies.filter((dependency) => dependency.to === id); - for (const dependency of edges) { + for (const dependency of configuration.dependencies.filter(({ to }) => to === id)) { const source = yield* node(dependency.from); for (const binding of dependency.bindings ?? []) { const output = source.outputs[binding.output]; @@ -421,337 +509,581 @@ export const make = Effect.fn("Orchestrator.make")(function* < return inputs; }); - const snapshotPlan = Effect.fn("Orchestrator.snapshotPlan")(function* ( + const launch = Effect.fn("Lifecycle.launch")(function* ( id: string, - ): Effect.fn.Return { - return yield* withGraph( - Effect.gen(function* () { - const configuration = yield* Ref.get(composition); - const configured = yield* configurationGraph(configuration); - const structure: Graph = configured.members.has(id) - ? configured - : { - members: new Map([...configured.members, [id, { id, activation: "eager" as const }]]), - prerequisites: configured.prerequisites, - dependents: configured.dependents, - }; - const closure = new Set(); - const visit = (member: string) => { - if (closure.has(member)) return; - closure.add(member); - for (const prerequisite of structure.prerequisites.get(member) ?? []) visit(prerequisite); - }; - visit(id); - const revisions = new Map(); - for (const member of closure) { - const observation = yield* (yield* node(member)).core.get; - revisions.set(member, observation.intentRevision); - } - return { - order: topo([...closure], structure.prerequisites), - revisions, - structure, - configuration, - }; - }), - ); + generation: number, + candidate: unknown, + ) { + yield* Effect.annotateCurrentSpan({ instance_id: id, generation }); + const started = yield* Effect.gen(function* () { + const entry = yield* node(id); + const inputs = yield* resolveInputs(id); + yield* entry.launch(generation, inputs, candidate); + }).pipe(Effect.exit); + if (Exit.isFailure(started) && !Cause.hasInterruptsOnly(started.cause)) + yield* report( + LifecycleEvent.LaunchFailed({ id, generation, cause: Cause.squash(started.cause) }), + ); }); - const prerequisiteGuard = Effect.fn("Orchestrator.prerequisiteGuard")(function* ( - plan: StartPlan, - id: string, - ): Effect.fn.Return { - if ((yield* Ref.get(composition)) !== plan.configuration) - return yield* serviceFailure("graph", "Composition changed before admission"); - const values = yield* Ref.get(registry); - for (const prerequisite of plan.structure.prerequisites.get(id) ?? []) { - const expected = plan.revisions.get(prerequisite); - const current = values.get(prerequisite); - if (expected === undefined || current === undefined) - return yield* serviceFailure("graph", `Missing prerequisite ${prerequisite}`); - const observation = yield* current.core.get; - if (observation.intentRevision !== expected) - return yield* serviceFailure( - "graph", - `Prerequisite ${prerequisite} changed before ${id} admission`, + // Interruptible so re-arming a key never waits out the timer it replaces. + const sleepThen = (delayMillis: number, event: LifecycleEvent) => + Effect.interruptible(Effect.sleep(`${delayMillis} millis`).pipe(Effect.andThen(report(event)))); + + /** Resolves waiters in place; every I/O command is forked onto the owner scope. */ + const execute = (command: LifecycleCommand, state: LifecycleState): Effect.Effect => { + switch (command._tag) { + case "AdmitConnection": + return settleWaiter(command.waiterId, Effect.void); + case "FailConnection": { + const cause = command.cause; + // A live session at failure time means the wait ended on readiness, not on the launch. + const operation = + state.services.get(command.id)?.phase._tag === "Running" + ? "readiness" + : cause instanceof ServiceError + ? cause.operation + : "wake"; + return settleWaiter( + command.waiterId, + Effect.fail( + new ServiceError({ + operation, + message: + cause instanceof Error + ? `${command.message}: ${errorChainMessage(cause)}` + : command.message, + ...(cause === undefined ? {} : { cause }), + }), + ), ); + } + case "Launch": + return fork(launch(command.id, command.generation, command.candidate)); + case "Stop": + return stopGeneration(command.id, command.generation, state); + case "Reprobe": { + const reprobe = node(command.id).pipe( + Effect.flatMap((entry) => entry.core.reprobe(command.generation)), + Effect.ignore, + Effect.withSpan("Lifecycle.reprobe", { + attributes: { instance_id: command.id, generation: command.generation }, + }), + ); + if (command.delayMillis === 0) return fork(reprobe); + // Started at once so the spacing is already timed when the transition is published. + return Effect.forkIn( + Effect.sleep(`${command.delayMillis} millis`).pipe(Effect.andThen(reprobe)), + owner, + { startImmediately: true, uninterruptible: false }, + ).pipe(Effect.asVoid); + } + case "ArmIdleTimer": + return FiberMap.run( + idleTimers, + command.id, + sleepThen( + command.delayMillis, + LifecycleEvent.IdleElapsed({ + id: command.id, + generation: command.generation, + epoch: command.epoch, + }), + ), + { startImmediately: true }, + ).pipe(Effect.asVoid); + case "ArmCooldownTimer": + return Effect.forkIn( + sleepThen( + command.delayMillis, + LifecycleEvent.CooldownElapsed({ id: command.id, openUntil: command.openUntil }), + ), + owner, + { startImmediately: true, uninterruptible: false }, + ).pipe(Effect.asVoid); + case "RequestRejected": + // The waiting or requesting caller reads these from the commands it dispatched. + return Effect.void; } + }; + + interface Wait { + readonly id: string; + readonly requireReady: boolean; + } + + /** The wake budget's failure, naming what the waiter was still blocked on. */ + const budgetExceeded = Effect.fnUntraced(function* (id: string, reached: Ref.Ref) { + const state = yield* SubscriptionRef.get(lifecycle); + const queued = yield* Ref.get(reached); + return yield* new ServiceError({ + operation: queued && state.services.get(id)?.phase._tag === "Running" ? "readiness" : "wake", + message: `${id} wake budget exceeded while waiting for ${queued ? blockingStage(state, id) : "admission"}`, + }); }); - const startNode = Effect.fn("Orchestrator.startNode")(function* ( - id: string, - wake = false, - awaitTarget = false, - savedPlan?: StartPlan, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const plan = savedPlan ?? (yield* snapshotPlan(id)); - for (const member of plan.order) { - const instance = yield* node(member); - const inputs = yield* resolveInputs(member, plan.configuration); - const revision = plan.revisions.get(member); - if (revision === undefined) - return yield* graphError("start", `Missing revision for ${member}`); - yield* instance.startAt(revision, inputs, wake, prerequisiteGuard(plan, member)); - yield* instance.bind; - if (member !== id || awaitTarget) yield* instance.core.ready; + interface Admission { + /** Explicit requests dispatched atomically before the waiters; a rejection fails the call. */ + readonly requests?: ReadonlyArray>; + readonly waits: ReadonlyArray; + /** + * Traffic waits carry a lease and the wake budget; explicit waits carry neither. Traffic that + * finds its service neither starting nor running is admitted as a `wake`. + */ + readonly traffic: boolean; + /** Checks the owner's admission guard under the same gate step as the requests. */ + readonly operation?: AdmittedOperation; + /** Runs right after the dispatch, before waiting. */ + readonly onApplied?: (applied: Applied) => Effect.Effect; + /** Runs uninterruptibly once every wait is admitted, so a lease can't be lost to cancellation. */ + readonly onAdmitted?: Effect.Effect; + } + + /** + * Dispatches requests together with one waiter per wait, atomically, then awaits them all. Any + * failure, rejection, timeout or cancellation withdraws the remaining waiters and releases + * admitted leases. + */ + const awaitAll = Effect.fnUntraced(function* (admission: Admission) { + const { waits } = admission; + // Registration and its unconditional cleanup are installed together, before any wait. + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const entries = yield* Effect.forEach(waits, (wait) => + Effect.all({ + waiterId: Ref.getAndUpdate(nextWaiterId, (value) => value + 1), + deferred: Deferred.make(), + }).pipe(Effect.map((created) => ({ ...wait, ...created }))), + ); + yield* Ref.update(waiters, (current) => { + const next = new Map(current); + for (const { waiterId, deferred } of entries) next.set(waiterId, { deferred }); + return next; + }); + const forget = Ref.update(waiters, (current) => { + const next = new Map(current); + for (const { waiterId } of entries) next.delete(waiterId); + return next; + }); + return yield* dispatchAndAwait(admission, entries, restore).pipe(Effect.ensuring(forget)); + }), + ); + }); + + /** + * Dispatches the requests and waiters, then awaits them. A traffic wait's one budget covers the + * gate wait, the dispatch and the wait for readiness. Every failure, rejection, timeout or + * cancellation after the dispatch withdraws the waiters and releases granted leases. + */ + const dispatchAndAwait = Effect.fnUntraced(function* ( + admission: Admission, + entries: ReadonlyArray< + Wait & { readonly waiterId: number; readonly deferred: Deferred.Deferred } + >, + restore: (effect: Effect.Effect) => Effect.Effect, + ) { + const { requests = [], traffic } = admission; + const withdraw = Effect.gen(function* () { + yield* dispatch( + entries.map(({ id, waiterId }) => LifecycleEvent.WaiterCancelled({ id, waiterId })), + ); + if (!traffic) return; + for (const { id, deferred } of entries) { + if (!(yield* Deferred.isDone(deferred))) continue; + if (Exit.isSuccess(yield* Effect.exit(Deferred.await(deferred)))) + yield* dispatch([LifecycleEvent.ConnectionClosed({ id })]); } }); + // Set inside the gate, so a failure knows whether there is anything to withdraw. + const reached = yield* Ref.make(false); + const admitUnderGate = Effect.gen(function* () { + if (admission.operation !== undefined) return yield* admit(admission.operation); + if (!traffic) return; + const state = yield* SubscriptionRef.get(lifecycle); + const wakes = entries.some(({ id }) => { + const phase = state.services.get(id)?.phase._tag; + return phase !== "Starting" && phase !== "Running"; + }); + if (wakes) yield* admit("wake"); + }); + const guarded = withGate( + admitUnderGate.pipe( + Effect.andThen(Ref.set(reached, true)), + Effect.andThen( + applyLocked([ + ...requests, + ...entries.map(({ id, waiterId, requireReady }) => + traffic + ? LifecycleEvent.ConnectionOpened({ id, waiterId, requireReady }) + : LifecycleEvent.ReadinessAwaited({ id, waiterId, requireReady }), + ), + ]), + ), + ), + ); + const admitted = Effect.gen(function* () { + const applied = yield* guarded; + for (const request of requests) { + const rejected = rejectionFor(applied.commands, request.id); + if (rejected !== undefined) return yield* rejected; + } + yield* admission.onApplied?.(applied) ?? Effect.void; + yield* Effect.forEach(entries, ({ deferred }) => Deferred.await(deferred), { + concurrency: "unbounded", + discard: true, + }).pipe( + Effect.withSpan("Lifecycle.awaitAdmission", { + attributes: { + instance_ids: entries.map(({ id }) => id).join(","), + traffic, + waiters: entries.length, + }, + }), + ); + return applied; + }); + const subject = traffic ? entries[0] : undefined; + const budgeted = + subject === undefined + ? admitted + : admitted.pipe( + Effect.timeoutOrElse({ + duration: `${waiterBudgetMillis} millis`, + orElse: () => budgetExceeded(subject.id, reached), + }), + ); + const outcome = yield* restore(budgeted).pipe(Effect.exit); + if (Exit.isFailure(outcome)) { + if (yield* Ref.get(reached)) yield* withdraw; + } else yield* admission.onAdmitted ?? Effect.void; + return yield* outcome; }); - const reschedulePrerequisites = Effect.fn("Orchestrator.reschedulePrerequisites")(function* ( - id: string, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const structure = yield* configurationGraph(yield* Ref.get(composition)); - for (const prerequisite of structure.prerequisites.get(id) ?? []) - yield* scheduleIdle(prerequisite); + /** The readiness waits that bound an explicit operation: every prerequisite, then the target. */ + const explicitWaits = Effect.fnUntraced(function* (id: string, targetReady: boolean) { + const state = yield* SubscriptionRef.get(lifecycle); + return [ + ...prerequisitesOf(state.graph, id).map((prerequisite) => ({ + id: prerequisite, + requireReady: true, + })), + { id, requireReady: targetReady }, + ]; + }); + + const bindClosure = Effect.fn("Orchestrator.bindClosure")(function* (id: string) { + const state = yield* SubscriptionRef.get(lifecycle); + for (const member of [...prerequisitesOf(state.graph, id), id]) + yield* (yield* node(member)).bind; + }); + + /** Dispatches one explicit request, failing with the reducer's rejection if it refused it. */ + const request = (event: Extract) => + awaitAll({ requests: [event], waits: [], traffic: false }); + + const status = Effect.fn("Orchestrator.status")(function* (id: string) { + const entry = yield* node(id); + return statusOf(yield* SubscriptionRef.get(lifecycle), id, yield* entry.core.get); + }); + + const changes = (id: string): Stream.Stream => + Stream.unwrap( + node(id).pipe( + Effect.map((entry) => + Stream.zipLatestWith( + SubscriptionRef.changes(lifecycle), + entry.core.observation, + (state, execution) => statusOf(state, id, execution), + ), + ), + ), + ); + + const start = Effect.fn("Orchestrator.start")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + yield* node(id); + yield* bindClosure(id); + yield* awaitAll({ + requests: [LifecycleEvent.StartRequested({ id })], + waits: yield* explicitWaits(id, false), + traffic: false, + operation: "start", }); }); - const scheduleIdle = Effect.fn("Orchestrator.scheduleIdle")(function* ( - id: string, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const configured = yield* Ref.get(composition); - const timeout = configured.members.find((member) => member.id === id)?.idleMillis; - if (timeout === undefined || timeout < 0) return; - const now = yield* Clock.currentTimeMillis; - const current = (yield* Ref.get(activity)).get(id); - if (current === undefined) return; - const delay = Math.max(0, timeout - (now - current.lastActivity)); - yield* FiberMap.run( - idleTimers, - id, - Effect.gen(function* () { - if (delay > 0) yield* Effect.sleep(`${delay} millis`); - const now = yield* Clock.currentTimeMillis; - const state = yield* withGraph( - Effect.gen(function* () { - const values = yield* Ref.get(activity); - const current = values.get(id); - return !( - current === undefined || - current.active > 0 || - now - current.lastActivity < timeout - ); - }), - ); - if (state) { - const result = yield* (yield* node(id)).core.sleep.pipe(Effect.exit); - if (Exit.isSuccess(result)) yield* reschedulePrerequisites(id); - } + const ready = Effect.fn("Orchestrator.ready")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + yield* node(id); + yield* awaitAll({ waits: yield* explicitWaits(id, true), traffic: false }); + }); + + /** Waits until `generation` is no longer stopping, failing while its cleanup is pending. */ + const awaitStopped = Effect.fnUntraced(function* (id: string, generation: number) { + const settled = yield* SubscriptionRef.changes(lifecycle).pipe( + Stream.map((state) => state.services.get(id)), + Stream.filter( + (service) => + service?.phase._tag !== "Stopping" || + service.phase.generation !== generation || + service.cleanupFailure !== undefined, + ), + Stream.runHead, + ); + const pending = Option.getOrUndefined(settled)?.cleanupFailure; + if (pending === undefined) return; + return yield* cleanupError(id, pending.cause); + }); + + /** Waits until a requested destroy holds the storage reservation, failing on pending cleanup. */ + const awaitDestroyReservation = Effect.fnUntraced(function* (id: string) { + const settled = yield* SubscriptionRef.changes(lifecycle).pipe( + Stream.map((state) => state.services.get(id)), + Stream.filter( + (service) => service?.destroy !== "requested" || service.cleanupFailure !== undefined, + ), + Stream.runHead, + ); + const service = Option.getOrUndefined(settled); + if (service?.destroy === "reserved") return; + if (service?.cleanupFailure !== undefined) + return yield* cleanupError(id, service.cleanupFailure.cause); + return yield* new ServiceError({ + operation: "destroy", + message: `${id} destroy was withdrawn`, + }); + }); + + const stop = Effect.fn("Orchestrator.stop")(function* (id: string) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + const entry = yield* node(id); + const applied = yield* request(LifecycleEvent.StopRequested({ id })); + const phase = applied.after.services.get(id)?.phase; + if (phase?._tag === "Stopping") + yield* awaitStopped(id, phase.generation).pipe( + Effect.withSpan("Lifecycle.awaitStopped", { + attributes: { instance_id: id, generation: phase.generation }, }), ); + yield* entry.close; + }); + + const restart = Effect.fn("Orchestrator.restart")(function* (id: string, config?: unknown) { + yield* Effect.annotateCurrentSpan({ instance_id: id }); + const entry = yield* node(id); + // A candidate is resolved and prepared before anything stops, so a bad one costs no downtime. + if (config !== undefined) yield* entry.prepare(yield* resolveInputs(id), config); + yield* bindClosure(id); + yield* awaitAll({ + requests: [LifecycleEvent.RestartRequested({ id, candidate: config })], + waits: yield* explicitWaits(id, false), + traffic: false, + operation: "restart", }); }); - const release = Effect.fn("Orchestrator.release")(function* ( + const storage = (id: string, operation: Effect.Effect) => + Effect.gen(function* () { + const entry = yield* node(id); + return yield* Effect.acquireUseRelease( + awaitAll({ + requests: [LifecycleEvent.StorageReserved({ id })], + waits: [], + traffic: false, + operation: "storage", + }), + () => entry.core.storage(operation), + () => dispatch([LifecycleEvent.StorageReleased({ id })]), + ); + }).pipe(Effect.withSpan("Orchestrator.storage", { attributes: { instance_id: id } })); + + const unregister = (id: string) => + withGate( + Effect.gen(function* () { + const values = new Map(yield* Ref.get(registry)); + values.delete(id); + const next = withoutMember(yield* Ref.get(composition), id); + // The graph update goes first: it removes the service while the destroy still holds it. + const applied = yield* applyLocked([ + LifecycleEvent.GraphUpdated({ graph: lifecycleGraph(values.keys(), next) }), + LifecycleEvent.DestroyReleased({ id }), + ]); + const rejected = rejectionFor(applied.commands, id); + if (rejected !== undefined) return yield* rejected; + yield* Ref.set(registry, values); + yield* Ref.set(composition, next); + }), + ); + + /** + * `ports` says what happens to the instance's saved port assignments once its data is removed: + * an individual destroy deletes them, a stack-wide destroy retains them for `Network.releaseStack` + * so a destroy that fails part-way keeps every assignment for the retry. + */ + const destroyInstance = Effect.fn("Orchestrator.destroy")(function* ( id: string, - ): Effect.fn.Return { - yield* Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - yield* withGraph( + ports: "delete" | "retain", + ) { + yield* Effect.annotateCurrentSpan({ instance_id: id, ports }); + const entry = yield* node(id); + const configured = yield* Ref.get(composition); + const dependent = configured.dependencies.find((dependency) => dependency.from === id); + if (dependent !== undefined) + return yield* new ServiceError({ + operation: "graph", + message: `Dependent ${dependent.to} blocks destroy of ${id}`, + }); + // Confirmed removal unregisters the instance even if the final close fails; any earlier exit + // gives the destroy up, so the service is usable again. + const removed = yield* Ref.make(false); + yield* Effect.uninterruptibleMask((restore) => + restore( Effect.gen(function* () { - const values = yield* Ref.get(activity); - const current = values.get(id); - if (current === undefined) return; - yield* Ref.set( - activity, - new Map(values).set(id, { - active: Math.max(0, current.active - 1), - lastActivity: now, + yield* request(LifecycleEvent.DestroyRequested({ id })); + yield* awaitDestroyReservation(id).pipe( + Effect.withSpan("Lifecycle.awaitDestroyReservation", { + attributes: { instance_id: id }, }), ); + yield* entry.close; + // `release` runs as part of the same `confirm` step `core.removeData` already + // serializes under its execution lock, strictly before the registration-removal + // publish: ports are released once data removal is confirmed, but before anything + // observes the instance as unregistered. + yield* entry.core.removeData( + entry.release.pipe( + Effect.andThen(ports === "delete" ? entry.releasePorts : Effect.void), + Effect.andThen(entry.confirmRemoved), + ), + ); + yield* Ref.set(removed, true); + yield* entry.close; }), - ); - yield* scheduleIdle(id); - }); + ).pipe( + Effect.onExit(() => + Ref.get(removed).pipe( + Effect.flatMap((confirmed) => + confirmed ? unregister(id) : dispatch([LifecycleEvent.DestroyReleased({ id })]), + ), + ), + ), + ), + ); }); - const settle = Effect.fn("Orchestrator.settle")(function* ( + const destroy = (id: string) => destroyInstance(id, "delete"); + + const settleOutcomes = Effect.fn("Orchestrator.settleOutcomes")(function* ( operation: "start" | "stop" | "destroy", ids: ReadonlyArray, - action: (id: string) => Effect.Effect, + action: (id: string) => Effect.Effect, concurrency: 1 | "unbounded" = 1, - ): Effect.fn.Return, OrchestratorError | LifecycleError> { - return yield* Effect.gen(function* () { - const outcomes = yield* Effect.forEach( - ids, - (id) => - action(id).pipe( - Effect.exit, - Effect.map((result) => ({ id, result })), - ), - { concurrency }, - ); - if (outcomes.some(({ result }) => Exit.isFailure(result))) - return yield* new OrchestratorError({ - operation, - message: `Composition ${operation} had failures`, - outcomes, - }); - return yield* Effect.forEach(ids, (id) => - node(id).pipe(Effect.flatMap((instance) => instance.core.get)), - ); - }); + ) { + const outcomes = yield* Effect.forEach( + ids, + (id) => + action(id).pipe( + Effect.exit, + Effect.map((result) => ({ id, result })), + ), + { concurrency }, + ); + if (outcomes.some(({ result }) => Exit.isFailure(result))) + return yield* new OrchestratorError({ + operation, + message: `Composition ${operation} had failures`, + outcomes, + }); + return yield* Effect.forEach(ids, status); }); - const startComposition = Effect.fn("Orchestrator.startComposition")( - function* (): Effect.fn.Return< - ReadonlyArray, - OrchestratorError | LifecycleError - > { - const configured = yield* Ref.get(composition); - const structure = yield* configurationGraph(configured); - const order = topo( - configured.members.map((member) => member.id), - structure.prerequisites, - ); - const eager = new Set( - topo( - [ - ...configured.members - .filter((member) => member.activation === "eager") - .map((member) => member.id), - ...order.filter((id) => !structure.members.has(id)), - ], - structure.prerequisites, + const startComposition = Effect.fn("Orchestrator.startComposition")(function* () { + const { graph } = yield* SubscriptionRef.get(lifecycle); + const memberIds = (yield* Ref.get(composition)).members.map((member) => member.id); + const order = inOrder(graph, [ + ...memberIds, + ...memberIds.flatMap((id) => prerequisitesOf(graph, id)), + ]); + const eager = new Set(order.filter((id) => graph.services.get(id)?.activation !== "lazy")); + const completions = new Map>(); + for (const id of order) completions.set(id, yield* Deferred.make()); + const bound = new Map>(); + for (const id of order) + bound.set( + id, + yield* node(id).pipe( + Effect.flatMap((entry) => entry.bind), + Effect.exit, ), ); - const plans = new Map(); - const bound = new Map>(); - const completions = new Map< - string, - Deferred.Deferred> - >(); - for (const id of order) plans.set(id, yield* snapshotPlan(id)); - for (const id of order) - completions.set( - id, - yield* Deferred.make>(), - ); - for (const id of order) - bound.set( - id, - yield* node(id).pipe( - Effect.flatMap((instance) => instance.bind), - Effect.exit, - ), - ); - return yield* settle( - "start", - order, - (id) => { - const completion = completions.get(id); - if (completion === undefined) - return Effect.fail(graphError("start", `Missing completion for ${id}`)); - const action = Effect.gen(function* () { - const prerequisites = structure.prerequisites.get(id) ?? []; - const prerequisiteOutcomes = yield* Effect.forEach(prerequisites, (prerequisite) => { - const dependency = completions.get(prerequisite); - return dependency === undefined - ? Effect.fail(graphError("start", `Missing completion for ${prerequisite}`)) - : Deferred.await(dependency).pipe( - Effect.map((result) => ({ id: prerequisite, result })), - ); + return yield* settleOutcomes( + "start", + order, + (id) => + Effect.gen(function* () { + const failedPrerequisites: Array = []; + for (const prerequisite of graph.services.get(id)?.prerequisites ?? []) { + const completion = completions.get(prerequisite); + if (completion !== undefined && !(yield* Deferred.await(completion))) + failedPrerequisites.push(prerequisite); + } + const binding = bound.get(id); + if (binding !== undefined) yield* binding; + if (!eager.has(id)) { + yield* awaitAll({ + requests: [LifecycleEvent.ArmRequested({ id })], + waits: [], + traffic: false, + operation: "start", }); - const binding = bound.get(id); - if (binding !== undefined) yield* binding; - const instance = yield* node(id); - const plan = plans.get(id); - const revision = plan?.revisions.get(id); - if (plan === undefined || revision === undefined) - return yield* graphError("start", `Missing plan for ${id}`); - const policy = structure.members.get(id); - if ( - policy?.activation === "lazy" || - (policy?.idleMillis !== undefined && instance.hasEndpoint) - ) - yield* instance.core.armAt(revision, prerequisiteGuard(plan, id)); - if (eager.has(id)) { - const failedPrerequisites = prerequisiteOutcomes - .filter(({ result }) => Exit.isFailure(result)) - .map(({ id: prerequisite }) => prerequisite) - .toSorted((left, right) => order.indexOf(left) - order.indexOf(right)); - if (failedPrerequisites.length > 0) - return yield* graphError( - "start", - `Blocked by prerequisites: ${failedPrerequisites.join(", ")}`, - ); - const inputs = yield* resolveInputs(id, configured); - yield* instance.startAt(revision, inputs, false, prerequisiteGuard(plan, id)); - yield* instance.bind; - yield* instance.core.ready; - } + return; + } + if (failedPrerequisites.length > 0) + return yield* graphError( + "start", + `Blocked by prerequisites: ${failedPrerequisites + .toSorted((left, right) => order.indexOf(left) - order.indexOf(right)) + .join(", ")}`, + ); + yield* awaitAll({ + requests: [LifecycleEvent.StartRequested({ id })], + waits: yield* explicitWaits(id, true), + traffic: false, + operation: "start", }); - return action.pipe( - Effect.onExit((result) => Deferred.succeed(completion, result).pipe(Effect.asVoid)), - ); - }, - "unbounded", - ); - }, - ); - - const stopIds = Effect.fn("Orchestrator.stopIds")(function* (ids: ReadonlyArray) { - const structure = yield* configurationGraph(yield* Ref.get(composition)); - const order = topo(ids, structure.prerequisites) - .filter((id) => ids.includes(id)) - .toReversed(); - return yield* settle("stop", order, (id) => - Effect.gen(function* () { - const instance = yield* node(id); - yield* instance.core.stop; - yield* instance.close; - yield* reschedulePrerequisites(id); - }), + }).pipe( + Effect.onExit((exit) => { + const completion = completions.get(id); + return completion === undefined + ? Effect.void + : Deferred.succeed(completion, Exit.isSuccess(exit)).pipe(Effect.asVoid); + }), + ), + "unbounded", ); }); - const destroyNode = Effect.fn("Orchestrator.destroyNode")(function* (id: string) { - const instance = yield* node(id); - yield* instance.core.destroy; - yield* instance.close; - yield* withGraph( - Effect.gen(function* () { - const values = yield* Ref.get(registry); - const next = new Map(values); - next.delete(id); - yield* Ref.set(registry, next); - const activities = new Map(yield* Ref.get(activity)); - activities.delete(id); - yield* Ref.set(activity, activities); - const configured = yield* Ref.get(composition); - yield* Ref.set(composition, { - members: configured.members.filter((member) => member.id !== id), - dependencies: configured.dependencies.filter( - (dependency) => dependency.from !== id && dependency.to !== id, - ), - }); - }), - ); + const stopIds = Effect.fn("Orchestrator.stopIds")(function* (ids: ReadonlyArray) { + const { graph } = yield* SubscriptionRef.get(lifecycle); + return yield* settleOutcomes("stop", inOrder(graph, ids).toReversed(), stop); }); const stopComposition = Effect.fn("Orchestrator.stopComposition")(function* () { - const configured = yield* Ref.get(composition); - return yield* stopIds(configured.members.map((member) => member.id)); + return yield* stopIds((yield* Ref.get(composition)).members.map((member) => member.id)); }); + const restartComposition = Effect.fn("Orchestrator.restartComposition")(function* () { - const configured = yield* Ref.get(composition); - yield* stopIds(configured.members.map((member) => member.id)); + yield* stopIds((yield* Ref.get(composition)).members.map((member) => member.id)); return yield* startComposition(); }); + const stopNamespace = Effect.fn("Orchestrator.stopNamespace")(function* () { - const values = yield* Ref.get(registry); - yield* stopIds([...values.keys()]); + yield* stopIds([...(yield* Ref.get(registry)).keys()]); }); + const destroyNamespace = Effect.fn("Orchestrator.destroyNamespace")(function* () { - const values = yield* Ref.get(registry); - const configured = yield* configurationGraph(yield* Ref.get(composition)); - const order = topo([...values.keys()], configured.prerequisites).toReversed(); + const order = (yield* SubscriptionRef.get(lifecycle)).graph.order.toReversed(); const outcomes = yield* Effect.forEach(order, (id) => - destroyNode(id).pipe( + destroyInstance(id, "retain").pipe( Effect.exit, Effect.map((result) => ({ id, result })), ), @@ -764,215 +1096,162 @@ export const make = Effect.fn("Orchestrator.make")(function* < }); }); + const acquire = Effect.fn("Orchestrator.acquire")(function* ( + id: string, + awaitReady = true, + trigger?: string, + ) { + yield* Effect.annotateCurrentSpan({ instance_id: id, await_ready: awaitReady }); + const entry = yield* node(id); + const scope = yield* Scope.Scope; + const initiated = yield* Ref.make(false); + // The caller that takes a sleeping or failed service out of rest names the wake once. + const announce = (applied: Applied) => { + const before = applied.before.services.get(id); + const after = applied.after.services.get(id); + const wakes = + before !== undefined && + (before.phase._tag === "Stopped" || before.phase._tag === "Failed") && + before.waiters.size === 0 && + (after?.waiters.size ?? 0) > 0; + return wakes + ? Ref.set(initiated, true).pipe( + Effect.andThen( + Effect.logInfo( + `Waking ${entry.service} ${id}${trigger === undefined ? "" : ` (${trigger})`}`, + ), + ), + ) + : Effect.void; + }; + yield* awaitAll({ + waits: [{ id, requireReady: awaitReady }], + traffic: true, + onApplied: announce, + onAdmitted: Scope.addFinalizer(scope, dispatch([LifecycleEvent.ConnectionClosed({ id })])), + }).pipe( + Effect.tapError((error) => + Ref.get(initiated).pipe( + Effect.flatMap((initiator) => + initiator + ? Effect.logError( + error.operation === "readiness" + ? `${entry.service} ${id} failed to become ready` + : `${entry.service} ${id} failed to wake`, + error, + ) + : Effect.void, + ), + ), + ), + ); + if (yield* Ref.get(initiated)) + yield* Effect.logInfo( + awaitReady + ? `${entry.service} ${id} is ready` + : `${entry.service} ${id} started (readiness not awaited)`, + ); + }); + const orchestrator: Interface = { - admissionFor, + report, register: Effect.fn("Orchestrator.register")(function* (instance) { yield* Effect.annotateCurrentSpan({ instance_id: instance.id }); - return yield* withGraph( + yield* withGate( Effect.gen(function* () { const values = yield* Ref.get(registry); if (values.has(instance.id)) return yield* graphError("register", `Duplicate instance ${instance.id}`); - yield* Ref.set(registry, new Map(values).set(instance.id, instance)); - const seenExit = yield* Ref.make(undefined); - const seenHealthy = yield* Ref.make(undefined); - yield* Effect.forkIn( - instance.core.observation.pipe( - Stream.takeUntil((observation) => !observation.registered), - Stream.runForEach((observation) => - Effect.gen(function* () { - if ( - observation.lifecycle === "running" && - observation.health === "healthy" && - observation.wakeEnabled && - observation.launchId !== (yield* Ref.get(seenHealthy)) - ) { - yield* Ref.set(seenHealthy, observation.launchId); - const now = yield* Clock.currentTimeMillis; - yield* withGraph( - Ref.update(activity, (values) => - new Map(values).set(instance.id, { - active: values.get(instance.id)?.active ?? 0, - lastActivity: now, - }), - ), - ); - yield* scheduleIdle(instance.id); - } - if (observation.exit === undefined) return; - if (observation.lifecycle === "stopped") { - yield* reschedulePrerequisites(instance.id); - return; - } - if (observation.wakeEnabled) return; - if ((yield* Ref.get(seenExit)) === observation.exit) return; - yield* Ref.set(seenExit, observation.exit); - yield* instance.close.pipe( - Effect.tapError((cause) => Effect.logError("Instance close failed", cause)), - Effect.ignore, - ); - }), - ), - ), - owner, - ); + const next = new Map(values).set(instance.id, instance); + yield* Ref.set(registry, next); + yield* applyLocked([ + LifecycleEvent.GraphUpdated({ + graph: lifecycleGraph(next.keys(), yield* Ref.get(composition)), + }), + ]); }), ); }), - get: Effect.fn("Orchestrator.get")((id) => node(id)), + get: (id) => node(id), + instances: Ref.get(registry).pipe(Effect.map((values) => [...values.values()])), + status, + changes, composition: Ref.get(composition), configure: (configuration: CompositionConfig, persist?: Effect.Effect) => - withGraph( + withGate( Effect.gen(function* () { const previous = yield* Ref.get(composition); - yield* configurationGraph(configuration); + const values = yield* Ref.get(registry); + const invalid = validateComposition(configuration, values); + if (invalid !== undefined) return yield* invalid; const affected = new Set([ ...previous.members.map((member) => member.id), ...configuration.members.map((member) => member.id), + ...[...previous.dependencies, ...configuration.dependencies].flatMap((dependency) => [ + dependency.from, + dependency.to, + ]), ]); - for (const dependency of [...previous.dependencies, ...configuration.dependencies]) { - affected.add(dependency.from); - affected.add(dependency.to); - } + const state = yield* SubscriptionRef.get(lifecycle); for (const affectedId of affected) { - const instance = yield* node(affectedId); - const observation = yield* instance.core.get; - if (observation.lifecycle !== "stopped" || observation.wakeEnabled) + if ( + state.services.has(affectedId) && + !isStoppedAndWakeDisabled({ + lifecycle: lifecycleOf(state, affectedId), + wakeEnabled: wakeEnabledOf(state, affectedId), + }) + ) return yield* graphError( "configure", `Instance ${affectedId} must be stopped and wake-disabled`, ); } + const update = LifecycleEvent.GraphUpdated({ + graph: lifecycleGraph(values.keys(), configuration), + }); + // The reducer is pure, so its verdict is known before anything is persisted. + const [, verdict] = reduce(state, update, yield* Clock.currentTimeMillis); + const rejected = verdict.find((command) => command._tag === "RequestRejected"); + if (rejected?._tag === "RequestRejected") + return yield* graphError("configure", rejected.message); if (persist !== undefined) yield* persist; yield* Ref.set(composition, configuration); + yield* applyLocked([update]); }), ).pipe(Effect.withSpan("Orchestrator.configure")), - start: Effect.fn("Orchestrator.start")((id) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const plan = yield* snapshotPlan(id); - for (const member of plan.order) yield* (yield* node(member)).bind; - yield* startNode(id, false, false, plan); - }), - ), - stop: Effect.fn("Orchestrator.stop")((id) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const instance = yield* node(id); - yield* instance.core.stop; - yield* instance.close; - yield* reschedulePrerequisites(id); - }), - ), - restart: Effect.fn("Orchestrator.restart")((id, config) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const plan = yield* snapshotPlan(id); - for (const member of plan.order) yield* (yield* node(member)).bind; - for (const member of plan.order.slice(0, -1)) { - const instance = yield* node(member); - const inputs = yield* resolveInputs(member, plan.configuration); - const revision = plan.revisions.get(member); - if (revision === undefined) - return yield* graphError("restart", `Missing revision for ${member}`); - yield* instance.startAt(revision, inputs, false, prerequisiteGuard(plan, member)); - yield* instance.bind; - yield* instance.core.ready; - } - const instance = yield* node(id); - const inputs = yield* resolveInputs(id, plan.configuration); - const revision = plan.revisions.get(id); - if (revision === undefined) - return yield* graphError("restart", `Missing revision for ${id}`); - yield* instance.restart(revision, inputs, config, prerequisiteGuard(plan, id)); - yield* instance.bind; - }), - ), - destroy: Effect.fn("Orchestrator.destroy")((id) => - Effect.annotateCurrentSpan({ instance_id: id }).pipe(Effect.andThen(destroyNode(id))), - ), + whileStopped: ( + ids: Iterable, + refuse: (id: string) => F, + effect: Effect.Effect, + ) => + withGate( + Effect.gen(function* () { + const state = yield* SubscriptionRef.get(lifecycle); + for (const id of ids) { + yield* node(id); + if ( + !isStoppedAndWakeDisabled({ + lifecycle: lifecycleOf(state, id), + wakeEnabled: wakeEnabledOf(state, id), + }) + ) + return yield* Effect.fail(refuse(id)); + } + return yield* effect; + }), + ).pipe(Effect.withSpan("Orchestrator.whileStopped")), + start, + ready, + stop, + restart, + destroy, + storage, startComposition: startComposition(), stopComposition: stopComposition(), restartComposition: restartComposition(), stopNamespace: stopNamespace(), destroyNamespace: destroyNamespace(), - acquire: Effect.fn("Orchestrator.acquire")((id, awaitReady = true, trigger) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan({ instance_id: id }); - const instance = yield* node(id); - const scope = yield* Scope.Scope; - const now = yield* Clock.currentTimeMillis; - const { wake, initiator } = yield* withGraph( - Effect.gen(function* () { - const observation = yield* instance.core.get; - if (observation.lifecycle === "stopped" && !observation.wakeEnabled) - return yield* serviceFailure("proxy", `Instance ${id} is explicitly stopped`); - const values = yield* Ref.get(activity); - yield* Ref.set( - activity, - new Map(values).set(id, { - active: (values.get(id)?.active ?? 0) + 1, - lastActivity: now, - }), - ); - yield* Scope.addFinalizer( - scope, - release(id).pipe( - Effect.tapError((cause) => Effect.logError("Instance release failed", cause)), - Effect.ignore, - ), - ); - // A "starting" observer is always joining an in-flight start (a wake or an - // eager/explicit start), never its initiator, regardless of the marker below. - if (observation.lifecycle === "running" || observation.lifecycle === "starting") - return { wake: observation.lifecycle === "starting", initiator: false } as const; - const inFlight = yield* Ref.get(wakes); - const initiator = !inFlight.has(id); - if (initiator) yield* Ref.set(wakes, new Set(inFlight).add(id)); - return { wake: true, initiator } as const; - }), - ); - if (wake) { - if (initiator) - yield* Effect.logInfo( - `Waking ${instance.service} ${id}${trigger === undefined ? "" : ` (${trigger})`}`, - ); - yield* Effect.gen(function* () { - const plan = yield* snapshotPlan(id); - for (const member of plan.order) yield* (yield* node(member)).bind; - yield* startNode(id, true, false, plan).pipe( - Effect.tapError((cause) => - initiator - ? Effect.logError(`${instance.service} ${id} failed to wake`, cause) - : Effect.void, - ), - ); - }).pipe( - Effect.ensuring( - initiator - ? Ref.update(wakes, (ids) => { - const next = new Set(ids); - next.delete(id); - return next; - }) - : Effect.void, - ), - ); - if (!awaitReady && initiator) - yield* Effect.logInfo(`${instance.service} ${id} started (readiness not awaited)`); - } - if (awaitReady) { - yield* instance.core.ready.pipe( - Effect.tapError((cause) => - wake && initiator - ? Effect.logError(`${instance.service} ${id} failed to become ready`, cause) - : Effect.void, - ), - ); - if (wake && initiator) yield* Effect.logInfo(`${instance.service} ${id} is ready`); - } - }), - ), + acquire, }; return orchestrator; }); diff --git a/packages/stack/src/Owner.analytics.integration.test.ts b/packages/stack/src/Owner.analytics.integration.test.ts index 20e44eeb87..06f7165728 100644 --- a/packages/stack/src/Owner.analytics.integration.test.ts +++ b/packages/stack/src/Owner.analytics.integration.test.ts @@ -19,11 +19,12 @@ import { import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { tmpdir } from "node:os"; import { logflareEvent, logflareSources } from "./host/LogflareEvents.ts"; -import * as State from "./State.ts"; -import type { SavedStack } from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack } from "./StackNamespace.ts"; import type { ServiceCreationInput } from "./services/Catalog.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; @@ -97,18 +98,18 @@ const startStack = ( const saved: SavedStack = { id: stackId, identity: { projectRoot: "/tmp/project", branchContext: name, stackName: stackId }, - runtime: "docker", + runtime: testEngine, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; + const stateRoot = path.dirname(path.dirname(root)); const state = Context.get( - yield* Layer.build(State.layer({ root: path.dirname(path.dirname(root)) })), - State.Service, + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, ); yield* state.save(saved); - const owner = yield* ownerFor({ saved, state, root, cacheRoot }); + const owner = yield* ownerFor({ saved, state, root, cacheRoot, engineTarget }); yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); const composed = yield* owner.rpc.supabaseComposition({ services }); @@ -143,7 +144,7 @@ const startStack = ( .pipe(Stream.filter(predicate), Stream.take(1), Stream.runDrain); return { owner, - state, + logsRoot: StackNamespace.stackLogsRoot(path, stateRoot, stackId), stackId, idOf, analyticsUrl, @@ -172,7 +173,7 @@ it.live( const path = yield* Path.Path; const client = yield* HttpClient.HttpClient; // A short idle timer lets the test observe Analytics sleeping while services keep logging. - const { owner, state, stackId, idOf, awaitAnalytics, wakeAnalytics, keepAwake } = + const { owner, logsRoot, stackId, idOf, awaitAnalytics, wakeAnalytics, keepAwake } = yield* startStack( "owner-logs", [ @@ -241,7 +242,7 @@ it.live( (generation === awakeRecord.position.generation && byteOffset >= awakeRecord.position.byteOffset)); const confirmed = yield* fs - .readFileString(path.join(state.logsRoot(stackId), "database", databaseId, "cursor.json")) + .readFileString(path.join(logsRoot, "database", databaseId, "cursor.json")) .pipe( Effect.flatMap(decodeConfirmed), Effect.filterOrFail(passedAwake), diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts index 029119d629..87c73cf9d0 100644 --- a/packages/stack/src/Owner.integration.test.ts +++ b/packages/stack/src/Owner.integration.test.ts @@ -1,25 +1,30 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { PgClient } from "@effect/sql-pg"; -import { expect, it } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Redacted, Ref, Schema } from "effect"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Effect, FileSystem, Layer, Redacted, Ref, Schema, Stream } from "effect"; import { HttpClient } from "effect/unstable/http"; -import { tmpdir } from "node:os"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { randomUUID } from "node:crypto"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "./Owner.ts"; import { OwnerRpc } from "./Rpc.ts"; -import * as State from "./State.ts"; -import type { SavedStack } from "./State.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack } from "./StackNamespace.ts"; import { DEFAULT_LOCAL_JWT_SECRET } from "./Defaults.ts"; -import { ServiceCreation, type ServiceCreationInput } from "./services/Catalog.ts"; +import type { ServiceCreationInput } from "./services/Catalog.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; +import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; const stateFor = (root: string) => Effect.gen(function* () { - const context = yield* Layer.build(State.layer({ root })); - return Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root })); + return Context.get(context, StackNamespace.Service); }); -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; const initial = (id: string): SavedStack => ({ id, @@ -28,7 +33,61 @@ const initial = (id: string): SavedStack => ({ instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], +}); + +const initialContainer = (id: string): SavedStack => ({ ...initial(id), runtime: testEngine }); + +/** + * Container ids the engine still reports for `instanceId`, by the labels an instance's own containers + * carry. Excludes a stack-wide shared volume helper (labeled only with the stack, never the + * instance), which outlives any one instance's destroy and is torn down on its own lifecycle. + */ +const containersForInstance = (stackId: string, instanceId: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make( + testEngine, + [ + ...engineTarget.argv, + "ps", + "-a", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--filter", + `label=com.supabase.instance=${instanceId}`, + "--format", + "{{.ID}} {{.Names}} {{.Image}} {{.Status}}", + ], + { stdout: "pipe", stderr: "pipe" }, + ), + ); + const output = yield* Stream.mkString(Stream.decodeText(child.stdout)); + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + }), + ).pipe(Effect.provide(NodeServices.layer)); + +/** Runs a Docker CLI command and returns its combined stdout, for assertions against the engine. */ +const runDocker = (args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("docker", args, { stdout: "pipe", stderr: "pipe" }), + ); + return yield* Stream.mkString(Stream.decodeText(child.stdout)); + }), + ).pipe(Effect.provide(NodeServices.layer)); + +/** The fields read directly off a database instance's own storage marker file. */ +const InstanceMarker = Schema.Struct({ + backend: Schema.Literals(["docker", "host"]), + volume: Schema.optionalKey(Schema.String), + namespace: Schema.String, }); const query = (url: string, statement: string) => @@ -142,7 +201,7 @@ it.live("forwards and rotates saved identity across composed services in one own endpoints: { http: { port: "auto" as const } }, }, ]; - const stackKeys = (suffix: string, gotrueJwtKeys: string): State.StackKeysInput => ({ + const stackKeys = (suffix: string, gotrueJwtKeys: string): StackNamespace.StackKeysInput => ({ publishableKey: `publishable-${suffix}`, secretKey: `secret-${suffix}`, anonKey: `anon-${suffix}`, @@ -291,7 +350,7 @@ it.live("forwards and rotates saved identity across composed services in one own const excludedStudio = (yield* owner.rpc.status({ id: studioId })).config; expect(excludedStudio.service).toBe("studio"); if (excludedStudio.service === "studio") - expect(excludedStudio.config.anonKey).toBe("anon-one"); + expect(excludedStudio.config.anonKey).toBe(secondCredentials.anonKey); for (const id of retainedIds) expect((yield* owner.rpc.status({ id: id })).lifecycle).toBe("stopped"); @@ -333,7 +392,7 @@ it.effect("publishes service removal and composition pruning together", () => const state = yield* stateFor(`${root}/state`); yield* state.save(stack); const removalWrite = yield* Ref.make(false); - const failingState: State.Interface = { + const failingState: StackNamespace.Interface = { ...state, save: (next) => Ref.get(removalWrite).pipe( @@ -341,7 +400,10 @@ it.effect("publishes service removal and composition pruning together", () => next.instances.length === 0 ? alreadyWritten ? Effect.fail( - new State.StateError({ operation: "write", message: "injected failure" }), + new StackNamespace.NamespaceError({ + operation: "write", + message: "injected failure", + }), ) : Ref.set(removalWrite, true).pipe(Effect.andThen(state.save(next))) : state.save(next), @@ -495,7 +557,7 @@ it.effect("isolates owner graphs built in one scope", () => saved, root: `${root}/data`, cacheRoot, - }).pipe(Layer.provide(Layer.succeed(State.Service, state))), + }).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), memoMap, scope, ).pipe( @@ -714,37 +776,192 @@ it.effect("lets a retry choose other credentials after the first database creati ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.effect("rejects a duplicate instance without releasing the existing instance's claims", () => +it.effect( + "rejects a duplicate instance without releasing the existing instance's reservations", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-duplicate-" }); + const state = yield* stateFor(`${root}/state`); + const stack = initial(`owner-duplicate-${randomUUID().slice(0, 8)}`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = yield* PortReservations.Service; + const endpoint = `${mail.id}:http`; + const assigned = yield* portReservations.find(realStateRoot, stack.id, endpoint); + expect(assigned).toBeDefined(); + const saved = yield* state.read(stack.id); + if (saved === undefined) return yield* Effect.die("saved state disappeared"); + const instance = saved.instances.find((entry) => entry.id === mail.id); + if (instance === undefined) return yield* Effect.die("mail instance was not saved"); + + const failure = yield* ownerFor({ + saved: { ...saved, instances: [instance, instance] }, + state, + root: `${root}/data`, + cacheRoot, + }).pipe(Effect.flip); + + expect(failure.message).toBe(`Duplicate instance ${mail.id}`); + expect(yield* portReservations.find(realStateRoot, stack.id, endpoint)).toBe(assigned); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live("destroying one service deletes its port reservation", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-duplicate-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-service-destroy-" }); + const stack = initial(`owner-service-destroy-${randomUUID().slice(0, 8)}`); const state = yield* stateFor(`${root}/state`); - const creation = yield* Schema.decodeEffect(ServiceCreation)({ + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + // Composing reserves every member's public port up front without starting any of them. + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = yield* PortReservations.Service; + expect( + yield* portReservations.find(realStateRoot, stack.id, `${mail.id}:http`), + ).toBeDefined(); + + yield* owner.rpc.destroyService({ id: mail.id }); + + expect( + yield* portReservations.find(realStateRoot, stack.id, `${mail.id}:http`), + ).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live("destroying a stack leaves its data root empty after an owner was killed mid-command", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-orphaned-job-" }); + const stack = initial(`owner-orphaned-job-${randomUUID().slice(0, 8)}`); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const orphanedJob = `${root}/data/jobs/0b1c2d3e-orphan`; + yield* fs.makeDirectory(orphanedJob, { recursive: true }); + yield* fs.writeFileString(`${orphanedJob}/config.json`, "{}"); + + yield* owner.namespace.destroy; + + expect(yield* fs.exists(`${root}/data`)).toBe(true); + expect(yield* fs.readDirectory(`${root}/data`)).toEqual([]); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live("refuses to destroy one service after its data directory became a symlink", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-service-symlink-" }); + const stack = initial(`owner-service-symlink-${randomUUID().slice(0, 8)}`); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const data = `${root}/state/${stack.id}/data`; + yield* fs.makeDirectory(data, { recursive: true }); + const owner = yield* ownerFor({ saved: stack, state, root: data, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const service = yield* owner.rpc.createService({ service: "mail", config: {}, - endpoints: { http: { port: "auto" } }, - }); - const instance = { id: "mail", creation }; - const claim = { key: "mail:http", host: "127.0.0.1", port: 54_321 }; - const stack: SavedStack = { - ...initial("owner-duplicate"), - instances: [instance], - ports: [claim], - }; + endpoints: { http: { port: "auto" }, smtp: { port: "auto" }, pop3: { port: "auto" } }, + }); + const outside = `${root}/outside`; + yield* fs.makeDirectory(`${outside}/${service.id}`, { recursive: true }); + yield* fs.writeFileString(`${outside}/${service.id}/precious.txt`, "keep"); + yield* fs.remove(data, { recursive: true }); + yield* fs.symlink(outside, data); + + const failure = yield* owner.rpc.destroyService({ id: service.id }).pipe(Effect.flip); + + expect(failure.message).toContain("symlink"); + expect(yield* fs.readFileString(`${outside}/${service.id}/precious.txt`)).toBe("keep"); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live("refuses to destroy a stack whose data directory is a symlink and keeps its target", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-symlinked-data-" }); + const stack = initial(`owner-symlinked-data-${randomUUID().slice(0, 8)}`); + const state = yield* stateFor(`${root}/state`); yield* state.save(stack); + const outside = `${root}/outside`; + yield* fs.makeDirectory(outside); + yield* fs.writeFileString(`${outside}/precious.txt`, "keep"); + const data = `${root}/state/${stack.id}/data`; + yield* fs.symlink(outside, data); + const owner = yield* ownerFor({ saved: stack, state, root: data, cacheRoot }); - const failure = yield* ownerFor({ - saved: { ...stack, instances: [instance, instance] }, - state, - root: `${root}/data`, - cacheRoot, - }).pipe(Effect.flip); + const failure = yield* owner.namespace.destroy.pipe(Effect.flip); - expect(failure.message).toBe("Duplicate instance mail"); - expect((yield* state.read(stack.id))?.ports).toEqual([claim]); + expect(failure.message).toContain("symlink"); + expect(yield* fs.readFileString(`${outside}/precious.txt`)).toBe("keep"); + expect(yield* state.read(stack.id), "the registration stays for a retry").toBeDefined(); }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), ); it.effect("refuses to generate credentials for a stack whose saved instances consume them", () => @@ -782,6 +999,104 @@ it.effect("refuses to generate credentials for a stack whose saved instances con ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); +it.live("blocks a credential change while a standalone credential consumer runs", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-credential-admission-", + }); + const stack = initial("owner-credential-admission"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const standalone = yield* owner.rpc.createService({ + service: "rest", + config: { databaseUrl: "postgresql://authenticator@127.0.0.1:1/postgres" }, + endpoints: {}, + }); + yield* owner.rpc.startService({ id: standalone.id }); + const before = yield* owner.getStackCredentials; + + const refused = yield* owner.rpc + .supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: {} }], + keys: { + publishableKey: "publishable-rotated", + secretKey: "secret-rotated", + anonKey: "anon-rotated", + serviceRoleKey: "service-role-rotated", + gotrueJwtKeys: "[]", + publicSigningKeys: "[]", + anonKeyIsOverride: true, + serviceRoleKeyIsOverride: true, + }, + }) + .pipe(Effect.flip); + + expect(refused.message).toContain( + `Service ${standalone.id} must be stopped with wake disabled before stack credentials change`, + ); + expect(yield* owner.getStackCredentials).toEqual(before); + yield* owner.rpc.stopService({ id: standalone.id }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "refreshes a stopped standalone credential consumer when the stack credentials change", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-credential-refresh-", + }); + const stack = initial("owner-credential-refresh"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const rest = yield* owner.rpc.createService({ + service: "rest", + config: { databaseUrl: "postgresql://authenticator@127.0.0.1:1/postgres" }, + endpoints: {}, + }); + const explicitKeys = '[{"kty":"oct","k":"explicit"}]'; + const auth = yield* owner.rpc.createService({ + service: "auth", + config: { gotrueJwtKeys: explicitKeys }, + endpoints: {}, + }); + const before = yield* owner.getStackCredentials; + + yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: {} }], + keys: { + gotrueJwtKeys: "[]", + publicSigningKeys: '[{"kty":"EC","kid":"rotated"}]', + }, + }); + + const after = yield* owner.getStackCredentials; + expect(after.jwks).not.toBe(before.jwks); + const observed = yield* owner.rpc.status({ id: rest.id }); + expect(observed.config).toMatchObject({ config: { jwks: after.jwks } }); + const saved = yield* state.read(stack.id); + expect(saved?.instances.find(({ id }) => id === rest.id)?.creation).toMatchObject({ + config: { jwks: after.jwks }, + }); + expect((yield* owner.rpc.status({ id: auth.id })).config).toMatchObject({ + config: { gotrueJwtKeys: explicitKeys }, + }); + + yield* owner.rpc.startService({ id: rest.id }); + yield* owner.rpc.stopService({ id: rest.id }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + it.live("rejects a missing required input before starting or stopping the service", () => Effect.scoped( Effect.gen(function* () { @@ -823,3 +1138,166 @@ it.live("rejects a missing required input before starting or stopping the servic }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + +it.live( + "destroys a database's container, storage namespace and port reservation after its registration file is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stackId = `owner-no-registration-${randomUUID().slice(0, 8)}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-owner-no-registration-", stackId); + const tempRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-no-registration-state-", + }); + const stack = initialContainer(stackId); + const state = yield* stateFor(`${tempRoot}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: dataRoot, + cacheRoot, + engineTarget, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const created = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-no-registration-password"), + jwtSecret: Redacted.make("owner-no-registration-jwt-secret-at-least-32-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: created.id }); + yield* owner.rpc.readyService({ id: created.id }); + + const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeDefined(); + expect(yield* containersForInstance(stackId, created.id)).not.toHaveLength(0); + + // The registration disappears from under the live owner (deleted externally, or by a + // concurrent process); its resource cleanup must still run to completion and release + // everything it owns, with the removal simply having nothing left to publish. + yield* fs.remove(`${tempRoot}/state/${stackId}/state.json`); + + yield* owner.rpc.destroyService({ id: created.id }); + + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + 120_000, +); + +// The Docker volume backend has no Podman counterpart: Podman keeps database data in host directories. +describe.runIf(testEngine === "docker")("Docker volume storage", () => { + it.live( + "destroys a database's container and storage namespace, and releases its port reservation, after the whole data root is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const stackId = `owner-no-data-root-${randomUUID().slice(0, 8)}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-owner-no-data-root-", stackId); + const tempRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-owner-no-data-root-state-", + }); + const stack = initialContainer(stackId); + const state = yield* stateFor(`${tempRoot}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: dataRoot, + cacheRoot, + engineTarget, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const created = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-no-data-root-password"), + jwtSecret: Redacted.make("owner-no-data-root-jwt-secret-at-least-32-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: created.id }); + yield* owner.rpc.readyService({ id: created.id }); + + const realStateRoot = yield* fs.realPath(`${tempRoot}/state`); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeDefined(); + expect(yield* containersForInstance(stackId, created.id)).not.toHaveLength(0); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(InstanceMarker))( + yield* fs.readFileString(`${dataRoot}/${created.id}/.supabase-database-storage.json`), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* Effect.die("Docker test selected host fallback"); + const volume = marker.volume; + + // The database sleeps; its storage object (inside the still-running owner) keeps the + // identity it already resolved in memory, then the whole owned data root disappears (the + // volume host, a disk failure, manual cleanup). + yield* owner.rpc.stopService({ id: created.id }); + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + yield* fs.remove(dataRoot, { recursive: true, force: true }); + + yield* owner.rpc.destroyService({ id: created.id }); + + expect(yield* containersForInstance(stackId, created.id)).toHaveLength(0); + expect( + yield* portReservations.find(realStateRoot, stackId, `${created.id}:sql`), + ).toBeUndefined(); + const remainingNamespace = yield* runDocker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + "docker.io/library/busybox:1.36", + "/bin/sh", + "-c", + `[ -e /store/${marker.namespace} ] && echo present || echo absent`, + ]); + expect(remainingNamespace.trim()).toBe("absent"); + yield* runDocker(["volume", "rm", volume]); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + 120_000, + ); +}); diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index c70e64f572..5dbbe5c022 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -16,12 +16,13 @@ import { } from "effect"; import { HttpClient } from "effect/unstable/http"; import { tmpdir } from "node:os"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; import { ownerFor } from "../tests/owner-rpc.ts"; import type { LogRecord } from "./host/LogRecord.ts"; import { StackError, streamStackLogs } from "./effect.ts"; import * as LogStore from "./host/LogStore.ts"; -import * as State from "./State.ts"; -import type { SavedStack } from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack } from "./StackNamespace.ts"; const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; const services = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -33,7 +34,6 @@ const stackFor = (id: string, runtime: SavedStack["runtime"]): SavedStack => ({ instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); /** Opens an owner that destroys its stack on scope exit unless `kept` preserves it for a later read. */ @@ -45,17 +45,24 @@ const openOwner = ( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const crypto = yield* Crypto.Crypto; + const path = yield* Path.Path; const root = kept?.root ?? (yield* fs.makeTempDirectoryScoped({ prefix })); const stack = stackFor( `owner-logs-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`, runtime, ); const state = Context.get( - yield* Layer.build(State.layer({ root: `${root}/state` })), - State.Service, + yield* Layer.build(StackNamespace.layer({ root: `${root}/state` })), + StackNamespace.Service, ); yield* state.save(stack); - const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + ...(runtime === "native" ? {} : { engineTarget }), + }); if (kept === undefined) yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); return { @@ -63,10 +70,26 @@ const openOwner = ( state, stack, stateRoot: `${root}/state`, - logsRoot: state.logsRoot(stack.id), + logsRoot: StackNamespace.stackLogsRoot(path, `${root}/state`, stack.id), }; }); +/** The port the shared API listener serves a service's `http` endpoint on. */ +const httpPort = ( + observation: Effect.Effect< + { readonly endpoints: ReadonlyArray<{ readonly name: string; readonly port: number }> }, + StackError + >, +) => + observation.pipe( + Effect.flatMap(({ endpoints }) => { + const http = endpoints.find(({ name }) => name === "http"); + return http === undefined + ? Effect.die("the service has no http endpoint") + : Effect.succeed(http.port); + }), + ); + const isOutput = (record: LogRecord) => record.kind === "stdout" || record.kind === "stderr"; const firstOutput = (records: Stream.Stream) => @@ -175,8 +198,8 @@ describe("owner persisted logs", () => { yield* owner.rpc.stopService({ id: mail.id }); yield* Scope.close(firstRun, Exit.void); const state = Context.get( - yield* Layer.build(State.layer({ root: owner.stateRoot })), - State.Service, + yield* Layer.build(StackNamespace.layer({ root: owner.stateRoot })), + StackNamespace.Service, ); const saved = yield* state.read(owner.stack.id); if (saved === undefined) return yield* Effect.die("stack state missing"); @@ -224,12 +247,9 @@ describe("owner persisted logs", () => { dependencies: [], }); yield* first.rpc.startComposition(); - const api = (yield* first.state.read(first.stack.id))?.ports.find( - ({ key }) => key === "api", - ); - if (api === undefined) return yield* Effect.die("the shared API port was not claimed"); + const api = yield* httpPort(first.rpc.status({ id: rest.id })); - const response = yield* client.get(`http://127.0.0.1:${api.port}/unrouted`); + const response = yield* client.get(`http://127.0.0.1:${api}/unrouted`); const [recorded] = yield* firstOutput(first.rpc.readLogs({ id: "gateway", follow: true })); expect(response.status).toBe(404); @@ -241,8 +261,8 @@ describe("owner persisted logs", () => { yield* fs.writeFileString(path.join(directory, "0000000002.log"), ""); const state = Context.get( - yield* Layer.build(State.layer({ root: first.stateRoot })), - State.Service, + yield* Layer.build(StackNamespace.layer({ root: first.stateRoot })), + StackNamespace.Service, ); const saved = yield* state.read(first.stack.id); if (saved === undefined) return yield* Effect.die("the stopped stack was not saved"); @@ -257,17 +277,14 @@ describe("owner persisted logs", () => { Effect.timeout("5 seconds"), ); yield* second.rpc.startComposition(); - const reopened = (yield* state.read(first.stack.id))?.ports.find( - ({ key }) => key === "api", - ); - if (reopened === undefined) return yield* Effect.die("the shared API port was not claimed"); + const reopened = yield* httpPort(second.rpc.status({ id: rest.id })); const again = yield* second.rpc.readLogs({ id: "gateway", follow: true }).pipe( Stream.filter((record) => isOutput(record) && record.text?.includes("/again") === true), Stream.take(1), Stream.runCollect, Effect.forkScoped({ startImmediately: true }), ); - yield* client.get(`http://127.0.0.1:${reopened.port}/again`); + yield* client.get(`http://127.0.0.1:${reopened}/again`); expect(history).toContainEqual(recorded); expect(begun).toHaveLength(1); @@ -278,56 +295,6 @@ describe("owner persisted logs", () => { ).pipe(Effect.provide(services)), ); - it.live("continues after the launch ids in its logs when its saved state has none", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "owner-logs-unsaved-launch-" }); - const firstRun = yield* Scope.make(); - const owner = yield* openOwner("owner-logs-unsaved-launch-", "native", { root }).pipe( - Scope.provide(firstRun), - ); - const mail = yield* owner.rpc.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* Scope.close(firstRun, Exit.void); - const state = Context.get( - yield* Layer.build(State.layer({ root: owner.stateRoot })), - State.Service, - ); - const saved = yield* state.read(owner.stack.id); - if (saved === undefined) return yield* Effect.die("stack state missing"); - // Logs a state saved before launch ids were persisted left behind at launch 3. - const directory = `${owner.logsRoot}/mail/${mail.id}`; - yield* fs.makeDirectory(directory, { recursive: true }); - yield* fs.writeFileString( - `${directory}/0000000001.log`, - "2026-01-01T00:00:00.000Z launch 3 | \n2026-01-01T00:00:00.001Z stdout 3 | earlier\n", - ); - const restarted = yield* ownerFor({ saved, state, root: `${root}/data`, cacheRoot }); - yield* Effect.addFinalizer(() => restarted.namespace.destroy.pipe(Effect.ignore)); - - const launched = yield* awaitLaunches( - restarted.rpc.readLogs({ id: mail.id, follow: true }), - 2, - ); - yield* restarted.rpc.startService({ id: mail.id }); - yield* restarted.rpc.readyService({ id: mail.id }); - yield* Fiber.join(launched); - const records = yield* restarted.rpc - .readLogs({ id: mail.id, follow: false }) - .pipe(Stream.runCollect); - - expect(saved.instances.find(({ id }) => id === mail.id)?.launchId).toBeUndefined(); - expect( - records.filter(({ kind }) => kind === "launch").map(({ launchId }) => launchId), - ).toEqual([3, 4]); - }), - ).pipe(Effect.provide(services)), - ); - it.live("resumes a follow at a record position without replaying earlier records", () => Effect.scoped( Effect.gen(function* () { @@ -404,7 +371,7 @@ describe("owner persisted logs", () => { () => Effect.scoped( Effect.gen(function* () { - const owner = yield* openOwner("owner-logs-docker-", "docker"); + const owner = yield* openOwner("owner-logs-docker-", testEngine); const mail = yield* owner.rpc.createService({ service: "mail", config: {}, diff --git a/packages/stack/src/Owner.native-socket-sweep.integration.test.ts b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts new file mode 100644 index 0000000000..753aef61cf --- /dev/null +++ b/packages/stack/src/Owner.native-socket-sweep.integration.test.ts @@ -0,0 +1,247 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { createHash } from "node:crypto"; +import { + Context, + Effect, + Exit, + FileSystem, + Layer, + Path, + PlatformError, + Redacted, + Ref, + Schema, +} from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as Owner from "./Owner.ts"; +import { NativeRuntimeRootBase, nativeRuntimeRootPath } from "./runtime/postgres-user.ts"; +import { ServiceCreation } from "./services/Catalog.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { reclaimStack } from "./Sweep.ts"; + +const databaseCreation = Schema.decodeEffect(ServiceCreation)({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("sweep-password"), + jwtSecret: Redacted.make("sweep-integration-jwt-secret-long-enough"), + jwtExpiry: 3600, + }, + endpoints: {}, +}); + +/** A fresh state namespace, a native runtime root base, and a stack with one saved database. */ +const arrange = Effect.fn("arrange")(function* ( + id: string, + lifetime: StackNamespace.SavedStack["lifetime"] = "detached", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs + .makeTempDirectoryScoped({ prefix: "stack-socket-sweep-" }) + .pipe(Effect.flatMap(fs.realPath)); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: `${root}/state` })), + StackNamespace.Service, + ); + const saved: StackNamespace.SavedStack = { + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: id }, + instances: [{ id: "database", creation: yield* databaseCreation }], + lifetime, + composition: { members: [], dependencies: [] }, + }; + yield* state.save(saved); + const runtimeRootBase = yield* fs.makeTempDirectoryScoped({ prefix: "stack-socket-sweep-base-" }); + const runtimeRoot = nativeRuntimeRootPath(path, runtimeRootBase, process.getuid?.() ?? 0); + const dataRoot = `${root}/state/${id}/data`; + return { root, state, saved, runtimeRootBase, runtimeRoot, dataRoot }; +}); + +type Arrangement = Effect.Success>; + +/** The socket directory a native database instance gets, derived independently of production code. */ +const socketDirectory = (arrangement: Arrangement, instanceId: string) => + `${arrangement.runtimeRoot}/pg-${createHash("sha256") + .update(`${arrangement.dataRoot}\0${instanceId}`) + .digest("hex") + .slice(0, 16)}`; + +/** A directory left by an owner that was SIGKILLed before its scope could remove it. */ +const leaveSocketDirectory = Effect.fn("leaveSocketDirectory")(function* ( + arrangement: Arrangement, + instanceId: string, +) { + const fs = yield* FileSystem.FileSystem; + const directory = socketDirectory(arrangement, instanceId); + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + yield* fs.writeFileString(`${directory}/pg_hba.conf`, "local all all trust\n"); + return directory; +}); + +const ownerFor = (arrangement: Arrangement, fileSystem?: Layer.Layer) => { + const owner = Owner.layer({ + saved: arrangement.saved, + root: arrangement.dataRoot, + cacheRoot: `${arrangement.root}/cache`, + }).pipe( + Layer.provide( + Layer.merge( + Layer.succeed(StackNamespace.Service, arrangement.state), + Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => Effect.die("Unexpected child process command")), + ), + ), + ), + ); + return Layer.build(fileSystem === undefined ? owner : owner.pipe(Layer.provide(fileSystem))).pipe( + Effect.map((context) => Context.get(context, Owner.Service)), + ); +}; + +/** Destroy runs in the caller's context, which names the runtime root base launch would use. */ +const inRuntimeRoot = (arrangement: Arrangement) => + Effect.provideService(NativeRuntimeRootBase, arrangement.runtimeRootBase); + +const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +it.live("destroy removes the socket directory a killed owner left behind", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-destroy"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const owner = yield* ownerFor(arrangement); + + yield* owner.namespace.destroy.pipe(inRuntimeRoot(arrangement)); + + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("a failed socket directory removal keeps the stack registered until a retry succeeds", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-retry"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const failing = yield* Ref.make(true); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + remove: (target: string, options?: Parameters[1]) => + // The runtime root is resolved to its real path, so match the derived directory name. + target.endsWith(`/${leftover.split("/").at(-1)}`) + ? Ref.get(failing).pipe( + Effect.flatMap((fail) => + fail + ? PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, + cause: new Error("injected removal failure"), + }) + : fs.remove(target, options), + ), + ) + : fs.remove(target, options), + }); + const owner = yield* ownerFor(arrangement, flakyFs); + + const failed = yield* Effect.exit(owner.namespace.destroy.pipe(inRuntimeRoot(arrangement))); + + expect(Exit.isFailure(failed)).toBe(true); + expect(yield* fs.exists(leftover)).toBe(true); + expect(yield* arrangement.state.read(arrangement.saved.id)).toBeDefined(); + + yield* Ref.set(failing, false); + yield* owner.namespace.destroy.pipe(inRuntimeRoot(arrangement)); + + expect(yield* fs.exists(leftover)).toBe(false); + expect(yield* arrangement.state.read(arrangement.saved.id)).toBeUndefined(); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("destroying one database instance removes its socket directory", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-instance"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + const owner = yield* ownerFor(arrangement); + + yield* owner.handlers.destroyService({ id: "database" }).pipe(inRuntimeRoot(arrangement)); + + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + +const reclaim = (arrangement: Arrangement) => + reclaimStack({ + state: arrangement.state, + stateRoot: `${arrangement.root}/state`, + cacheRoot: `${arrangement.root}/cache`, + id: arrangement.saved.id, + }).pipe(Effect.provide(Layer.succeed(NativeRuntimeRootBase, arrangement.runtimeRootBase))); + +it.live( + "an orphan reclaim removes the socket directory of a session stack whose owner is gone", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-orphan", "session"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + + expect(yield* reclaim(arrangement)).toBe(true); + + expect(yield* fs.exists(leftover)).toBe(false); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("removes only the socket directory of the stack being cleaned up", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cleaned = yield* arrange("socket-cleaned"); + const sibling = yield* arrange("socket-sibling"); + const cleanedLeftover = yield* leaveSocketDirectory(cleaned, "database"); + // Same instance id and runtime root, different stack: a distinct derived name. + const siblingLeftover = yield* leaveSocketDirectory( + { ...sibling, runtimeRoot: cleaned.runtimeRoot }, + "database", + ); + const owner = yield* ownerFor(cleaned); + + yield* owner.namespace.destroy.pipe(inRuntimeRoot(cleaned)); + + expect(yield* fs.exists(cleanedLeftover)).toBe(false); + expect(yield* fs.exists(siblingLeftover)).toBe(true); + }), + ).pipe(Effect.provide(platform)), +); + +it.live("leaves a socket directory under a runtime root another uid could have taken over", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const arrangement = yield* arrange("socket-takeover"); + const leftover = yield* leaveSocketDirectory(arrangement, "database"); + // Tests cannot create another uid's files, so a world-writable root stands in for one + // recreated by a foreign uid. + yield* fs.chmod(arrangement.runtimeRoot, 0o777); + const owner = yield* ownerFor(arrangement); + + yield* owner.namespace.destroy.pipe(inRuntimeRoot(arrangement)); + + expect(yield* fs.exists(leftover)).toBe(true); + }), + ).pipe(Effect.provide(platform)), +); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 4312ed4e8b..2e76827097 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -12,6 +12,7 @@ import { Scope, Semaphore, Stream, + SubscriptionRef, } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; @@ -24,10 +25,8 @@ import type { CompositionConfig } from "./Orchestrator.ts"; import { makeService, ServiceError, - type ServiceAdmission, type ServiceInstance, type ServiceInstanceContext, - type ServiceObservation, } from "./Service.ts"; import { ProxyError } from "./Proxy.ts"; import { @@ -45,6 +44,7 @@ import { CredentialError, credentialOverrides, nextCredentials, + refreshCredentials, withCredentials, withoutUnusedCredentials, } from "./host/Credentials.ts"; @@ -64,8 +64,9 @@ import type { CatalogError } from "./services/Recipe.ts"; import * as Container from "./runtime/Container.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; -import * as State from "./State.ts"; -import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; +import { confirmStackDataRoot } from "./namespace/Paths.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import type { SavedStack, StackCredentials, StackKeysInput } from "./StackNamespace.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; import * as GatewayLog from "./host/GatewayLog.ts"; import * as LogForwarder from "./host/LogForwarder.ts"; @@ -74,11 +75,18 @@ import * as LogStore from "./host/LogStore.ts"; export interface OwnerOptions { readonly saved: SavedStack; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly root: string; readonly cacheRoot: string; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: Container.HostGateway; + /** The engine endpoint and identity resolved once at startup; absent for a native stack. */ + readonly engineTarget?: Container.EngineTarget; + /** + * Reads whether shutdown has begun. The host owns that one-way fact; an owner built without a + * host (for example a sweep) never drains and so admits all work. + */ + readonly draining?: Effect.Effect; } type OwnerRpcs = RpcGroup.Rpcs; @@ -90,36 +98,59 @@ type Handlers = { ) => Rpc.ResultFrom; }; -/** Removes containers labeled with this stack and data root; native stacks own none. */ +/** + * Removes every container carrying this stack's identity and data-root labels through the + * owner's pinned engine target. A container that survives removal fails the sweep, so the caller + * (startup, stop, or destroy) can report it and retry instead of proceeding as if the stack were + * fully torn down. + */ export const sweepContainers = Effect.fn("Owner.sweepContainers")(function* ( - saved: Pick, + saved: Pick, root: string, + target: Container.EngineTarget | undefined, ) { - if (saved.runtime !== "native") - yield* Container.removeStackContainers({ engine: saved.runtime, stackId: saved.id, root }); + const path = yield* Path.Path; + if (target === undefined) return; + const remaining = yield* Container.removeStackContainers({ + target, + stackId: saved.id, + stackRoot: path.resolve(root), + }); + if (remaining.length > 0) + return yield* new StackNamespace.NamespaceError({ + operation: "cleanup", + message: `Containers remain: ${remaining.join(", ")}`, + }); }); type NamespaceError = | Orchestrator.OrchestratorError - | Orchestrator.LifecycleError + | ServiceError | Network.NetworkError - | State.StateError + | StackNamespace.NamespaceError | Effect.Error>; export interface Interface { readonly handlers: Handlers; - readonly getStackCredentials: Effect.Effect; + readonly getStackCredentials: Effect.Effect< + StackCredentials, + StackNamespace.NamespaceError | CredentialError + >; readonly namespace: { /** Stops every instance after in-flight definition changes settle, then removes containers. */ readonly stop: Effect.Effect; /** * Destroys every instance once in-flight definition changes settle, then releases owned - * containers, claims and saved state. + * containers, native socket directories and saved state. */ readonly destroy: Effect.Effect; + /** + * Stops what this owner launched once in-flight definition changes settle, for an owner whose + * claim on the stack ended: containers, data, ports and saved state may now belong to a + * successor and stay untouched. + */ + readonly release: Effect.Effect; }; - readonly setDraining: (draining: boolean) => Effect.Effect; - readonly getServing: Effect.Effect; } export class Service extends Context.Service()("@supabase/stack/Owner") {} @@ -163,16 +194,9 @@ const withoutInstance = (current: SavedStack, id: string): SavedStack => withoutUnusedCredentials({ ...current, instances: current.instances.filter((instance) => instance.id !== id), - composition: { - members: current.composition.members.filter((member) => member.id !== id), - dependencies: current.composition.dependencies.filter( - (dependency) => dependency.from !== id && dependency.to !== id, - ), - }, + composition: Orchestrator.withoutMember(current.composition, id), }); -const drainingBlocks: ReadonlyArray = ["start", "arm", "restart", "storage"]; - const makeOwner = Effect.fn("Owner.make")(function* ( options: OwnerOptions, gateway: GatewayLog.GatewayLog, @@ -188,12 +212,33 @@ const makeOwner = Effect.fn("Owner.make")(function* ( const ownerScope = Context.get(services, Scope.Scope); const crypto = Context.get(services, Crypto.Crypto); const path = Context.get(services, Path.Path); + const fs = Context.get(services, FileSystem.FileSystem); const network = yield* Network.Service; - const orchestrator = yield* Orchestrator.make(); - const helpers = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4); - const logStore = yield* LogStore.make({ root: options.state.logsRoot(options.saved.id) }).pipe( - Effect.provideContext(services), + // Everything under the data root belongs to the stack, including what a killed owner left + // mid-command (`jobs/`); the directory itself stays for `Registry.pruneDestroyed`. + const removeDataRootEntries = fs.exists(options.root).pipe( + Effect.flatMap((exists) => (exists ? fs.readDirectory(options.root) : Effect.succeed([]))), + Effect.flatMap((names) => + Effect.forEach( + names, + (name) => fs.remove(path.join(options.root, name), { recursive: true, force: true }), + { discard: true }, + ), + ), + Effect.mapError(serviceError("cleanup")), ); + const rejectWhileDraining = (options.draining ?? Effect.succeed(false)).pipe( + Effect.flatMap((isDraining) => + isDraining + ? Effect.fail(new ServiceError({ operation: "draining", message: "Owner is draining" })) + : Effect.void, + ), + ); + const orchestrator = yield* Orchestrator.make({ admit: () => rejectWhileDraining }); + const helpers = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4); + const logStore = yield* LogStore.make({ + root: StackNamespace.stackLogsRoot(path, options.state.root, options.saved.id), + }).pipe(Effect.provideContext(services)); /** The database Analytics stores events in, as the host reaches it. */ const analyticsDatabase = (analyticsId: string) => Effect.gen(function* () { @@ -223,7 +268,7 @@ const makeOwner = Effect.fn("Owner.make")(function* ( yield* logStore.attach({ ...GatewayLog.gatewayLog, logs: gateway.logs, - observation: gateway.observation, + launches: gateway.launches, }); yield* gateway.begin( Option.getOrElse(yield* logStore.latestLaunchId(GatewayLog.gatewayLog), () => 0) + 1, @@ -233,7 +278,6 @@ const makeOwner = Effect.fn("Owner.make")(function* ( service: GatewayLog.gatewayLog.service, }); const definitionGate = yield* Semaphore.make(1); - const draining = yield* Ref.make(false); const { id: stackId, runtime } = options.saved; const project = projectSegmentFor(options.saved.identity, path); const routeKeys = { @@ -243,70 +287,84 @@ const makeOwner = Effect.fn("Owner.make")(function* ( serviceRoleKey: options.saved.credentials?.serviceRoleKey ?? "", }; - const rejectWhileDraining = Ref.get(draining).pipe( - Effect.flatMap((isDraining) => - isDraining - ? Effect.fail(new ServiceError({ operation: "draining", message: "Owner is draining" })) - : Effect.void, - ), - ); - - // Mask only the permit handoff; admitted definition changes belong to the owner scope. A change - // arriving while draining fails before waiting behind the shutdown that holds the permit. + // Admitted definition changes run in the owner scope, so a caller cancelled after admission + // detaches from them; one still queued for the permit is withdrawn. A change arriving while + // draining fails before waiting behind the shutdown that holds the permit. const definitionChange = (work: Effect.Effect) => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - yield* rejectWhileDraining; - yield* restore(definitionGate.take(1)); - const fiber = yield* Effect.forkIn( - rejectWhileDraining.pipe( - Effect.andThen(work), - Effect.ensuring(definitionGate.release(1)), - ), - ownerScope, - { uninterruptible: false }, - ); - return yield* restore(Fiber.join(fiber)); - }), - ); + Effect.gen(function* () { + yield* rejectWhileDraining; + const claim = yield* Ref.make<"queued" | "admitted" | "withdrawn">("queued"); + const settle = (to: "admitted" | "withdrawn") => + Ref.modify(claim, (current) => (current === "queued" ? [true, to] : [false, current])); + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const worker = yield* Effect.forkIn( + definitionGate.withPermit( + settle("admitted").pipe( + Effect.flatMap((admitted) => + admitted + ? rejectWhileDraining.pipe(Effect.andThen(work), Effect.map(Option.some)) + : Effect.succeed(Option.none()), + ), + ), + ), + ownerScope, + { uninterruptible: false }, + ); + const outcome = yield* restore(Fiber.join(worker)).pipe( + Effect.onInterrupt(() => + settle("withdrawn").pipe( + Effect.flatMap((withdrawn) => (withdrawn ? Fiber.interrupt(worker) : Effect.void)), + ), + ), + ); + return Option.isSome(outcome) ? outcome.value : yield* Effect.interrupt; + }), + ); + }); - const readSaved = options.state - .read(stackId) - .pipe( - Effect.flatMap((saved) => - saved === undefined - ? Effect.fail( - new State.StateError({ operation: "read", message: "Saved stack is missing" }), - ) - : Effect.succeed(saved), - ), - ); + const readSaved = options.state.read(stackId).pipe( + Effect.flatMap((saved) => + saved === undefined + ? Effect.fail( + new StackNamespace.NamespaceError({ + operation: "read", + message: "Saved stack is missing", + }), + ) + : Effect.succeed(saved), + ), + ); const updateState = (update: (current: SavedStack) => SavedStack) => options.state.withLock( readSaved.pipe(Effect.flatMap((current) => options.state.save(update(current)))), ); - - const requireStopped = (configuration: CompositionConfig) => - Effect.forEach( - new Set([ - ...configuration.members.map(({ id }) => id), - ...configuration.dependencies.flatMap(({ from, to }) => [from, to]), - ]), - (id) => - orchestrator.get(id).pipe( - Effect.flatMap((entry) => entry.core.get), - Effect.flatMap(({ lifecycle, wakeEnabled }) => - lifecycle === "stopped" && !wakeEnabled - ? Effect.void - : Effect.fail( - new CredentialError({ - message: `Service ${id} must be stopped with wake disabled before stack credentials change`, - }), - ), + // Idempotent on an already-missing registration: the registration, or the whole state root + // with its registry lock, may vanish under destroy, so a failure counts only while it remains. + const removeInstanceRegistration = (id: string) => + options.state + .withLock( + options.state + .read(stackId) + .pipe( + Effect.flatMap((current) => + current === undefined + ? Effect.void + : options.state.save(withoutInstance(current, id)), + ), ), + ) + .pipe( + Effect.catch((error) => + options.state + .read(stackId) + .pipe( + Effect.flatMap((registered) => + registered === undefined ? Effect.void : Effect.fail(error), + ), + ), ), - { discard: true }, - ); + ); const resolveStackCredentials = Effect.fn("Owner.resolveStackCredentials")(function* ( overrides: Effect.Success>, @@ -317,8 +375,49 @@ const makeOwner = Effect.fn("Owner.make")(function* ( const current = yield* readSaved; const next = yield* nextCredentials(current, overrides, keys); if (next === current.credentials) return next; - if (current.credentials !== undefined) yield* requireStopped(current.composition); - yield* options.state.save({ ...current, credentials: next }); + const previous = current.credentials; + if (previous === undefined) { + yield* options.state.save({ ...current, credentials: next }); + return next; + } + const consumers = current.instances.filter(({ creation }) => consumesCredentials(creation)); + const refreshed = new Map( + yield* Effect.forEach(consumers, ({ id, creation }) => + refreshCredentials(creation, previous, next).pipe( + Effect.map((refreshedCreation) => [id, refreshedCreation] as const), + ), + ), + ); + yield* orchestrator.whileStopped( + new Set([ + ...current.composition.members.map(({ id }) => id), + ...current.composition.dependencies.flatMap(({ from, to }) => [from, to]), + ...consumers.map(({ id }) => id), + ]), + (id) => + new CredentialError({ + message: `Service ${id} must be stopped with wake disabled before stack credentials change`, + }), + Effect.gen(function* () { + yield* options.state.save({ + ...current, + credentials: next, + instances: current.instances.map((instance) => { + const creation = refreshed.get(instance.id); + return creation === undefined ? instance : { ...instance, creation }; + }), + }); + yield* Effect.forEach( + refreshed, + ([id, creation]) => + orchestrator + .get(id) + .pipe(Effect.flatMap((entry) => Ref.set(entry.creation, creation))), + { discard: true }, + ); + }), + ); + yield* Effect.annotateCurrentSpan("refreshed_instances", refreshed.size); return next; }), ); @@ -336,109 +435,76 @@ const makeOwner = Effect.fn("Owner.make")(function* ( }); const recipeFor = (creation: ServiceCreation, id: string) => - makeServiceRecipe( - creation, - { - stackId, - instanceId: id, - project, - root: options.root, - cacheRoot: options.cacheRoot, - runtime, - helpers, - ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), - }, - options.state.claims, - ).pipe(Effect.provideContext(services)); - - const persistCreation = ( - entry: Pick, - creation: ServiceCreation, - launchId?: number, - ) => - updateState((current) => ({ - ...current, - instances: current.instances.map((instance) => - instance.id === entry.id - ? { ...instance, creation, ...(launchId === undefined ? {} : { launchId }) } - : instance, + makeServiceRecipe(creation, { + stackId, + instanceId: id, + project, + root: options.root, + cacheRoot: options.cacheRoot, + runtime, + helpers, + ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), + }).pipe(Effect.provideContext(services)); + + const sameCreation = Schema.toEquivalence(ServiceCreation); + const persistCreation = (entry: Pick, creation: ServiceCreation) => + Ref.get(entry.creation).pipe( + Effect.flatMap((saved) => + sameCreation(saved, creation) + ? Effect.void + : updateState((current) => ({ + ...current, + instances: current.instances.map((instance) => + instance.id === entry.id ? { ...instance, creation } : instance, + ), + })).pipe(Effect.andThen(Ref.set(entry.creation, creation))), ), - })).pipe(Effect.andThen(Ref.set(entry.creation, creation))); + ); - const register = Effect.fn("Owner.register")(function* ( - id: string, - recipe: CatalogRecipe, - lastLaunchId?: number, - ) { + const register = Effect.fn("Owner.register")(function* (id: string, recipe: CatalogRecipe) { if (Option.isSome(yield* orchestrator.get(id).pipe(Effect.option))) return yield* new Orchestrator.OrchestratorError({ operation: "register", message: `Duplicate instance ${id}`, }); const initial = recipe.creation; - // A state saved before launch ids were persisted continues after the ids its logs hold. - const resumeAfter = - lastLaunchId ?? - Option.getOrUndefined( - yield* logStore.latestLaunchId({ service: initial.service, instanceId: id }), - ); + const resumeAfter = Option.getOrUndefined( + yield* logStore.latestLaunchId({ service: initial.service, instanceId: id }), + ); + const launches = yield* SubscriptionRef.make(undefined); const creation = yield* Ref.make(initial); - const namespaceRef = yield* Ref.make(undefined); const core = yield* makeService( { ...recipe.definition, launch: (context) => - persistCreation({ id, creation }, context.config, context.launchId).pipe( - Effect.mapError(serviceError("state")), - Effect.andThen(recipe.definition.launch(context)), - ), - removeData: (context) => - recipe.definition.removeData(context).pipe( + SubscriptionRef.set(launches, context.launchId).pipe( Effect.andThen( - Ref.get(namespaceRef).pipe( - Effect.flatMap((namespace) => namespace?.release ?? Effect.void), - Effect.mapError(serviceError("release")), - ), - ), - Effect.andThen( - updateState((current) => withoutInstance(current, id)).pipe( - Effect.mapError(serviceError("state")), - ), - ), - // Shipping writes its cursor into the instance's logs, so it stops before they go. - Effect.andThen(forwarder.detach(id)), - Effect.andThen( - logStore - .remove({ service: initial.service, instanceId: id }) - .pipe( - Effect.catch((cause) => - Effect.logWarning( - `${cause.message}; the next owner start or stack destroy retries`, - cause, - ), - ), - ), + Scope.addFinalizer(context.scope, SubscriptionRef.set(launches, undefined)), ), + Effect.andThen(persistCreation({ id, creation }, context.config)), + Effect.mapError(serviceError("state")), + Effect.andThen(recipe.definition.launch(context)), ), }, { id, config: initial, + report: orchestrator.report, ...(resumeAfter === undefined ? {} : { lastLaunchId: resumeAfter }), - coordinate: (operation, transition) => - (drainingBlocks.includes(operation) ? rejectWhileDraining : Effect.void).pipe( - Effect.andThen(orchestrator.admissionFor(id)(operation, transition)), - ), }, ).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const enabled = core.get.pipe( - Effect.map( - (observation) => - observation.registered && - observation.lifecycle !== "stopped" && - !(observation.exit !== undefined && !observation.wakeEnabled), - ), + // Listeners stay open while the service runs or demand can still wake it. + const enabled = orchestrator.status(id).pipe( + Effect.map((status) => !Orchestrator.isStoppedAndWakeDisabled(status)), + Effect.orElseSucceed(() => false), ); + const configFor = (inputs: Record, candidate: unknown) => + Ref.get(creation).pipe( + Effect.flatMap((current) => restartCreation(current, candidate)), + Effect.flatMap((next) => mergeInputs(next, inputs)), + Effect.flatMap(requireInputs), + ); const endpoints = Object.fromEntries( endpointNames(initial).map((name) => { const shared = sharedRoutes(initial, name, routeKeys); @@ -450,7 +516,7 @@ const makeOwner = Effect.fn("Owner.make")(function* ( .acquire(id, name !== "inspector", `traffic on endpoint ${name}`) .pipe( Effect.andThen(recipe.endpoint(name)), - Effect.flatMap(backendAddress), + Effect.map(backendAddress), Effect.mapError((cause) => cause instanceof ProxyError ? cause @@ -465,7 +531,6 @@ const makeOwner = Effect.fn("Owner.make")(function* ( }), ); const namespace = yield* network.register({ id, endpoints }); - yield* Ref.set(namespaceRef, namespace); const entry: Entry = { id, service: initial.service, @@ -473,19 +538,31 @@ const makeOwner = Effect.fn("Owner.make")(function* ( recipe, creation, namespace, - startAt: (revision, inputs, wake, guard) => - Ref.get(creation).pipe( - Effect.flatMap((current) => mergeInputs(current, inputs)), - Effect.flatMap(requireInputs), - Effect.flatMap((candidate) => core.startAt(revision, candidate, wake, guard)), + confirmRemoved: removeInstanceRegistration(id).pipe( + Effect.mapError(serviceError("state")), + // Shipping writes its cursor into the instance's logs, so it stops before they go. + Effect.andThen(forwarder.detach(id)), + Effect.andThen( + logStore + .remove({ service: initial.service, instanceId: id }) + .pipe( + Effect.catch((cause) => + Effect.logWarning( + `${cause.message}; the next owner start or stack destroy retries`, + cause, + ), + ), + ), ), - restart: (revision, inputs, candidate, guard) => - Ref.get(creation).pipe( - Effect.flatMap((current) => restartCreation(current, candidate)), - Effect.flatMap((next) => mergeInputs(next, inputs)), - Effect.flatMap(requireInputs), - Effect.flatMap((next) => core.restart(next, revision, guard)), + ), + release: namespace.release.pipe(Effect.mapError(serviceError("release"))), + releasePorts: namespace.releasePorts.pipe(Effect.mapError(serviceError("release"))), + launch: (generation, inputs, candidate) => + configFor(inputs, candidate).pipe( + Effect.flatMap((config) => core.launch(generation, config)), ), + prepare: (inputs, candidate) => + configFor(inputs, candidate).pipe(Effect.flatMap((config) => core.prepare(config))), bind: namespace.bind.pipe(Effect.mapError(serviceError("bind"))), close: namespace.close.pipe(Effect.mapError(serviceError("close"))), hasEndpoint: endpointNames(initial).length > 0, @@ -503,19 +580,29 @@ const makeOwner = Effect.fn("Owner.make")(function* ( service: initial.service, instanceId: id, logs: recipe.logs, - observation: core.observation, + launches: SubscriptionRef.changes(launches), }); yield* forwarder.attach({ id, service: initial.service, endpoint: recipe.endpoint, creation: Ref.get(creation), - observation: core.observation, + serving: orchestrator.changes(id).pipe( + Stream.mapEffect((status) => + SubscriptionRef.get(launches).pipe( + Effect.map((launchId) => ({ + serving: status.lifecycle === "running" && status.health === "healthy", + launchId, + })), + ), + ), + Stream.catch(() => Stream.empty), + ), }); }); for (const saved of options.saved.instances) - yield* register(saved.id, yield* recipeFor(saved.creation, saved.id), saved.launchId); + yield* register(saved.id, yield* recipeFor(saved.creation, saved.id)); yield* logStore.removeOrphans.pipe( Effect.catch((cause) => Effect.logWarning("Orphaned instance logs were not removed", cause)), ); @@ -537,7 +624,7 @@ const makeOwner = Effect.fn("Owner.make")(function* ( type ComposeError = | Effect.Error> - | Orchestrator.LifecycleError + | ServiceError | Network.NetworkError; const configure = (configuration: CompositionConfig) => @@ -573,7 +660,7 @@ const makeOwner = Effect.fn("Owner.make")(function* ( Effect.flatMap((entry) => Ref.get(entry.creation)), Effect.map((creation) => ({ id, creation })), ), - status: (id) => orchestrator.get(id).pipe(Effect.flatMap((entry) => entry.core.get)), + status: orchestrator.status, create: addInstance, destroy: orchestrator.destroy, bind: (id) => orchestrator.get(id).pipe(Effect.flatMap((entry) => entry.bind)), @@ -636,18 +723,14 @@ const makeOwner = Effect.fn("Owner.make")(function* ( return yield* orchestrator.restart(id, next); }); - const observe = (entry: Entry, value: ServiceObservation) => + const observe = (entry: Entry, value: Orchestrator.Status) => Effect.all({ config: Ref.get(entry.creation), endpoints: entry.namespace.bindings }).pipe( Effect.map((current) => ({ ...value, ...current })), ); const observation = (id: string) => - orchestrator - .get(id) - .pipe( - Effect.flatMap((entry) => - entry.core.get.pipe(Effect.flatMap((value) => observe(entry, value))), - ), - ); + Effect.all([orchestrator.get(id), orchestrator.status(id)]).pipe( + Effect.flatMap(([entry, value]) => observe(entry, value)), + ); const observeAll = (values: ReadonlyArray<{ readonly id: string }>) => Effect.forEach(values, ({ id }) => observation(id)); @@ -666,7 +749,8 @@ const makeOwner = Effect.fn("Owner.make")(function* ( operation: "storage", message: "Snapshots and data reset are only supported for databases", }); - return yield* entry.core.storage( + return yield* orchestrator.storage( + id, Effect.acquireUseRelease( Scope.make("parallel"), (scope) => @@ -696,11 +780,7 @@ const makeOwner = Effect.fn("Owner.make")(function* ( }), ).pipe(rpcError("createService")), startService: ({ id }) => orchestrator.start(id).pipe(rpcError("startService")), - readyService: ({ id }) => - orchestrator.get(id).pipe( - Effect.flatMap((entry) => entry.core.ready), - rpcError("readyService"), - ), + readyService: ({ id }) => orchestrator.ready(id).pipe(rpcError("readyService")), stopService: ({ id }) => orchestrator.stop(id).pipe(rpcError("stopService")), // A config-changing restart can adopt saved credentials, so it serializes with definition // changes that introduce or roll them back. @@ -709,7 +789,12 @@ const makeOwner = Effect.fn("Owner.make")(function* ( rpcError("restartService"), ), destroyService: ({ id }) => - definitionChange(orchestrator.destroy(id)).pipe(rpcError("destroyService")), + definitionChange( + confirmStackDataRoot(options.state.root, stackId).pipe( + Effect.provideContext(services), + Effect.andThen(orchestrator.destroy(id)), + ), + ).pipe(rpcError("destroyService")), prepareService: ({ id }) => orchestrator.get(id).pipe( Effect.flatMap((entry) => @@ -728,7 +813,7 @@ const makeOwner = Effect.fn("Owner.make")(function* ( .get(id) .pipe( Effect.map((entry) => - entry.core.observation.pipe(Stream.mapEffect((value) => observe(entry, value))), + orchestrator.changes(id).pipe(Stream.mapEffect((value) => observe(entry, value))), ), ), ).pipe(Stream.mapError((cause) => stackError("followStatus", cause))), @@ -779,7 +864,9 @@ const makeOwner = Effect.fn("Owner.make")(function* ( ), }; - const sweep = sweepContainers(options.saved, options.root).pipe(Effect.provideContext(services)); + const sweep = sweepContainers(options.saved, options.root, options.engineTarget).pipe( + Effect.provideContext(services), + ); const getStackCredentials = readSaved.pipe( Effect.flatMap(({ credentials }) => credentials === undefined @@ -795,22 +882,37 @@ const makeOwner = Effect.fn("Owner.make")(function* ( handlers, getStackCredentials, namespace: { + // Services stop in reverse dependency order with their listeners open, so a dependent's + // graceful stop can still reach its prerequisites through the proxy. stop: orchestrator.stopNamespace.pipe( Effect.andThen(sweep), definitionGate.withPermits(1), Effect.withSpan("Owner.stopNamespace"), ), - destroy: orchestrator.destroyNamespace.pipe( - Effect.andThen(network.release), + // Instance teardown retains port rows until the registration is gone; the release after + // that is best-effort because `isGone` reclaims a gone stack's rows lazily. + destroy: confirmStackDataRoot(options.state.root, stackId).pipe( + Effect.provideContext(services), + Effect.andThen(orchestrator.destroyNamespace), Effect.andThen(sweep), + Effect.andThen(removeDataRootEntries), Effect.andThen(logStore.close), Effect.andThen(options.state.remove(stackId)), + Effect.andThen( + network.releaseStack.pipe( + Effect.catch((cause) => + Effect.logWarning("Destroyed stack could not release its port reservations", cause), + ), + ), + ), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), ), + release: orchestrator.stopNamespace.pipe( + definitionGate.withPermits(1), + Effect.withSpan("Owner.releaseNamespace"), + ), }, - setDraining: (value) => Ref.set(draining, value), - getServing: Ref.get(draining).pipe(Effect.map((isDraining) => !isDraining)), } satisfies Interface; }); @@ -821,7 +923,7 @@ export const layer = (options: Omit) => Layer.effect( Service, Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; return Service.of(yield* makeOwner({ ...options, state }, gateway)); }), ).pipe( diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts index c7f4df0938..bd75ac0008 100644 --- a/packages/stack/src/Ports.integration.test.ts +++ b/packages/stack/src/Ports.integration.test.ts @@ -1,511 +1,660 @@ -import { NodeServices, NodeSocketServer } from "@effect/platform-node"; +import { NodeFileSystem, NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Cause, Context, Effect, Exit, FileSystem, Layer, Option, Ref, Scope } from "effect"; -import { makePorts, portBase, portSpan, PortError, reserveNativePort } from "./Ports.ts"; -import * as State from "./State.ts"; +import { + Context, + DateTime, + Effect, + Exit, + FileSystem, + Fiber, + Hash, + Layer, + Path, + Schedule, + Scope, +} from "effect"; +import * as Net from "node:net"; +import { userInfo } from "node:os"; +import { randomUUID } from "node:crypto"; +import { + makePorts, + nativePortBase, + nativePortSpan, + PortError, + probeVacant, + reserveNativePort, +} from "./Ports.ts"; +import { systemError } from "effect/PlatformError"; +import { bindTcp } from "./Proxy.ts"; +import { CONTAINER_ENV_DIRNAME } from "./namespace/Paths.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { ownerFor } from "../tests/owner-rpc.ts"; + +/** + * Scenarios the public `effect.ts` facade cannot reach deterministically: forcing an + * auto-allocation retry onto a known candidate, which needs the exact first port a real round trip + * picked; a real TCP bind failure, which a probe-equipped `Network.ts` callback would otherwise + * intercept first on a platform where overlapping binds succeed; and a stack-wide destroy's + * retained row release through a real, in-process `Owner`, whose first attempt a concrete sweep + * failure leaves uncertain. All exercise the real per-user registry and real sockets; see + * `port-reservation.e2e.test.ts` for the end-to-end scenarios, and `Network.integration.test.ts`'s + * "keeps shared routes independent and retains the shared claim" for the shared-listener + * self-reuse case. + */ const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); -const bind = (host: string, port: number) => - NodeSocketServer.make({ host, port }).pipe( - Effect.mapError((cause) => new PortError({ key: "sql", message: "Cannot bind", cause })), - ); +const saveStack = (state: StackNamespace.Interface, root: string, id: string) => + state.save({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "test", stackName: id }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); -it.live("retains distinct claims for stopped stacks and rebinds the original public port", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - for (const id of ["first", "second"]) - yield* state.save({ +/** Binds like a brand-new listener Network.ts would create: probed first, then really bound, so + * the assertion exercises Linux's real EADDRINUSE path and macOS's probe alike. */ +const probedBind = (key: string) => (host: string, port: number) => + probeVacant(process.platform)(key, host, port).pipe(Effect.andThen(bindTcp(host, port))); + +/** + * Binds a real wildcard listener directly in the native span, skipping any already-occupied + * candidate, instead of reserving a port through the allocator and racing to rebind it as a + * blocker afterward. + */ +const bindNativeWildcard = (): Effect.Effect<{ + readonly port: number; + readonly server: Net.Server; +}> => + Effect.gen(function* () { + for (let offset = 0; offset < nativePortSpan; offset++) { + const port = nativePortBase + offset; + const attempt = yield* Effect.exit( + Effect.callback((resume) => { + const server = Net.createServer(); + server.once("error", (cause) => resume(Effect.fail(cause))); + server.listen(port, "0.0.0.0", () => resume(Effect.succeed(server))); + }), + ); + if (Exit.isSuccess(attempt)) return { port, server: attempt.value }; + } + return yield* Effect.die("No port in the native span was free for the fixture"); + }); + +/** + * Brute-forces a key whose hash-seeded first scan candidate is exactly `port`, so a test can plant + * a blocker there ahead of time and assert the scan skips it, rather than merely missing it by + * chance. + */ +const keyWithFirstCandidate = (port: number): string => { + const offset = port - nativePortBase; + for (let attempt = 0; attempt < nativePortSpan * 20; attempt++) { + const key = `native-fixture-${attempt}`; + if (Math.abs(Hash.string(key)) % nativePortSpan === offset) return key; + } + throw new Error(`No key found whose first candidate is port ${port}`); +}; + +it.live( + "on this platform, auto allocation skips a port a foreign listener occupies, even where overlapping binds would otherwise succeed", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const id = `auto-skip-${randomUUID()}`; + const state = yield* makeTestState(root); + yield* saveStack(state, root, id); + const ports = yield* makePorts(state); + const request = { stackId: id, key: "api", host: "127.0.0.1", port: "auto" as const }; + const probed = probedBind("api"); + + // The scan's first candidate is already reserved when it reaches `bind`; occupying it + // there leaves no gap for a parallel test to take the port before the foreign listener. + const foreign = Net.createServer(); + let firstCandidate: number | undefined; + let foreignBound = false; + const bind = (host: string, port: number) => + Effect.gen(function* () { + if (firstCandidate === undefined) { + firstCandidate = port; + foreignBound = yield* Effect.callback((resume) => { + foreign.once("error", (cause: NodeJS.ErrnoException) => + resume(cause.code === "EADDRINUSE" ? Effect.succeed(false) : Effect.die(cause)), + ); + foreign.listen(port, "127.0.0.1", () => resume(Effect.succeed(true))); + }); + // An unrelated process already holds the candidate: report the collision it caused + // rather than probing a listener this test doesn't own. + if (!foreignBound) + return yield* new PortError({ + key: "api", + message: `Port ${port} is already in use`, + conflict: { port, endpoint: "api", holder: "foreign" }, + }); + } + return yield* probed(host, port); + }); + yield* Effect.addFinalizer(() => + Effect.callback((resume) => { + if (foreignBound) foreign.close(() => resume(Effect.void)); + else resume(Effect.void); + }), + ); + + const acquired = yield* ports.acquire(request, bind); + expect(firstCandidate).toBeDefined(); + expect(acquired.port).not.toBe(firstCandidate); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live( + "a fixed TCP request surfaces a structured, foreign conflict when a real bind fails with EADDRINUSE", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const id = `tcp-foreign-${randomUUID()}`; + const state = yield* makeTestState(root); + yield* saveStack(state, root, id); + const ports = yield* makePorts(state); + + const foreign = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const server = Net.createServer(); + server.once("error", (cause) => resume(Effect.fail(cause))); + server.listen(0, "127.0.0.1", () => resume(Effect.succeed(server))); + }), + (server) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }), + ); + const address = foreign.address(); + if (address === null || typeof address === "string") + return yield* Effect.die("Unable to reserve a test port"); + + // No probe wrapper here, unlike `Network.ts`'s own callback: this exercises the real TCP + // bind failure (EADDRINUSE) directly, which the probe would otherwise intercept first on a + // platform where overlapping binds succeed. + const failure = yield* ports + .acquire({ stackId: id, key: "api", host: "127.0.0.1", port: address.port }, bindTcp) + .pipe(Effect.flip); + if (!(failure instanceof PortError)) return yield* Effect.die("expected a PortError"); + expect(failure.conflict?.holder).toBe("foreign"); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live( + "interrupting an initial auto acquisition while its bind attempt is pending leaves no row behind", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const id = `auto-interrupt-${randomUUID()}`; + const state = yield* makeTestState(root); + yield* saveStack(state, root, id); + const ports = yield* makePorts(state); + const realRoot = yield* fs.realPath(root); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + + // Stands in for a probe or bind that never settles: the first candidate's row is + // committed, then the acquisition is interrupted while still inside it. + const bind = (_host: string, _port: number) => Effect.never; + const fiber = yield* ports + .acquire({ stackId: id, key: "api", host: "127.0.0.1", port: "auto" }, bind) + .pipe(Effect.forkChild); + const scanned = yield* portReservations.find(realRoot, id, "api").pipe( + Effect.repeat({ + while: (port) => port === undefined, + schedule: Schedule.spaced("5 millis"), + }), + Effect.timeout("5 seconds"), + ); + expect(scanned).toBeDefined(); + + yield* Fiber.interrupt(fiber); + expect(yield* portReservations.find(realRoot, id, "api")).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), +); + +it.live( + "a stack-wide destroy retains a dedicated row; a cleanup failure keeps it, and the next destroy, once resolved, releases it", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-defer-destroy-" }); + const id = `defer-destroy-${randomUUID()}`; + const state = yield* makeTestState(`${root}/state`); + const saved = { id, - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "test", stackName: id }, instances: [], - lifetime: "detached", + lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], + }; + yield* state.save(saved); + + // Fails removing a leftover scratch directory under the data root on the first attempt + // only, standing in for any concrete cleanup failure (an unreachable engine, a locked + // file, and so on); resolved for the retry once the uncertainty would realistically have + // cleared. + const scratch = path.join(root, "data", CONTAINER_ENV_DIRNAME); + let resolved = false; + const unreliableFileSystem = Layer.effect( + FileSystem.FileSystem, + Effect.map(FileSystem.FileSystem, (real) => + FileSystem.FileSystem.of({ + ...real, + remove: (target, options) => + Effect.suspend(() => + target === scratch && !resolved + ? Effect.fail( + systemError({ + _tag: "PermissionDenied", + module: "test", + method: "remove", + description: "Injected cleanup failure", + }), + ) + : real.remove(target, options), + ), + }), + ), + ).pipe(Layer.provide(NodeFileSystem.layer)); + + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: `${root}/cache`, + }).pipe(Effect.provide(unreliableFileSystem)); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + resolved = true; + yield* owner.namespace.destroy; + }).pipe(Effect.ignore), + ); + // Composing reserves every member's public port up front (including a lazy one like + // mail), with no member ever started: a real reservation with no native artifact needed, + // so this runs on every platform, Windows included. + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], }); - const ports = yield* makePorts(state); - const firstScope = yield* Scope.make(); - const request = { stackId: "first", key: "db/sql", host: "127.0.0.1", port: "auto" as const }; - const first = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, firstScope)); - yield* Scope.close(firstScope, Exit.void); - const second = yield* ports.acquire({ ...request, stackId: "second" }, bind); - expect(second.port).not.toBe(first.port); - const reopened = yield* makePorts(yield* makeTestState(root)); - const again = yield* reopened.acquire(request, bind); - expect(again.port).toBe(first.port); - expect((yield* state.read("first"))?.ports).toEqual([ - { key: "db/sql", host: "127.0.0.1", port: first.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); -it.live("reports an occupied saved port without moving its assignment", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - const scope = yield* Scope.make(); - const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; - const first = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, scope)); - yield* Scope.close(scope, Exit.void); - yield* bind("127.0.0.1", first.port); - const failure = yield* ports.acquire(request, bind).pipe(Effect.flip); - expect(failure).toBeInstanceOf(PortError); - expect(failure.message).toContain(`api at 127.0.0.1:${first.port}`); - expect(failure.message).not.toContain("claims this port"); - expect((yield* state.read("stack"))?.ports[0]?.port).toBe(first.port); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); -it.live("allocates an auto port outside a contiguous range that refuses to bind", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - // Reserves most of the span contiguously, as Windows excluded ranges do. - const reservedBelow = 30000; - const acquired = yield* ports.acquire( - { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" }, - (host, port) => - port < reservedBelow - ? Effect.fail(new PortError({ key: "api", message: `bind EACCES ${host}:${port}` })) - : Effect.succeed(port), - ); - expect(acquired.port).toBeGreaterThanOrEqual(reservedBelow); - expect((yield* state.read("stack"))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: acquired.port }, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); + yield* fs.makeDirectory(scratch, { recursive: true }); + const firstAttempt = yield* owner.namespace.destroy.pipe(Effect.exit); + expect(firstAttempt._tag).toBe("Failure"); + // Retained: the row survives a destroy that could not confirm nothing remains. + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeDefined(); -it.live("reassigns the same auto port after its claim is released", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }); - const ports = yield* makePorts(state); - const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; - const accept = (_host: string, port: number) => Effect.succeed(port); - const first = yield* ports.acquire(request, accept); - yield* ports.release("stack", "api"); - const again = yield* ports.acquire(request, accept); - expect(again.port).toBe(first.port); - }), - ).pipe(Effect.provide(NodeServices.layer)), + resolved = true; + yield* owner.namespace.destroy; + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBeUndefined(); + }), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + Layer.merge( + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + ), ); -it.live("names the last bind failure when no public port is available", () => +it.live("a destroy that cannot remove the registration keeps every reservation", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* state.save({ - id: "stack", - runtime: "native", - identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-remove-fails-" }); + const id = `destroy-remove-fails-${randomUUID()}`; + const state = yield* makeTestState(`${root}/state`); + const saved = { + id, + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "test", stackName: id }, instances: [], - lifetime: "detached", + lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], + }; + yield* state.save(saved); + let removable = false; + const unremovableState: StackNamespace.Interface = { + ...state, + remove: (target) => + Effect.suspend(() => + removable + ? state.remove(target) + : Effect.fail( + new StackNamespace.NamespaceError({ + operation: "remove", + message: "injected failure", + }), + ), + ), + }; + const owner = yield* ownerFor({ + saved, + state: unremovableState, + root: `${root}/data`, + cacheRoot: `${root}/cache`, }); - const ports = yield* makePorts(state); - const failure = yield* ports - .acquire({ stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" }, (host, port) => - Effect.fail(new PortError({ key: "api", message: `bind EACCES ${host}:${port}` })), - ) - .pipe(Effect.flip); - expect(failure.message).toContain("No public port is available"); - expect(failure.message).toContain("bind EACCES 127.0.0.1:"); - expect((yield* state.read("stack"))?.ports).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); - -const saveStack = ( - state: State.Interface, - root: string, - id: string, - ports: State.SavedStack["ports"] = [], -) => - state.save({ - id, - runtime: "native", - identity: { projectRoot: root, branchContext: `branch-${id}`, stackName: id }, - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports, - }); - -it.live("allocates past siblings whose state is unreadable or from a newer format", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "healthy"); - yield* fs.makeDirectory(`${root}/broken`); - yield* fs.writeFileString(`${root}/broken/state.json`, "{broken"); - yield* fs.makeDirectory(`${root}/newer`); - yield* fs.writeFileString( - `${root}/newer/state.json`, - '{"id":"newer","runtime":"future","ports":[]}', + yield* Effect.addFinalizer(() => + Effect.suspend(() => { + removable = true; + return owner.namespace.destroy; + }).pipe(Effect.ignore), ); - const ports = yield* makePorts(state); - const acquired = yield* ports.acquire( - { stackId: "healthy", key: "sql", host: "127.0.0.1", port: "auto" }, - bind, + const definitions = yield* owner.rpc.supabaseComposition({ + services: [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + }); + const mail = definitions.find((entry) => entry.creation.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const realStateRoot = yield* fs.realPath(`${root}/state`); + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, ); - expect((yield* state.read("healthy"))?.ports).toEqual([ - { key: "sql", host: "127.0.0.1", port: acquired.port }, - ]); - expect(yield* fs.readFileString(`${root}/broken/state.json`)).toBe("{broken"); - }), - ).pipe(Effect.provide(NodeServices.layer)), -); + const assigned = yield* portReservations.find(realStateRoot, id, `${mail.id}:http`); + expect(assigned).toBeDefined(); -it.live("keeps auto allocation off ports claimed by a sibling in a newer format", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "stack"); - const ports = yield* makePorts(state); - const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; - const accept = (_host: string, port: number) => Effect.succeed(port); - const preferred = yield* ports.acquire(request, accept); - yield* ports.release("stack", "api"); - yield* fs.makeDirectory(`${root}/newer`); - yield* fs.writeFileString( - `${root}/newer/state.json`, - `{"id":"newer","runtime":"future","ports":[{"key":"api","host":"127.0.0.1","port":${preferred.port}}]}`, - ); - const moved = yield* ports.acquire(request, accept); - expect(moved.port).not.toBe(preferred.port); + const failed = yield* owner.namespace.destroy.pipe(Effect.exit); + + expect(failed._tag).toBe("Failure"); + expect(yield* portReservations.find(realStateRoot, id, `${mail.id}:http`)).toBe(assigned); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe( + Effect.provide( + Layer.merge( + NodeServices.layer, + Layer.merge( + NodeHttpClient.layerNodeHttp, + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), + ), + ), + ), + ), ); -it.live("lets a stack bind an explicit port that a stopped stack still claims", () => +it.live("reserveNativePort skips a port a wildcard listener already holds", () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "stopped"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state); - const stoppedScope = yield* Scope.make(); - const stopped = yield* ports - .acquire({ stackId: "stopped", key: "db/sql", host: "127.0.0.1", port: "auto" }, bind) - .pipe(Effect.provideService(Scope.Scope, stoppedScope)); - yield* Scope.close(stoppedScope, Exit.void); - - const current = yield* ports.acquire( - { stackId: "current", key: "db/sql", host: "127.0.0.1", port: stopped.port }, - bind, + const blocked = yield* bindNativeWildcard(); + yield* Effect.addFinalizer(() => + Effect.callback((resume) => { + blocked.server.close(() => resume(Effect.void)); + }), ); - expect(current.port).toBe(stopped.port); - expect((yield* state.read("stopped"))?.ports).toEqual([ - { key: "db/sql", host: "127.0.0.1", port: stopped.port }, - ]); - expect((yield* state.read("current"))?.ports).toEqual([ - { key: "db/sql", host: "127.0.0.1", port: stopped.port }, - ]); + // A key whose hash-seeded first candidate is exactly the blocked port, so the scan must skip + // past the wildcard listener instead of merely missing it by chance. + const key = keyWithFirstCandidate(blocked.port); + + const reserved = yield* reserveNativePort(key, new Set()); + expect(reserved.port).not.toBe(blocked.port); }), ).pipe(Effect.provide(NodeServices.layer)), ); -it.live("names the stack claiming an explicit port that a live listener holds", () => +it.live("reserveNativePort excludes a port a previous attempt lost from the next reservation", () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "holder"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state); - const held = yield* ports.acquire( - { stackId: "holder", key: "db/sql", host: "127.0.0.1", port: "auto" }, - bind, + const key = `native-exclude-${randomUUID()}`; + const firstScope = yield* Scope.make(); + const first = yield* reserveNativePort(key, new Set()).pipe( + Effect.provideService(Scope.Scope, firstScope), ); + yield* Scope.close(firstScope, Exit.void); - const failure = yield* ports - .acquire({ stackId: "current", key: "db/sql", host: "127.0.0.1", port: held.port }, bind) - .pipe(Effect.flip); - expect(failure).toBeInstanceOf(PortError); - expect(failure.message).toContain(`db/sql at 127.0.0.1:${held.port}`); - expect(failure.message).toContain(`stack "holder" on branch-holder in ${root}`); - expect((yield* state.read("current"))?.ports).toEqual([]); + const second = yield* reserveNativePort(key, new Set([first.port])); + expect(second.port).not.toBe(first.port); }), ).pipe(Effect.provide(NodeServices.layer)), ); -for (const [held, requested] of [ - ["127.0.0.1", "0.0.0.0"], - ["::1", "127.0.0.1"], -] as const) - it.live(`rejects ${requested} on a port a ${held} listener holds where binds can overlap`, () => +// Windows has no POSIX mode bits and no stepped-down native database. +it.live.skipIf(process.platform === "win32")( + "opening the port registry keeps the traverse bit a stepped-down database needs on its directory", + () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "holder"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state, "darwin"); - const listener = yield* ports.acquire( - { stackId: "holder", key: "api", host: held, port: "auto" }, - bind, + const path = yield* Path.Path; + const openRegistry = Layer.build( + PortReservations.layer.pipe(Layer.provide(NodeServices.layer)), ); - const overlappingBinds = yield* Ref.make(0); - - const failure = yield* ports - .acquire({ stackId: "current", key: "api", host: requested, port: listener.port }, () => - Ref.update(overlappingBinds, (count) => count + 1), - ) - .pipe(Effect.flip); - expect(failure.message).toContain("already in use"); - expect(failure.message).toContain(`stack "holder"`); - expect(yield* Ref.get(overlappingBinds)).toBe(0); - expect((yield* state.read("current"))?.ports).toEqual([]); + const directory = path.join(userInfo().homedir, ".supabase"); + yield* openRegistry; + // The registry directory is an ancestor of native data, so a root-run Postgres that + // stepped down to another user traverses it through o+x granted by the runtime. + const original = (yield* fs.stat(directory)).mode & 0o7777; + yield* Effect.addFinalizer(() => fs.chmod(directory, original).pipe(Effect.ignore)); + yield* fs.chmod(directory, 0o701); + + yield* openRegistry; + + expect(yield* fs.exists(path.join(directory, "ports.sqlite"))).toBe(true); + expect((yield* fs.stat(directory)).mode & 0o777).toBe(0o701); }), ).pipe(Effect.provide(NodeServices.layer)), - ); +); -it.live("leaves a fixed port to the bind where overlapping binds are rejected", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "holder"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state, "linux"); - const listener = yield* ports.acquire( - { stackId: "holder", key: "api", host: "127.0.0.1", port: "auto" }, - bind, - ); - const binds = yield* Ref.make(0); +/** A pinned port above the auto range, so it never meets a registry row from an auto allocation. */ +const pinnedPort = () => 33000 + (Math.abs(Hash.string(randomUUID())) % 16000); - const acquired = yield* ports.acquire( - { stackId: "current", key: "api", host: "0.0.0.0", port: listener.port }, - () => Ref.update(binds, (count) => count + 1), - ); - expect(acquired.port).toBe(listener.port); - expect(yield* Ref.get(binds)).toBe(1); - }), - ).pipe(Effect.provide(NodeServices.layer)), +const withRegistry = Effect.provide( + Layer.merge(NodeServices.layer, PortReservations.layer.pipe(Layer.provide(NodeServices.layer))), ); -it.live("lets a stack bind a port that a running stack claims but does not listen on", () => +/** Two stacks of separate projects registered in one state root. */ +const twoProjects = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const state = yield* makeTestState(root); + const projectA = `${root}/project-a`; + const projectB = `${root}/project-b`; + const a = `pinned-a-${randomUUID()}`; + const b = `pinned-b-${randomUUID()}`; + yield* saveStack(state, projectA, a); + yield* saveStack(state, projectB, b); + const ports = yield* makePorts(state); + const registry = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + return { state, ports, registry, stateRoot: yield* fs.realPath(root), a, b, projectA }; +}); + +it.live("a stopped stack's pinned port is free for another project's stack", () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "running"); - yield* saveStack(state, root, "current"); - const ports = yield* makePorts(state); - yield* ports.acquire( - { stackId: "running", key: "admin", host: "127.0.0.1", port: "auto" }, - bind, + const { ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + const bind = (_host: string, bound: number) => Effect.succeed(bound); + + yield* Effect.scoped( + ports.acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, bind), ); - const restScope = yield* Scope.make(); - const rest = yield* ports - .acquire({ stackId: "running", key: "api", host: "127.0.0.1", port: "auto" }, bind) - .pipe(Effect.provideService(Scope.Scope, restScope)); - yield* Scope.close(restScope, Exit.void); - - const current = yield* ports.acquire( - { stackId: "current", key: "api", host: "127.0.0.1", port: rest.port }, + + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); + const started = yield* ports.acquire( + { stackId: b, key: "db:sql", host: "127.0.0.1", port }, bind, ); - expect(current.port).toBe(rest.port); - expect((yield* state.read("current"))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: rest.port }, - ]); + expect(started.port).toBe(port); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe(withRegistry), ); -it.live("lets exactly one of two stacks sharing a saved port bind it when both start at once", () => +it.live("a running stack's pinned port conflict names the port, its project, and the fix", () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "first"); - const ports = yield* makePorts(state); - const request = { stackId: "first", key: "api", host: "127.0.0.1", port: "auto" as const }; - const seedScope = yield* Scope.make(); - const seed = yield* ports - .acquire(request, bind) - .pipe(Effect.provideService(Scope.Scope, seedScope)); - yield* Scope.close(seedScope, Exit.void); - yield* saveStack(state, root, "second", [{ key: "api", host: "127.0.0.1", port: seed.port }]); - - const [first, second] = yield* Effect.all( - [ - Effect.exit(ports.acquire(request, bind)), - Effect.exit(ports.acquire({ ...request, stackId: "second" }, bind)), - ], - { concurrency: "unbounded" }, - ); - const outcomes = [ - { claimant: "second", exit: first }, - { claimant: "first", exit: second }, - ]; - expect(outcomes.filter(({ exit }) => Exit.isSuccess(exit))).toHaveLength(1); - const loser = outcomes.find(({ exit }) => Exit.isFailure(exit)); - const failure = - loser !== undefined && Exit.isFailure(loser.exit) - ? Cause.findErrorOption(loser.exit.cause) - : Option.none(); - if (Option.isNone(failure)) return yield* Effect.die("expected a port conflict"); - expect(failure.value).toBeInstanceOf(PortError); - expect(failure.value.message).toContain(`127.0.0.1:${seed.port}`); - expect(failure.value.message).toContain(`stack "${loser?.claimant}"`); - for (const id of ["first", "second"]) - expect((yield* state.read(id))?.ports).toEqual([ - { key: "api", host: "127.0.0.1", port: seed.port }, - ]); + const { state, ports, a, b, projectA } = yield* twoProjects; + const port = pinnedPort(); + const bind = (_host: string, bound: number) => Effect.succeed(bound); + yield* state.acquireLease(a); + yield* ports.acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, bind); + + const failure = yield* ports + .acquire({ stackId: b, key: "db:sql", host: "127.0.0.1", port }, bind) + .pipe(Effect.flip); + + expect(failure.message).toContain(String(port)); + expect(failure.message).toContain(projectA); + expect(failure.message).toContain("supabase stack stop"); + expect(failure.message).not.toContain("destroy"); + if (!(failure instanceof PortError)) return yield* Effect.die("expected a PortError"); + expect(failure.conflict?.holder).toMatchObject({ stackId: a }); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe(withRegistry), ); -// Fixed so a test can force two reservations to the same candidate; production uses randomPortSpanStart. -const fixedStart = Effect.succeed(0); +it.live( + "a pinned reservation left behind by a dead owner is released when the next owner stops", + () => + Effect.scoped( + Effect.gen(function* () { + const { ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + const bind = (_host: string, bound: number) => Effect.succeed(bound); + yield* registry.reserve(stateRoot, a, "db:sql", port); -// Binds a real listener directly in the native-reservation span, retrying past occupied -// candidates, instead of reserving then releasing a port that something else could grab meanwhile. -const bindBlockingPort = (host: string) => - Effect.gen(function* () { - for (let offset = 0; offset < portSpan; offset++) { - const attempt = yield* Effect.exit(bind(host, portBase + offset)); - if (Exit.isSuccess(attempt)) return { port: portBase + offset, listener: attempt.value }; - } - return yield* Effect.die("No port in the native reservation span was free for the fixture"); - }); + yield* Effect.scoped( + ports.acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, bind), + ); -it.live("skips a native backend port claimed by another saved stack", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); + expect( + (yield* ports.acquire({ stackId: b, key: "db:sql", host: "127.0.0.1", port }, bind)).port, + ).toBe(port); + }), + ).pipe(withRegistry), +); - const probeScope = yield* Scope.make(); - const probe = yield* reserveNativePort([], "pooler", fixedStart).pipe( - Effect.provideService(Scope.Scope, probeScope), - ); - yield* Scope.close(probeScope, Exit.void); +it.live( + "a pinned port held by an owner that died without closing is free for another project", + () => + Effect.scoped( + Effect.gen(function* () { + const { ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + yield* registry.reserve(stateRoot, a, "db:sql", port); - yield* saveStack(state, root, "claimer", [ - { key: "db/sql", host: "127.0.0.1", port: probe.port }, - ]); + const started = yield* ports.acquire( + { stackId: b, key: "db:sql", host: "127.0.0.1", port }, + (_host, bound) => Effect.succeed(bound), + ); - const reserved = yield* reserveNativePort(yield* state.claims, "pooler", fixedStart); - expect(reserved.port).not.toBe(probe.port); - }), - ).pipe(Effect.provide(NodeServices.layer)), + expect(started.port).toBe(port); + expect(yield* registry.find(stateRoot, b, "db:sql")).toBe(port); + }), + ).pipe(withRegistry), ); -it.live("skips a native backend port a wildcard listener holds", () => +it.live("a configured port takes over a reservation whose stack is only held by a sweeper", () => Effect.scoped( Effect.gen(function* () { - // A wildcard bind is reachable through loopback, so a loopback-only probe would miss it. - const blocked = yield* bindBlockingPort("0.0.0.0"); - const forcedStart = Effect.succeed(blocked.port - portBase); + const { state, ports, registry, stateRoot, a, b } = yield* twoProjects; + const port = pinnedPort(); + yield* registry.reserve(stateRoot, a, "db:sql", port); + const lease = yield* state.acquireLease(a); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: DateTime.formatIso(yield* DateTime.now), + }); - const reserved = yield* reserveNativePort([], "pooler", forcedStart); - expect(reserved.port).not.toBe(blocked.port); + const started = yield* ports.acquire( + { stackId: b, key: "db:sql", host: "127.0.0.1", port }, + (_host, bound) => Effect.succeed(bound), + ); + + expect(started.port).toBe(port); + expect(yield* registry.find(stateRoot, b, "db:sql")).toBe(port); + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe(withRegistry), ); -it.live("excludes a port a previous attempt lost from the next reservation", () => +it.live("a pinned port that fails to bind is not left reserved", () => Effect.scoped( Effect.gen(function* () { - const firstScope = yield* Scope.make(); - const first = yield* reserveNativePort([], "pooler", fixedStart).pipe( - Effect.provideService(Scope.Scope, firstScope), - ); - yield* Scope.close(firstScope, Exit.void); + const { ports, registry, stateRoot, a } = yield* twoProjects; + const port = pinnedPort(); - const second = yield* reserveNativePort([], "pooler", fixedStart, new Set([first.port])); - expect(second.port).not.toBe(first.port); + yield* ports + .acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port }, () => + Effect.fail(new PortError({ key: "db:sql", message: "bind failed" })), + ) + .pipe(Effect.flip); + + expect(yield* registry.find(stateRoot, a, "db:sql")).toBeUndefined(); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe(withRegistry), ); -it.live("skips a public auto candidate a loopback listener already holds", () => +it.live("a pinned port inside the native backend range names the range and the endpoint", () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped(); - const state = yield* makeTestState(root); - yield* saveStack(state, root, "stack"); - const ports = yield* makePorts(state); - // A container-runtime stack's bind to the wildcard host would otherwise succeed here too. - const request = { stackId: "stack", key: "api", host: "0.0.0.0", port: "auto" as const }; - const accept = (_host: string, port: number) => Effect.succeed(port); - - // A loopback-only listener keeps occupying the port while only its saved claim is released, - // so the next auto allocation has to skip it for real via loopbackOccupied, not a real bind. - const probe = yield* ports.acquire(request, (_host, port) => bind("127.0.0.1", port)); - yield* ports.release("stack", "api"); - - const acquired = yield* ports.acquire(request, accept); - expect(acquired.port).not.toBe(probe.port); + const { ports, a } = yield* twoProjects; + + const failure = yield* ports + .acquire({ stackId: a, key: "db:sql", host: "127.0.0.1", port: 15432 }, (_host, bound) => + Effect.succeed(bound), + ) + .pipe(Effect.flip); + + expect(failure.message).toContain("db:sql"); + expect(failure.message).toContain("native service backends"); }), - ).pipe(Effect.provide(NodeServices.layer)), + ).pipe(withRegistry), ); diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts index 8b55842335..025ad27fe0 100644 --- a/packages/stack/src/Ports.ts +++ b/packages/stack/src/Ports.ts @@ -1,17 +1,40 @@ -import { Cause, Crypto, Data, Effect, Exit, Hash, Option, Scope } from "effect"; +import { + Cause, + Data, + Effect, + Exit, + FileSystem, + Hash, + Option, + Path, + Predicate, + Schema, + Scope, +} from "effect"; import * as Net from "node:net"; -import type * as State from "./State.ts"; +import type * as StackNamespace from "./StackNamespace.ts"; +import * as Lease from "./namespace/Lease.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import type { Holder } from "./namespace/PortReservations.ts"; -export const portBase = 20000; +const portBase = 20000; /** Stays below the Linux ephemeral range, per the [architecture ADR](../../../docs/adr/0017-simplified-managed-stack-architecture.md). */ -export const portSpan = 12768; +const portSpan = 12768; /** Co-prime with the span, so the scan visits every port once and steps past reserved ranges. */ const portStride = 257; +/** Why a port could not be used: another stack's live reservation, or a process outside the registry. */ +export interface PortConflict { + readonly port: number; + readonly endpoint: string; + readonly holder: Holder | "foreign"; +} + export class PortError extends Data.TaggedError("PortError")<{ readonly key: string; readonly message: string; readonly cause?: unknown; + readonly conflict?: PortConflict; }> {} export interface PortRequest { @@ -22,7 +45,7 @@ export interface PortRequest { } /** Spreads the scan across the span so separate checkouts, stacks, and keys start apart. */ -const scanStart = (stack: State.SavedStack, key: string) => +const scanStart = (stack: StackNamespace.SavedStack, key: string) => Math.abs(Hash.string(`${stack.identity.projectRoot}:${stack.id}:${key}`)) % portSpan; /** A wildcard listener is reachable through loopback, where a same-port loopback listener answers too. */ @@ -47,46 +70,78 @@ export const accepts = (host: string, port: number) => /** Linux rejects a bind that overlaps a same-family listener on the port; macOS, BSD, and Windows accept it. */ const bindsCanOverlap = (platform: NodeJS.Platform) => platform !== "linux"; -const loopbackOccupied = (port: number) => - Effect.forEach(["127.0.0.1", "::1"], (host) => accepts(host, port), { +const isRefused = (cause: unknown) => + Predicate.hasProperty(cause, "code") && cause.code === "ECONNREFUSED"; + +/** A refused loopback connect can take seconds on Windows; darwin and linux answer almost immediately. */ +const refusalTimeout = (platform: NodeJS.Platform) => + platform === "win32" ? "3 seconds" : "250 millis"; + +/** + * Resolves `true` only on a definite `ECONNREFUSED`, which is the one outcome that proves nothing + * answers `host:port`. An accepted connection, a timeout, or any other error (for example + * `ECONNRESET`) all count as occupied: this is the vacancy probe's own, stricter semantics, kept + * separate from {@link accepts}'s generous "did anything answer" check that `ProcessRecipe` uses + * for readiness. + */ +const refused = (platform: NodeJS.Platform) => (host: string, port: number) => + Effect.callback((resume) => { + const socket = Net.connect({ host: probeHost(host), port }); + const settle = (value: boolean) => { + socket.destroy(); + resume(Effect.succeed(value)); + }; + socket.once("connect", () => settle(false)); + socket.on("error", (cause: unknown) => settle(isRefused(cause))); + return Effect.sync(() => { + socket.destroy(); + }); + }).pipe( + Effect.timeoutOrElse({ + duration: refusalTimeout(platform), + orElse: () => Effect.succeed(false), + }), + ); + +const loopbackVacant = (platform: NodeJS.Platform) => (port: number) => + Effect.forEach(["127.0.0.1", "::1"], (host) => refused(platform)(host, port), { concurrency: "unbounded", - }).pipe(Effect.map((answers) => answers.some(Boolean))); + }).pipe(Effect.map((refusals) => refusals.every(Boolean))); -const claimantOf = (stacks: ReadonlyArray, stackId: string, port: number) => - stacks.find((other) => other.id !== stackId && other.ports.some((claim) => claim.port === port)) - ?.id; +/** The OS error `bindTcp`/`makeHttpProxy` surface when another process already owns the port. */ +const isAddressInUse = (cause: unknown) => + Predicate.hasProperty(cause, "code") && cause.code === "EADDRINUSE"; -const resolveRequest = ( - stack: State.SavedStack, - request: PortRequest, -): Effect.Effect< - { readonly saved: State.PortClaim | undefined; readonly requested: number | "auto" }, - PortError -> => { - const saved = stack.ports.find((entry) => entry.key === request.key); - if ( - saved !== undefined && - (saved.host !== request.host || (request.port !== "auto" && saved.port !== request.port)) - ) - return Effect.fail( - new PortError({ - key: request.key, - message: "The requested listener differs from its saved assignment", - }), - ); - const requested = saved?.port ?? request.port; - if (requested === "auto") return Effect.succeed({ saved, requested }); - if (!Number.isInteger(requested) || requested < 1 || requested > 65535) - return Effect.fail(new PortError({ key: request.key, message: "Invalid public port" })); - if (stack.ports.some((claim) => claim.key !== request.key && claim.port === requested)) - return Effect.fail( - new PortError({ - key: request.key, - message: `Public port ${requested} is claimed by another listener of this stack`, - }), - ); - return Effect.succeed({ saved, requested }); -}; +/** + * Guards a brand-new physical listener only; reusing an existing one never probes. On platforms + * where overlapping binds silently succeed, only `ECONNREFUSED` on both loopback families proves + * the port vacant, so anything else (an answer, a timeout, another error) counts as occupied. This + * narrows, but cannot close, the race between the probe and the real bind that follows it: the OS + * still allows a foreign process to bind in between on these platforms. + */ +export const probeVacant = + (platform: NodeJS.Platform) => + (key: string, host: string, port: number): Effect.Effect => + Effect.gen(function* () { + if (!bindsCanOverlap(platform)) return; + if (yield* loopbackVacant(platform)(port)) return; + return yield* new PortError({ + key, + message: `Port ${port} for ${key} at ${host}:${port} is in use by another process; free it or configure a different port for ${key}`, + conflict: { port, endpoint: key, holder: "foreign" }, + }); + }); + +/** + * Disjoint from the public auto range (`portBase`..`portBase + portSpan`) and contiguous below it, + * and a pinned public port inside it is rejected, so a native backend's direct bind can never land + * on a port the per-user registry is reserving for a public listener. Both ranges stay below the + * OS ephemeral range (ADR 0017). + */ +export const nativePortBase = 10000; +export const nativePortSpan = portBase - nativePortBase; +/** Co-prime with the span, matching the public scan's stride. */ +const nativePortStride = 257; export interface NativePortReservation { readonly port: number; @@ -130,36 +185,35 @@ const bindNativePort = ( }); }); -const emptyPortSet: ReadonlySet = new Set(); - -/** Random, so a reopened stack doesn't retry a backend port still in server-side `TIME_WAIT`. */ -export const randomPortSpanStart = (crypto: Crypto.Crypto): Effect.Effect => - crypto.randomIntBetween(0, portSpan, { halfOpen: true }); - /** - * Scans the below-ephemeral span for a backend port, skipping claimed and excluded ports; reuses - * `loopbackOccupied` so a wildcard listener a loopback-only bind would miss on macOS, BSD or - * Windows still rules out the candidate. Never persists one of its own. + * Reserves a native backend's private port, hash-seeding the scan from `key` so a reopened + * instance starts from the same candidate while separate keys spread out (mirrors the public auto + * scan's hash seed). Probes with {@link probeVacant} before the real bind, since a loopback-only + * bind can silently coexist with a wildcard listener on macOS, BSD, and Windows. A port a prior + * attempt in this batch lost is passed in `excluded` so a retry advances instead of repeating it. + * Never saved: backend ports are private and are not stable across restarts. */ export const reserveNativePort = Effect.fn("Ports.reserveNativePort")( ( - claims: ReadonlyArray, key: string, - randomStart: Effect.Effect, - excluded: ReadonlySet = emptyPortSet, + excluded: ReadonlySet, + platform: NodeJS.Platform = process.platform, ): Effect.Effect => Effect.acquireRelease( Effect.gen(function* () { - const claimed = new Set(claims.flatMap((stack) => stack.ports.map((claim) => claim.port))); - const start = yield* randomStart; + const probe = probeVacant(platform); + const start = Math.abs(Hash.string(key)) % nativePortSpan; let failures = 0; let lastFailure: PortError | undefined; - for (let attempt = 0; attempt < portSpan && failures < 64; attempt++) { - const port = portBase + ((start + attempt * portStride) % portSpan); - if (claimed.has(port) || excluded.has(port)) continue; - if (yield* loopbackOccupied(port)) { + for (let attempt = 0; attempt < nativePortSpan && failures < 64; attempt++) { + const port = nativePortBase + ((start + attempt * nativePortStride) % nativePortSpan); + if (excluded.has(port)) continue; + const probed = yield* Effect.exit(probe(key, "127.0.0.1", port)); + if (Exit.isFailure(probed)) { + const probeError = Cause.findErrorOption(probed.cause); + if (Option.isNone(probeError)) return yield* Effect.failCause(probed.cause); failures++; - lastFailure = new PortError({ key, message: `Port ${port} is already in use` }); + lastFailure = probeError.value; continue; } const result = yield* Effect.exit(bindNativePort(key, port)); @@ -182,133 +236,213 @@ export const reserveNativePort = Effect.fn("Ports.reserveNativePort")( ), ); -/** Claims steer auto allocation away from saved stacks; live listeners and binds decide conflicts for fixed ports. */ -export const makePorts = (state: State.Interface, platform: NodeJS.Platform = process.platform) => - Effect.sync(() => { - const describe = (id: string) => - state.read(id).pipe( - Effect.map((saved) => - saved === undefined - ? id - : `"${saved.identity.stackName}" on ${saved.identity.branchContext} in ${saved.identity.projectRoot}`, - ), - Effect.orElseSucceed(() => id), +const resolveRequest = ( + request: PortRequest, + owned: number | undefined, +): Effect.Effect => { + if (request.port !== "auto" && owned !== undefined && owned !== request.port) + return Effect.fail( + new PortError({ + key: request.key, + message: "The requested listener differs from its reserved assignment", + }), + ); + const requested = owned ?? request.port; + if (requested === "auto") return Effect.succeed(requested); + if (!Number.isInteger(requested) || requested < 1 || requested > 65535) + return Effect.fail(new PortError({ key: request.key, message: "Invalid public port" })); + if (requested >= nativePortBase && requested < portBase) + return Effect.fail( + new PortError({ + key: request.key, + message: `Public port ${requested} for ${request.key} is inside ${nativePortBase}-${portBase - 1}, which is reserved for native service backends; configure a port outside that range`, + }), + ); + return Effect.succeed(requested); +}; + +const RegisteredProject = Schema.fromJsonString( + Schema.Struct({ identity: Schema.Struct({ projectRoot: Schema.String }) }), +); + +/** + * One stack's public ports, reserved through the per-user registry before any physical listener + * exists: a reservation is committed first, then the listener is created. An auto port stays + * reserved while its stack is stopped; a pinned port is reserved only while its listener is open. + */ +export const makePorts = (state: StackNamespace.Interface) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const portReservations = yield* PortReservations.Service; + // The registry's state_root is the realpath, so a symlinked or relative root still matches + // the identity another process derives from the same stack. + const stateRoot = yield* fs.realPath(state.root); + const services = yield* Effect.context(); + + /** The holder's project directory, read from its own registration; undefined when unreadable. */ + const projectOf = (holder: Holder) => + fs.readFileString(path.join(holder.stateRoot, holder.stackId, "state.json")).pipe( + Effect.flatMap(Schema.decodeEffect(RegisteredProject)), + Effect.map((saved) => saved.identity.projectRoot), + Effect.orElseSucceed(() => undefined), ); - const claimedBy = (stacks: ReadonlyArray, stackId: string, port: number) => { - const claimant = claimantOf(stacks, stackId, port); - return claimant === undefined - ? Effect.succeed("") - : describe(claimant).pipe(Effect.map((stack) => `; stack ${stack} claims this port`)); - }; + /** Only a confirmed `ENOENT` on the holder's own registration makes its reservation stale. */ + const isGone = (holder: Holder) => + fs.stat(path.join(holder.stateRoot, holder.stackId, "state.json")).pipe( + Effect.as(false), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(true), + ), + Effect.orElseSucceed(() => false), + ); - // A caller may serve several claims of one stack from a listener it acquired here, so that - // listener is not a foreign occupant. - const held = new Map(); - const hold = (scope: Scope.Scope, stackId: string, port: number) => { - const key = `${stackId}:${port}`; - return Effect.sync(() => held.set(key, (held.get(key) ?? 0) + 1)).pipe( - Effect.andThen( - Scope.addFinalizer( - scope, - Effect.sync(() => { - const remaining = (held.get(key) ?? 1) - 1; - if (remaining > 0) held.set(key, remaining); - else held.delete(key); - }), - ), + /** + * Whether a process holds the holder stack's owner lease; an unreadable lease counts as held. + * A sweeper holding it does not count, since a sweeper never uses the stack's public ports. + */ + const isLeased = (holder: Holder) => + Lease.make({ root: holder.stateRoot, isRegistered: () => Effect.succeed(true) }).pipe( + Effect.flatMap((lease) => + lease + .leased(holder.stackId) + .pipe( + Effect.flatMap((leased) => + leased + ? lease.readHolder(holder.stackId).pipe(Effect.map((h) => h?.role !== "sweeper")) + : Effect.succeed(false), + ), + ), ), + Effect.provideContext(services), + Effect.orElseSucceed(() => true), ); - }; - /** Advisory and outside the registry lock; the bind under the lock still decides. */ - const rejectOccupied = Effect.fn("Ports.rejectOccupied")(function* (request: PortRequest) { - const preview = yield* state.read(request.stackId); - if (preview === undefined) return; - const { requested } = yield* resolveRequest(preview, request); - if ( - requested === "auto" || - !bindsCanOverlap(platform) || - held.has(`${request.stackId}:${requested}`) || - !(yield* loopbackOccupied(requested)) - ) - return; - return yield* new PortError({ - key: request.key, - message: `Public port ${requested} for ${request.key} at ${request.host}:${requested} is already in use${yield* claimedBy(yield* state.claims, request.stackId, requested)}`, + /** + * Reserves `port` for `(stackId, key)`, running the lazy-reclamation check against a + * conflict's own registration before giving up on it. A pinned request also reclaims a + * holder with no live owner, since the owner that died never closed its listener. Resolves + * to the still-live holder on conflict, or `undefined` once the row is ours. Untraced: an + * auto scan calls this per candidate port, and the enclosing `Ports.acquire` span already + * records the attempt count. + */ + const reserveCandidate = Effect.fnUntraced(function* ( + stackId: string, + key: string, + port: number, + pinned: boolean, + ) { + const holder = yield* portReservations.reserve(stateRoot, stackId, key, port); + if (holder === undefined) return undefined; + if (!(yield* isGone(holder)) && !(pinned && !(yield* isLeased(holder)))) return holder; + const reclaimed = yield* portReservations.reclaim(port, holder, { + stateRoot, + stackId, + endpoint: key, }); + return reclaimed ? undefined : holder; + }); + + const holderMessage = Effect.fnUntraced(function* ( + key: string, + port: number, + stackId: string, + holder: Holder, + ) { + if (holder.stackId === stackId) + return `Public port ${port} for ${key} is claimed by another listener of this stack`; + const project = yield* projectOf(holder); + const where = + project === undefined + ? `stack ${holder.stackId}` + : `the stack of project ${project} (stack ${holder.stackId})`; + return `Public port ${port} for ${key} is in use by ${where}; run \`supabase stack stop\` in that project, or configure a different port for ${key}`; }); const acquire = Effect.fn("Ports.acquire")(function* ( request: PortRequest, bind: (host: string, port: number) => Effect.Effect, ) { - yield* rejectOccupied(request); return yield* state.withLock( Effect.gen(function* () { const stack = yield* state.read(request.stackId); if (stack === undefined) return yield* new PortError({ key: request.key, message: "Stack is not registered" }); - const { saved, requested } = yield* resolveRequest(stack, request); - const others = yield* state.claims; - const claimed = new Set([ - ...stack.ports.filter((claim) => claim.key !== request.key).map((claim) => claim.port), - ...others - .filter((other) => other.id !== request.stackId) - .flatMap((other) => other.ports.map((claim) => claim.port)), - ]); - + const owned = yield* portReservations.find(stateRoot, request.stackId, request.key); + const requested = yield* resolveRequest(request, owned); const owner = yield* Scope.Scope; - const start = scanStart(stack, request.key); - let failures = 0; - let lastFailure: PortError | undefined; - for (let attempt = 0; attempt < portSpan && failures < 64; attempt++) { - const port = - requested === "auto" - ? portBase + ((start + attempt * portStride) % portSpan) - : requested; - if (requested === "auto" && claimed.has(port)) continue; - if (requested === "auto" && (yield* loopbackOccupied(port))) { - failures++; - lastFailure = new PortError({ - key: request.key, - message: `Port ${port} is already in use`, - }); - continue; - } - const result = yield* Effect.uninterruptibleMask((restore) => + yield* Effect.annotateCurrentSpan({ + "stack.endpoint": request.key, + "stack.port.mode": + requested !== "auto" ? (owned !== undefined ? "owned" : "fixed") : "auto", + }); + + /** Binds `port` against whatever scope the caller provides. */ + const bindListener = (port: number) => + bind(request.host, port).pipe( + Effect.map((listener) => ({ port, listener })), + Effect.mapError( + (cause) => + new PortError({ + key: request.key, + message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, + cause: cause.cause, + conflict: + cause.conflict ?? + (isAddressInUse(cause.cause) + ? { port, endpoint: request.key, holder: "foreign" as const } + : undefined), + }), + ), + ); + + /** + * One bind attempt at `port` in its own forked scope; a failure closes the scope, a + * success keeps it open. A pinned request's row lives exactly as long as that scope, so a + * stopped stack holds none; an auto assignment's row outlives it and is kept across stops. + */ + const attemptBind = (port: number) => + Effect.uninterruptibleMask((restore) => Effect.gen(function* () { + if (owned === undefined) { + const holder = yield* reserveCandidate( + request.stackId, + request.key, + port, + request.port !== "auto", + ); + if (holder !== undefined) + return Exit.fail( + new PortError({ + key: request.key, + message: yield* holderMessage(request.key, port, request.stackId, holder), + conflict: { port, endpoint: request.key, holder }, + }), + ); + } const scope = yield* Scope.fork(owner, "sequential"); + if (request.port !== "auto") + yield* Scope.addFinalizer( + scope, + portReservations + .release(stateRoot, request.stackId, request.key) + .pipe(Effect.orDie), + ); return yield* restore( - Effect.gen(function* () { - const listener = yield* bind(request.host, port).pipe( - Effect.mapError( - (cause) => - new PortError({ - key: request.key, - message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, - cause, - }), - ), - Effect.provideService(Scope.Scope, scope), - ); - if (saved === undefined) - yield* state.save({ - ...stack, - ports: [...stack.ports, { key: request.key, host: request.host, port }], - }); - return { port, listener }; - }), + bindListener(port).pipe(Effect.provideService(Scope.Scope, scope)), ).pipe( Effect.onExit((exit) => - Exit.isFailure(exit) - ? Scope.close(scope, exit) - : hold(scope, request.stackId, port), + Exit.isFailure(exit) ? Scope.close(scope, exit) : Effect.void, ), Effect.exit, ); }), ); + + if (requested !== "auto") { + const result = yield* attemptBind(requested); if (Exit.isSuccess(result)) return result.value; const error = Cause.findErrorOption(result.cause); if ( @@ -318,15 +452,61 @@ export const makePorts = (state: State.Interface, platform: NodeJS.Platform = pr !(error.value instanceof PortError) ) return yield* Effect.failCause(result.cause); - if (requested !== "auto") - return yield* new PortError({ - key: request.key, - message: `${error.value.message}${yield* claimedBy(others, request.stackId, requested)}`, - cause: error.value.cause, - }); + return yield* error.value; + } + + // A freshly auto-reserved port that fails to bind (including by interruption mid-probe) + // has no restart stickiness to protect, so it must release the row again. Reserving, + // forking the listener's scope, attempting the bind, and rolling both back on failure all + // happen in one uninterruptible mask, with only the bind attempt itself restored to + // interruptible, mirroring `attemptBind` exactly: a pending interrupt delivered during the + // probe surfaces inside that single mask, where rollback can never be skipped. + const attemptCandidate = (port: number) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const holder = yield* reserveCandidate(request.stackId, request.key, port, false); + if (holder !== undefined) return undefined; + const scope = yield* Scope.fork(owner, "sequential"); + return yield* restore( + bindListener(port).pipe(Effect.provideService(Scope.Scope, scope)), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) + ? Scope.close(scope, exit).pipe( + Effect.andThen( + portReservations.release(stateRoot, request.stackId, request.key), + ), + ) + : Effect.void, + ), + Effect.exit, + ); + }), + ); + + const start = scanStart(stack, request.key); + let failures = 0; + let lastFailure: PortError | undefined; + for (let attempt = 0; attempt < portSpan && failures < 64; attempt++) { + const port = portBase + ((start + attempt * portStride) % portSpan); + const result = yield* attemptCandidate(port); + if (result === undefined) continue; + if (Exit.isSuccess(result)) { + yield* Effect.annotateCurrentSpan({ "stack.port.attempts": attempt + 1 }); + return result.value; + } + const error = Cause.findErrorOption(result.cause); + if ( + Exit.hasInterrupts(result) || + Exit.hasDies(result) || + Option.isNone(error) || + !(error.value instanceof PortError) + ) + return yield* Effect.failCause(result.cause); failures++; lastFailure = error.value; } + yield* Effect.annotateCurrentSpan({ "stack.port.failures": failures }); return yield* new PortError({ key: request.key, message: @@ -339,18 +519,19 @@ export const makePorts = (state: State.Interface, platform: NodeJS.Platform = pr ); }); + /** The port this stack's reservation holds for `key`, or `undefined` when it holds none. */ + const assigned = Effect.fn("Ports.assigned")((stackId: string, key: string) => + portReservations.find(stateRoot, stackId, key), + ); + const release = Effect.fn("Ports.release")(function* (stackId: string, key: string) { - yield* state.withLock( - Effect.gen(function* () { - const stack = yield* state.read(stackId); - if (stack !== undefined) - yield* state.save({ - ...stack, - ports: stack.ports.filter((entry) => entry.key !== key), - }); - }), - ); + yield* state.withLock(portReservations.release(stateRoot, stackId, key)); + }); + + /** Releases every reservation a stack holds, regardless of endpoint; used at destroy's success boundary. */ + const releaseStack = Effect.fn("Ports.releaseStack")(function* (stackId: string) { + yield* portReservations.releaseStack(stateRoot, stackId); }); - return { acquire, release }; + return { acquire, assigned, release, releaseStack }; }); diff --git a/packages/stack/src/Proxy.integration.test.ts b/packages/stack/src/Proxy.integration.test.ts index e08b81eaa2..3f3b396b80 100644 --- a/packages/stack/src/Proxy.integration.test.ts +++ b/packages/stack/src/Proxy.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeHttpServer } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Deferred, Effect, Fiber, Predicate } from "effect"; +import { Deferred, Effect, Exit, Fiber, Predicate, Ref, Scope } from "effect"; import { HttpClient, HttpClientRequest, @@ -149,3 +149,133 @@ it.live("does not log an error when a backend copy resets after a successful con }), ).pipe(Effect.provide(captureErrors(logs))); }); + +/** Starts a raw TCP echo backend and returns its address, retained until the scope closes. */ +const echoBackend = () => + Effect.acquireRelease( + Effect.callback< + { readonly server: Server; readonly host: string; readonly port: number }, + never + >((resume) => { + const server = createServer((socket) => socket.pipe(socket)); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + resume( + Effect.succeed({ + server, + host: "127.0.0.1", + port: typeof address === "object" && address !== null ? address.port : 0, + }), + ); + }); + return Effect.void; + }), + ({ server }) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + return Effect.void; + }), + ); + +const connectAndWrite = (port: number, message: string) => + Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }), + (socket) => Effect.sync(() => socket.destroy()), + ).pipe(Effect.tap((socket) => Effect.sync(() => socket.write(message)))); + +const readOnceTcp = (socket: Socket) => + Effect.callback((resume) => { + const onData = (chunk: Buffer) => resume(Effect.succeed(chunk.toString())); + socket.once("data", onData); + return Effect.sync(() => socket.off("data", onData)); + }).pipe(Effect.timeout("5 seconds")); + +it.live( + "releases the scoped target and keeps serving the next connection after a client resets while readiness is pending", + () => + Effect.scoped( + Effect.gen(function* () { + const listener = yield* bindTcp("127.0.0.1", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const port = listener.address.port; + const backend = yield* echoBackend(); + const acquired = yield* Deferred.make(); + const released = yield* Deferred.make(); + const gate = yield* Deferred.make(); + const connections = yield* Ref.make(0); + // Only the first connection's target acquisition is held behind the gate; a later one + // (the "listener still serves" check below) resolves immediately against the real backend. + const target = Ref.updateAndGet(connections, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 1 + ? Effect.acquireRelease(Deferred.succeed(acquired, undefined), () => + Deferred.succeed(released, undefined), + ).pipe( + Effect.andThen(Deferred.await(gate)), + Effect.as({ host: backend.host, port: backend.port }), + ) + : Effect.succeed({ host: backend.host, port: backend.port }), + ), + ); + yield* serveTcp(listener, target, "cold-wake").pipe(Effect.forkScoped); + + const client = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + yield* Deferred.await(acquired); + // Readiness is still pending behind `gate`; an unhandled reset here would otherwise crash + // the owning process before the fix. + yield* Effect.sync(() => client.resetAndDestroy()); + yield* Deferred.await(released); + + // The listener (and the process running it) survived the reset and still serves a fresh + // connection normally. + const probe = yield* connectAndWrite(port, "ping"); + expect(yield* readOnceTcp(probe)).toBe("ping"); + }), + ), +); + +it.live("tears down a connection that arrives before run installs its handler", () => + Effect.gen(function* () { + const listenerScope = yield* Scope.make(); + const listener = yield* bindTcp("127.0.0.1", 0).pipe(Scope.provide(listenerScope)); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + const port = listener.address.port; + + const connect = Effect.callback((resume) => { + const socket = new Socket(); + socket.once("connect", () => resume(Effect.succeed(socket))); + socket.connect(port, "127.0.0.1"); + return Effect.void; + }).pipe(Effect.timeout("5 seconds")); + const awaitClose = (socket: Socket) => + Effect.callback((resume) => { + const onClose = () => resume(Effect.void); + socket.once("close", onClose); + if (socket.destroyed) onClose(); + return Effect.sync(() => socket.off("close", onClose)); + }).pipe(Effect.timeout("5 seconds")); + + // `run` is never called on this listener: the connection below stays queued. Closing the + // listener's own scope destroys it and completes rather than hanging on `server.close()` + // waiting for a socket nothing ever destroyed. + const queued = yield* connect; + const closed = yield* awaitClose(queued).pipe(Effect.forkScoped); + yield* Scope.close(listenerScope, Exit.void).pipe(Effect.timeout("5 seconds")); + yield* Fiber.join(closed); + }).pipe(Effect.scoped), +); diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts index 9ac7018056..5aaa583abb 100644 --- a/packages/stack/src/Proxy.ts +++ b/packages/stack/src/Proxy.ts @@ -1,6 +1,5 @@ -import { NodeSink, NodeSocket, NodeSocketServer, NodeStream } from "@effect/platform-node"; -import { Data, Effect, Option, Stream } from "effect"; -import type { Scope } from "effect"; +import { NodeSink, NodeStream } from "@effect/platform-node"; +import { Data, Effect, Exit, FiberSet, Scope, Stream } from "effect"; import type { SocketServer } from "effect/unstable/socket"; import * as Net from "node:net"; import { PortError } from "./Ports.ts"; @@ -17,13 +16,116 @@ export class ProxyError extends Data.TaggedError("ProxyError")<{ const proxyError = (cause: unknown) => new ProxyError({ message: cause instanceof Error ? cause.message : String(cause), cause }); -/** Binds a dedicated public listener and retains the socket until its scope closes. */ -export const bindTcp = (host: string, port: number) => - NodeSocketServer.make({ host, port, allowHalfOpen: true }).pipe( - Effect.mapError( - (cause) => new PortError({ key: "tcp", message: "Cannot bind TCP listener", cause }), - ), - ); +export interface TcpListener { + readonly address: SocketServer.Address; + /** Installs the per-connection handler; keeps the listener alive until interrupted. */ + readonly run: ( + handler: (socket: Net.Socket) => Effect.Effect<_, E, R>, + ) => Effect.Effect; +} + +/** + * Binds a dedicated public listener and retains the socket until its scope closes, which also + * destroys every established connection. + */ +export const bindTcp = ( + host: string, + port: number, +): Effect.Effect => + Effect.gen(function* () { + const services = yield* Effect.context(); + // Every accepted socket, queued or dispatched, from acceptance until close, so teardown + // destroys all of them. + const sockets = new Set(); + // A connection that arrives before `run` installs its handler is queued, the same as + // `NodeSocketServer`'s own listener, so none are dropped in that gap. + const pending = new Set(); + let onConnection: (conn: Net.Socket) => void = (conn) => { + pending.add(conn); + }; + const server = Net.createServer({ allowHalfOpen: true }, (conn) => { + // Attached synchronously, before any dispatch, so a reset while a handler is still cold + // (queued, or waiting on scoped target acquisition) never surfaces as an unhandled error. + conn.on("error", () => conn.destroy()); + sockets.add(conn); + conn.once("close", () => { + sockets.delete(conn); + pending.delete(conn); + }); + onConnection(conn); + }); + yield* Effect.acquireRelease( + Effect.callback((resume) => { + const onError = (cause: Error) => + // Unwrapped so `Ports.ts`'s conflict detection sees the same shape it gets from + // `HttpProxy` (for example EADDRINUSE). + resume( + Effect.fail(new PortError({ key: "tcp", message: "Cannot bind TCP listener", cause })), + ); + server.once("error", onError); + server.listen(port, host, () => { + // Accept errors after binding (for example fd exhaustion) are logged for the listener's + // whole life rather than left unhandled to crash the owner. + server.off("error", onError); + server.on("error", (cause) => + Effect.runSyncWith(services)(Effect.logError("TCP listener error", cause)), + ); + resume(Effect.void); + }); + return Effect.sync(() => server.off("error", onError)); + }), + () => + Effect.callback((resume) => { + pending.clear(); + for (const socket of sockets) socket.destroy(); + server.close(() => resume(Effect.void)); + return Effect.void; + }), + ); + const bound = server.address(); + if (bound === null) + return yield* new PortError({ key: "tcp", message: "TCP listener has no address" }); + const address: SocketServer.Address = + typeof bound === "string" + ? { _tag: "UnixAddress", path: bound } + : { _tag: "TcpAddress", hostname: bound.address, port: bound.port }; + + const run = ( + handler: (socket: Net.Socket) => Effect.Effect<_, E, R>, + ): Effect.Effect => + Effect.gen(function* () { + const connectionScope = yield* Scope.make(); + const runFiber = yield* FiberSet.makeRuntime().pipe( + Effect.provideService(Scope.Scope, connectionScope), + ); + const previous = onConnection; + onConnection = (conn) => { + runFiber( + handler(conn).pipe( + Effect.catchCause((cause) => + Effect.logError("Unhandled TCP connection failure", cause), + ), + ), + ); + }; + for (const conn of pending) onConnection(conn); + pending.clear(); + return yield* Effect.onExit(Effect.never, () => + Scope.close(connectionScope, Exit.void).pipe( + Effect.andThen( + Effect.sync(() => { + onConnection = previous; + }), + ), + ), + ); + }); + + return { + address, + run, + }; + }); const connect = Effect.fn("Proxy.connect")((address: BackendAddress) => Effect.gen(function* () { @@ -57,22 +159,13 @@ const copy = (source: Net.Socket, destination: Net.Socket) => /** The scoped target acquisition owns request activity and performs orchestrated wake/readiness. */ export const serveTcp = Effect.fn("Proxy.serveTcp")( ( - listener: SocketServer.SocketServer["Service"], + listener: TcpListener, target: Effect.Effect, label: string, ) => - listener.run(() => + listener.run((incoming) => Effect.scoped( Effect.gen(function* () { - // rc.112 supplies this service to handlers but does not remove it from run's requirements. - const incoming = yield* Effect.serviceOption(NodeSocket.NetSocket).pipe( - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(proxyError("TCP listener supplied no connection")), - onSome: Effect.succeed, - }), - ), - ); yield* Effect.addFinalizer(() => Effect.sync(() => { incoming.destroy(); diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 873f1b7b70..f2dda81329 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -1,11 +1,12 @@ -import { Exit, Schema } from "effect"; +import { Cause, Exit, Option, Predicate, Schema } from "effect"; import { Rpc, RpcGroup } from "effect/unstable/rpc"; import { ServiceCreation, ServiceCreationInput } from "./services/Catalog.ts"; import { snapshotScopes } from "./services/DatabaseSnapshot.ts"; import { causeMessage, CompositionConfig, OrchestratorError } from "./Orchestrator.ts"; import { CommandInvocation } from "./Commands.ts"; -import { StackKeysInput } from "./State.ts"; +import { StackKeysInput } from "./StackNamespace.ts"; import { failureMessage } from "./internal/failure-message.ts"; +import type { PortConflict } from "./Ports.ts"; import { LogPosition, LogRecord } from "./host/LogRecord.ts"; const Outcome = Schema.Struct({ @@ -14,6 +15,16 @@ const Outcome = Schema.Struct({ error: Schema.optionalKey(Schema.String), }); +const ConflictHolder = Schema.Union([ + Schema.Struct({ stackId: Schema.String, stateRoot: Schema.String }), + Schema.Literal("foreign"), +]); +const Conflict = Schema.Struct({ + port: Schema.Int, + endpoint: Schema.String, + holder: ConflictHolder, +}); + /** A typed failure returned by the stack owner. */ export class StackError extends Schema.TaggedError()("StackError", { operation: Schema.String, @@ -27,12 +38,54 @@ export class StackError extends Schema.TaggedError()("StackError", { reason: Schema.optionalKey( Schema.Literals(["owner-unavailable", "release-mismatch", "runtime-unavailable"]), ), + /** The contested public port, when the failure is a port reservation conflict. */ + conflict: Schema.optionalKey(Conflict), }) {} +const isHolder = ( + value: unknown, +): value is { readonly stackId: string; readonly stateRoot: string } => + Predicate.hasProperty(value, "stackId") && + typeof value.stackId === "string" && + Predicate.hasProperty(value, "stateRoot") && + typeof value.stateRoot === "string"; + +const isPortConflict = (value: unknown): value is PortConflict => + Predicate.hasProperty(value, "port") && + typeof value.port === "number" && + Predicate.hasProperty(value, "endpoint") && + typeof value.endpoint === "string" && + Predicate.hasProperty(value, "holder") && + (value.holder === "foreign" || isHolder(value.holder)); + +/** The first `conflict` found by walking a failure's `cause` chain, if any carries one. */ +const findConflict = (cause: unknown, depth = 0): PortConflict | undefined => { + if (depth > 10 || typeof cause !== "object" || cause === null) return undefined; + if ("conflict" in cause && isPortConflict(cause.conflict)) return cause.conflict; + if ("cause" in cause) return findConflict(cause.cause, depth + 1); + return undefined; +}; + +/** A composition failure's member outcomes, each a plain Effect `Exit`, not just this cause's own. */ +const conflictFromOutcomes = ( + outcomes: ReadonlyArray<{ readonly result: Exit.Exit }> | undefined, +): PortConflict | undefined => { + if (outcomes === undefined) return undefined; + for (const { result } of outcomes) { + if (!Exit.isFailure(result)) continue; + const failure = Cause.findErrorOption(result.cause); + if (Option.isNone(failure)) continue; + const conflict = findConflict(failure.value); + if (conflict !== undefined) return conflict; + } + return undefined; +}; + /** Maps an owner failure to the RPC error, preserving per-member composition outcomes. */ export const stackError = (operation: string, cause: unknown): StackError => { if (Schema.is(StackError)(cause)) return cause; - if (cause instanceof OrchestratorError && cause.outcomes !== undefined) + if (cause instanceof OrchestratorError && cause.outcomes !== undefined) { + const conflict = findConflict(cause) ?? conflictFromOutcomes(cause.outcomes); return new StackError({ operation, message: failureMessage(cause), @@ -41,8 +94,15 @@ export const stackError = (operation: string, cause: unknown): StackError => { succeeded: Exit.isSuccess(result), ...(Exit.isFailure(result) ? { error: causeMessage(result.cause) } : {}), })), + ...(conflict === undefined ? {} : { conflict }), }); - return new StackError({ operation, message: failureMessage(cause) }); + } + const conflict = findConflict(cause); + return new StackError({ + operation, + message: failureMessage(cause), + ...(conflict === undefined ? {} : { conflict }), + }); }; const ServiceErrorSchema = Schema.TaggedStruct("ServiceError", { @@ -64,15 +124,11 @@ export const Observation = Schema.Struct({ lifecycle: Schema.Literals(["stopped", "starting", "running", "stopping"]), health: Schema.UndefinedOr(Schema.Literals(["starting", "healthy", "unhealthy"])), error: Schema.UndefinedOr(ServiceErrorSchema), - cleanupError: Schema.UndefinedOr(ServiceErrorSchema), exit: Schema.UndefinedOr(Schema.Exit(Schema.Void, ServiceErrorSchema, Schema.Defect())), currentOperation: Schema.UndefinedOr( Schema.Literals(["start", "stop", "restart", "storage", "destroy", "sleep"]), ), - launchId: Schema.UndefinedOr(Schema.Int), - intentRevision: Schema.Int, wakeEnabled: Schema.Boolean, - registered: Schema.Boolean, }); export interface Observation extends Schema.Schema.Type {} diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts index 2769658da9..fdab67ce9c 100644 --- a/packages/stack/src/Service.integration.test.ts +++ b/packages/stack/src/Service.integration.test.ts @@ -1,18 +1,18 @@ import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, Fiber, Queue, Ref, Scope, Stream } from "effect"; +import { Deferred, Effect, Exit, Fiber, Option, Queue, Ref, Scope, Stream } from "effect"; +import { makeStandaloneService } from "../tests/standalone-service.ts"; +import { LifecycleEvent } from "./Lifecycle.ts"; +import type { Status } from "./Orchestrator.ts"; import { makeService, - ServiceDestroyed, ServiceError, ServiceLaunchError, - ServiceStaleLaunch, type RuntimeSession, type ServiceDefinition, - type ServiceInstance, - type ServiceObservation, } from "./Service.ts"; type Config = { readonly version: number }; +type Standalone = Effect.Success>>; interface ResourceState { readonly phase: "created" | "launched" | "stopping" | "stopped"; @@ -39,7 +39,6 @@ interface RuntimePlan { interface PreparationPlan { readonly gate: Deferred.Deferred; readonly started: Deferred.Deferred; - readonly completed: Deferred.Deferred; } const makeRuntimePlan = Effect.gen(function* () { @@ -68,76 +67,48 @@ const makePreparationPlan = Effect.gen(function* () { return { gate: yield* Deferred.make(), started: yield* Deferred.make(), - completed: yield* Deferred.make(), } satisfies PreparationPlan; }); const open = (deferred: Deferred.Deferred) => Deferred.succeed(deferred, undefined); -const waitForObservation = ( - service: ServiceInstance, - predicate: (observation: ServiceObservation) => boolean, -) => +const waitForStatus = (service: Standalone, predicate: (status: Status) => boolean) => service.observation.pipe( Stream.filter(predicate), Stream.runHead, Effect.asVoid, - Effect.forkScoped, + Effect.forkScoped({ startImmediately: true }), ); const makeDefinition = ( plans: Queue.Queue, - preparations?: Queue.Queue, - prepared?: Ref.Ref>, - invalid?: Ref.Ref, - launchFailure?: Ref.Ref, + options: { + readonly preparations?: Queue.Queue; + readonly invalid?: Ref.Ref; + } = {}, ): ServiceDefinition => ({ - prepare: - preparations === undefined && prepared === undefined && invalid === undefined - ? undefined - : (config) => { - const queue = preparations; - return Effect.gen(function* () { - if (invalid !== undefined && (yield* Ref.get(invalid))) { - return yield* new ServiceError({ - operation: "prepare", - message: "invalid configuration", - }); - } - if (prepared !== undefined) - yield* Ref.update(prepared, (values) => [...values, config]); - if (queue !== undefined) { - const preparation = yield* Queue.take(queue); - yield* Deferred.succeed(preparation.started, undefined); - yield* Deferred.await(preparation.gate); - yield* Deferred.succeed(preparation.completed, undefined); - } - }); - }, + prepare: () => + Effect.gen(function* () { + if (options.invalid !== undefined && (yield* Ref.get(options.invalid))) + return yield* new ServiceError({ operation: "prepare", message: "invalid configuration" }); + if (options.preparations === undefined) return; + const preparation = yield* Queue.take(options.preparations); + yield* open(preparation.started); + yield* Deferred.await(preparation.gate); + }), launch: (_context) => Effect.gen(function* () { const plan = yield* Queue.take(plans); - yield* Deferred.succeed(plan.launchStarted, undefined); + yield* open(plan.launchStarted); yield* Deferred.await(plan.launchGate); - if (launchFailure !== undefined && (yield* Ref.get(launchFailure))) { - yield* Ref.set(launchFailure, false); - return yield* new ServiceError({ operation: "launch", message: "launch failed" }); - } yield* Ref.update(plan.state, (state): ResourceState => ({ ...state, phase: "launched" })); const session: RuntimeSession = { health: Effect.gen(function* () { - const state = yield* Ref.get(plan.state); - if (state.health !== "not-started") { - return yield* new ServiceError({ - operation: "health", - message: "health started twice", - }); - } yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, health: "starting", })); - yield* Deferred.succeed(plan.healthStarted, undefined); + yield* open(plan.healthStarted); yield* Deferred.await(plan.healthGate); yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, @@ -150,24 +121,20 @@ const makeDefinition = ( ...value, phase: "stopping", })); - yield* Deferred.succeed(plan.stopStarted, undefined); - if (yield* Ref.get(plan.stopFailure)) { - yield* Ref.set(plan.stopFailure, false); + yield* open(plan.stopStarted); + if (yield* Ref.getAndSet(plan.stopFailure, false)) return yield* new ServiceError({ operation: "stop", message: "stop failed" }); - } - yield* Deferred.succeed(plan.stopRetryStarted, undefined); + yield* open(plan.stopRetryStarted); yield* Deferred.await(plan.stopGate); yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, phase: "stopped" })); }), remove: Effect.gen(function* () { - yield* Deferred.succeed(plan.removeStarted, undefined); - if (yield* Ref.get(plan.removeFailure)) { - yield* Ref.set(plan.removeFailure, false); + yield* open(plan.removeStarted); + if (yield* Ref.getAndSet(plan.removeFailure, false)) return yield* new ServiceError({ operation: "remove", message: "exact cleanup failed", }); - } yield* Deferred.await(plan.removeGate); yield* Ref.update(plan.state, (value) => ({ ...value, removed: true })); }), @@ -177,61 +144,108 @@ const makeDefinition = ( removeData: () => Effect.void, }); -const makeFixture = Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const service = yield* makeService(makeDefinition(plans), { - id: "database-1", - config: { version: 17 }, +const makeFixture = (id: string, definition = makeDefinition) => + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const service = yield* makeStandaloneService(definition(plans), { + id, + config: { version: 17 }, + }); + return { plans, service }; }); - return { plans, service }; -}); -const stopFixture = (service: ServiceInstance, plan: RuntimePlan) => +const partialFailure = new ServiceError({ operation: "launch", message: "initialization failed" }); + +/** Acquires a runtime, then fails the launch with it retained for cleanup. */ +const partialDefinition = (plans: Queue.Queue): ServiceDefinition => { + const base = makeDefinition(plans); + return { + launch: (context) => + base + .launch(context) + .pipe( + Effect.flatMap((runtime) => + Effect.fail(new ServiceLaunchError({ failure: partialFailure, runtime })), + ), + ), + removeData: base.removeData, + }; +}; + +/** + * Resolves once the next change after the current status is published. The subscription is live + * when this returns, so a destroy queued behind a held storage operation, the only change in that + * window, can't be missed. + */ +const awaitNextChange = (service: Standalone) => + Effect.gen(function* () { + const subscribed = yield* Deferred.make(); + const change = yield* service.observation.pipe( + Stream.zipWithIndex, + Stream.tap(([, index]) => + index === 0 ? Deferred.succeed(subscribed, undefined) : Effect.void, + ), + Stream.drop(1), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + return change; + }); + +const stopFixture = (service: Standalone, plan: RuntimePlan) => Effect.gen(function* () { yield* open(plan.stopGate); yield* open(plan.removeGate); yield* service.stop; }); -describe("service kernel", () => { - it.live("submits stop while launch is delayed and completes it after launch", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - const starting = yield* fixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(plan.launchStarted); - - const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); - expect((yield* fixture.service.get).lifecycle).toBe("starting"); - yield* open(plan.launchGate); - yield* Fiber.join(starting); - yield* Deferred.await(plan.stopStarted); - yield* open(plan.stopGate); - yield* Deferred.await(plan.removeStarted); - yield* open(plan.removeGate); - yield* Fiber.join(stopping); +describe("service execution", () => { + it.live( + "stops a launch requested to stop mid-launch once its runtime exists, then cleans it", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture("database-mid-launch"); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + const starting = yield* fixture.service.start.pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(plan.launchStarted); + + const stopRequested = yield* waitForStatus( + fixture.service, + (status) => status.lifecycle === "stopping", + ); + const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Fiber.join(stopRequested); + yield* open(plan.launchGate); + yield* Deferred.await(plan.stopStarted); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + yield* Fiber.join(stopping); - expect((yield* fixture.service.get).lifecycle).toBe("stopped"); - expect((yield* Ref.get(plan.state)).removed).toBe(true); - }), - ), + expect(Exit.isFailure(yield* Fiber.join(starting))).toBe(true); + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), ); - it.live("shares one health program and a canceled readiness waiter does not poison it", () => + it.live("shares one health check and a cancelled readiness caller does not poison it", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const fixture = yield* makeFixture("database-shared-health"); const plan = yield* makeRuntimePlan; yield* Queue.offer(fixture.plans, plan); yield* open(plan.launchGate); yield* fixture.service.start; yield* Deferred.await(plan.healthStarted); - const canceled = yield* fixture.service.ready.pipe(Effect.forkScoped); + const cancelled = yield* fixture.service.ready.pipe(Effect.forkScoped); const ready = yield* fixture.service.ready.pipe(Effect.forkScoped); - yield* Fiber.interrupt(canceled); + yield* Fiber.interrupt(cancelled); yield* open(plan.healthGate); expect(Exit.isSuccess(yield* Fiber.await(ready))).toBe(true); expect((yield* Ref.get(plan.state)).health).toBe("healthy"); @@ -241,116 +255,75 @@ describe("service kernel", () => { ), ); - it.live("cancels after preparation while waiting for the lifecycle gate", () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const preparation = yield* makePreparationPlan; - const retryPreparation = yield* makePreparationPlan; - yield* Queue.offer(preparations, preparation); - yield* Queue.offer(preparations, retryPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-gate-wait", - config: { version: 17 }, - }); - const plan = yield* makeRuntimePlan; - yield* Queue.offer(plans, plan); - - const storageStarted = yield* Deferred.make(); - const storageGate = yield* Deferred.make(); - const storage = yield* service - .storage( - Effect.gen(function* () { - yield* Deferred.succeed(storageStarted, undefined); - yield* Deferred.await(storageGate); - }), - ) - .pipe(Effect.forkScoped); - yield* Deferred.await(storageStarted); - - const start = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(preparation.started); - yield* open(preparation.gate); - yield* Deferred.await(preparation.completed); - yield* Fiber.interrupt(start); - yield* open(storageGate); - yield* Fiber.join(storage); - - expect(yield* Deferred.isDone(plan.launchStarted)).toBe(false); - const retry = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(retryPreparation.started); - yield* open(retryPreparation.gate); - yield* open(plan.launchGate); - yield* Fiber.join(retry); - expect((yield* service.get).lifecycle).toBe("running"); - yield* stopFixture(service, plan); - }), - ), - ); - - it.live("records a failed wake preparation on the observation", () => + it.live("records a failed preparation on the observation and stays wakeable", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const invalid = yield* Ref.make(true); - const service = yield* makeService(makeDefinition(plans, undefined, undefined, invalid), { - id: "database-wake-prepare", + const service = yield* makeStandaloneService(makeDefinition(plans, { invalid }), { + id: "database-prepare-failure", config: { version: 17 }, }); - yield* service.arm; - const failure = yield* service.startAt(0, undefined, true).pipe(Effect.flip); - expect(failure).toBeInstanceOf(ServiceError); - const observation = yield* service.get; - expect(observation.lifecycle).toBe("stopped"); - expect(observation.wakeEnabled).toBe(true); - expect(observation.error?.message).toBe("invalid configuration"); + const failure = yield* service.start.pipe(Effect.flip); + expect(failure.message).toContain("invalid configuration"); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopped", + wakeEnabled: true, + error: { message: "invalid configuration" }, + }); }), ), ); - it.live("keeps a stale preparation failure off a relaunched observation", () => + it.live("keeps a stopped launch's late preparation failure off its successor", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const staleStarted = yield* Deferred.make(); const staleGate = yield* Deferred.make(); const attempts = yield* Ref.make(0); - const service = yield* makeService( + const service = yield* makeStandaloneService( { ...makeDefinition(plans), + // Uninterruptible, so the stop has to wait it out and its failure arrives late. prepare: () => Effect.gen(function* () { if ((yield* Ref.updateAndGet(attempts, (value) => value + 1)) > 1) return; - yield* Deferred.succeed(staleStarted, undefined); + yield* open(staleStarted); yield* Deferred.await(staleGate); return yield* new ServiceError({ operation: "prepare", message: "stale preparation", }); - }), + }).pipe(Effect.uninterruptible), }, { id: "database-stale-failure", config: { version: 17 } }, ); const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); yield* open(plan.launchGate); - const stale = yield* service.start.pipe(Effect.forkScoped); + const stale = yield* service.start.pipe(Effect.exit, Effect.forkScoped); yield* Deferred.await(staleStarted); - yield* service.start; - expect((yield* service.get).lifecycle).toBe("running"); - + // The stop of generation 1 is admitted, and waiting on its preparation, before it fails. + const stopAdmitted = yield* waitForStatus( + service, + (status) => status.lifecycle === "stopping", + ); + const restarted = yield* service.restart().pipe(Effect.forkScoped); + yield* Fiber.join(stopAdmitted); yield* open(staleGate); - expect(Exit.isFailure(yield* Fiber.await(stale))).toBe(true); - const observation = yield* service.get; - expect(observation.lifecycle).toBe("running"); - expect(observation.error).toBeUndefined(); + yield* Fiber.join(restarted); + + // The earlier start is satisfied by the restarted generation. + expect(Exit.isSuccess(yield* Fiber.join(stale))).toBe(true); + expect(yield* service.get).toMatchObject({ lifecycle: "running", error: undefined }); + expect((yield* Ref.get(plan.state)).phase).toBe("launched"); yield* stopFixture(service, plan); }), ), ); - it.live("does not prepare concurrent starts more than once", () => + it.live("prepares and launches once for concurrent starts", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); @@ -359,8 +332,8 @@ describe("service kernel", () => { const duplicatePreparation = yield* makePreparationPlan; yield* Queue.offer(preparations, preparation); yield* Queue.offer(preparations, duplicatePreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-concurrent-prepare", + const service = yield* makeStandaloneService(makeDefinition(plans, { preparations }), { + id: "database-concurrent-start", config: { version: 17 }, }); const plan = yield* makeRuntimePlan; @@ -368,103 +341,97 @@ describe("service kernel", () => { const first = yield* service.start.pipe(Effect.forkScoped); yield* Deferred.await(preparation.started); - yield* open(preparation.gate); - yield* Deferred.await(preparation.completed); - yield* Deferred.await(plan.launchStarted); const second = yield* service.start.pipe(Effect.forkScoped({ startImmediately: true })); + yield* open(preparation.gate); yield* open(plan.launchGate); yield* Fiber.join(first); - expect(yield* Deferred.isDone(duplicatePreparation.started)).toBe(false); yield* Fiber.join(second); + expect(yield* Deferred.isDone(duplicatePreparation.started)).toBe(false); expect((yield* service.get).lifecycle).toBe("running"); yield* stopFixture(service, plan); }), ), ); - it.live("reprepares a queued candidate after the current launch fails", () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const firstPreparation = yield* makePreparationPlan; - const secondPreparation = yield* makePreparationPlan; - yield* Queue.offer(preparations, firstPreparation); - yield* Queue.offer(preparations, secondPreparation); - const launchFailure = yield* Ref.make(true); - const service = yield* makeService( - makeDefinition(plans, preparations, undefined, undefined, launchFailure), - { id: "database-retry-prepare", config: { version: 17 } }, - ); - const firstPlan = yield* makeRuntimePlan; - const secondPlan = yield* makeRuntimePlan; - yield* Queue.offer(plans, firstPlan); - yield* Queue.offer(plans, secondPlan); - - const first = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(firstPreparation.started); - yield* open(firstPreparation.gate); - yield* Deferred.await(firstPlan.launchStarted); - const second = yield* service - .startAt(0, { version: 18 }, false, Effect.void) - .pipe(Effect.forkScoped({ startImmediately: true })); - yield* open(firstPlan.launchGate); - expect(Exit.isFailure(yield* Fiber.await(first))).toBe(true); - yield* Deferred.await(secondPreparation.started); - yield* open(secondPreparation.gate); - yield* open(secondPlan.launchGate); - yield* Fiber.join(second); - expect((yield* service.get).config.version).toBe(18); - yield* stopFixture(service, secondPlan); - }), - ), - ); - - it.live("invalidates a prepared start when stop is admitted before preparation finishes", () => + it.live("skips the launch when a stop arrives during preparation", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const preparations = yield* Queue.unbounded(); const preparation = yield* makePreparationPlan; yield* Queue.offer(preparations, preparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-stale-preparation", + const service = yield* makeStandaloneService(makeDefinition(plans, { preparations }), { + id: "database-stop-in-preparation", config: { version: 17 }, }); const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); - const start = yield* service.start.pipe(Effect.forkScoped); + const start = yield* service.start.pipe(Effect.exit, Effect.forkScoped); yield* Deferred.await(preparation.started); yield* service.stop; yield* open(preparation.gate); - const result = yield* Fiber.await(start); - expect(Exit.isFailure(result)).toBe(true); + expect(Exit.isFailure(yield* Fiber.join(start))).toBe(true); expect(yield* Deferred.isDone(plan.launchStarted)).toBe(false); expect((yield* service.get).lifecycle).toBe("stopped"); }), ), ); - it.live("records a launch failure as stopped and can launch successfully on retry", () => + it.live("never launches a restart whose service is destroyed while it prepares", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const initialPreparation = yield* makePreparationPlan; + const restartPreparation = yield* makePreparationPlan; + yield* open(initialPreparation.gate); + yield* Queue.offer(preparations, initialPreparation); + yield* Queue.offer(preparations, restartPreparation); + const service = yield* makeStandaloneService(makeDefinition(plans, { preparations }), { + id: "database-late-restart", + config: { version: 17 }, + }); + const original = yield* makeRuntimePlan; + const superseded = yield* makeRuntimePlan; + yield* Queue.offer(plans, original); + yield* Queue.offer(plans, superseded); + yield* open(original.launchGate); + yield* service.start; + + const restarting = yield* service.restart({ version: 18 }).pipe(Effect.forkScoped); + yield* Deferred.await(restartPreparation.started); + yield* open(original.stopGate); + yield* open(original.removeGate); + yield* service.destroy; + expect(yield* service.listed).toBe(false); + expect((yield* Ref.get(original.state)).removed).toBe(true); + + yield* open(restartPreparation.gate); + expect(Exit.isFailure(yield* Fiber.await(restarting))).toBe(true); + expect(yield* Deferred.isDone(superseded.launchStarted)).toBe(false); + }), + ), + ); + + it.live("records a launch failure as stopped and launches successfully on retry", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const failLaunch = yield* Ref.make(true); const definition = makeDefinition(plans); - const service = yield* makeService( + const service = yield* makeStandaloneService( { launch: (context) => - Effect.gen(function* () { - if (yield* Ref.get(failLaunch)) { - yield* Ref.set(failLaunch, false); - return yield* new ServiceError({ - operation: "launch", - message: "binary unavailable", - }); - } - return yield* definition.launch(context); - }), + Ref.getAndSet(failLaunch, false).pipe( + Effect.flatMap((fail) => + fail + ? Effect.fail( + new ServiceError({ operation: "launch", message: "binary unavailable" }), + ) + : definition.launch(context), + ), + ), removeData: definition.removeData, }, { id: "database-launch-retry", config: { version: 17 } }, @@ -472,10 +439,11 @@ describe("service kernel", () => { const plan = yield* makeRuntimePlan; yield* Queue.offer(plans, plan); - const failed = yield* service.start.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopped"); - expect((yield* service.get).error?.operation).toBe("launch"); + expect(Exit.isFailure(yield* service.start.pipe(Effect.exit))).toBe(true); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopped", + error: { operation: "launch" }, + }); expect((yield* Ref.get(plan.state)).phase).toBe("created"); yield* open(plan.launchGate); @@ -486,392 +454,348 @@ describe("service kernel", () => { ), ); - it.live("retains a runtime acquired before launch failure for exact cleanup", () => + it.live("retains a runtime acquired before a launch failure for exact cleanup", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const failure = partialFailure; + const partial = yield* makeFixture("database-partial", partialDefinition); const plan = yield* makeRuntimePlan; - const base = makeDefinition(fixture.plans); - const failure = new ServiceError({ operation: "launch", message: "initialization failed" }); - const partial = yield* makeService( - { - launch: (context) => - base - .launch(context) - .pipe( - Effect.flatMap((runtime) => - Effect.fail(new ServiceLaunchError({ failure, runtime })), - ), - ), - removeData: base.removeData, - }, - { id: "database-partial", config: { version: 17 } }, - ); - yield* Queue.offer(fixture.plans, plan); + yield* Queue.offer(partial.plans, plan); yield* open(plan.launchGate); yield* Ref.set(plan.stopFailure, true); - const failed = yield* partial.start.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* partial.get).lifecycle).toBe("stopping"); - expect((yield* partial.get).error).toEqual(failure); - expect((yield* partial.get).cleanupError).toBeDefined(); + expect(Exit.isFailure(yield* partial.service.start.pipe(Effect.exit))).toBe(true); + expect(yield* partial.service.get).toMatchObject({ lifecycle: "stopping", error: failure }); expect((yield* Ref.get(plan.state)).removed).toBe(false); - const retry = yield* partial.stop.pipe(Effect.forkScoped); + const retry = yield* partial.service.stop.pipe(Effect.forkScoped); yield* Deferred.await(plan.stopRetryStarted); yield* open(plan.stopGate); yield* Deferred.await(plan.removeStarted); yield* open(plan.removeGate); yield* Fiber.join(retry); - expect((yield* partial.get).lifecycle).toBe("stopped"); + expect((yield* partial.service.get).lifecycle).toBe("stopped"); expect((yield* Ref.get(plan.state)).removed).toBe(true); - const plans2 = yield* Queue.unbounded(); - const plan2 = yield* makeRuntimePlan; - const base2 = makeDefinition(plans2); - const cleanFailure = new ServiceError({ - operation: "launch", - message: "clean partial failure", - }); - const cleaned = yield* makeService( - { - launch: (context) => - base2 - .launch(context) - .pipe( - Effect.flatMap((runtime) => - Effect.fail(new ServiceLaunchError({ failure: cleanFailure, runtime })), - ), - ), - removeData: base2.removeData, - }, - { id: "database-partial-clean", config: { version: 17 } }, - ); - yield* Queue.offer(plans2, plan2); - yield* open(plan2.launchGate); - yield* open(plan2.stopGate); - yield* open(plan2.removeGate); - expect(Exit.isFailure(yield* cleaned.start.pipe(Effect.exit))).toBe(true); - expect((yield* cleaned.get).lifecycle).toBe("stopped"); - expect((yield* Ref.get(plan2.state)).removed).toBe(true); + const cleaned = yield* makeFixture("database-partial-clean", partialDefinition); + const cleanPlan = yield* makeRuntimePlan; + yield* Queue.offer(cleaned.plans, cleanPlan); + yield* open(cleanPlan.launchGate); + yield* open(cleanPlan.stopGate); + yield* open(cleanPlan.removeGate); + expect(Exit.isFailure(yield* cleaned.service.start.pipe(Effect.exit))).toBe(true); + expect((yield* cleaned.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(cleanPlan.state)).removed).toBe(true); }), ), ); - it.live("admits a prepared start after restart and does not relaunch", () => - Effect.scoped( - Effect.gen(function* () { - const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const queuedStartPreparation = yield* makePreparationPlan; - const restartPreparation = yield* makePreparationPlan; - yield* Queue.offer(preparations, queuedStartPreparation); - yield* Queue.offer(preparations, restartPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-start-after-restart", - config: { version: 17 }, - }); - const first = yield* makeRuntimePlan; - const unused = yield* makeRuntimePlan; - yield* Queue.offer(plans, first); - yield* Queue.offer(plans, unused); + it.live( + "keeps a retained runtime pending cleanup, refusing storage, until a stop retry confirms its removal", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture("database-retained", partialDefinition); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* Ref.set(plan.stopFailure, true); + expect(Exit.isFailure(yield* fixture.service.start.pipe(Effect.exit))).toBe(true); + + yield* Ref.set(plan.stopFailure, true); + expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); + expect( + Exit.isFailure(yield* fixture.service.storage(Effect.void).pipe(Effect.exit)), + ).toBe(true); + + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* fixture.service.stop; + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), + ); - const queuedStart = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(queuedStartPreparation.started); + it.live( + "fails a restart whose stop fails instead of waiting, and retries the cleanup on stop", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture("database-restart-stop-failure"); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* Ref.set(plan.stopFailure, true); - const restart = yield* service.restart().pipe(Effect.forkScoped); - yield* Deferred.await(restartPreparation.started); - yield* open(restartPreparation.gate); - yield* open(first.launchGate); - yield* Fiber.join(restart); - expect((yield* service.get).lifecycle).toBe("running"); + const failed = yield* fixture.service + .restart() + .pipe(Effect.flip, Effect.timeoutOption("5 seconds")); + expect(Option.getOrUndefined(failed)?.message).toContain("stop failed"); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); - yield* open(queuedStartPreparation.gate); - expect(Exit.isSuccess(yield* Fiber.await(queuedStart))).toBe(true); - expect((yield* service.get).launchId).toBe(1); - expect(yield* Deferred.isDone(unused.launchStarted)).toBe(false); - yield* stopFixture(service, first); - }), - ), + yield* stopFixture(fixture.service, plan); + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), ); - it.live("accepts a prepared start queued during restart once restart is running", () => + it.live("never lets a late stop of an older generation touch a newer session", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); - const preparations = yield* Queue.unbounded(); - const initialPreparation = yield* makePreparationPlan; - const queuedStartPreparation = yield* makePreparationPlan; - const restartPreparation = yield* makePreparationPlan; - yield* open(initialPreparation.gate); - yield* Queue.offer(preparations, initialPreparation); - yield* Queue.offer(preparations, restartPreparation); - yield* Queue.offer(preparations, queuedStartPreparation); - const service = yield* makeService(makeDefinition(plans, preparations), { - id: "database-restart-queue", + const service = yield* makeService(makeDefinition(plans), { + id: "database-late-stop", config: { version: 17 }, + report: () => Effect.void, }); const first = yield* makeRuntimePlan; const second = yield* makeRuntimePlan; yield* Queue.offer(plans, first); yield* Queue.offer(plans, second); - yield* open(first.launchGate); - yield* service.start; - - const restartRunning = yield* waitForObservation( - service, - (value) => value.currentOperation === "restart", - ); - const restart = yield* service.restart().pipe(Effect.forkScoped); - yield* Deferred.await(restartPreparation.started); - yield* open(restartPreparation.gate); - yield* Fiber.join(restartRunning); - const queuedStart = yield* service.start.pipe(Effect.forkScoped); - yield* Deferred.await(queuedStartPreparation.started); - yield* Deferred.await(first.stopStarted); - yield* open(first.stopGate); - yield* Deferred.await(first.removeStarted); - yield* open(first.removeGate); - yield* Deferred.await(second.launchStarted); - yield* open(queuedStartPreparation.gate); - yield* open(second.launchGate); - yield* Fiber.join(restart); - expect(Exit.isSuccess(yield* Fiber.await(queuedStart))).toBe(true); - expect((yield* service.get).lifecycle).toBe("running"); - yield* stopFixture(service, second); - }), - ), - ); - - it.live("keeps an admitted launch alive after its caller is canceled", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - const running = yield* waitForObservation( - fixture.service, - (value) => value.lifecycle === "running", - ); - const caller = yield* fixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(plan.launchStarted); - yield* Fiber.interrupt(caller); - yield* open(plan.launchGate); - yield* Fiber.join(running); - expect((yield* Ref.get(plan.state)).phase).toBe("launched"); - yield* stopFixture(fixture.service, plan); + for (const plan of [first, second]) + for (const gate of [plan.launchGate, plan.healthGate, plan.stopGate, plan.removeGate]) + yield* open(gate); + const stopOptions = { operation: "stop", discard: false } as const; + + yield* service.launch(1, { version: 17 }); + yield* service.stop(1, stopOptions); + yield* service.launch(2, { version: 17 }); + yield* service.stop(1, stopOptions); + + expect((yield* Ref.get(second.state)).phase).toBe("launched"); + yield* service.stop(2, stopOptions); + expect((yield* Ref.get(second.state)).removed).toBe(true); }), ), ); - it.live("lets independent instances progress concurrently", () => - Effect.scoped( - Effect.gen(function* () { - const leftFixture = yield* makeFixture; - const rightFixture = yield* makeFixture; - const leftPlan = yield* makeRuntimePlan; - const rightPlan = yield* makeRuntimePlan; - yield* Queue.offer(leftFixture.plans, leftPlan); - yield* Queue.offer(rightFixture.plans, rightPlan); - const leftRunning = yield* waitForObservation( - leftFixture.service, - (value) => value.lifecycle === "running", - ); - const rightRunning = yield* waitForObservation( - rightFixture.service, - (value) => value.lifecycle === "running", - ); - const leftStart = yield* leftFixture.service.start.pipe(Effect.forkScoped); - const rightStart = yield* rightFixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(leftPlan.launchStarted); - yield* Deferred.await(rightPlan.launchStarted); - yield* open(leftPlan.launchGate); - yield* Fiber.join(leftRunning); - expect((yield* leftFixture.service.get).lifecycle).toBe("running"); - expect((yield* rightFixture.service.get).lifecycle).toBe("starting"); - yield* open(rightPlan.launchGate); - yield* Fiber.join(leftStart); - yield* Fiber.join(rightRunning); - yield* Fiber.join(rightStart); - yield* stopFixture(leftFixture.service, leftPlan); - yield* stopFixture(rightFixture.service, rightPlan); - }), - ), + it.live( + "keeps a launch, and the resources its scope owns, alive after its caller is cancelled", + () => + Effect.scoped( + Effect.gen(function* () { + const resourceClosed = yield* Deferred.make(); + const launchStarted = yield* Deferred.make(); + const launchGate = yield* Deferred.make(); + const service = yield* makeStandaloneService( + { + launch: (context) => + Effect.gen(function* () { + yield* Scope.addFinalizer( + context.scope, + open(resourceClosed).pipe(Effect.asVoid), + ); + yield* open(launchStarted); + yield* Deferred.await(launchGate); + return { + health: Effect.void, + exit: Effect.never, + stop: Effect.void, + remove: Effect.void, + } satisfies RuntimeSession; + }), + removeData: () => Effect.void, + }, + { id: "cold-launch-resource", config: { version: 1 } }, + ); + const caller = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(launchStarted); + yield* Fiber.interrupt(caller); + expect(yield* Deferred.isDone(resourceClosed)).toBe(false); + yield* open(launchGate); + yield* service.ready; + expect((yield* service.get).lifecycle).toBe("running"); + expect(yield* Deferred.isDone(resourceClosed)).toBe(false); + yield* service.stop; + expect(yield* Deferred.isDone(resourceClosed)).toBe(true); + }), + ), ); - it.live("keeps exact cleanup authority after failure and retries only remaining cleanup", () => + it.live("keeps exact cleanup authority after a failed stop and retries only what remains", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const fixture = yield* makeFixture("database-cleanup-retry"); const plan = yield* makeRuntimePlan; yield* Queue.offer(fixture.plans, plan); yield* open(plan.launchGate); yield* fixture.service.start; yield* open(plan.stopGate); yield* Ref.set(plan.removeFailure, true); - const failed = yield* fixture.service.stop.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.service.get).lifecycle).toBe("stopping"); + expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "stopping", + error: { operation: "remove" }, + }); expect((yield* Ref.get(plan.state)).removed).toBe(false); + + const retry = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Deferred.await(plan.removeStarted); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); yield* open(plan.removeGate); - yield* fixture.service.stop; + yield* Fiber.join(retry); expect((yield* fixture.service.get).lifecycle).toBe("stopped"); expect((yield* Ref.get(plan.state)).removed).toBe(true); }), ), ); - it.live("keeps registration after data removal failure", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - const dataFailure = yield* Ref.make(true); - const service = yield* makeService( - { - launch: makeDefinition(fixture.plans).launch, - removeData: () => - Effect.gen(function* () { - if (yield* Ref.get(dataFailure)) { - yield* Ref.set(dataFailure, false); - return yield* new ServiceError({ operation: "data", message: "retry" }); - } - }), - }, - { id: "database-data", config: { version: 17 } }, - ); - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); - yield* service.start; - yield* stopFixture(service, plan); - expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); - expect((yield* service.get).registered).toBe(true); - yield* service.destroy; - expect((yield* service.get).registered).toBe(false); - }), - ), + it.live( + "reports confirmed termination when removeData cleans up a session a failed stop retained", + () => + Effect.scoped( + Effect.gen(function* () { + // Direct `makeService`, not `makeStandaloneService`: the orchestrator's own dispatch + // already retries and reports a failed stop correctly (`Lifecycle.ts`'s `retryCleanup`) + // and so never exercises this path. Only a direct `removeData` call does. + const plans = yield* Queue.unbounded(); + const events = yield* Ref.make>([]); + const service = yield* makeService(makeDefinition(plans), { + id: "database-removedata-retry", + config: { version: 17 }, + report: (event) => Ref.update(events, (current) => [...current, event]), + }); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* open(plan.healthGate); + yield* service.launch(1, { version: 17 }); + + yield* Ref.set(plan.stopFailure, true); + yield* service.stop(1, { operation: "stop", discard: true }); + expect((yield* Ref.get(events)).some((event) => event._tag === "StopFailed")).toBe(true); + + // The retained session's retry, through `removeData` alone this time. + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* service.removeData(); + + expect( + (yield* Ref.get(events)).some((event) => event._tag === "Exited"), + "the retry's confirmed termination must reach the reducer, or it stays Stopping forever", + ).toBe(true); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), + ); + + it.live( + "keeps the registration and data of a running instance whose destroy fails to stop it", + () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const dataRemoved = yield* Ref.make(false); + const service = yield* makeStandaloneService( + { launch: makeDefinition(plans).launch, removeData: () => Ref.set(dataRemoved, true) }, + { id: "database-destroy-stop-failure", config: { version: 17 } }, + ); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* service.start; + yield* Ref.set(plan.stopFailure, true); + + expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); + expect(yield* service.get).toMatchObject({ lifecycle: "stopping" }); + expect(yield* service.listed).toBe(true); + expect(yield* Ref.get(dataRemoved)).toBe(false); + + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* service.destroy; + expect(yield* Ref.get(dataRemoved)).toBe(true); + expect(yield* service.listed).toBe(false); + }), + ), ); - it.live("destroys a running instance and retains registration after stop failure", () => + it.live("destroys an instance once the storage operation that owns it finishes", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); - yield* fixture.service.start; - yield* open(plan.stopGate); - yield* open(plan.removeGate); - yield* fixture.service.destroy; - expect((yield* fixture.service.get).registered).toBe(false); - expect((yield* Ref.get(plan.state)).removed).toBe(true); - const plans = yield* Queue.unbounded(); - const dataRemoved = yield* Ref.make(false); - const service = yield* makeService( + const storageStarted = yield* Deferred.make(); + const storageGate = yield* Deferred.make(); + const removalStarted = yield* Deferred.make(); + const removalGate = yield* Deferred.make(); + const service = yield* makeStandaloneService( { launch: makeDefinition(plans).launch, - removeData: () => Ref.set(dataRemoved, true), + removeData: () => + open(removalStarted).pipe(Effect.andThen(Deferred.await(removalGate))), }, - { id: "database-destroy-stop-failure", config: { version: 17 } }, + { id: "database-storage", config: { version: 17 } }, ); - const failedPlan = yield* makeRuntimePlan; - yield* Queue.offer(plans, failedPlan); - yield* open(failedPlan.launchGate); - yield* service.start; - yield* Ref.set(failedPlan.stopFailure, true); - - expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopping"); - expect((yield* service.get).registered).toBe(true); - expect(yield* Ref.get(dataRemoved)).toBe(false); - - yield* open(failedPlan.stopGate); - yield* open(failedPlan.removeGate); - yield* service.stop; - yield* service.destroy; - expect(yield* Ref.get(dataRemoved)).toBe(true); - expect((yield* service.get).registered).toBe(false); - }), - ), - ); + const storage = yield* service + .storage(open(storageStarted).pipe(Effect.andThen(Deferred.await(storageGate)))) + .pipe(Effect.forkScoped); + yield* Deferred.await(storageStarted); + expect((yield* service.get).currentOperation).toBe("storage"); - it.live("reports a destroyed instance as ServiceDestroyed to readiness callers", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - yield* fixture.service.destroy; - const result = yield* fixture.service.ready.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = result.cause.reasons.find(Cause.isFailReason); - expect(failure !== undefined && failure.error instanceof ServiceDestroyed).toBe(true); - } - }), - ), - ); + const destroyRequested = yield* awaitNextChange(service); + const destroyed = yield* service.destroy.pipe(Effect.forkScoped); + yield* Fiber.join(destroyRequested); + expect(yield* service.listed).toBe(true); + yield* open(storageGate); + yield* Fiber.join(storage); + yield* Deferred.await(removalStarted); + const refused = yield* service.storage(Effect.void).pipe(Effect.flip); + expect(refused.message).toBe("database-storage is being destroyed"); - it.live("waits for a pending launch before checking readiness", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - const start = yield* fixture.service.start.pipe(Effect.forkScoped); - yield* Deferred.await(plan.launchStarted); - const ready = yield* fixture.service.ready.pipe( - Effect.forkScoped({ startImmediately: true }), - ); - yield* open(plan.launchGate); - yield* Deferred.await(plan.healthStarted); - yield* open(plan.healthGate); - expect(Exit.isSuccess(yield* Fiber.await(ready))).toBe(true); - yield* Fiber.join(start); - yield* stopFixture(fixture.service, plan); + yield* open(removalGate); + yield* Fiber.join(destroyed); + expect(yield* service.listed).toBe(false); }), ), ); - it.live("serializes storage and destroy through the same gate", () => + it.live("keeps an instance usable when a destroy waiting behind storage is interrupted", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); const storageStarted = yield* Deferred.make(); const storageGate = yield* Deferred.make(); - const service = yield* makeService( - { - launch: makeDefinition(plans).launch, - removeData: () => Effect.void, - }, - { id: "database-storage", config: { version: 17 } }, + const dataRemoved = yield* Ref.make(false); + const service = yield* makeStandaloneService( + { launch: makeDefinition(plans).launch, removeData: () => Ref.set(dataRemoved, true) }, + { id: "database-storage-interrupted", config: { version: 17 } }, ); const storage = yield* service - .storage( - Effect.gen(function* () { - yield* Deferred.succeed(storageStarted, undefined); - yield* Deferred.await(storageGate); - }), - ) + .storage(open(storageStarted).pipe(Effect.andThen(Deferred.await(storageGate)))) .pipe(Effect.forkScoped); yield* Deferred.await(storageStarted); - const destroying = yield* service.destroy.pipe(Effect.forkScoped); - expect((yield* service.get).currentOperation).toBe("storage"); + + const destroyRequested = yield* awaitNextChange(service); + const destroyed = yield* service.destroy.pipe(Effect.forkScoped); + yield* Fiber.join(destroyRequested); + yield* Fiber.interrupt(destroyed); + expect(Exit.hasInterrupts(yield* Fiber.await(destroyed))).toBe(true); yield* open(storageGate); yield* Fiber.join(storage); - yield* Fiber.join(destroying); - expect((yield* service.get).registered).toBe(false); + + expect(yield* service.storage(Effect.succeed("stored"))).toBe("stored"); + expect(yield* Ref.get(dataRemoved)).toBe(false); + expect(yield* service.listed).toBe(true); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* service.start; + expect((yield* service.get).lifecycle).toBe("running"); + yield* stopFixture(service, plan); }), ), ); - it.live("prepares invalid restart configuration before stopping current runtime", () => + it.live("prepares an invalid restart configuration before stopping the current runtime", () => Effect.scoped( Effect.gen(function* () { const plans = yield* Queue.unbounded(); - const prepared = yield* Ref.make>([]); const invalid = yield* Ref.make(false); - const service = yield* makeService(makeDefinition(plans, undefined, prepared, invalid), { + const service = yield* makeStandaloneService(makeDefinition(plans, { invalid }), { id: "database-restart", config: { version: 17 }, }); @@ -883,234 +807,90 @@ describe("service kernel", () => { expect(Exit.isFailure(yield* service.restart({ version: 18 }).pipe(Effect.exit))).toBe( true, ); - expect((yield* service.get).lifecycle).toBe("running"); - expect((yield* service.get).config.version).toBe(17); + expect(yield* service.get).toMatchObject({ lifecycle: "running", config: { version: 17 } }); expect((yield* Ref.get(plan.state)).removed).toBe(false); yield* stopFixture(service, plan); }), ), ); - it.live("fails readiness with the crash error and cleans the exact runtime", () => + it.live("fails readiness with the crash error, cleans the exact runtime and stays wakeable", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; + const fixture = yield* makeFixture("database-crash"); const plan = yield* makeRuntimePlan; yield* Queue.offer(fixture.plans, plan); yield* open(plan.launchGate); yield* fixture.service.start; yield* Deferred.await(plan.healthStarted); - const ready = yield* fixture.service.ready.pipe(Effect.forkScoped); - const stopping = yield* waitForObservation( - fixture.service, - (value) => value.lifecycle === "stopping", + const ready = yield* fixture.service.ready.pipe( + Effect.flip, + Effect.forkScoped({ startImmediately: true }), ); - const stopped = yield* waitForObservation( + const stopped = yield* waitForStatus( fixture.service, - (value) => value.lifecycle === "stopped", + (value) => value.lifecycle === "stopped" && value.currentOperation === undefined, ); yield* Deferred.succeed( plan.exit, Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), ); yield* Deferred.await(plan.stopStarted); + expect((yield* fixture.service.get).currentOperation).toBe("stop"); yield* open(plan.stopGate); yield* Deferred.await(plan.removeStarted); yield* open(plan.removeGate); - yield* Fiber.join(stopping); yield* Fiber.join(stopped); - const result = yield* Fiber.await(ready); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = result.cause.reasons.find(Cause.isFailReason); - expect(failure?.error).toBeInstanceOf(ServiceError); - expect(failure?.error).toMatchObject({ operation: "process", message: "crashed" }); - } - expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + + expect((yield* Fiber.join(ready)).message).toContain("crashed"); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "stopped", + wakeEnabled: true, + error: { operation: "process", message: "crashed" }, + }); expect((yield* Ref.get(plan.state)).removed).toBe(true); - expect((yield* fixture.service.get).wakeEnabled).toBe(false); }), ), ); - it.live("reports stop as the current operation during exit cleanup", () => + it.live("fails an old readiness wait on stop and lets the next launch become ready", () => Effect.scoped( Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); + const fixture = yield* makeFixture("database-relaunch"); + const first = yield* makeRuntimePlan; + const second = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, first); + yield* Queue.offer(fixture.plans, second); + yield* open(first.launchGate); yield* fixture.service.start; - yield* Deferred.succeed( - plan.exit, - Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), - ); - yield* Deferred.await(plan.stopStarted); - expect((yield* fixture.service.get).currentOperation).toBe("stop"); - yield* open(plan.stopGate); - yield* Deferred.await(plan.removeStarted); - yield* open(plan.removeGate); - }), - ), - ); + yield* Deferred.await(first.healthStarted); - it.live("preserves cleanupError while a retry is admitted and blocked", () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const plan = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, plan); - yield* open(plan.launchGate); - yield* fixture.service.start; - yield* Ref.set(plan.stopFailure, true); - expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); - expect((yield* fixture.service.get).cleanupError).toBeDefined(); + const oldReady = yield* fixture.service.ready.pipe(Effect.exit, Effect.forkScoped); + const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Deferred.await(first.stopStarted); + yield* open(first.stopGate); + yield* Deferred.await(first.removeStarted); + yield* open(first.removeGate); + yield* Fiber.join(stopping); + expect(Exit.isFailure(yield* Fiber.join(oldReady))).toBe(true); - const retry = yield* fixture.service.stop.pipe(Effect.forkScoped); - yield* Deferred.await(plan.stopRetryStarted); - const exited = yield* waitForObservation( - fixture.service, - (value) => value.exit !== undefined, - ); - yield* Deferred.succeed( - plan.exit, - Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), - ); - yield* Fiber.join(exited); - expect((yield* fixture.service.get).cleanupError).toBeDefined(); - yield* open(plan.stopGate); - yield* Deferred.await(plan.removeStarted); - yield* open(plan.removeGate); - yield* Fiber.join(retry); - expect((yield* fixture.service.get).cleanupError).toBeUndefined(); + yield* open(second.launchGate); + yield* fixture.service.start; + yield* Deferred.await(second.healthStarted); + const newReady = yield* fixture.service.ready.pipe(Effect.forkScoped); + yield* open(second.healthGate); + expect(Exit.isSuccess(yield* Fiber.await(newReady))).toBe(true); + expect(yield* fixture.service.get).toMatchObject({ + lifecycle: "running", + health: "healthy", + }); + expect((yield* Ref.get(second.state)).health).toBe("healthy"); + yield* stopFixture(fixture.service, second); }), ), ); - - it.live( - "makes an old readiness wait stale across stop and lets the new launch become ready", - () => - Effect.scoped( - Effect.gen(function* () { - const fixture = yield* makeFixture; - const first = yield* makeRuntimePlan; - const second = yield* makeRuntimePlan; - yield* Queue.offer(fixture.plans, first); - yield* Queue.offer(fixture.plans, second); - yield* open(first.launchGate); - yield* fixture.service.start; - yield* Deferred.await(first.healthStarted); - - const oldReady = yield* fixture.service.ready.pipe(Effect.forkScoped); - const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); - yield* Deferred.await(first.stopStarted); - yield* open(first.stopGate); - yield* Deferred.await(first.removeStarted); - yield* open(first.removeGate); - yield* Fiber.join(stopping); - const stale = yield* Fiber.await(oldReady); - expect(Exit.isFailure(stale)).toBe(true); - if (Exit.isFailure(stale)) - expect(stale.cause.reasons.find(Cause.isFailReason)?.error).toBeInstanceOf( - ServiceStaleLaunch, - ); - - yield* open(second.launchGate); - yield* fixture.service.start; - yield* Deferred.await(second.healthStarted); - const newReady = yield* fixture.service.ready.pipe(Effect.forkScoped); - yield* open(second.healthGate); - expect(Exit.isSuccess(yield* Fiber.await(newReady))).toBe(true); - expect((yield* fixture.service.get).launchId).toBe(2); - yield* stopFixture(fixture.service, second); - }), - ), - ); }); -it.live("retains armed wake after a partially launched process exits during cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const exited = yield* Deferred.make>(); - const stopping = yield* Deferred.make(); - const finishStop = yield* Deferred.make(); - const failure = new ServiceError({ operation: "launch", message: "initialization failed" }); - const service = yield* makeService( - { - launch: () => - Effect.fail( - new ServiceLaunchError({ - failure, - runtime: { - health: Effect.never, - exit: Deferred.await(exited), - stop: Deferred.succeed(stopping, undefined).pipe( - Effect.andThen(Deferred.await(finishStop)), - ), - remove: Effect.void, - }, - }), - ), - removeData: () => Effect.void, - }, - { id: "armed-partial", config: {} }, - ); - yield* service.arm; - const observedExit = yield* waitForObservation(service, (state) => state.exit !== undefined); - const starting = yield* service.start.pipe(Effect.forkChild); - yield* Deferred.await(stopping); - yield* Deferred.succeed(exited, Exit.void); - yield* Fiber.join(observedExit); - yield* Deferred.succeed(finishStop, undefined); - expect(Exit.isFailure(yield* Fiber.await(starting))).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopped"); - expect((yield* service.get).wakeEnabled).toBe(true); - }), - ), -); - -it.live("keeps a sleeping instance armed when exit observation retries failed removal", () => - Effect.scoped( - Effect.gen(function* () { - const exited = yield* Deferred.make>(); - const failRemoval = yield* Ref.make(true); - const service = yield* makeService( - { - launch: () => - Effect.succeed({ - health: Effect.void, - exit: Deferred.await(exited), - stop: Effect.void, - remove: Ref.getAndSet(failRemoval, false).pipe( - Effect.flatMap((fail) => - fail - ? Effect.fail( - new ServiceError({ - operation: "remove", - message: "temporary removal failure", - }), - ) - : Effect.void, - ), - ), - }), - removeData: () => Effect.void, - }, - { id: "sleep-cleanup-retry", config: {} }, - ); - yield* service.arm; - yield* service.start; - yield* service.ready; - expect(Exit.isFailure(yield* Effect.exit(service.sleep))).toBe(true); - expect((yield* service.get).wakeEnabled).toBe(true); - const stopped = yield* waitForObservation(service, (state) => state.lifecycle === "stopped"); - yield* Deferred.succeed(exited, Exit.void); - yield* Fiber.join(stopped); - expect((yield* service.get).wakeEnabled).toBe(true); - }), - ), -); - const unhealthy = (message: string) => new ServiceError({ operation: "health", message }); const makeProbedService = ( @@ -1119,7 +899,7 @@ const makeProbedService = ( ) => Effect.gen(function* () { const launches = yield* Ref.make(0); - const service = yield* makeService( + const service = yield* makeStandaloneService( { launch: () => Ref.updateAndGet(launches, (count) => count + 1).pipe( @@ -1138,41 +918,14 @@ const makeProbedService = ( return { service, launches }; }); -const startUnhealthy = (service: ServiceInstance) => +const startUnhealthy = (service: Standalone) => Effect.gen(function* () { - const failed = yield* waitForObservation(service, (value) => value.health === "unhealthy"); + const failed = yield* waitForStatus(service, (value) => value.health === "unhealthy"); yield* service.start; yield* Fiber.join(failed); }); describe("service readiness recovery", () => { - it.live("serves readiness once a running launch recovers from a failed health check", () => - Effect.scoped( - Effect.gen(function* () { - const healthy = yield* Ref.make(false); - const { service, launches } = yield* makeProbedService(() => - Ref.get(healthy).pipe( - Effect.flatMap((ok) => (ok ? Effect.void : Effect.fail(unhealthy("still booting")))), - ), - ); - yield* startUnhealthy(service); - expect(yield* Effect.flip(service.ready)).toMatchObject({ message: "still booting" }); - - yield* Ref.set(healthy, true); - yield* service.ready; - - expect(yield* service.get).toMatchObject({ - lifecycle: "running", - health: "healthy", - error: undefined, - launchId: 1, - }); - expect(yield* Ref.get(launches)).toBe(1); - yield* service.stop; - }), - ), - ); - it.live("reports the initial check's failure to callers waiting on it without re-probing", () => Effect.scoped( Effect.gen(function* () { @@ -1199,9 +952,7 @@ describe("service readiness recovery", () => { yield* open(checkGate); - expect(yield* Fiber.join(waiting)).toMatchObject({ - message: "rest HTTP readiness timed out", - }); + expect((yield* Fiber.join(waiting)).message).toContain("rest HTTP readiness timed out"); expect(yield* Ref.get(checks)).toBe(1); expect(yield* service.get).toMatchObject({ health: "unhealthy", @@ -1227,8 +978,8 @@ describe("service readiness recovery", () => { ); yield* startUnhealthy(service); - expect(yield* Effect.flip(service.ready)).toMatchObject({ message: "setup failed" }); - expect(yield* Effect.flip(service.ready)).toMatchObject({ message: "setup failed" }); + expect((yield* Effect.flip(service.ready)).message).toContain("setup failed"); + expect((yield* Effect.flip(service.ready)).message).toContain("setup failed"); expect(yield* Ref.get(checks)).toBe(1); yield* service.stop; }), @@ -1266,8 +1017,8 @@ describe("service readiness recovery", () => { yield* Deferred.await(probeStarted); yield* open(probeGate); - expect(yield* Fiber.join(first)).toMatchObject({ message: "still booting" }); - expect(yield* Fiber.join(second)).toMatchObject({ message: "still booting" }); + expect((yield* Fiber.join(first)).message).toContain("still booting"); + expect((yield* Fiber.join(second)).message).toContain("still booting"); expect(yield* Ref.get(checks)).toBe(2); yield* service.ready; expect(yield* Ref.get(checks)).toBe(3); @@ -1282,7 +1033,7 @@ describe("service readiness recovery", () => { const firstLaunchChecks = yield* Ref.make(0); const probeStarted = yield* Deferred.make(); const probeInterrupted = yield* Deferred.make(); - const { service } = yield* makeProbedService((launch) => + const { service, launches } = yield* makeProbedService((launch) => launch === 1 ? Ref.updateAndGet(firstLaunchChecks, (count) => count + 1).pipe( Effect.flatMap((count) => @@ -1297,26 +1048,27 @@ describe("service readiness recovery", () => { : Effect.fail(unhealthy("second launch unhealthy")), ); yield* startUnhealthy(service); - const stale = yield* Effect.forkChild(service.ready, { startImmediately: true }); + const stale = yield* Effect.forkChild(Effect.exit(service.ready), { + startImmediately: true, + }); yield* Deferred.await(probeStarted); - const relaunched = yield* waitForObservation( + const relaunched = yield* waitForStatus( service, - (value) => value.launchId === 2 && value.health === "unhealthy", + (value) => + value.health === "unhealthy" && value.error?.message === "second launch unhealthy", ); yield* service.restart(); yield* Fiber.join(relaunched); yield* Deferred.await(probeInterrupted); - expect(Exit.isFailure(yield* Fiber.await(stale))).toBe(true); + expect(Exit.isFailure(yield* Fiber.join(stale))).toBe(true); expect(yield* service.get).toMatchObject({ - launchId: 2, health: "unhealthy", error: { message: "second launch unhealthy" }, }); - expect(yield* Effect.flip(service.ready)).toMatchObject({ - message: "second launch unhealthy", - }); + expect(yield* Ref.get(launches)).toBe(2); + expect((yield* Effect.flip(service.ready)).message).toContain("second launch unhealthy"); expect(yield* Ref.get(firstLaunchChecks)).toBe(2); yield* service.stop; }), @@ -1346,7 +1098,7 @@ describe("service readiness recovery", () => { yield* Scope.close(owner, Exit.void); - expect(yield* Fiber.join(waiting)).toMatchObject({ message: "Session stopped" }); + expect((yield* Fiber.join(waiting)).message).toBe("Lifecycle owner stopped"); }), ), ); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 17e0e0883a..4c9d8e0330 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -1,7 +1,6 @@ import { Cause, Data, - Deferred, Effect, Exit, Fiber, @@ -12,25 +11,21 @@ import { SubscriptionRef, } from "effect"; import type { Stream } from "effect"; +import { LifecycleEvent } from "./Lifecycle.ts"; -type ServiceLifecycle = "stopped" | "starting" | "running" | "stopping"; -type ServiceHealth = "starting" | "healthy" | "unhealthy"; +/** The execution step a service is performing right now, for status reporting. */ type ServiceOperation = "start" | "stop" | "restart" | "storage" | "destroy" | "sleep"; -export type ServiceAdmission = ServiceOperation | "arm"; +/** + * Execution facts about one service: its configuration and the outcome of its last launch, check, + * exit or cleanup. Lifecycle phase and health come from the lifecycle reducer, not from here. + */ export interface ServiceObservation { readonly id: string; readonly config: Config; - readonly lifecycle: ServiceLifecycle; - readonly health: ServiceHealth | undefined; readonly error: ServiceError | undefined; - readonly cleanupError: ServiceError | undefined; readonly exit: Exit.Exit | undefined; readonly currentOperation: ServiceOperation | undefined; - readonly launchId: number | undefined; - readonly intentRevision: number; - readonly wakeEnabled: boolean; - readonly registered: boolean; } export class ServiceError extends Data.TaggedError("ServiceError")<{ @@ -39,26 +34,6 @@ export class ServiceError extends Data.TaggedError("ServiceError")<{ readonly cause?: unknown; }> {} -export class ServiceDestroyed extends Data.TaggedError("ServiceDestroyed")<{ - readonly id: string; -}> {} - -export class ServiceNotRunning extends Data.TaggedError("ServiceNotRunning")<{ - readonly id: string; -}> {} - -class ServiceNotStopped extends Data.TaggedError("ServiceNotStopped")<{ - readonly id: string; - readonly lifecycle: ServiceLifecycle; - readonly message: string; -}> {} - -export class ServiceStaleLaunch extends Data.TaggedError("ServiceStaleLaunch")<{ - readonly id: string; - readonly launchId: number; - readonly message: string; -}> {} - /** The exact runtime resources owned by one service launch. */ export interface RuntimeSession { /** The initial readiness program for this session. */ @@ -80,7 +55,7 @@ export class ServiceLaunchError extends Data.TaggedError("ServiceLaunchError")<{ }> {} export interface ServiceDefinition { - /** Preparation is performed before lifecycle admission. */ + /** Idempotent preparation (artifacts, images) run at the start of every launch. */ readonly prepare?: (config: Config) => Effect.Effect; /** Failures after resource acquisition carry the session for ordinary cleanup. */ readonly launch: ( @@ -103,64 +78,56 @@ export interface ServiceLaunchContext extends ServiceInstanceContext { readonly id: string; readonly observation: Stream.Stream>; readonly get: Effect.Effect>; - readonly start: Effect.Effect; - readonly startAt: ( - revision: number, - config?: Config, - wake?: boolean, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly arm: Effect.Effect; - readonly armAt: ( - revision: number, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly sleep: Effect.Effect; - readonly stop: Effect.Effect; - readonly restart: ( - config?: Config, - revision?: number, - guard?: Effect.Effect, - ) => Effect.Effect; - readonly ready: Effect.Effect< - void, - ServiceError | ServiceDestroyed | ServiceNotRunning | ServiceStaleLaunch - >; - /** Runs instance-owned storage work only while the instance is stopped. */ + readonly prepare: (config: Config) => Effect.Effect; + /** Runs generation `generation`'s launch and readiness check; outcomes are reported, never returned. */ + readonly launch: (generation: number, config: Config) => Effect.Effect; + /** + * Stops `generation`'s session, never a newer one, and reports `Exited` once cleanup is + * confirmed or `StopFailed` while its resources are still held. + */ + readonly stop: ( + generation: number, + options: { readonly operation: ServiceOperation; readonly discard: boolean }, + ) => Effect.Effect; + /** Re-runs a failed readiness check of `generation`'s live session and reports the result. */ + readonly reprobe: (generation: number) => Effect.Effect; + /** Runs instance-owned storage work; the lifecycle reservation keeps the service stopped. */ readonly storage: ( operation: Effect.Effect, - ) => Effect.Effect; - readonly destroy: Effect.Effect; + ) => Effect.Effect; + /** + * Removes the instance's data; the service must already be stopped. + * `confirm`, when given, runs inside the same execution lock once resources are confirmed + * removed — destroy's registration publication. + */ + readonly removeData: ( + confirm?: Effect.Effect, + ) => Effect.Effect; } -type HealthCheck = Deferred.Deferred>; - interface SessionRecord { - readonly launchId: number; + readonly generation: number; readonly runtime: RuntimeSession; + /** The execution handle's scope, closed only after the runtime's stop and removal succeed. */ readonly scope: Scope.Closeable; readonly healthScope: Scope.Closeable; - /** The latest readiness check of this launch; a settled failure is replaced by an on-demand probe. */ - readonly health: Ref.Ref; readonly stopped: Ref.Ref; readonly removed: Ref.Ref; } -const sessionStopped = () => new ServiceError({ operation: "health", message: "Session stopped" }); - -const contextFor = ( - id: string, - config: Config, - scope: Scope.Closeable, -): ServiceInstanceContext => ({ - id, - config, - scope, -}); +interface Attempt { + readonly generation: number; + readonly fiber: Fiber.Fiber; +} const exitError = ( operation: string, @@ -178,588 +145,383 @@ const exitError = ( }), ); -/** Creates one independently serialized service instance. */ +/** Creates one service's execution boundary; launches and cleanups run in the current scope. */ export const makeService = ( definition: ServiceDefinition, options: { readonly id: string; readonly config: Config; + /** Delivers an execution outcome to the lifecycle authority. */ + readonly report: (event: LifecycleEvent) => Effect.Effect; /** Launch ids continue after this one. */ readonly lastLaunchId?: number; - readonly coordinate?: ( - operation: ServiceAdmission, - transition: Effect.Effect, - ) => Effect.Effect; }, ): Effect.Effect, never, Scope.Scope> => Effect.gen(function* () { const owner = yield* Scope.Scope; - const gate = yield* Semaphore.make(1); - const revision = yield* Ref.make(0); + const { id, report } = options; + // Serializes every step that touches runtime resources: a launch's acquisition, a stop, an + // exit's cleanup, storage work and data removal. + const execution = yield* Semaphore.make(1); const launchCounter = yield* Ref.make(options.lastLaunchId ?? 0); const current = yield* Ref.make(undefined); - // stopNow clears observation.launchId while keeping the error, so readiness must not read that field. - const launchError = yield* Ref.make< - { readonly launchId: number; readonly error: ServiceError } | undefined - >(undefined); + const attempt = yield* Ref.make(undefined); + // The highest generation already stopped: a launch for it, admitted late, must never run. + const stoppedThrough = yield* Ref.make(0); const config = yield* Ref.make(options.config); const observations = yield* SubscriptionRef.make>({ - id: options.id, + id, config: options.config, - lifecycle: "stopped", - health: undefined, error: undefined, - cleanupError: undefined, exit: undefined, currentOperation: undefined, - launchId: undefined, - intentRevision: 0, - wakeEnabled: false, - registered: true, }); const update = (change: Partial>) => SubscriptionRef.update(observations, (value) => ({ ...value, ...change })); + const recordCleanupError = (error: ServiceError) => + SubscriptionRef.update(observations, (value) => ({ + ...value, + error: value.error ?? error, + })); + + /** Halts and removes one session, retrying only the steps a previous attempt didn't finish. */ + const stopRecord = Effect.fn("Service.stopRecord")(function* ( + record: SessionRecord, + discard: boolean, + ) { + yield* Effect.annotateCurrentSpan({ member_id: id, generation: record.generation }); + yield* Scope.close(record.healthScope, Exit.void); + if (!(yield* Ref.get(record.stopped))) { + const halt = + discard && record.runtime.discard !== undefined + ? record.runtime.discard + : record.runtime.stop; + yield* halt.pipe(Effect.tapError(recordCleanupError)); + yield* Ref.set(record.stopped, true); + } + if (!(yield* Ref.get(record.removed))) { + yield* record.runtime.remove.pipe(Effect.tapError(recordCleanupError)); + yield* Ref.set(record.removed, true); + } + yield* Scope.close(record.scope, Exit.void); + yield* Ref.update(current, (value) => (value === record ? undefined : value)); + }); - const setOperation = (operation: ServiceOperation | undefined) => - update({ currentOperation: operation }); - const coordinate = - options.coordinate ?? - ((_operation: ServiceAdmission, transition: Effect.Effect) => transition); - - // Health and exit settlement each write the observation and launchError as one step. - const settlement = yield* Semaphore.make(1); + /** Runs one readiness program in the session's health scope; interruption yields `undefined`. */ + const check = (record: SessionRecord, program: Effect.Effect) => + Effect.forkIn(program, record.healthScope, { startImmediately: true }).pipe( + Effect.flatMap(Fiber.await), + Effect.map((exit) => + Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) ? undefined : exit, + ), + ); const settleCheck = Effect.fn("Service.settleCheck")(function* ( record: SessionRecord, exit: Exit.Exit, + recovered: boolean, ) { const error = exitError("health", exit); - yield* settlement.withPermit( - Effect.gen(function* () { - const owned = yield* SubscriptionRef.modify( - observations, - (observation): [boolean, ServiceObservation] => - observation.launchId === record.launchId && observation.lifecycle === "running" - ? [ - true, - { - ...observation, - health: error === undefined ? "healthy" : "unhealthy", - error, - }, - ] - : [false, observation], - ); - if (owned) - yield* Ref.set( - launchError, - error === undefined ? undefined : { launchId: record.launchId, error }, - ); - }), + yield* Effect.annotateCurrentSpan({ healthy: error === undefined }); + if ((yield* Ref.get(current)) !== record) return; + yield* update({ error }); + const generation = record.generation; + yield* report( + error !== undefined + ? LifecycleEvent.ReadinessLost({ id, generation, cause: error }) + : recovered + ? LifecycleEvent.ReadinessRecovered({ id, generation }) + : LifecycleEvent.LaunchSucceeded({ id, generation }), ); }); - /** Runs one readiness check whose result is shared by every `ready` caller awaiting it. */ - const runCheck = Effect.fn("Service.runCheck")( - (record: SessionRecord, check: Effect.Effect, result: HealthCheck) => - Effect.forkIn( - check.pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) - ? Deferred.succeed(result, Exit.fail(sessionStopped())) - : settleCheck(record, exit).pipe(Effect.andThen(Deferred.succeed(result, exit))), - ), - ), - record.healthScope, - // Starting immediately installs the settlement before a closing scope can interrupt it. - { startImmediately: true, uninterruptible: false }, - ), - ); - - // Mask only the permit handoff; admitted work belongs to the host scope. - const run = Effect.fn("Service.run")(function* < - A, - E extends ServiceError | ServiceDestroyed | ServiceNotStopped, - >(operation: Effect.Effect) { - return yield* Effect.uninterruptibleMask((restore) => + /** + * Reports a session whose runtime exited on its own as lost before anything else, so traffic + * stops routing to it while its cleanup runs; then cleans it and reports the exit. + */ + const observeExit = Effect.fn("Service.observeExit")(function* ( + record: SessionRecord, + exit: Exit.Exit, + ) { + yield* Effect.annotateCurrentSpan({ member_id: id, generation: record.generation }); + const error = + exitError("exit", exit) ?? + new ServiceError({ operation: "exit", message: "Runtime exited unexpectedly" }); + if ((yield* Ref.get(current)) !== record) return; + yield* report( + LifecycleEvent.SessionLost({ id, generation: record.generation, cause: error }), + ); + const cleaned = yield* execution.withPermit( Effect.gen(function* () { - yield* restore(gate.take(1)); - const fiber = yield* Effect.forkIn( - operation.pipe(Effect.ensuring(gate.release(1))), - owner, - { uninterruptible: false }, + if ((yield* Ref.get(current)) !== record) return undefined; + yield* update({ error, exit, currentOperation: "stop" }); + return yield* stopRecord(record, false).pipe( + Effect.exit, + Effect.ensuring(update({ currentOperation: undefined })), ); - return yield* restore(Fiber.join(fiber)); }), ); + if (cleaned === undefined) return; + const generation = record.generation; + yield* report( + Exit.isSuccess(cleaned) + ? LifecycleEvent.Exited({ id, generation, requested: false }) + : LifecycleEvent.StopFailed({ + id, + generation, + cause: Cause.squash(cleaned.cause), + failure: error, + }), + ); }); - const stopNow = Effect.fn("Service.stopNow")(function* ( - expected?: SessionRecord, - retainWake = false, - discard = false, - ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* Effect.gen(function* () { - const record = yield* Ref.get(current); - if (expected !== undefined && record !== expected) return; - if (record === undefined) { - yield* update({ lifecycle: "stopped", health: undefined }); - return; - } - - yield* update({ lifecycle: "stopping", health: undefined, wakeEnabled: retainWake }); - yield* Deferred.succeed(yield* Ref.get(record.health), Exit.fail(sessionStopped())); - yield* Scope.close(record.healthScope, Exit.void); - if (!(yield* Ref.get(record.stopped))) { - const halt = - discard && record.runtime.discard !== undefined - ? record.runtime.discard - : record.runtime.stop; - yield* halt.pipe( - Effect.tapError((error) => - SubscriptionRef.update(observations, (value) => ({ - ...value, - error: value.error ?? error, - cleanupError: error, - })), - ), - ); - yield* Ref.set(record.stopped, true); - } - if (!(yield* Ref.get(record.removed))) { - yield* record.runtime.remove.pipe( - Effect.tapError((error) => - SubscriptionRef.update(observations, (value) => ({ - ...value, - error: value.error ?? error, - cleanupError: error, - })), - ), - ); - yield* Ref.set(record.removed, true); - } - yield* Scope.close(record.scope, Exit.void); - - yield* Ref.set(current, undefined); - yield* update({ - lifecycle: "stopped", - health: undefined, - launchId: undefined, - cleanupError: undefined, - }); - }); - }); - - const launchNow = Effect.fn("Service.launchNow")(function* ( - guard: Effect.Effect, + /** Acquires one generation's runtime; on failure, its retained resources are cleaned first. */ + const acquire = Effect.fn("Service.acquire")(function* ( + generation: number, + launchConfig: Config, + handle: Scope.Closeable, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* Effect.gen(function* () { - let observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; - if (observation.lifecycle === "stopping") { - yield* stopNow(undefined, observation.wakeEnabled); - observation = yield* SubscriptionRef.get(observations); - if (observation.lifecycle !== "stopped") return; - } - - const launchId = yield* Ref.updateAndGet(launchCounter, (value) => value + 1); - yield* coordinate( - "start", - guard.pipe( - Effect.andThen( - Ref.set(launchError, undefined).pipe( - Effect.andThen( - update({ - lifecycle: "starting", - health: "starting", - error: undefined, - cleanupError: undefined, - exit: undefined, - wakeEnabled: observation.wakeEnabled, - launchId, - }), - ), - ), - ), + if (generation <= (yield* Ref.get(stoppedThrough))) return { _tag: "Fenced" } as const; + // The lifecycle launches only once the previous session's cleanup is confirmed. + if ((yield* Ref.get(current)) !== undefined) + return { + _tag: "Failed", + error: new ServiceError({ + operation: "launch", + message: `Service ${id} still holds a previous session`, + }), + } as const; + yield* Ref.set(config, launchConfig); + yield* update({ config: launchConfig, exit: undefined, currentOperation: "start" }); + const launchId = yield* Ref.updateAndGet(launchCounter, (value) => value + 1); + const launched = yield* Effect.exit( + definition.launch({ id, config: launchConfig, scope: handle, launchId }).pipe( + Effect.map((runtime) => ({ runtime, failure: undefined })), + Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => + Effect.succeed({ runtime, failure }), ), - ); - const runtimeScope = yield* Scope.fork(owner, "parallel"); - const launchExit = yield* Effect.exit( - definition - .launch({ - ...contextFor(options.id, yield* Ref.get(config), runtimeScope), - launchId, - }) - .pipe( - Effect.map((runtime) => ({ runtime, failure: undefined })), - Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => - Effect.succeed({ runtime, failure }), - ), + ), + ); + if (Exit.isFailure(launched)) { + yield* Scope.close(handle, launched); + return { + _tag: "Failed", + error: + exitError("launch", launched) ?? + new ServiceError({ operation: "launch", message: "Launch failed" }), + } as const; + } + const { runtime, failure } = launched.value; + const record: SessionRecord = { + generation, + runtime, + scope: handle, + healthScope: yield* Scope.fork(handle, "parallel"), + stopped: yield* Ref.make(false), + removed: yield* Ref.make(false), + }; + yield* Ref.set(current, record); + if (failure !== undefined) { + const cleaned = yield* stopRecord(record, false).pipe(Effect.result); + return cleaned._tag === "Failure" + ? ({ _tag: "Retained", error: failure, cleanup: cleaned.failure } as const) + : ({ _tag: "Failed", error: failure } as const); + } + yield* Effect.forkIn( + runtime.exit.pipe( + // Recorded at once, so a requested stop's exit is visible when the stop completes. + Effect.tap((exit) => + Ref.get(current).pipe( + Effect.flatMap((live) => (live === record ? update({ exit }) : Effect.void)), ), - ); - if (Exit.isFailure(launchExit)) { - yield* Scope.close(runtimeScope, launchExit); - const error = exitError("launch", launchExit); - if (error !== undefined) yield* Ref.set(launchError, { launchId, error }); - yield* update({ - lifecycle: "stopped", - health: undefined, - wakeEnabled: observation.wakeEnabled, - error, - launchId: undefined, - }); - return yield* Effect.failCause(launchExit.cause); - } - const { runtime, failure: launchFailure } = launchExit.value; - const health = yield* Deferred.make>(); - const record: SessionRecord = { - launchId, - runtime, - scope: runtimeScope, - healthScope: yield* Scope.fork(runtimeScope, "parallel"), - health: yield* Ref.make(health), - stopped: yield* Ref.make(false), - removed: yield* Ref.make(false), - }; - yield* Ref.set(current, record); - if (launchFailure !== undefined) - yield* Ref.set(launchError, { launchId, error: launchFailure }); - yield* update({ - lifecycle: launchFailure === undefined ? "running" : "stopping", - health: launchFailure === undefined ? "starting" : undefined, - error: launchFailure, - launchId, - config: yield* Ref.get(config), - }); - - const observeExit = record.runtime.exit.pipe( + ), + // Cleanup closes this session's scope, so it runs on the owner's fiber instead. Effect.flatMap((exit) => - Effect.gen(function* () { - const owned = yield* settlement.withPermit( - Effect.gen(function* () { - if ((yield* Ref.get(current)) !== record) return false; - const observation = yield* SubscriptionRef.get(observations); - const error = - observation.lifecycle === "stopping" - ? observation.error - : (exitError("exit", exit) ?? - new ServiceError({ - operation: "exit", - message: "Runtime exited unexpectedly", - })); - if (error !== undefined) - yield* Ref.set(launchError, { launchId: record.launchId, error }); - yield* update({ - lifecycle: "stopping", - health: undefined, - error, - exit, - wakeEnabled: observation.lifecycle === "stopping" && observation.wakeEnabled, - }); - return true; - }), - ); - if (!owned) return; - yield* run( - Effect.gen(function* () { - if ((yield* Ref.get(current)) !== record) return; - yield* setOperation("stop"); - const live = yield* SubscriptionRef.get(observations); - yield* stopNow(record, live.wakeEnabled).pipe( - Effect.ensuring(setOperation(undefined)), - ); - }), - ); - }), + Effect.forkIn(observeExit(record, exit), owner, { startImmediately: true }), ), - ); - if (launchFailure === undefined) yield* runCheck(record, runtime.health, health); - yield* Effect.forkIn(observeExit, runtimeScope, { uninterruptible: false }); - if (launchFailure !== undefined) { - yield* Deferred.succeed(health, Exit.fail(launchFailure)); - yield* stopNow(undefined, observation.wakeEnabled); - return yield* launchFailure; - } - }); - }); - - /** A wake has no caller to receive a preparation failure, so an idle observation records it. */ - const recordPreparationFailure = Effect.fn("Service.recordPreparationFailure")(function* ( - expectedRevision: number, - error: ServiceError, - ) { - if ((yield* Ref.get(revision)) !== expectedRevision) return; - yield* SubscriptionRef.update(observations, (value) => - value.registered && value.lifecycle === "stopped" ? { ...value, error } : value, + ), + handle, + { startImmediately: true }, ); + return { _tag: "Live", record } as const; }); - const startAt = Effect.fn("Service.startAt")(function* ( - expectedRevision: number, - candidate?: Config, - wake = false, - guard: Effect.Effect = Effect.void, + const run = Effect.fn("Service.launch")(function* ( + generation: number, + launchConfig: Config, + handle: Scope.Closeable, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - let existing = yield* SubscriptionRef.get(observations); - if (!existing.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (existing.lifecycle === "running") return; - if (existing.lifecycle === "starting") { - yield* run(Effect.void); - existing = yield* SubscriptionRef.get(observations); - if (!existing.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (existing.lifecycle === "running") return; + yield* Effect.annotateCurrentSpan({ member_id: id, generation }); + yield* update({ error: undefined }); + const prepared = yield* (definition.prepare?.(launchConfig) ?? Effect.void).pipe(Effect.exit); + if (Exit.isFailure(prepared)) { + // A stopped generation's late failure must not mark the observation of its successor. + if (generation <= (yield* Ref.get(stoppedThrough))) return; + const error = exitError("prepare", prepared); + yield* update({ error }); + yield* report(LifecycleEvent.LaunchFailed({ id, generation, cause: error })); + return; } - const nextConfig = candidate ?? (yield* Ref.get(config)); - if (definition.prepare !== undefined) - yield* definition - .prepare(nextConfig) - .pipe(Effect.tapError((error) => recordPreparationFailure(expectedRevision, error))); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; - if ((yield* Ref.get(revision)) !== expectedRevision || (wake && !observation.wakeEnabled)) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* setOperation("start"); - yield* launchNow( - guard.pipe( - Effect.andThen(Ref.set(config, nextConfig)), - Effect.andThen(update({ config: nextConfig })), - ), - ).pipe(Effect.ensuring(setOperation(undefined))); - }), + yield* report(LifecycleEvent.StageChanged({ id, generation, stage: "launching" })); + const acquired = yield* execution.withPermit( + acquire(generation, launchConfig, handle).pipe( + Effect.ensuring(update({ currentOperation: undefined })), + ), ); - }); - const start = Ref.get(revision).pipe(Effect.flatMap((revision) => startAt(revision))); - - const invalidate = Effect.fn("Service.invalidate")(function* () { - const next = yield* Ref.updateAndGet(revision, (value) => value + 1); - const record = yield* Ref.get(current); - yield* update({ - intentRevision: next, - wakeEnabled: false, - lifecycle: record === undefined ? "stopped" : "stopping", - health: undefined, - }); + if (acquired._tag === "Fenced") return; + if (acquired._tag === "Retained") { + yield* update({ error: acquired.error }); + yield* report( + LifecycleEvent.StopFailed({ + id, + generation, + cause: acquired.cleanup, + failure: acquired.error, + }), + ); + return; + } + if (acquired._tag === "Failed") { + yield* update({ error: acquired.error }); + yield* report(LifecycleEvent.LaunchFailed({ id, generation, cause: acquired.error })); + return; + } + yield* report(LifecycleEvent.SessionAvailable({ id, generation })); + const healthy = yield* check(acquired.record, acquired.record.runtime.health); + if (healthy !== undefined) yield* settleCheck(acquired.record, healthy, false); }); - const armAt = Effect.fn("Service.armAt")(function* ( - expectedRevision: number, - guard: Effect.Effect = Effect.void, + const launch = Effect.fn("Service.launchAttempt")(function* ( + generation: number, + launchConfig: Config, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* run( - Effect.gen(function* () { - if (!(yield* SubscriptionRef.get(observations)).registered) - return yield* new ServiceDestroyed({ id: options.id }); - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* coordinate("arm", guard.pipe(Effect.andThen(update({ wakeEnabled: true })))); - }), - ); - }); - const arm = Ref.get(revision).pipe(Effect.flatMap((revision) => armAt(revision))); - - const sleep = Effect.fn("Service.sleep")(function* () { - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle !== "running" || !observation.wakeEnabled) return; - yield* coordinate( - "sleep", - update({ lifecycle: "stopping", health: undefined, currentOperation: "sleep" }), - ); - yield* stopNow(undefined, true).pipe(Effect.ensuring(setOperation(undefined))); - }), + // Created before preparation so a stopped or abandoned attempt still has a scope to close; + // once a session holds it, only `stopRecord` closes it. + const handle = yield* Scope.fork(owner, "parallel"); + const releaseUnlessHeld = Ref.get(current).pipe( + Effect.flatMap((record) => + record?.scope === handle ? Effect.void : Scope.close(handle, Exit.void), + ), ); - }); - - const stop = Effect.fn("Service.stop")(function* () { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - yield* coordinate("stop", invalidate()); - yield* setOperation("stop"); - yield* stopNow().pipe(Effect.ensuring(setOperation(undefined))); - }), + const fiber = yield* Effect.forkIn( + run(generation, launchConfig, handle).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.void + : report(LifecycleEvent.LaunchFailed({ id, generation, cause: Cause.squash(cause) })), + ), + Effect.ensuring(releaseUnlessHeld), + Effect.ensuring( + Ref.update(attempt, (value) => (value?.generation === generation ? undefined : value)), + ), + ), + owner, + { startImmediately: true, uninterruptible: false }, ); + yield* Ref.set(attempt, { generation, fiber }); + yield* Fiber.await(fiber); }); - const restart = Effect.fn("Service.restart")(function* ( - candidate?: Config, - requestedRevision?: number, - guard: Effect.Effect = Effect.void, + const stop = Effect.fn("Service.stop")(function* ( + generation: number, + stopOptions: { readonly operation: ServiceOperation; readonly discard: boolean }, ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - const expectedRevision = requestedRevision ?? (yield* Ref.get(revision)); - const nextConfig = candidate ?? (yield* Ref.get(config)); - if (definition.prepare !== undefined) yield* definition.prepare(nextConfig); - yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceError({ - operation: "admission", - message: "Service intent changed before admission", - }); - yield* coordinate("restart", invalidate()); - yield* setOperation("restart"); - yield* Effect.gen(function* () { - yield* stopNow(); - yield* Ref.set(config, nextConfig); - yield* update({ config: nextConfig }); - yield* launchNow(guard); - }).pipe(Effect.ensuring(setOperation(undefined))); - }), + yield* Effect.annotateCurrentSpan({ member_id: id, generation }); + // Fencing under the execution lock waits out an acquisition already in progress, so its + // session is visible below, and makes any later acquisition for this generation a no-op. + // Only then is the attempt interrupted, while it prepares or checks readiness. + yield* execution.withPermit( + Ref.update(stoppedThrough, (value) => Math.max(value, generation)), ); - }); - - const staleLaunch = (launchId: number, message: string) => - new ServiceStaleLaunch({ id: options.id, launchId, message }); - const diedBeforeReady = (launchId: number) => - Effect.gen(function* () { - const owned = yield* Ref.get(launchError); - if (owned !== undefined && owned.launchId === launchId) return yield* owned.error; - return yield* staleLaunch(launchId, `Service ${options.id} stopped before it was ready`); - }); - const isLive = (record: SessionRecord) => - Effect.gen(function* () { - if ((yield* Ref.get(current)) !== record) return false; - return (yield* SubscriptionRef.get(observations)).lifecycle === "running"; - }); - - /** Concurrent callers share one probe per settled failure of a launch. */ - const reprobe = Effect.fn("Service.reprobe")(function* ( - record: SessionRecord, - failed: HealthCheck, - probe: Effect.Effect, - ) { - const next = yield* Deferred.make>(); - const check = yield* Ref.modify(record.health, (latest): [HealthCheck, HealthCheck] => - latest === failed ? [next, next] : [latest, latest], + const pending = yield* Ref.get(attempt); + if (pending !== undefined && pending.generation <= generation) + yield* Fiber.interrupt(pending.fiber); + const stopped = yield* execution + .withPermit( + Effect.gen(function* () { + const record = yield* Ref.get(current); + // A newer session belongs to a later generation's own stop. + if (record === undefined || record.generation > generation) return; + yield* update({ currentOperation: stopOptions.operation }); + yield* stopRecord(record, stopOptions.discard).pipe( + Effect.ensuring(update({ currentOperation: undefined })), + ); + }), + ) + .pipe(Effect.exit); + yield* report( + Exit.isSuccess(stopped) + ? LifecycleEvent.Exited({ id, generation, requested: true }) + : LifecycleEvent.StopFailed({ id, generation, cause: Cause.squash(stopped.cause) }), ); - if (check === next) { - // stopNow fails the latest check after leaving "running", so a probe admitted here is always settled. - if (yield* isLive(record)) yield* runCheck(record, probe, next); - else yield* Deferred.succeed(next, Exit.fail(sessionStopped())); - } - return yield* Deferred.await(check); }); - const ready = Effect.fn("Service.ready")(function* () { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - const initial = yield* SubscriptionRef.get(observations); - if (!initial.registered) return yield* new ServiceDestroyed({ id: options.id }); - const expectedRevision = yield* Ref.get(revision); - const expectedLaunchId = initial.lifecycle === "starting" ? initial.launchId : undefined; - let record = yield* Ref.get(current); - if (record === undefined && initial.lifecycle === "starting") { - yield* run(Effect.void); - record = yield* Ref.get(current); - } - if (record === undefined) { - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* staleLaunch( - yield* Ref.get(launchCounter), - `Service ${options.id} changed before it was ready`, - ); - return yield* new ServiceNotRunning({ id: options.id }); - } - const launchId = record.launchId; - if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); - if (expectedLaunchId !== undefined && record.launchId !== expectedLaunchId) - return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); - if ((yield* SubscriptionRef.get(observations)).lifecycle === "stopping") { - return yield* diedBeforeReady(launchId); - } - const check = yield* Ref.get(record.health); - const settledOnArrival = yield* Deferred.isDone(check); - const healthResult = yield* Deferred.await(check); - if (!(yield* isLive(record))) return yield* diedBeforeReady(launchId); + const reprobe = Effect.fn("Service.reprobe")(function* (generation: number) { + yield* Effect.annotateCurrentSpan({ member_id: id, generation }); + const record = yield* Ref.get(current); + if (record?.generation !== generation) return; const probe = record.runtime.probe; - if (Exit.isSuccess(healthResult) || !settledOnArrival || probe === undefined) - return yield* healthResult; - const reprobed = yield* reprobe(record, check, probe); - if (!(yield* isLive(record))) return yield* diedBeforeReady(launchId); - return yield* reprobed; + if (probe === undefined) { + const error = + (yield* SubscriptionRef.get(observations)).error ?? + new ServiceError({ operation: "health", message: `Service ${id} is not ready` }); + yield* report(LifecycleEvent.ReadinessLost({ id, generation, cause: error })); + return; + } + const result = yield* check(record, probe); + if (result !== undefined) yield* settleCheck(record, result, true); }); - const storage = Effect.fn("Service.storage")(function* ( - operation: Effect.Effect, - ) { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - return yield* run( - Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); - if (observation.lifecycle !== "stopped" || observation.wakeEnabled) { - return yield* new ServiceNotStopped({ - id: options.id, - lifecycle: observation.lifecycle, - message: `Service ${options.id} must be stopped with wake disabled before modifying data`, - }); - } - yield* coordinate("storage", setOperation("storage")); - return yield* operation.pipe(Effect.ensuring(setOperation(undefined))); - }), - ); - }); + const storage = (operation: Effect.Effect) => + execution + .withPermit( + update({ currentOperation: "storage" }).pipe( + Effect.andThen(operation), + Effect.ensuring(update({ currentOperation: undefined })), + ), + ) + .pipe(Effect.withSpan("Service.storage", { attributes: { member_id: id } })); - const destroy = Effect.fn("Service.destroy")(function* () { - yield* Effect.annotateCurrentSpan({ member_id: options.id }); - yield* run( + const removeData = Effect.fn("Service.removeData")(function* ( + confirm: Effect.Effect = Effect.void, + ) { + yield* Effect.annotateCurrentSpan({ member_id: id }); + yield* execution.withPermit( Effect.gen(function* () { - const observation = yield* SubscriptionRef.get(observations); - if (!observation.registered) return; - yield* coordinate("destroy", invalidate()); - yield* setOperation("destroy"); + yield* update({ currentOperation: "destroy" }); yield* Effect.gen(function* () { - yield* stopNow(undefined, false, true); - { - const dataScope = yield* Scope.fork(owner, "parallel"); - yield* definition - .removeData(contextFor(options.id, yield* Ref.get(config), dataScope)) - .pipe( - Effect.tapError((error) => update({ error })), - Effect.ensuring(Scope.close(dataScope, Exit.void)), - ); + const leftover = yield* Ref.get(current); + if (leftover !== undefined) { + yield* stopRecord(leftover, true); + // A previous failed stop can leave the reducer in `Stopping` forever; report this + // retry's termination too, or `NetworkNamespace.release`'s endpoint check never learns. + yield* report( + LifecycleEvent.Exited({ id, generation: leftover.generation, requested: true }), + ); } - yield* update({ registered: false, lifecycle: "stopped" }); - }).pipe(Effect.ensuring(setOperation(undefined))); + const dataScope = yield* Scope.fork(owner, "parallel"); + yield* definition + .removeData({ id, config: yield* Ref.get(config), scope: dataScope }) + .pipe( + Effect.tapError((error) => update({ error })), + Effect.ensuring(Scope.close(dataScope, Exit.void)), + ); + yield* confirm; + }).pipe(Effect.ensuring(update({ currentOperation: undefined }))); }), ); }); return { - id: options.id, + id, observation: SubscriptionRef.changes(observations), get: SubscriptionRef.get(observations), - start, - startAt, - arm, - armAt, - sleep: sleep(), - stop: stop(), - restart, - ready: ready(), + prepare: (candidate) => definition.prepare?.(candidate) ?? Effect.void, + launch, + stop, + reprobe, storage, - destroy: destroy(), + removeData, }; }); diff --git a/packages/stack/src/StackHost.container-shutdown.integration.test.ts b/packages/stack/src/StackHost.container-shutdown.integration.test.ts index 4c0971f97c..7825d523e1 100644 --- a/packages/stack/src/StackHost.container-shutdown.integration.test.ts +++ b/packages/stack/src/StackHost.container-shutdown.integration.test.ts @@ -1,28 +1,58 @@ +import { PgClient } from "@effect/sql-pg"; import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Crypto, Effect, FileSystem, Layer, Path, Redacted, Stream } from "effect"; +import { + Context, + Crypto, + Effect, + Exit, + FileSystem, + Layer, + Path, + Redacted, + Schedule, + Stream, +} from "effect"; +import * as Reactivity from "effect/unstable/reactivity/Reactivity"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "./State.ts"; +import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- holds a raw TCP connection open across the owner stop. +import { fileURLToPath } from "node:url"; +import * as StackNamespace from "./StackNamespace.ts"; import { hasReason, launchHost, ownerClient, ownerExitProbe, waitForOwnerExit, + type HostAccess, } from "./HostProcess.ts"; import { makeContainerRuntime } from "./runtime/Container.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; -import { shutdownOwner } from "../tests/owner.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; +import { holdReleaseFifo } from "../tests/release-fifo.ts"; +import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; + +const shortRegistrationPollFixture = fileURLToPath( + new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), +); +const gatedDockerStopFixture = fileURLToPath( + new URL("../tests/gated-docker-stop-fixture.ts", import.meta.url), +); const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; -const docker = Effect.fn("StackHostContainerShutdownTest.docker")((args: ReadonlyArray) => +const engine = Effect.fn("StackHostContainerShutdownTest.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, [...engineTarget.argv, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -33,19 +63,19 @@ const docker = Effect.fn("StackHostContainerShutdownTest.docker")((args: Readonl { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`docker ${args.join(" ")} failed: ${stderr}`); + return yield* Effect.die(`${testEngine} ${args.join(" ")} failed: ${stderr}`); return stdout.trim(); }), ), ); const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const containers = (stackId: string, dataRoot: string) => - docker([ + engine([ "ps", "--all", "--quiet", @@ -59,25 +89,13 @@ const containers = (stackId: string, dataRoot: string) => const removeContainers = (stackId: string, dataRoot: string) => containers(stackId, dataRoot).pipe( Effect.flatMap((ids) => - Effect.forEach(ids, (id) => docker(["rm", "--force", id]).pipe(Effect.ignore), { + Effect.forEach(ids, (id) => engine(["rm", "--force", id]).pipe(Effect.ignore), { concurrency: 1, discard: true, }), ), ); -const createOwnedContainer = (name: string, stackId: string, dataRoot: string) => - docker([ - "create", - "--name", - name, - "--label", - `com.supabase.stack=${stackId}`, - "--label", - `com.supabase.stack-root=${dataRoot}`, - helperImage, - ]); - const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projectRoot: string) => Effect.gen(function* () { const state = yield* stateFor(stateRoot); @@ -85,18 +103,17 @@ const startHost = (stateRoot: string, cacheRoot: string, stackId: string, projec if (current === undefined) yield* state.save({ id: stackId, - runtime: "docker", + runtime: testEngine, identity: { projectRoot, branchContext: "container-shutdown-test", stackName: stackId }, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); return yield* launchHost(state, { stateRoot, cacheRoot, stackId }); }); it.live.skipIf(process.platform === "win32")( - "stops only containers owned by the same stack id and data root", + "stops, destroys, and releases only containers owned by the same stack id and data root", () => Effect.scoped( Effect.gen(function* () { @@ -114,7 +131,10 @@ it.live.skipIf(process.platform === "win32")( const rootA = path.dirname(path.dirname(dataA)); const rootB = path.dirname(path.dirname(dataB)); const cacheRoot = `${base}/cache`; - const helper = yield* makeContainerRuntime({ engine: "docker", root: dataA }); + const helper = yield* makeContainerRuntime({ + target: engineTarget, + root: dataA, + }); yield* helper.prepare(helperImage); let activeA: { readonly pid: number; readonly port: number } | undefined; let activeB: { readonly pid: number; readonly port: number } | undefined; @@ -141,41 +161,16 @@ it.live.skipIf(process.platform === "win32")( yield* removeContainers(stackId, dataB).pipe(Effect.ignore); }), ); - const staleAtStartup = yield* createOwnedContainer( - `stack-stale-${stackId}-startup`, - stackId, - dataA, - ); const accessA = yield* startHost(rootA, cacheRoot, stackId, `${base}/project-a`); const endpointA = accessA.endpoint; activeA = endpointA; stoppedA = false; - expect(staleAtStartup.length).toBeGreaterThan(0); - expect( - yield* containers(stackId, dataA), - "startup sweep removes A stale container", - ).toEqual([]); - const staleForA = yield* createOwnedContainer( - `stack-stale-${stackId}-parallel`, - stackId, - dataA, - ); - const staleForB = yield* createOwnedContainer( - `stack-stale-${stackId}-other-root`, - stackId, - dataB, - ); const accessB = yield* startHost(rootB, cacheRoot, stackId, `${base}/project-b`); const endpointB = accessB.endpoint; activeB = endpointB; stoppedB = false; - expect(staleForA.length).toBeGreaterThan(0); - expect(staleForB.length).toBeGreaterThan(0); - expect(yield* containers(stackId, dataA)).toEqual([staleForA]); - expect( - yield* containers(stackId, dataB), - "startup sweep removes B stale container", - ).toEqual([]); + expect(yield* containers(stackId, dataA)).toEqual([]); + expect(yield* containers(stackId, dataB)).toEqual([]); const clientA = yield* ownerClient(accessA); const clientB = yield* ownerClient(accessB); @@ -252,6 +247,555 @@ it.live.skipIf(process.platform === "win32")( activeB = undefined; expect(yield* containers(stackId, dataB), "destroy removes B containers").toEqual([]); expect(yield* (yield* stateFor(rootB)).read(stackId)).toBeUndefined(); + + // Registration-loss phase: two fresh owners share this id again, each under its own root. + // Ending A's ownership must only ever reach containers carrying A's own `stack-root` label. + const stateA3 = yield* stateFor(rootA); + const stateB3 = yield* stateFor(rootB); + yield* stateA3.save({ + id: stackId, + runtime: testEngine, + identity: { + projectRoot: `${base}/project-a`, + branchContext: "container-shutdown-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + yield* stateB3.save({ + id: stackId, + runtime: testEngine, + identity: { + projectRoot: `${base}/project-b`, + branchContext: "container-shutdown-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + const accessA3 = yield* launchHost(stateA3, { + stateRoot: rootA, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + activeA = accessA3.endpoint; + stoppedA = false; + const accessB3 = yield* launchHost(stateB3, { stateRoot: rootB, cacheRoot, stackId }); + activeB = accessB3.endpoint; + stoppedB = false; + + const clientA3 = yield* ownerClient(accessA3); + const clientB3 = yield* ownerClient(accessB3); + const mailA3 = yield* clientA3.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* clientA3.startService({ id: mailA3.id }); + yield* clientA3.readyService({ id: mailA3.id }); + const mailB3 = yield* clientB3.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* clientB3.startService({ id: mailB3.id }); + yield* clientB3.readyService({ id: mailB3.id }); + + expect( + (yield* containers(stackId, dataA)).length, + "A runs its mail container", + ).toBeGreaterThan(0); + const idsB3 = yield* containers(stackId, dataB); + expect(idsB3.length, "B runs its mail container").toBeGreaterThan(0); + + const leaseReleasedA3 = yield* watchLeaseRelease(rootA, stackId); + yield* fs.remove(`${rootA}/${stackId}/state.json`); + yield* leaseReleasedA3; + yield* waitForOwnerExit(accessA3.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("30 seconds"), + ); + stoppedA = true; + activeA = undefined; + + expect( + yield* containers(stackId, dataA), + "ending A's ownership stops only A's containers", + ).toEqual([]); + expect( + yield* containers(stackId, dataB), + "ending A's ownership leaves B's containers running", + ).toEqual(idsB3); + // B's own container still answers a real readiness probe, proving A's owner never + // touched it. + yield* clientB3.readyService({ id: mailB3.id }); + + yield* shutdownOwner(accessB3, true); + yield* waitForOwnerExit(accessB3.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("15 seconds"), + ); + stoppedB = true; + activeB = undefined; + expect(yield* containers(stackId, dataB), "destroy removes B containers").toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "stops its containers, keeps its data and exits when its registration is confirmed gone", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registration-lost-" }); + const stackId = `registration-lost-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot( + "stack-registration-lost-data-", + stackId, + ).pipe(Effect.flatMap(fs.realPath)); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: testEngine, + identity: { + projectRoot: `${base}/project`, + branchContext: "registration-lost-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + // The shortened poll interval comes only from this dedicated test entrypoint, through + // the internal `Context.Reference`; production startup never reads an env var or `Config`. + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + const client = yield* ownerClient(access); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("registration-lost-password"), + jwtSecret: Redacted.make("registration-lost-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + expect( + (yield* containers(stackId, dataRoot)).length, + "the owner runs the database container", + ).toBeGreaterThan(0); + const containerEnvRoot = `${dataRoot}/.container-env`; + expect( + yield* fs.exists(containerEnvRoot), + "the stack's shared container-env scratch directory exists", + ).toBe(true); + + // Subscribes to the owner's own exit signal (its lease release) before triggering the + // deletion, rather than polling for it afterward. + const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* leaseReleased; + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("30 seconds"), + ); + + expect(yield* containers(stackId, dataRoot), "the owner stopped its containers").toEqual( + [], + ); + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + expect( + yield* fs.exists(containerEnvRoot), + "the data root is left in place for a successor or `destroy`", + ).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +/** + * Installs an engine CLI shim the owner finds first on its PATH: the first `stop` announces + * itself by creating `waiting` in `gateDir` and blocks until the test writes the `release` FIFO, + * then every command runs the real engine. The gate only holds once the test has created `armed`, + * so a one-shot container stopped during startup never takes it. Returns the idempotent, + * scope-owned `release`. + */ +const holdDockerStops = (gateDir: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + yield* fs.makeDirectory(`${gateDir}/bin`, { recursive: true }); + yield* spawner.spawn(ChildProcess.make("mkfifo", [`${gateDir}/release`])).pipe( + Effect.flatMap((child) => child.exitCode), + Effect.scoped, + ); + yield* fs.writeFileString( + `${gateDir}/bin/${testEngine}`, + [ + "#!/bin/sh", + `gate='${gateDir}'`, + 'if [ -e "$gate/armed" ] && [ ! -e "$gate/passed" ]; then', + ' for arg in "$@"; do', + ' if [ "$arg" = stop ]; then', + ' : > "$gate/waiting"', + ' read -r _ < "$gate/release"', + ' : > "$gate/passed"', + " break", + " fi", + " done", + "fi", + 'PATH="${PATH#"$gate/bin:"}"', + `exec ${testEngine} "$@"`, + "", + ].join("\n"), + { mode: 0o755 }, + ); + return yield* holdReleaseFifo(`${gateDir}/release`); + }); + +type StopTrigger = "signal" | "http"; + +/** + * Starts a data-preserving stop through `trigger` without waiting for it to finish: a + * signal-driven stop runs in the owner process on its own, while an HTTP-driven stop is forked + * so the held container stop it blocks on never blocks the caller. + */ +const triggerStop = (trigger: StopTrigger, access: HostAccess) => + trigger === "signal" + ? Effect.sync(() => process.kill(access.endpoint.pid, "SIGTERM")) + : Effect.forkScoped(shutdownOwner(access, false).pipe(Effect.ignore)); + +const losesRegistrationWhileStopRuns = (trigger: StopTrigger) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registration-lost-gated-" }); + const stackId = `registration-lost-gated-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot( + "stack-registration-lost-gated-data-", + stackId, + ).pipe(Effect.flatMap(fs.realPath)); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const gateDir = `${base}/gate`; + const hold = yield* holdDockerStops(gateDir); + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: testEngine, + identity: { + projectRoot: `${base}/project`, + branchContext: "registration-lost-gated-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: gatedDockerStopFixture, + entrypointArgs: [gateDir], + }); + // Runs before the container and gate-directory cleanup: frees the held stop, signals the + // owner (a signal during a running stop stays queued), and requires its exit. + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* hold.release; + yield* Effect.try(() => process.kill(access.endpoint.pid, "SIGTERM")).pipe(Effect.ignore); + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + }).pipe(Effect.orDie), + ); + const client = yield* ownerClient(access); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("registration-lost-gated-password"), + jwtSecret: Redacted.make( + "registration-lost-gated-jwt-secret-at-least-thirty-two-characters", + ), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const status = yield* client.status({ id: database.id }); + if (!status.endpoints.some((endpoint) => endpoint.name === "sql")) + return yield* Effect.die("Missing database sql endpoint"); + const containerEnvRoot = `${dataRoot}/.container-env`; + expect( + yield* fs.exists(containerEnvRoot), + "the stack's shared container-env scratch directory exists", + ).toBe(true); + + // Subscribes to the held engine stop's own marker before triggering the stop, so the + // registration deletion below never races the gate itself. + const waiting = yield* watchEntry(gateDir, "waiting", true); + yield* fs.writeFileString(`${gateDir}/armed`, ""); + yield* triggerStop(trigger, access); + yield* waiting.pipe(Effect.timeout("30 seconds")); + + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* hold.release; + + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + expect( + yield* fs.exists(containerEnvRoot), + "the stop that was running when the registration disappeared leaves the data root in place", + ).toBe(true); + expect(yield* containers(stackId, dataRoot), "the held stop still completed").toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))); + +it.live.skipIf(process.platform === "win32")( + "finishes a signal-driven stop and exits when the registration disappears mid-stop", + () => losesRegistrationWhileStopRuns("signal"), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "finishes an HTTP-driven stop and exits when the registration disappears mid-stop", + () => losesRegistrationWhileStopRuns("http"), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "stops its containers and exits when its whole state root is confirmed gone", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-root-lost-" }); + const stackId = `state-root-lost-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-state-root-lost-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: testEngine, + identity: { + projectRoot: `${base}/project`, + branchContext: "state-root-lost-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + const client = yield* ownerClient(access); + // `mail` rather than `database`: its container holds no host-mounted data volume, so + // deleting the state root out from under it doesn't also disrupt its own stop path — this + // test is about registration-independent cleanup, not about surviving every workload's + // reaction to losing its mounted storage. + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + expect( + (yield* containers(stackId, dataRoot)).length, + "the owner runs the mail container", + ).toBeGreaterThan(0); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); + // Deletes the actual `` itself, not just the entries inside + // `/`: its sibling `.registry-lock.sqlite` (`Ports.ts`'s per-root + // reservation registry) is gone too, so the stop can only come from the owner's in-memory + // resources, and must never open that registry at all. The lease file goes with it, so + // this relies on budgeted polling rather than a lease-release subscription. + yield* fs.remove(stateRoot, { recursive: true, force: true }); + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ + schedule: Schedule.spaced("1 second"), + while: (failure) => + failure.reason === "owner-exit-pending" || failure.reason === "owner-exit-zombie", + }), + Effect.timeout("30 seconds"), + ); + + expect( + yield* containers(stackId, dataRoot), + "the owner stops its service containers with the registry gone", + ).toEqual([]); + + // The vanished registry cannot release the port reservation: a fresh stack can still + // claim the exact same port, through `Ports.ts`'s own lazy reclamation once the former + // holder's registration is confirmed gone. + const reclaimedBase = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-state-root-lost-reclaim-", + }); + const reclaimedStackId = `state-root-lost-reclaim-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const reclaimedDataRoot = yield* makeDockerDatabaseRoot( + "stack-state-root-lost-reclaim-data-", + reclaimedStackId, + ).pipe(Effect.flatMap(fs.realPath)); + const reclaimedStateRoot = path.dirname(path.dirname(reclaimedDataRoot)); + yield* Effect.addFinalizer(() => + removeContainers(reclaimedStackId, reclaimedDataRoot).pipe(Effect.ignore), + ); + const reclaimedAccess = yield* startHost( + reclaimedStateRoot, + cacheRoot, + reclaimedStackId, + `${reclaimedBase}/project`, + ); + const reclaimedClient = yield* ownerClient(reclaimedAccess); + const reclaimedMail = yield* reclaimedClient.createService({ + service: "mail", + config: {}, + endpoints: { http: { port } }, + }); + yield* reclaimedClient.startService({ id: reclaimedMail.id }); + yield* reclaimedClient.readyService({ id: reclaimedMail.id }); + const reclaimedStatus = yield* reclaimedClient.status({ id: reclaimedMail.id }); + expect(reclaimedStatus.endpoints.find((endpoint) => endpoint.name === "http")?.port).toBe( + port, + ); + yield* shutdownOwner(reclaimedAccess, true); + yield* waitForOwnerExit(reclaimedAccess.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("15 seconds"), + ); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "closes a pinned postgres connection and exits a real owner promptly", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "stack-pinned-postgres-" }); + const stackId = `pinned-postgres-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`; + const dataRoot = yield* makeDockerDatabaseRoot("stack-pinned-postgres-data-", stackId).pipe( + Effect.flatMap(fs.realPath), + ); + const stateRoot = path.dirname(path.dirname(dataRoot)); + const cacheRoot = `${base}/cache`; + const state = yield* stateFor(stateRoot); + yield* state.save({ + id: stackId, + runtime: testEngine, + identity: { + projectRoot: `${base}/project`, + branchContext: "pinned-postgres-test", + stackName: stackId, + }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + yield* Effect.addFinalizer(() => removeContainers(stackId, dataRoot).pipe(Effect.ignore)); + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + }); + const client = yield* ownerClient(access); + const password = Redacted.make("pinned-postgres-password"); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: password, + jwtSecret: Redacted.make("pinned-postgres-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const status = yield* client.status({ id: database.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "sql")?.port; + if (port === undefined) return yield* Effect.die("Missing database sql endpoint"); + + const services = yield* Layer.build( + Layer.effect( + PgClient.PgClient, + PgClient.makeClient({ + host: "127.0.0.1", + port, + database: "postgres", + username: "supabase_admin", + password, + }), + ).pipe(Layer.provide(Reactivity.layer)), + ); + const sql = Context.get(services, PgClient.PgClient); + yield* sql.unsafe("SELECT 1"); + + // The owner stops services without waiting on clients, so the pinned connection closes + // with the database container. + yield* Effect.callback((resume) => { + const probe = Net.createConnection({ host: "127.0.0.1", port }); + probe.once("connect", () => { + probe.once("close", () => resume(Effect.void)); + process.kill(access.endpoint.pid, "SIGTERM"); + }); + probe.once("error", (cause) => resume(Effect.die(cause))); + return Effect.sync(() => probe.destroy()); + }); + + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.retry({ while: hasReason("owner-exit-pending") }), + Effect.timeout("30 seconds"), + ); + const cut = yield* sql.unsafe("SELECT 1").pipe(Effect.timeout("5 seconds"), Effect.exit); + expect(Exit.isFailure(cut), "the pinned connection is gone with the owner").toBe(true); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, diff --git a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts b/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts deleted file mode 100644 index 5e893aaa75..0000000000 --- a/packages/stack/src/StackHost.container-sweep-retry.integration.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { NodeHttpClient, NodeServices } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; -import { Context, Deferred, Effect, FileSystem, Layer, Sink, Stream } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "./State.ts"; -import * as Owner from "./Owner.ts"; -import * as CommandRunner from "./host/CommandRunner.ts"; -import { bindControl, makeRuntime } from "./StackHost.ts"; -import { shutdownOwner } from "../tests/owner.ts"; - -const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), - ); - -it.live("retains saved state when destroy sweep fails and retries after engine recovery", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-container-sweep-retry-" }); - const state = yield* stateFor(`${root}/state`); - const saved = { - id: "stack", - runtime: "docker" as const, - identity: { projectRoot: root, branchContext: "main", stackName: "sweep-retry" }, - instances: [], - lifetime: "detached" as const, - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - let listCalls = 0; - const engine = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") - return Effect.die("Unexpected child process command"); - const isStackSweep = - command.args[0] === "ps" && - command.args.includes(`label=com.supabase.stack=${saved.id}`) && - command.args.includes(`label=com.supabase.stack-root=${root}/data`); - const failList = isStackSweep && ++listCalls === 1; - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(failList ? 1 : 0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: failList - ? Stream.succeed(new TextEncoder().encode("temporary engine failure")) - : Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ); - }); - const ownerLayer = Owner.layer({ - saved, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - }).pipe( - Layer.provide( - Layer.merge( - Layer.succeed(State.Service, state), - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine), - ), - ), - ); - const baseOwner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - const owner = baseOwner; - const acquired = yield* bindControl(); - const runnerLayer = yield* Layer.build( - CommandRunner.layer({ - stackId: saved.id, - root: `${root}/data`, - cacheRoot: `${root}/cache`, - runtime: "docker", - }).pipe(Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, engine))), - ); - const runtime = yield* makeRuntime( - owner, - { - endpoint: { - stackId: saved.id, - identity: saved.identity, - pid: process.pid, - port: acquired.port, - release: "test", - }, - secret: "test-secret", - }, - acquired.server, - acquired.closeConnections, - ).pipe( - Effect.provideService( - CommandRunner.Service, - Context.get(runnerLayer, CommandRunner.Service), - ), - ); - yield* runtime.serve; - const failure = yield* shutdownOwner(runtime.access, true).pipe(Effect.flip); - expect(listCalls, failure.message).toBe(3); - expect(yield* (yield* stateFor(`${root}/state`)).read(saved.id)).toBeDefined(); - yield* Deferred.await(runtime.exit).pipe(Effect.timeout("10 seconds")); - const retryOwner = yield* Layer.build(ownerLayer).pipe( - Effect.map((context) => Context.get(context, Owner.Service)), - ); - yield* retryOwner.namespace.destroy; - expect(listCalls).toBe(5); - expect(yield* (yield* stateFor(`${root}/state`)).read(saved.id)).toBeUndefined(); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), -); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 0bb65d4a65..8ac184fac6 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -20,6 +20,7 @@ import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; import * as HttpClient from "effect/unstable/http/HttpClient"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- integration observes exact listener closure. import * as Net from "node:net"; +import { fileURLToPath } from "node:url"; import { launchHost, ownerAuthorization, @@ -31,18 +32,23 @@ import { import * as Owner from "./Owner.ts"; import { OrchestratorError } from "./Orchestrator.ts"; import { CommandEvent, StackError } from "./Rpc.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { bindControl, makeRuntime } from "./StackHost.ts"; -import { shutdownOwner } from "../tests/owner.ts"; +import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; import { postgres } from "./Commands.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; + +const shortRegistrationPollFixture = fileURLToPath( + new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), +); class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} const stateFor = (root: string) => Effect.gen(function* () { - const context = yield* Layer.build(State.layer({ root })); - return Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root })); + return Context.get(context, StackNamespace.Service); }); const hostTestError = (cause: unknown) => @@ -50,16 +56,19 @@ const hostTestError = (cause: unknown) => const ownerFor = (options: { readonly saved: Parameters[0]["saved"]; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly root: string; readonly cacheRoot: string; }) => { const { state, ...layerOptions } = options; return Effect.gen(function* () { + const draining = yield* Deferred.make(); const context = yield* Layer.build( - Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(State.Service, state))), + Owner.layer({ ...layerOptions, draining: Deferred.isDone(draining) }).pipe( + Layer.provide(Layer.succeed(StackNamespace.Service, state)), + ), ); - return Context.get(context, Owner.Service); + return { ...Context.get(context, Owner.Service), draining }; }); }; @@ -134,8 +143,8 @@ const awaitClosed = (socket: Net.Socket) => }); const inProcessRuntime = ( - owner: Parameters[0], - state: State.Interface, + owner: Parameters[0] & { readonly draining: Deferred.Deferred }, + state: StackNamespace.Interface, root: string, ) => Effect.gen(function* () { @@ -144,7 +153,7 @@ const inProcessRuntime = ( CommandRunner.layer({ stackId: "stack", root, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, runtime: "native", }), ); @@ -162,6 +171,7 @@ const inProcessRuntime = ( }, acquired.server, acquired.closeConnections, + owner.draining, ).pipe( Effect.provideService(CommandRunner.Service, Context.get(toolContext, CommandRunner.Service)), ); @@ -225,14 +235,13 @@ it.live("preserves composition outcomes over RPC", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const { runtime } = yield* inProcessRuntime(owner, state, root); const client = yield* ownerClient(runtime.access); @@ -268,7 +277,7 @@ it.live("preserves composition outcomes over RPC", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("keeps serving when namespace shutdown fails", () => +it.live("stops serving after a failed shutdown and keeps reporting that failure", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -281,14 +290,13 @@ it.live("keeps serving when namespace shutdown fails", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const failedOwner = { ...owner, @@ -300,15 +308,17 @@ it.live("keeps serving when namespace shutdown fails", () => }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); - const client = yield* ownerClient(runtime.access); const failure = yield* shutdownOwner(runtime.access, false).pipe(Effect.flip); expect(failure.message).toContain("cleanup failed"); - yield* client.configureComposition({ members: [], dependencies: [] }); + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); + expect(yield* Deferred.isDone(owner.draining)).toBe(true); + const again = yield* runtime.shutdown(true).pipe(Effect.flip); + expect(again.message).toContain("Shutdown mode is already selected"); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("reports destroy and fallback stop failures together", () => +it.live("reports a destroy failure and the owner exits", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -321,79 +331,52 @@ it.live("reports destroy and fallback stop failures together", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); - const failureWithOutcome = ( - operation: "destroy" | "stop", - message: string, - id: string, - reason: string, - ) => - new OrchestratorError({ - operation, - message, - outcomes: [ - { - id, - result: Exit.fail(new OrchestratorError({ operation, message: reason })), - }, - ], - }); const failedOwner = { ...owner, namespace: { ...owner.namespace, destroy: Effect.fail( - failureWithOutcome( - "destroy", - "Namespace destroy had failures", - "database-destroy", - "data removal refused", - ), - ), - stop: Effect.fail( - failureWithOutcome( - "stop", - "Composition stop had failures", - "database-stop", - "process stop refused", - ), + new OrchestratorError({ + operation: "destroy", + message: "Namespace destroy had failures", + outcomes: [ + { + id: "database-destroy", + result: Exit.fail( + new OrchestratorError({ + operation: "destroy", + message: "data removal refused", + }), + ), + }, + ], + }), ), }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); const failure = yield* shutdownOwner(runtime.access, true).pipe(Effect.flip); expect(failure.message).toContain("Namespace destroy had failures"); - expect(failure.message).toContain("database-destroy:"); - expect(failure.message).toContain("data removal refused"); - expect(failure.message).toContain("fallback stop failed: Composition stop had failures"); - expect(failure.message).toContain("database-stop:"); - expect(failure.message).toContain("process stop refused"); expect("outcomes" in failure).toBe(true); if (!("outcomes" in failure)) return yield* Effect.die("shutdown outcomes were missing"); - expect(failure.outcomes).toEqual( - expect.arrayContaining([ - { - id: "database-destroy", - succeeded: false, - error: expect.stringContaining("data removal refused"), - }, - { - id: "database-stop", - succeeded: false, - error: expect.stringContaining("process stop refused"), - }, - ]), - ); - expect(yield* Deferred.isDone(runtime.exit)).toBe(false); - expect(yield* owner.getServing).toBe(true); + expect(failure.outcomes).toEqual([ + { + id: "database-destroy", + succeeded: false, + error: expect.stringContaining("data removal refused"), + }, + ]); + // A failed destroy does not fall back to stop: the owner exits and the registered stack + // is reclaimed by the next stop or destroy. + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); @@ -411,14 +394,13 @@ it.live("rejects destroy while stop is in flight", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const entered = yield* Deferred.make(); const release = yield* Deferred.make(); @@ -444,7 +426,7 @@ it.live("rejects destroy while stop is in flight", () => ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("retains ownership when namespace shutdown defects and retries cleanup", () => +it.live("exits after a namespace shutdown defect without rerunning cleanup", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -457,14 +439,13 @@ it.live("retains ownership when namespace shutdown defects and retries cleanup", instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const failStop = yield* Ref.make(true); const failedOwner = { @@ -478,18 +459,12 @@ it.live("retains ownership when namespace shutdown defects and retries cleanup", }, }; const { runtime } = yield* inProcessRuntime(failedOwner, state, root); - const client = yield* ownerClient(runtime.access); const failure = yield* runtime.shutdown(false).pipe(Effect.exit); expect(Exit.isFailure(failure)).toBe(true); if (Exit.isFailure(failure)) expect(Cause.pretty(failure.cause)).toContain("cleanup failed"); - const http = yield* HttpClient.HttpClient; - const identity = yield* http.get(`http://127.0.0.1:${runtime.endpoint.port}/identity`, { - headers: { authorization: ownerAuthorization(runtime.access.secret) }, - }); - expect(identity.status).toBe(200); - expect(yield* owner.getServing).toBe(true); - yield* client.configureComposition({ members: [], dependencies: [] }); - yield* runtime.shutdown(false); + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); + // The stop would succeed if it ran again; the failure is final for this owner. + expect(Exit.isFailure(yield* runtime.shutdown(false).pipe(Effect.exit))).toBe(true); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); @@ -509,11 +484,10 @@ it.live( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const access = yield* launchHost(state, { stateRoot: `${root}/state`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, stackId: "stack", }); const { endpoint } = access; @@ -710,7 +684,6 @@ it.live("finishes detached shutdown after the caller disconnects", () => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const owner = yield* ownerFor({ saved: { @@ -720,11 +693,10 @@ it.live("finishes detached shutdown after the caller disconnects", () => instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const entered = yield* Deferred.make(); const allow = yield* Deferred.make(); @@ -762,14 +734,13 @@ it.live("withdraws a command waiting for its prerequisite", () => instances: [], lifetime: "detached" as const, composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); const owner = yield* ownerFor({ saved, state, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const entered = yield* Deferred.make(); const cancelled = yield* Deferred.make(); @@ -814,14 +785,13 @@ const disconnectFixture = (prefix: string) => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix }); const state = yield* stateFor(`${root}/state`); - const saved: State.SavedStack = { + const saved: StackNamespace.SavedStack = { id: "stack", runtime: "native", identity: { projectRoot: root, branchContext: "main", stackName: prefix }, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; yield* state.save(saved); return { root, state, saved }; @@ -852,7 +822,7 @@ it.live("finishes a service creation after its caller disconnects", () => ), }, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const interrupted = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( @@ -914,7 +884,7 @@ it.live("persists a composition change after its caller disconnects", () => ).pipe(Effect.andThen(state.save(next))), }, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const interrupted = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( @@ -980,17 +950,12 @@ const abandonedComposition = (prefix: string, destroy: boolean) => ), }, root: `${root}/data`, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }); const interrupted = yield* Deferred.make(); - const draining = yield* Deferred.make(); const { runtime } = yield* inProcessRuntime( { ...owner, - setDraining: (value: boolean) => - owner - .setDraining(value) - .pipe(Effect.andThen(value ? Deferred.succeed(draining, undefined) : Effect.void)), handlers: { ...owner.handlers, supabaseComposition: (input: Parameters[0]) => @@ -1023,7 +988,7 @@ const abandonedComposition = (prefix: string, destroy: boolean) => yield* Fiber.interrupt(composition); yield* Deferred.await(interrupted); const shutdown = yield* Effect.forkScoped(shutdownOwner(runtime.access, destroy)); - yield* Deferred.await(draining); + yield* Deferred.await(owner.draining); yield* Deferred.succeed(allow, undefined); yield* Fiber.join(shutdown); yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); @@ -1058,3 +1023,97 @@ it.live("destroys a stack only after an abandoned composition settles", () => }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + +const endedOwnership = (prefix: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix }); + const stateRoot = `${root}/state`; + const state = yield* stateFor(stateRoot); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "host-ownership" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + }; + yield* state.save(saved); + // The shortened poll interval comes only from this dedicated test entrypoint. + const launch = launchHost(state, { + stateRoot, + cacheRoot: testArtifactCacheRoot, + stackId: saved.id, + entrypoint: shortRegistrationPollFixture, + }); + const access = yield* launch; + yield* Effect.addFinalizer(() => + shutdownOwner(access, true).pipe( + Effect.ignore, + Effect.andThen( + waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe(Effect.ignore), + ), + ), + ); + return { fs, stateRoot, state, saved, launch, access }; + }); + +it.live.skipIf(process.platform === "win32")( + "exits when its stack directory is deleted and leaves a restarted stack's owner discoverable", + () => + Effect.scoped( + Effect.gen(function* () { + const { fs, stateRoot, state, saved, launch, access } = + yield* endedOwnership("stack-host-dir-deleted-"); + const exited = yield* watchLeaseRelease(stateRoot, saved.id); + yield* fs.remove(`${stateRoot}/${saved.id}`, { recursive: true }); + yield* state.save(saved); + const successor = yield* launch; + yield* Effect.addFinalizer(() => + shutdownOwner(successor, true).pipe( + Effect.ignore, + Effect.andThen( + waitForOwnerExit(successor.endpoint.pid, ownerExitProbe(fs)).pipe(Effect.ignore), + ), + ), + ); + yield* exited; + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("10 seconds"), + ); + + expect(successor.endpoint.pid, "the restart started its own owner").not.toBe( + access.endpoint.pid, + ); + expect((yield* state.readHolder(saved.id))?.role === "owner").toBe(true); + expect((yield* launch).endpoint.pid, "the successor is still discoverable").toBe( + successor.endpoint.pid, + ); + yield* shutdownOwner(successor, false); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 60_000 }, +); + +it.live.skipIf(process.platform === "win32")( + "exits without deleting data when only its registration is deleted", + () => + Effect.scoped( + Effect.gen(function* () { + const { fs, stateRoot, saved, access } = yield* endedOwnership( + "stack-host-registration-deleted-", + ); + const kept = `${stateRoot}/${saved.id}/data/kept.txt`; + yield* fs.writeFileString(kept, "data"); + const exited = yield* watchLeaseRelease(stateRoot, saved.id); + yield* fs.remove(`${stateRoot}/${saved.id}/state.json`); + yield* exited; + yield* waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("10 seconds"), + ); + + expect(yield* fs.readFileString(kept)).toBe("data"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 60_000 }, +); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index 23f2a5b813..51eb4d2e39 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -11,6 +11,7 @@ import { Data, DateTime, Deferred, + type Duration, Effect, Exit, FileSystem, @@ -38,21 +39,31 @@ import { type HostEndpoint, type ShutdownFailure, } from "./HostProcess.ts"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { projectSegmentFor } from "./identity/Identity.ts"; import * as Owner from "./Owner.ts"; import { StackError, stackError, StackRpc, type RunCommandPayload } from "./Rpc.ts"; -import { makeHostGateway } from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import { engineUnreachable, makeHostGateway, resolveEngineTarget } from "./runtime/Container.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { sweepOrphans } from "./Sweep.ts"; import { makeCommandAttachments } from "./host/CommandAttachments.ts"; import * as CommandRunner from "./host/CommandRunner.ts"; +/** + * How often an owner confirms its own registration still exists. Internal only: tests shorten + * it through a dedicated entrypoint that overrides this reference, never through env or `Config`. + */ +export const RegistrationCheckInterval = Context.Reference( + "@supabase/stack/RegistrationCheckInterval", + { defaultValue: () => "30 seconds" }, +); + export interface StackHostOptions { readonly stateRoot: string; readonly cacheRoot: string; readonly stackId: string; /** Registers this definition once the owner holds the lease; the stack must not exist. */ - readonly register?: State.SavedStack; + readonly register?: StackNamespace.SavedStack; readonly release?: string; readonly onReady?: (access: HostAccess) => Effect.Effect; } @@ -116,10 +127,14 @@ const creatorGone = Effect.callback((resume) => { }); }); -const isOpen = (value: boolean): Effect.Effect => - value - ? Effect.void - : Effect.fail(new StackError({ operation: "host", message: "Stack host is draining" })); +const rejectWhileDraining = (draining: Deferred.Deferred): Effect.Effect => + Deferred.isDone(draining).pipe( + Effect.flatMap((isDraining) => + isDraining + ? Effect.fail(new StackError({ operation: "host", message: "Stack host is draining" })) + : Effect.void, + ), + ); const requestSignal = () => Effect.callback<"SIGTERM" | "SIGINT", never>((resume) => { @@ -157,21 +172,32 @@ export interface StackHostRuntime { destroy: boolean, response?: ReturnType, ) => Effect.Effect; + /** + * Ends ownership that was lost to a deleted registration or a replaced lease file: stops only + * what this owner launched and leaves data, ports, registration and any successor's containers + * alone. It joins a shutdown already running. Failures are logged, never raised. + */ + readonly release: Effect.Effect; readonly exit: Deferred.Deferred; } +/** + * Serves the host. `draining` is the one source of truth for whether shutdown has begun: the host + * completes it when a shutdown begins and never resets it, and the owner reads it for admission. + */ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( ( owner: Owner.Interface, access: HostAccess, server: HttpServer.HttpServer["Service"], closeConnections: Effect.Effect, + draining: Deferred.Deferred, ): Effect.Effect => Effect.gen(function* () { const scope = yield* Scope.Scope; const runner = yield* CommandRunner.Service; const attachments = yield* makeCommandAttachments({ - admit: owner.getServing.pipe(Effect.flatMap(isOpen)), + admit: rejectWhileDraining(draining), run: (input) => "stdin" in input ? runner.run({ @@ -197,8 +223,9 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); const exit = yield* Deferred.make(); const gate = yield* Semaphore.make(1); + type ShutdownMode = "stop" | "destroy" | "release"; const current = yield* Ref.make< - { destroy: boolean; fiber: Fiber.Fiber } | undefined + { readonly mode: ShutdownMode; readonly fiber: Fiber.Fiber } | undefined >(undefined); const watchResponse = ( response: ReturnType, @@ -208,123 +235,83 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( responseClosed(response).pipe(Effect.andThen(Deferred.succeed(closed, undefined))), scope, ); + // Admits `mode`'s cleanup when nothing else is in flight: begins draining, forks the cleanup + // and tracks it as `current`. Draining never reverses: a failed shutdown leaves its failed + // fiber in `current` (repeat calls rejoin the same failure), and the owner exits once the + // response has been delivered. + const begin = ( + mode: ShutdownMode, + response?: ReturnType, + ): Effect.Effect> => + Effect.gen(function* () { + yield* Deferred.succeed(draining, undefined); + const responseClosedSignal = + response === undefined ? undefined : yield* Deferred.make(); + if (response !== undefined && responseClosedSignal !== undefined) + yield* watchResponse(response, responseClosedSignal); + const finish = Effect.gen(function* () { + if (response !== undefined) { + if (responseClosedSignal === undefined) return; + yield* Effect.forkIn( + Deferred.await(responseClosedSignal).pipe( + Effect.andThen(closeConnections), + Effect.andThen(Deferred.succeed(exit, undefined)), + ), + scope, + ); + } else { + yield* closeConnections; + yield* Deferred.succeed(exit, undefined); + } + }); + const body = Effect.gen(function* () { + yield* attachments.stopAll; + yield* runner.cleanup; + yield* owner.namespace[mode]; + }).pipe( + Effect.onError(() => finish), + Effect.andThen(finish), + Effect.mapError((cause) => stackError("shutdown", cause)), + ); + const fiber = yield* Effect.forkIn(body, scope); + yield* Ref.set(current, { mode, fiber }); + return fiber; + }); - const shutdown = Effect.fn("StackHost.shutdown")( - (destroy: boolean, response?: ReturnType) => - Effect.gen(function* () { - const fiber = yield* gate.withPermits(1)( - Effect.uninterruptibleMask(() => - Effect.gen(function* () { - const existing = yield* Ref.get(current); - if (existing !== undefined) { - if (existing.destroy && !destroy) return existing.fiber; - if (!existing.destroy && destroy) - return yield* new StackError({ - operation: "shutdown", - message: "Shutdown mode is already selected", - }); - return existing.fiber; - } - yield* owner.setDraining(true); - const responseClosedSignal = - response === undefined ? undefined : yield* Deferred.make(); - if (response !== undefined && responseClosedSignal !== undefined) - yield* watchResponse(response, responseClosedSignal); - let retiringAfterDestroyFailure = false; - const stopOwned = Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.stop; + // The one shutdown pipeline: every mode is claimed through the same gate and `current` + // record. A repeat request rejoins the shutdown in flight (a stop after a destroy, or the + // same mode again), and a destroy after a stop is rejected. + const claim = ( + mode: ShutdownMode, + response?: ReturnType, + ): Effect.Effect => + gate + .withPermits(1)( + Effect.uninterruptibleMask(() => + Effect.gen(function* () { + const existing = yield* Ref.get(current); + if (existing === undefined) return yield* begin(mode, response); + if (existing.mode === "stop" && mode === "destroy") + return yield* new StackError({ + operation: "shutdown", + message: "Shutdown mode is already selected", }); - const finish = Effect.gen(function* () { - if (response !== undefined) { - if (responseClosedSignal === undefined) return; - yield* Effect.forkIn( - Deferred.await(responseClosedSignal).pipe( - Effect.andThen(closeConnections), - Effect.andThen(Deferred.succeed(exit, undefined)), - ), - scope, - ); - } else { - yield* closeConnections; - yield* Deferred.succeed(exit, undefined); - } - }); - const cleanup = Effect.gen(function* () { - if (!destroy) { - yield* stopOwned; - yield* finish; - return; - } - const destroyExit = yield* Effect.gen(function* () { - yield* attachments.stopAll; - yield* runner.cleanup; - yield* owner.namespace.destroy; - }).pipe(Effect.exit); - if (Exit.isSuccess(destroyExit)) { - yield* finish; - return; - } - const stopExit = yield* stopOwned.pipe(Effect.exit); - if (Exit.isFailure(stopExit)) { - const describeCause = (cause: Cause.Cause) => { - const error = Option.match(Cause.findErrorOption(cause), { - onNone: () => - new StackError({ - operation: "shutdown", - message: Cause.pretty(cause), - }), - onSome: (value) => stackError("shutdown", value), - }); - const failedOutcomes = error.outcomes - ?.filter(({ succeeded }) => !succeeded) - .map(({ id, error: reason }) => `${id}: ${reason ?? "failed"}`) - .join("; "); - return { - error, - message: - failedOutcomes === undefined || failedOutcomes.length === 0 - ? error.message - : `${error.message} (${failedOutcomes})`, - }; - }; - const destroyFailure = describeCause(destroyExit.cause); - const stopFailure = describeCause(stopExit.cause); - const outcomes = [ - ...(destroyFailure.error.outcomes ?? []), - ...(stopFailure.error.outcomes ?? []), - ]; - return yield* new StackError({ - operation: "shutdown", - message: `${destroyFailure.message}; fallback stop failed: ${stopFailure.message}`, - ...(outcomes.length === 0 ? {} : { outcomes }), - }); - } - retiringAfterDestroyFailure = true; - yield* finish; - return yield* Effect.failCause(destroyExit.cause); - }).pipe( - Effect.mapError((cause) => stackError("shutdown", cause)), - Effect.catchCause((cause) => - retiringAfterDestroyFailure - ? Effect.failCause(cause) - : owner - .setDraining(false) - .pipe( - Effect.andThen(gate.withPermits(1)(Ref.set(current, undefined))), - Effect.andThen(Effect.failCause(cause)), - ), - ), - ); - const fiber = yield* Effect.forkIn(cleanup, scope); - yield* Ref.set(current, { destroy, fiber }); - return fiber; - }), - ), - ); - yield* Fiber.join(fiber); - }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))), + return existing.fiber; + }), + ), + ) + .pipe(Effect.flatMap(Fiber.join)); + const shutdown = ( + destroy: boolean, + response?: ReturnType, + ) => + claim(destroy ? "destroy" : "stop", response).pipe( + Effect.mapError((cause) => stackError("shutdown", cause)), + ); + // The ownership poll drives this, never an RPC caller, so there is no response to watch. + const release = claim("release").pipe( + Effect.tapCause((cause) => Effect.logError("Releasing the stack failed", cause)), + Effect.ignore, ); const handlers = StackRpc.of({ ...owner.handlers, @@ -375,11 +362,55 @@ export const makeRuntime = Effect.fn("StackHost.makeRuntime")( }); const serve: Effect.Effect = server.serve(application); - return { endpoint: access.endpoint, access, serve, shutdown, closeConnections, exit }; + return { + endpoint: access.endpoint, + access, + serve, + shutdown, + release, + closeConnections, + exit, + }; }), ); -type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone"; +type HostEvent = "SIGTERM" | "SIGINT" | "creator-gone" | "ownership-ended"; + +/** + * Ownership holds while the stack is registered and the lease path still names the file this + * owner locked. Only a confirmed ENOENT registration or a confirmed missing or replaced lease + * file ends it; any other error (permissions, an unmounted root mid-read) keeps the owner running. + */ +const ownershipEnded = Effect.fn("StackHost.ownershipEnded")(function* ( + state: StackNamespace.Interface, + id: string, + pathHeld: Effect.Effect, +) { + return yield* Effect.all([state.read(id), pathHeld]).pipe( + Effect.map(([saved, held]) => saved === undefined || !held), + Effect.tapError((cause) => + Effect.logWarning("Could not confirm the stack ownership; keeping the owner running", cause), + ), + Effect.orElseSucceed(() => false), + ); +}); + +/** Polls ownership and offers `ownership-ended` once it ends. */ +const pollOwnership = Effect.fn("StackHost.pollOwnership")(function* ( + state: StackNamespace.Interface, + id: string, + pathHeld: Effect.Effect, + events: Queue.Queue, +) { + const interval = yield* RegistrationCheckInterval; + while (true) { + yield* Effect.sleep(interval); + if (yield* ownershipEnded(state, id, pathHeld)) { + yield* Queue.offer(events, "ownership-ended"); + return; + } + } +}); export const runStackHost = Effect.fn("StackHost.run")( (options: StackHostOptions): Effect.Effect => @@ -393,20 +424,25 @@ export const runStackHost = Effect.fn("StackHost.run")( ), ), ); - const stateContext = yield* Layer.build(State.layer({ root: options.stateRoot })); - const state = Context.get(stateContext, State.Service); + const stateContext = yield* Layer.build(StackNamespace.layer({ root: options.stateRoot })); + const state = Context.get(stateContext, StackNamespace.Service); const id = options.stackId; // The lease is released last, after every owned process and listener has closed. - if (!(yield* state.lease(id))) - return yield* new StackHostError({ - operation: "lease", - message: `Another owner holds the lease of stack ${id}`, - reason: "lease-held", - }); + const lease = yield* state.acquireLease(id).pipe( + Effect.catchTag( + "Namespace.LeaseHeldError", + () => + new StackHostError({ + operation: "lease", + message: `Another owner holds the lease of stack ${id}`, + reason: "lease-held", + }), + ), + ); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fs.truncate(state.ownerLog(id)).pipe(Effect.ignore); - yield* state.retractHolder(id); + yield* lease.retractHolder; const register = options.register; if (register !== undefined) yield* state.withLock( @@ -420,40 +456,48 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* state.save(register); }), ); + const draining = yield* Deferred.make(); const started = yield* Effect.gen(function* () { const saved = yield* state.read(id); if (saved === undefined) return yield* hostError("startup", "Stack is not registered"); const control = yield* bindControl(); - const dataRootPath = path.join(options.stateRoot, saved.id, "data"); - yield* fs.makeDirectory(dataRootPath, { recursive: true }); - const dataRoot = yield* fs.realPath(dataRootPath); + const dataRoot = yield* StackNamespace.resolveStackDataRoot(options.stateRoot, saved.id); const project = projectSegmentFor(saved.identity, path); - yield* Owner.sweepContainers(saved, dataRoot).pipe( + // Resolved once, here, for this owner's whole lifetime: the container runtime, the + // storage helpers, the host-gateway probes and reconcile below all share this one + // target instead of each resolving (and so potentially disagreeing on) their own. + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const engineTarget = + saved.runtime === "native" + ? undefined + : yield* resolveEngineTarget(spawner, saved.runtime).pipe( + Effect.mapError((cause) => + hostError( + "startup-cleanup", + cause, + engineUnreachable(cause) ? "runtime-unavailable" : undefined, + ), + ), + ); + yield* Owner.sweepContainers(saved, dataRoot, engineTarget).pipe( Effect.mapError((cause) => hostError( "startup-cleanup", cause, - cause.reason === "engine-unavailable" ? "runtime-unavailable" : undefined, + engineUnreachable(cause) ? "runtime-unavailable" : undefined, ), ), ); - // After the sweep, so no leftover Vector container still mounts the files it removes. - // The loaded state already omits Vector, so a failed migration waits for a later start. - yield* state - .migrate(id) - .pipe( - Effect.catch((error) => - Effect.logWarning(`Unable to migrate the saved state of stack ${id}`, error), - ), - ); const hostGateway = yield* makeHostGateway; const services = yield* Layer.build( Layer.merge( Owner.layer({ + draining: Deferred.isDone(draining), saved, root: dataRoot, cacheRoot: options.cacheRoot, hostGateway, + engineTarget, }), CommandRunner.layer({ stackId: saved.id, @@ -462,8 +506,9 @@ export const runStackHost = Effect.fn("StackHost.run")( cacheRoot: options.cacheRoot, runtime: saved.runtime, hostGateway, + engineTarget, }), - ).pipe(Layer.provide(Layer.succeed(State.Service, state))), + ).pipe(Layer.provide(Layer.succeed(StackNamespace.Service, state))), ); const owner = Context.get(services, Owner.Service); const endpoint: HostEndpoint = { @@ -483,6 +528,7 @@ export const runStackHost = Effect.fn("StackHost.run")( access, control.server, control.closeConnections, + draining, ).pipe( Effect.provideService( CommandRunner.Service, @@ -490,8 +536,8 @@ export const runStackHost = Effect.fn("StackHost.run")( ), ); yield* runtime.serve; - yield* Effect.addFinalizer(() => state.retractHolder(id).pipe(Effect.ignore)); - yield* state.publishHolder(id, { + yield* Effect.addFinalizer(() => lease.retractHolder.pipe(Effect.ignore)); + yield* lease.publishHolder({ role: "owner", secret, port: endpoint.port, @@ -504,6 +550,7 @@ export const runStackHost = Effect.fn("StackHost.run")( yield* Effect.forkScoped( creatorGone.pipe(Effect.andThen(Queue.offer(events, "creator-gone"))), ); + yield* Effect.forkScoped(pollOwnership(state, id, lease.pathHeld, events)); yield* options.onReady?.(access) ?? Effect.void; yield* Effect.forkScoped( sweepOrphans({ @@ -511,6 +558,7 @@ export const runStackHost = Effect.fn("StackHost.run")( stateRoot: options.stateRoot, cacheRoot: options.cacheRoot, ownerId: id, + engineTarget, }), ); return runtime; @@ -532,25 +580,26 @@ export const runStackHost = Effect.fn("StackHost.run")( register === undefined ? Effect.void : state.remove(id).pipe(Effect.ignore), ), ); - while (true) { - const event = yield* Deferred.await(started.exit).pipe( - Effect.map(() => "done" as const), - Effect.raceFirst(Queue.take(events)), + const event = yield* Deferred.await(started.exit).pipe( + Effect.map(() => "done" as const), + Effect.raceFirst(Queue.take(events)), + ); + if (event === "creator-gone") { + yield* started.shutdown(true).pipe( + Effect.tapCause((cause) => Effect.logError("Session stack destroy failed", cause)), + Effect.ignore, ); - if (event === "done") break; - if (event === "creator-gone") { - yield* started.shutdown(true).pipe( - Effect.tapCause((cause) => Effect.logError("Session stack destroy failed", cause)), - Effect.ignore, - ); - break; - } - const shutdownSucceeded = yield* started.shutdown(false).pipe( + return; + } + if (event === "ownership-ended") { + yield* started.release; + return; + } + if (event !== "done") + yield* started.shutdown(false).pipe( Effect.tapCause((cause) => Effect.logError("Stack shutdown failed", cause)), - Effect.matchCause({ onSuccess: () => true, onFailure: () => false }), + Effect.ignore, ); - if (shutdownSucceeded) break; - } }), ).pipe( Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), diff --git a/packages/stack/src/StackNamespace.ts b/packages/stack/src/StackNamespace.ts new file mode 100644 index 0000000000..f7511a0deb --- /dev/null +++ b/packages/stack/src/StackNamespace.ts @@ -0,0 +1,62 @@ +import { Context, Effect, FileSystem, Layer, Path } from "effect"; +import { namespaceError, type NamespaceError } from "./namespace/Capabilities.ts"; +import * as Lease from "./namespace/Lease.ts"; +import * as Registry from "./namespace/Registry.ts"; +import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; + +export { NamespaceError } from "./namespace/Capabilities.ts"; +export { LeaseHeldError, LeaseHolder } from "./namespace/Lease.ts"; +export { resolveStackDataRoot, stackDataRootLabel, stackLogsRoot } from "./namespace/Paths.ts"; +export { publish } from "./namespace/Publication.ts"; +export { + SavedStack, + StackCredentials, + StackKeysInput, + StackLifetime, +} from "./namespace/Registry.ts"; + +export interface Interface extends Registry.Interface, Lease.Interface { + /** The namespace's own root directory. */ + readonly root: string; +} + +export class Service extends Context.Service()( + "@supabase/stack/StackNamespace", +) {} + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; + readonly onInvalidState?: (id: string, error: NamespaceError) => Effect.Effect; + /** Observes a lease request that found the lease held and is waiting for it. */ + readonly onLeaseContended?: (id: string) => Effect.Effect; +} + +const make = Effect.fn("Namespace.acquire")(function* ( + options: Options, +): Effect.fn.Return { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = path.normalize(options.root); + yield* fs + .makeDirectory(root, { recursive: true }) + .pipe(Effect.mapError((cause) => namespaceError("root", cause))); + yield* restrictDirectoryToOwner(fs, root).pipe( + Effect.mapError((cause) => namespaceError("root", cause)), + ); + const registry = yield* Registry.make({ + root, + platform: options.platform, + onInvalidState: options.onInvalidState, + }); + const lease = yield* Lease.make({ + root, + platform: options.platform, + onLeaseContended: options.onLeaseContended, + isRegistered: (id) => registry.read(id).pipe(Effect.map((saved) => saved !== undefined)), + }); + return { ...registry, ...lease, root }; +}); + +export const layer = (options: Options) => + Layer.effect(Service, make(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/State.process.integration.test.ts b/packages/stack/src/State.process.integration.test.ts deleted file mode 100644 index 90dec13572..0000000000 --- a/packages/stack/src/State.process.integration.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; -import { - Context, - Data, - Deferred, - Effect, - Fiber, - FileSystem, - Layer, - Ref, - Schema, - Stream, -} from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { fileURLToPath } from "node:url"; -import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; -import * as State from "./State.ts"; - -class FixtureError extends Data.TaggedError("StateLockFixtureError")<{ - readonly message: string; -}> {} -const fixture = fileURLToPath(new URL("../tests/state-lock-fixture.ts", import.meta.url)); -const readOutput = (child: ChildProcessSpawner.ChildProcessHandle) => - Effect.all( - [ - child.stdout.pipe(Stream.decodeText, Stream.mkString), - child.stderr.pipe(Stream.decodeText, Stream.mkString), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - -for (const compiled of [false, true]) { - it.live( - `serializes writers and releases a killed holder while its child survives (${compiled ? "compiled" : "source"})`, - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-process-" }); - const executable = compiled - ? `${root}/fixture${process.platform === "win32" ? ".exe" : ""}` - : process.execPath; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const spawn = (command: string, args: ReadonlyArray) => - spawner.spawn( - ChildProcess.make(command, args, { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - detached: true, - forceKillAfter: "1 second", - }), - ); - if (compiled) { - const build = yield* spawn("bun", [ - "build", - fixture, - "--compile", - `--outfile=${executable}`, - ]); - const [, stderr, code] = yield* readOutput(build); - expect(code, stderr).toBe(0); - } - const stateRoot = `${root}/state`; - const context = yield* Layer.build(State.layer({ root: stateRoot })); - const state = Context.get(context, State.Service); - const saved: State.SavedStack = { - id: "stack-main", - identity: { projectRoot: root, branchContext: "test", stackName: "lock" }, - runtime: "native", - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }; - yield* state.save(saved); - const args = (mode: string, id?: string) => [ - ...(compiled ? [] : [fixture]), - mode, - stateRoot, - ...(id === undefined ? [] : [id]), - ]; - yield* Effect.forEach( - ["first", "second", "third"], - (id) => - Effect.scoped( - Effect.gen(function* () { - const writer = yield* spawn(executable, args("write", id)); - const [, stderr, code] = yield* readOutput(writer); - expect(code, stderr).toBe(0); - }), - ), - { concurrency: "unbounded" }, - ); - expect((yield* state.read(saved.id))?.instances.map(({ id }) => id).sort()).toEqual([ - "first", - "second", - "third", - ]); - - const holder = yield* spawn(executable, args("hold")); - const locked = yield* Deferred.make(); - const childReady = yield* Deferred.make(); - const observed = yield* Ref.make>([]); - const output = yield* holder.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - Ref.update(observed, (lines) => [...lines, line]).pipe( - Effect.andThen( - line === "locked" - ? Deferred.succeed(locked, undefined).pipe(Effect.asVoid) - : line.startsWith("child:") - ? Deferred.succeed(childReady, line.slice(6)).pipe(Effect.asVoid) - : Effect.void, - ), - ), - ), - Effect.forkScoped, - ); - const stderr = yield* Ref.make(""); - const diagnostics = yield* holder.stderr.pipe( - Stream.decodeText, - Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), - Effect.forkScoped, - ); - yield* Effect.all([Deferred.await(locked), Deferred.await(childReady)]).pipe( - Effect.raceFirst( - holder.exitCode.pipe( - Effect.matchEffect({ - onFailure: (cause) => - Ref.get(stderr).pipe( - Effect.flatMap((text) => - Effect.fail( - new FixtureError({ - message: `Holder exited before readiness: ${String(cause)}\n${text}`, - }), - ), - ), - ), - onSuccess: (code) => - Ref.get(stderr).pipe( - Effect.flatMap((text) => - Effect.fail( - new FixtureError({ - message: `Holder exited before readiness (${code}): ${text}`, - }), - ), - ), - ), - }), - ), - ), - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => - Effect.all([Ref.get(stderr), Ref.get(observed)]).pipe( - Effect.flatMap(([text, lines]) => - Effect.fail( - new FixtureError({ - message: `Holder readiness timed out: ${lines.join("\n")}\n${text}`, - }), - ), - ), - ), - }), - ); - const childPid = yield* Schema.decodeEffect(Schema.FiniteFromString)( - yield* Deferred.await(childReady), - ); - const kill = (pid: number) => - Effect.try({ - try: () => process.kill(pid, "SIGKILL"), - catch: (cause) => new FixtureError({ message: String(cause) }), - }); - yield* Effect.addFinalizer(() => - kill(childPid).pipe( - Effect.ignore, - Effect.andThen(Fiber.join(output)), - Effect.andThen(Fiber.join(diagnostics)), - // Windows keeps the compiled child's image in the temp root locked until it exits. - Effect.andThen(waitForOwnerExit(childPid, ownerExitProbe(fs)).pipe(Effect.ignore)), - Effect.timeout("10 seconds"), - Effect.orDie, - ), - ); - yield* kill(Number(holder.pid)); - yield* holder.exitCode.pipe(Effect.exit); - yield* state.withLock( - state.save({ ...saved, identity: { ...saved.identity, stackName: "after-crash" } }), - ); - expect((yield* state.read(saved.id))?.identity.stackName).toBe("after-crash"); - yield* Effect.try({ - try: () => process.kill(childPid, 0), - catch: (cause) => - new FixtureError({ message: `Child did not survive holder: ${String(cause)}` }), - }); - expect((yield* fs.stat(`${stateRoot}/.registry-lock.sqlite`)).size).toBe(0n); - expect((yield* fs.readDirectory(stateRoot)).sort()).toEqual([ - ".registry-lock.sqlite", - saved.id, - ]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); -} diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts deleted file mode 100644 index 01fa3d67cf..0000000000 --- a/packages/stack/src/State.ts +++ /dev/null @@ -1,791 +0,0 @@ -import { - Clock, - Data, - Effect, - Exit, - FileSystem, - Context, - Layer, - Option, - Path, - Predicate, - Schedule, - Schema, - Scope, -} from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. -import { rmdir } from "node:fs/promises"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no OS-owned cross-process lock primitive. -import { DatabaseSync } from "node:sqlite"; -import { pathToFileURL } from "node:url"; -import { CompositionConfig } from "./Orchestrator.ts"; -import { errorCode, retrySharingViolation } from "./internal/sharing-violation.ts"; -import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; -import { ServiceCreation } from "./services/Catalog.ts"; - -const SafeId = Schema.String.pipe( - Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { - identifier: "SafeStateId", - message: "Expected a safe state id", - }), -); - -const SavedInstance = Schema.Struct({ - id: SafeId, - creation: Schema.toCodecJson(ServiceCreation), - /** The latest launch id; the next owner continues after it, so launch ids keep increasing. */ - launchId: Schema.optionalKey(Schema.Int), -}); -interface SavedInstance extends Schema.Schema.Type {} - -/** API and JWT signing keys that override the keys a stack derives by default. */ -export const StackKeysInput = Schema.Struct({ - publishableKey: Schema.optionalKey(Schema.String), - secretKey: Schema.optionalKey(Schema.String), - anonKey: Schema.optionalKey(Schema.String), - anonKeyIsOverride: Schema.optionalKey(Schema.Boolean), - serviceRoleKey: Schema.optionalKey(Schema.String), - serviceRoleKeyIsOverride: Schema.optionalKey(Schema.Boolean), - gotrueJwtKeys: Schema.optionalKey(Schema.String), - publicSigningKeys: Schema.optionalKey(Schema.String), - remoteJwks: Schema.optionalKey(Schema.String), -}); -export interface StackKeysInput extends Schema.Schema.Type {} - -export const StackCredentials = Schema.Struct({ - jwtSecret: Schema.String, - postgresRootKey: Schema.String, - databasePassword: Schema.String, - publishableKey: Schema.String, - secretKey: Schema.String, - anonKey: Schema.String, - serviceRoleKey: Schema.String, - jwks: Schema.String, - gotrueJwtKeys: Schema.String, - remoteJwks: Schema.String, - anonKeyIsOverride: Schema.Boolean, - serviceRoleKeyIsOverride: Schema.Boolean, -}); -export interface StackCredentials extends Schema.Schema.Type {} - -const PortClaim = Schema.Struct({ - key: Schema.String, - host: Schema.String, - port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), -}); -export interface PortClaim extends Schema.Schema.Type {} - -/** A session stack is destroyed when its creator exits; a detached stack outlives it. */ -export const StackLifetime = Schema.Literals(["session", "detached"]); -export type StackLifetime = Schema.Schema.Type; - -export const SavedStack = Schema.Struct({ - id: SafeId, - lifetime: StackLifetime, - identity: Schema.Struct({ - projectRoot: Schema.String, - branchContext: Schema.String, - stackName: Schema.String, - }), - runtime: Schema.Literals(["native", "docker", "podman"]), - instances: Schema.Array(SavedInstance).check( - Schema.makeFilter((instances) => - new Set(instances.map(({ id }) => id)).size === instances.length - ? undefined - : "Expected unique instance ids", - ), - ), - composition: CompositionConfig, - credentials: Schema.optionalKey(StackCredentials), - ports: Schema.Array(PortClaim), -}); -export interface SavedStack extends Schema.Schema.Type {} - -const ClaimsDocument = Schema.Struct({ ports: Schema.Array(PortClaim) }); - -/** What the current lease holder publishes: an owner's control endpoint, or a sweeper's marker. */ -const LeaseHolder = Schema.Union([ - Schema.Struct({ - role: Schema.Literal("owner"), - port: PortClaim.fields.port, - pid: Schema.Int, - release: Schema.String, - lifetime: StackLifetime, - startedAt: Schema.String, - secret: Schema.String, - }), - Schema.Struct({ role: Schema.Literal("sweeper"), pid: Schema.Int, startedAt: Schema.String }), -]); -type LeaseHolder = Schema.Schema.Type; - -export interface StackClaims { - readonly id: string; - readonly ports: ReadonlyArray; -} - -export class StateError extends Data.TaggedError("StateError")<{ - readonly operation: string; - readonly message: string; - readonly cause?: unknown; -}> {} - -export interface Interface { - /** Omits saved Vector instances and their composition entries without rewriting the file. */ - readonly read: (id: string) => Effect.Effect; - /** - * Persists the removal of saved Vector instances and deletes their stack-owned files. Call while - * holding the stack's lease, before any of its services run. - */ - readonly migrate: (id: string) => Effect.Effect; - /** Skips each stack entry that stays unreadable after transient retries, reporting it to `onInvalidState`. */ - readonly list: Effect.Effect, StateError>; - /** Decodes only each stack's port claims and silently skips entries that cannot provide them. */ - readonly claims: Effect.Effect, StateError>; - readonly save: (state: SavedStack) => Effect.Effect; - readonly remove: (id: string) => Effect.Effect; - /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ - readonly withLock: ( - effect: Effect.Effect, - ) => Effect.Effect; - /** - * Holds the stack's owner lease for the enclosing scope, or returns `false` while another - * process holds it. Releasing the lease of a removed stack deletes its directory. - */ - readonly lease: (id: string) => Effect.Effect; - /** Reports whether any process currently holds the stack's owner lease. */ - readonly leased: (id: string) => Effect.Effect; - /** Only meaningful while the lease is held; a record left by a dead holder is stale. */ - readonly readHolder: (id: string) => Effect.Effect; - /** Written by the lease holder with owner-only permissions, since it carries the owner secret. */ - readonly publishHolder: (id: string, record: LeaseHolder) => Effect.Effect; - readonly retractHolder: (id: string) => Effect.Effect; - /** The file that receives the stack owner's stdout and stderr. */ - readonly ownerLog: (id: string) => string; - /** The directory that holds the stack's persisted service logs. */ - readonly logsRoot: (id: string) => string; -} - -export class Service extends Context.Service()("@supabase/stack/State") {} - -const stateError = (operation: string, cause: unknown): StateError => - new StateError({ - operation, - message: cause instanceof Error ? cause.message : String(cause), - cause, - }); - -const checkId = (id: string): Effect.Effect => - Schema.is(SafeId)(id) - ? Effect.void - : Effect.fail(stateError("identity", `Invalid state id: ${id}`)); - -const retainedEntries = ( - entries: unknown, - retired: (entry: Readonly>) => boolean, -): unknown => - Array.isArray(entries) - ? entries.filter((entry: unknown) => !(Predicate.isReadonlyObject(entry) && retired(entry))) - : entries; - -/** - * Drops saved Vector instances, which the stack no longer runs, with every composition member, - * dependency and port claim that references them. An instance with an unsafe id stays, so decoding - * rejects the document. - */ -const withoutVectorInstances = ( - document: unknown, -): { readonly document: unknown; readonly removed: ReadonlyArray } => { - if (!Predicate.isReadonlyObject(document) || !Array.isArray(document.instances)) - return { document, removed: [] }; - const removed = document.instances.flatMap((instance: unknown) => - Predicate.isReadonlyObject(instance) && - Schema.is(SafeId)(instance.id) && - Predicate.isReadonlyObject(instance.creation) && - instance.creation.service === "vector" - ? [instance.id] - : [], - ); - if (removed.length === 0) return { document, removed }; - const ids: ReadonlySet = new Set(removed); - const { composition } = document; - return { - removed, - document: { - ...document, - instances: retainedEntries(document.instances, (instance) => ids.has(instance.id)), - composition: Predicate.isReadonlyObject(composition) - ? { - ...composition, - members: retainedEntries(composition.members, (member) => ids.has(member.id)), - dependencies: retainedEntries( - composition.dependencies, - (dependency) => ids.has(dependency.from) || ids.has(dependency.to), - ), - } - : composition, - ports: retainedEntries( - document.ports, - ({ key }) => typeof key === "string" && removed.some((id) => key.startsWith(`${id}:`)), - ), - }, - }; -}; - -const decodeState = ( - text: string, - id: string, - target: string, -): Effect.Effect => - Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text).pipe( - Effect.flatMap((document) => - Schema.decodeUnknownEffect(SavedStack)(withoutVectorInstances(document).document), - ), - Effect.mapError( - (cause) => - new StateError({ - operation: "decode", - message: `Unable to decode state ${target} for stack ${id}: ${cause.message}`, - cause, - }), - ), - ); - -interface Options { - readonly root: string; - readonly platform?: NodeJS.Platform; - readonly onInvalidState?: (id: string, error: StateError) => Effect.Effect; - /** Observes a lease request that found the lease held and is waiting for it. */ - readonly onLeaseContended?: (id: string) => Effect.Effect; -} - -const logsDirectory = "logs"; - -/** Resolves the persisted service logs directory of a stack without opening the state root. */ -export const stackLogsRoot = ( - path: Path.Path, - stateRoot: string, - id: string, -): Effect.Effect => - checkId(id).pipe(Effect.as(path.join(path.normalize(stateRoot), id, logsDirectory))); - -const stateWritePrefix = ".state-write-"; -/** A state write takes milliseconds, so a temporary directory this old belongs to a dead writer. */ -const staleWriteAgeMillis = 24 * 60 * 60 * 1000; - -/** Best-effort removal of temporary write directories a killed writer left behind. */ -export const reapStaleWrites = (fs: FileSystem.FileSystem, path: Path.Path, root: string) => - Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - const entries = yield* fs.readDirectory(root); - yield* Effect.forEach( - entries.filter((entry) => entry.startsWith(stateWritePrefix)), - (entry) => { - const candidate = path.join(root, entry); - return fs.stat(candidate).pipe( - Effect.flatMap((info) => - Option.exists(info.mtime, (mtime) => now - mtime.getTime() > staleWriteAgeMillis) - ? fs.remove(candidate, { recursive: true, force: true }) - : Effect.void, - ), - Effect.ignore, - ); - }, - { discard: true }, - ); - }).pipe(Effect.ignore); - -/** - * Replaces `target` with `content` (mode 0600) through a temporary file in a fresh directory under - * `directory`, which must share the target's file system, retrying Windows sharing violations. - */ -export const writeFileAtomically = ( - fs: FileSystem.FileSystem, - path: Path.Path, - options: { - readonly directory: string; - readonly target: string; - readonly content: string; - readonly platform?: NodeJS.Platform; - }, -): Effect.Effect => - Effect.acquireUseRelease( - fs - .makeTempDirectory({ directory: options.directory, prefix: stateWritePrefix }) - .pipe(Effect.mapError((cause) => stateError("write", cause))), - (directory) => - Effect.gen(function* () { - const temporary = path.join(directory, path.basename(options.target)); - yield* fs - .writeFileString(temporary, options.content, { mode: 0o600 }) - .pipe(Effect.mapError((cause) => stateError("write", cause))); - yield* fs.rename(temporary, options.target).pipe( - retrySharingViolation(options.platform), - Effect.mapError( - (cause) => - new StateError({ - operation: "publish", - message: `Unable to publish state to ${options.target}${errorCode(cause) ? ` (${errorCode(cause)})` : ""}: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - Effect.withSpan("State.publish"), - ); - }), - (directory) => - fs - .remove(directory, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => stateError("cleanup", cause))), - ); - -const makeState = ( - options: Options, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.normalize(options.root); - // SQLite alone opens this persistent file; other descriptors can invalidate its POSIX locks. - const lock = path.join(root, ".registry-lock.sqlite"); - yield* fs - .makeDirectory(root, { recursive: true }) - .pipe(Effect.mapError((cause) => stateError("root", cause))); - yield* restrictDirectoryToOwner(fs, root).pipe( - Effect.mapError((cause) => stateError("root", cause)), - ); - yield* reapStaleWrites(fs, path, root); - - const stackRoot = (id: string) => path.join(root, id); - const statePath = (id: string) => path.join(stackRoot(id), "state.json"); - // Only lease code opens a lease file, for the same reason as the registry lock. - const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); - const ownerPath = (id: string) => path.join(stackRoot(id), "owner.json"); - const ownerLog = (id: string) => path.join(stackRoot(id), "owner.log"); - const logsRoot = (id: string) => path.join(stackRoot(id), logsDirectory); - const retryShared = retrySharingViolation(options.platform); - const retryTransientRead = (effect: Effect.Effect) => - effect.pipe( - retryShared, - Effect.mapError((cause) => stateError("read", cause)), - ); - const removeEmptyDirectory = (directory: string) => - Effect.tryPromise({ - try: () => rmdir(directory), - catch: (cause) => stateError("cleanup", cause), - }).pipe( - Effect.catch((cause) => { - const code = - typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause - ? cause.cause.code - : undefined; - return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" - ? Effect.void - : Effect.fail(cause); - }), - ); - const read = Effect.fn("State.read")(function* (id: string) { - yield* checkId(id); - const target = statePath(id); - const exists = yield* fs.exists(target).pipe(retryTransientRead); - if (!exists) return undefined; - const text = yield* fs.readFileString(target).pipe(retryTransientRead); - const state = yield* decodeState(text, id, target); - if (state.id !== id) { - return yield* stateError("identity", "State document identity does not match its path"); - } - return state; - }); - const stackIds = fs.readDirectory(root).pipe( - Effect.map((entries) => entries.filter(Schema.is(SafeId))), - Effect.mapError((cause) => stateError("list", cause)), - ); - const readEntries = ( - readEntry: (id: string) => Effect.Effect, - onSkipped: (id: string, error: StateError) => Effect.Effect, - ) => - Effect.gen(function* () { - const entries: Array = []; - for (const id of yield* stackIds) { - const value = yield* readEntry(id).pipe( - Effect.catch((error) => onSkipped(id, error).pipe(Effect.as(undefined))), - ); - if (value !== undefined) entries.push(value); - } - return entries; - }); - const list = Effect.fn("State.list")(() => - readEntries(read, (id, error) => options.onInvalidState?.(id, error) ?? Effect.void), - ); - const decodeClaims = Schema.decodeEffect(Schema.fromJsonString(ClaimsDocument)); - const readClaims = (id: string) => - fs.readFileString(statePath(id)).pipe( - retryTransientRead, - Effect.flatMap((text) => - decodeClaims(text).pipe(Effect.mapError((cause) => stateError("decode", cause))), - ), - Effect.map(({ ports }): StackClaims => ({ id, ports })), - ); - const claims = Effect.fn("State.claims")(() => readEntries(readClaims, () => Effect.void)); - const writeAtomically = (id: string, target: string, serialized: string) => - fs.makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }).pipe( - Effect.mapError((cause) => stateError("write", cause)), - Effect.andThen( - writeFileAtomically(fs, path, { - directory: root, - target, - content: serialized, - ...(options.platform === undefined ? {} : { platform: options.platform }), - }), - ), - ); - const save = Effect.fn("State.save")(function* (state: SavedStack) { - yield* checkId(state.id); - const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(SavedStack))(state).pipe( - Effect.mapError((cause) => stateError("encode", cause)), - ); - yield* writeAtomically(state.id, statePath(state.id), serialized); - }); - const remove = Effect.fn("State.remove")(function* (id: string) { - yield* checkId(id); - // Logs go before the state file, so a stack whose logs remain stays listed for another removal. - yield* fs.remove(logsRoot(id), { recursive: true, force: true }).pipe( - retryShared, - Effect.mapError((cause) => stateError("remove", cause)), - ); - for (const file of [statePath(id), ownerPath(id), ownerLog(id)]) - yield* fs - .remove(file, { force: true }) - .pipe(Effect.mapError((cause) => stateError("remove", cause))); - yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); - yield* removeEmptyDirectory(stackRoot(id)); - }); - const openLock = (file: string) => - Effect.try({ - try: () => new DatabaseSync(file), - catch: (cause) => stateError("lock", cause), - }); - /** Opens an existing lock file without creating a stray one. */ - const openExistingLock = (file: string) => - Effect.try({ - try: () => new DatabaseSync(new URL(`${pathToFileURL(file).href}?mode=rw`)), - catch: (cause) => stateError("lock", cause), - }); - const closeLock = (connection: DatabaseSync) => - Effect.try({ - try: () => connection.close(), - catch: (cause) => stateError("unlock", cause), - }); - const hasErrcode = (code: number) => (error: StateError) => - Predicate.hasProperty(error.cause, "errcode") && error.cause.errcode === code; - const isBusy = hasErrcode(5); - const isMissing = hasErrcode(14); - /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ - const isMoved = (error: StateError) => hasErrcode(6922)(error) || hasErrcode(1802)(error); - /** Takes the file's SQLite write lock on this connection, retrying contention on `schedule`. */ - const takeLock = ( - connection: DatabaseSync, - schedule: Schedule.Schedule, - onContended: Effect.Effect = Effect.void, - ) => - Effect.gen(function* () { - yield* Effect.try({ - try: () => connection.exec("PRAGMA busy_timeout = 0"), - catch: (cause) => stateError("lock", cause), - }); - let contended = false; - return yield* Effect.try({ - try: () => connection.exec("BEGIN IMMEDIATE"), - catch: (cause) => stateError("lock", cause), - }).pipe( - Effect.tapError((error) => - isBusy(error) && !contended - ? Effect.sync(() => { - contended = true; - }).pipe(Effect.andThen(onContended)) - : Effect.void, - ), - Effect.retry({ schedule, while: isBusy }), - Effect.as(true), - Effect.catchIf(isBusy, () => Effect.succeed(false)), - ); - }); - const withLock = Effect.fn("State.withLock")((effect: Effect.Effect) => - Effect.acquireUseRelease( - openLock(lock), - (connection) => - Effect.gen(function* () { - const held = yield* takeLock( - connection, - Schedule.spaced("50 millis").pipe(Schedule.upTo({ duration: "5 seconds" })), - ); - if (!held) - return yield* new StateError({ - operation: "lock", - message: "Stack registry is locked by another operation; retry shortly", - }); - return yield* effect; - }), - closeLock, - ), - ); - /** - * Deletes an unregistered stack's lease file while it is still the file locked as `identity`; - * `false` means it is still in place. - */ - const removeLeaseFile = (id: string, identity: string) => - Effect.all([fs.exists(statePath(id)), fileIdentity(leasePath(id))]).pipe( - Effect.flatMap(([registered, current]) => - registered || !Option.contains(current, identity) - ? Effect.void - : fs.remove(leasePath(id), { force: true }), - ), - Effect.as(true), - Effect.orElseSucceed(() => false), - ); - const fileIdentity = (file: string) => - fs.stat(file).pipe( - Effect.map((info) => `${info.dev}:${Option.getOrElse(info.ino, () => "")}`), - Effect.option, - ); - /** - * A waiter can open the lease file just before its holder unlinks it, and then lock the - * unlinked file; comparing the path's file before opening and after locking rejects that. - * Unlinking under the lock keeps new openers off a doomed file. Windows refuses to unlink an - * open file, so only there does a second attempt follow the close. - */ - const lease = Effect.fn("State.lease")(function* (id: string) { - yield* checkId(id); - const target = leasePath(id); - const scope = yield* Scope.Scope; - let held: string | undefined; - let unlinked = false; - yield* Effect.addFinalizer(() => - held !== undefined - ? (unlinked ? Effect.void : removeLeaseFile(id, held)).pipe( - Effect.andThen(removeEmptyDirectory(stackRoot(id))), - Effect.ignore, - ) - : Effect.void, - ); - const onContended = options.onLeaseContended?.(id) ?? Effect.void; - for (let attempt = 0; attempt < 8; attempt++) { - // A releasing holder may remove the empty stack directory at any moment. - yield* fs - .makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => stateError("lease", cause))); - const before = yield* fileIdentity(target); - const attemptScope = yield* Scope.fork(scope, "sequential"); - const connection = yield* Effect.acquireRelease(openLock(target), (connection) => - closeLock(connection).pipe( - Effect.catch((error) => - Effect.logWarning(`Unable to release stack lease ${id}`, error), - ), - ), - ).pipe( - Scope.provide(attemptScope), - Effect.map(Option.some), - Effect.catchIf(isMissing, () => Effect.succeed(Option.none())), - ); - if (Option.isNone(connection)) { - yield* Scope.close(attemptScope, Exit.void); - continue; - } - const acquired = yield* takeLock( - connection.value, - Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "500 millis" })), - onContended, - ).pipe( - Effect.map((held) => (held ? "held" : "busy")), - Effect.catchIf(isMoved, () => Effect.succeed("moved" as const)), - ); - if (acquired === "busy") { - yield* Scope.close(attemptScope, Exit.void); - return false; - } - const after = acquired === "moved" ? Option.none() : yield* fileIdentity(target); - if (Option.isSome(before) && Option.isSome(after) && before.value === after.value) { - held = after.value; - yield* Scope.addFinalizer( - attemptScope, - removeLeaseFile(id, after.value).pipe( - Effect.map((removed) => { - unlinked = removed; - }), - ), - ); - return true; - } - yield* Scope.close(attemptScope, Exit.void); - } - return yield* stateError("lease", `The lease file of stack ${id} keeps changing`); - }); - const leased = Effect.fn("State.leased")(function* (id: string) { - yield* checkId(id); - return yield* Effect.acquireUseRelease( - openExistingLock(leasePath(id)), - (connection) => - takeLock(connection, Schedule.recurs(0)).pipe(Effect.map((acquired) => !acquired)), - closeLock, - ).pipe( - // A lease file that is missing, or that its holder unlinked meanwhile, is not held. - Effect.catchIf( - (error) => isMissing(error) || isMoved(error), - () => Effect.succeed(false), - ), - ); - }); - const decodeHolder = Schema.decodeEffect(Schema.fromJsonString(LeaseHolder)); - const readHolder = Effect.fn("State.readHolder")(function* (id: string) { - yield* checkId(id); - const target = ownerPath(id); - // The holder may retract its record at any moment, so a missing record is not an error. - const text = yield* fs.readFileString(target).pipe( - Effect.map(Option.some), - Effect.catchIf( - (error) => error.reason._tag === "NotFound", - () => Effect.succeed(Option.none()), - ), - retryTransientRead, - ); - if (Option.isNone(text)) return undefined; - return yield* decodeHolder(text.value).pipe( - Effect.mapError((cause) => stateError("decode", `Unable to decode ${target}: ${cause}`)), - ); - }); - const publishHolder = Effect.fn("State.publishHolder")(function* ( - id: string, - record: LeaseHolder, - ) { - yield* checkId(id); - const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(LeaseHolder))( - record, - ).pipe(Effect.mapError((cause) => stateError("encode", cause))); - yield* writeAtomically(id, ownerPath(id), serialized); - }); - const retractHolder = Effect.fn("State.retractHolder")(function* (id: string) { - yield* checkId(id); - yield* fs - .remove(ownerPath(id), { force: true }) - .pipe(Effect.mapError((cause) => stateError("remove", cause))); - }); - const vectorConfigRoot = (instanceRoot: string) => path.join(instanceRoot, "runtime", "vector"); - const isVectorRecipe = (entry: string) => - entry === "vector.yaml" || - entry === "vector-api.yaml" || - entry === "vector.rendered.yaml" || - entry.startsWith(".vector-write-"); - /** - * Resolved instance data roots that still hold Vector recipe files. Found by layout rather than - * by saved ids, so a cleanup that failed is retried after the document no longer names Vector. - * A root whose config directory resolves elsewhere is skipped, so removal stays in the stack. - */ - const vectorDataRoots = (id: string) => - Effect.gen(function* () { - const dataRoot = path.join(stackRoot(id), "data"); - const entries = yield* fs.readDirectory(dataRoot).pipe( - Effect.catchIf( - (error) => error.reason._tag === "NotFound", - () => Effect.succeed([]), - ), - retryTransientRead, - ); - if (entries.length === 0) return []; - const realDataRoot = path.join( - yield* fs.realPath(stackRoot(id)).pipe(retryTransientRead), - "data", - ); - const roots: Array = []; - for (const entry of entries.filter(Schema.is(SafeId))) { - const instanceRoot = path.join(realDataRoot, entry); - const configRoot = vectorConfigRoot(instanceRoot); - // Another service's data directory may be unreadable to this process. - const configEntries = yield* fs - .realPath(vectorConfigRoot(path.join(dataRoot, entry))) - .pipe( - Effect.flatMap((resolved) => - resolved === configRoot ? fs.readDirectory(configRoot) : Effect.succeed([]), - ), - Effect.orElseSucceed((): ReadonlyArray => []), - ); - if (configEntries.some(isVectorRecipe)) roots.push(instanceRoot); - } - return roots; - }); - // A caller's Vector configPath may live under the instance root, so only recipe files and - // empty directories go. - const removeVectorData = (instanceRoot: string) => - Effect.gen(function* () { - const configRoot = vectorConfigRoot(instanceRoot); - for (const entry of yield* fs.readDirectory(configRoot)) - if (isVectorRecipe(entry)) - yield* fs.remove(path.join(configRoot, entry), { recursive: true, force: true }); - for (const directory of [configRoot, path.dirname(configRoot), instanceRoot]) - yield* removeEmptyDirectory(directory); - }).pipe( - Effect.catchCause((cause) => - Effect.logWarning( - `Unable to remove the Vector files in ${instanceRoot}; the next owner start retries`, - cause, - ), - ), - ); - /** Whether the saved document still holds a Vector instance. */ - const holdsVector = (target: string) => - Effect.gen(function* () { - if (!(yield* fs.exists(target).pipe(retryTransientRead))) return false; - const text = yield* fs.readFileString(target).pipe(retryTransientRead); - const { removed } = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( - text, - ).pipe( - Effect.map(withoutVectorInstances), - Effect.orElseSucceed(() => ({ removed: [] })), - ); - return removed.length > 0; - }); - /** What a migration still has to do, or `undefined` when the stack no longer holds Vector. */ - const vectorLeftovers = (id: string) => - Effect.gen(function* () { - const legacy = yield* holdsVector(statePath(id)); - const candidates = yield* vectorDataRoots(id); - if (!legacy && candidates.length === 0) return undefined; - const saved = yield* read(id); - // Vector files in a saved instance's root belong to that instance, e.g. a caller's config. - const live = new Set(saved?.instances.map((instance) => instance.id)); - const roots = candidates.filter((root) => !live.has(path.basename(root))); - return legacy || roots.length > 0 - ? { saved: legacy ? saved : undefined, roots } - : undefined; - }); - const migrate = Effect.fn("State.migrate")(function* (id: string) { - yield* checkId(id); - // Only a stack that still holds Vector takes the registry lock, to migrate it. - if ((yield* vectorLeftovers(id)) === undefined) return; - yield* withLock( - Effect.gen(function* () { - const leftovers = yield* vectorLeftovers(id); - if (leftovers === undefined) return; - yield* Effect.annotateCurrentSpan({ vector_data_roots: leftovers.roots.length }); - yield* Effect.forEach(leftovers.roots, removeVectorData, { discard: true }); - if (leftovers.saved !== undefined) yield* save(leftovers.saved); - }), - ); - }); - return { - read, - migrate, - list: list(), - claims: claims(), - save, - remove, - withLock, - lease, - leased, - readHolder, - publishHolder, - retractHolder, - ownerLog, - logsRoot, - }; - }); - -export const layer = (options: Options) => - Layer.effect(Service, makeState(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/State.vector-migration.integration.test.ts b/packages/stack/src/State.vector-migration.integration.test.ts deleted file mode 100644 index 29dca51126..0000000000 --- a/packages/stack/src/State.vector-migration.integration.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -import { NodeHttpClient, NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Context, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Layer, - Path, - PlatformError, - Ref, - Schema, -} from "effect"; -import { create, open } from "./effect.ts"; -import * as State from "./State.ts"; -import { destroyTestStack } from "../tests/stack-cleanup.ts"; - -const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -const Json = Schema.fromJsonString(Schema.Unknown); -const savedServices = (text: string) => - Schema.decodeEffect( - Schema.fromJsonString( - Schema.Struct({ - instances: Schema.Array( - Schema.Struct({ creation: Schema.Struct({ service: Schema.String }) }), - ), - }), - ), - )(text).pipe(Effect.map(({ instances }) => instances.map(({ creation }) => creation.service))); - -const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), - ); - -/** A state document saved before the stack dropped its Vector service kind. */ -const legacyDocument = (id: string, projectRoot: string) => ({ - id, - lifetime: "detached", - identity: { projectRoot, branchContext: "main", stackName: "local" }, - runtime: "native", - instances: [ - { id: "analytics", creation: { service: "analytics", config: { backend: "postgres" } } }, - { id: "mail", creation: { service: "mail", config: {} } }, - { - id: "vector", - creation: { - service: "vector", - config: { analyticsUrl: "http://127.0.0.1:1", configPath: `${projectRoot}/vector.yaml` }, - }, - }, - { id: "vector-owned", creation: { service: "vector", config: {} } }, - ], - composition: { - members: [ - { id: "analytics", activation: "eager" }, - { id: "mail", activation: "eager" }, - { id: "vector", activation: "eager" }, - { id: "vector-owned", activation: "eager" }, - ], - dependencies: [ - { - from: "analytics", - to: "vector", - bindings: [{ output: "url", input: "analyticsUrl" }], - }, - { from: "analytics", to: "vector-owned" }, - { from: "mail", to: "analytics" }, - ], - }, - ports: [ - { key: "vector:http", host: "127.0.0.1", port: 24_501 }, - { key: "mail:http", host: "127.0.0.1", port: 24_502 }, - ], -}); - -const writeLegacyState = Effect.fn("test.writeLegacyState")(function* ( - stateRoot: string, - id: string, - projectRoot: string, -) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const data = path.join(stateRoot, id, "data"); - yield* fs.makeDirectory(path.join(data, "vector", "runtime", "vector"), { recursive: true }); - yield* fs.writeFileString(path.join(data, "vector", "runtime", "vector", "vector.yaml"), ""); - yield* fs.writeFileString(path.join(data, "vector", "runtime", "vector", "vector-api.yaml"), ""); - yield* fs.writeFileString( - path.join(data, "vector", "runtime", "vector", "vector.rendered.yaml"), - "rendered", - ); - yield* fs.writeFileString(path.join(data, "vector", "pipeline.yaml"), "caller"); - const owned = path.join(data, "vector-owned", "runtime", "vector"); - yield* fs.makeDirectory(path.join(owned, ".vector-write-1"), { recursive: true }); - yield* fs.writeFileString(path.join(owned, "vector-api.yaml"), ""); - const file = path.join(stateRoot, id, "state.json"); - yield* fs.writeFileString( - file, - yield* Schema.encodeEffect(Json)(legacyDocument(id, projectRoot)), - ); - return file; -}); - -describe("saved Vector instance migration", () => { - it.live("reads a legacy state without Vector and leaves the file for the lease holder", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-read-" }); - const state = yield* stateFor(root); - const file = yield* writeLegacyState(root, "legacy", root); - const before = yield* fs.readFileString(file); - - const saved = yield* state.read("legacy"); - - expect(saved?.instances.map(({ id }) => id)).toEqual(["analytics", "mail"]); - expect(saved?.composition).toEqual({ - members: [ - { id: "analytics", activation: "eager" }, - { id: "mail", activation: "eager" }, - ], - dependencies: [{ from: "mail", to: "analytics" }], - }); - expect(saved?.ports.map(({ key }) => key)).toEqual(["mail:http"]); - expect((yield* state.list).map(({ id }) => id)).toEqual(["legacy"]); - expect(yield* fs.readFileString(file)).toBe(before); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live("persists the migration once and keeps files a caller owns", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-migrate-" }); - const state = yield* stateFor(root); - const file = yield* writeLegacyState(root, "legacy", root); - const data = path.join(root, "legacy", "data"); - - yield* state.migrate("legacy"); - - const migrated = yield* fs.readFileString(file); - expect(yield* savedServices(migrated)).toEqual(["analytics", "mail"]); - expect(yield* Schema.decodeEffect(Schema.fromJsonString(State.SavedStack))(migrated)).toEqual( - yield* state.read("legacy"), - ); - expect(yield* fs.exists(path.join(data, "vector", "runtime"))).toBe(false); - expect(yield* fs.readFileString(path.join(data, "vector", "pipeline.yaml"))).toBe("caller"); - expect(yield* fs.exists(path.join(data, "vector-owned"))).toBe(false); - - yield* state.migrate("legacy"); - expect(yield* fs.readFileString(file)).toBe(migrated); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live( - "retries removing Vector files that a failed migration left after the state is saved again", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-retry-" }); - const failNext = yield* Ref.make(true); - const injected: FileSystem.FileSystem = { - ...fs, - remove: (target, options) => - Effect.gen(function* () { - if ( - target.endsWith("vector.rendered.yaml") && - (yield* Ref.getAndSet(failNext, false)) - ) - return yield* PlatformError.systemError({ - _tag: "PermissionDenied", - module: "FileSystem", - method: "remove", - pathOrDescriptor: target, - }); - return yield* fs.remove(target, options); - }), - }; - const state = yield* stateFor(root).pipe( - Effect.provideService(FileSystem.FileSystem, injected), - ); - const file = yield* writeLegacyState(root, "legacy", root); - const runtime = path.join(root, "legacy", "data", "vector", "runtime"); - - yield* state.migrate("legacy"); - const keptFiles = yield* fs.exists(runtime); - const saved = yield* state.read("legacy"); - if (saved === undefined) return yield* Effect.die("the legacy stack is not saved"); - yield* state.save(saved); - yield* state.migrate("legacy"); - - expect(keptFiles).toBe(true); - expect(yield* savedServices(yield* fs.readFileString(file))).toEqual(["analytics", "mail"]); - expect(yield* fs.exists(runtime)).toBe(false); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live.skipIf(process.platform === "win32")( - "keeps Vector-named files that a symlinked data directory reaches outside the stack", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-symlink-" }); - const state = yield* stateFor(root); - yield* writeLegacyState(root, "legacy", root); - const outside = path.join(root, "outside", "runtime", "vector"); - yield* fs.makeDirectory(outside, { recursive: true }); - yield* fs.writeFileString(path.join(outside, "vector.yaml"), "outside"); - const data = path.join(root, "legacy", "data"); - yield* fs.symlink(path.join(root, "outside"), path.join(data, "linked-root")); - yield* fs.makeDirectory(path.join(data, "linked-config", "runtime"), { recursive: true }); - yield* fs.symlink(outside, path.join(data, "linked-config", "runtime", "vector")); - - yield* state.migrate("legacy"); - - expect(yield* fs.readFileString(path.join(outside, "vector.yaml"))).toBe("outside"); - expect(yield* fs.exists(path.join(data, "vector", "runtime"))).toBe(false); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live("keeps Vector-named files in the data directory of a saved instance", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-live-" }); - const state = yield* stateFor(root); - yield* writeLegacyState(root, "legacy", root); - const mailConfig = path.join(root, "legacy", "data", "mail", "runtime", "vector"); - yield* fs.makeDirectory(mailConfig, { recursive: true }); - yield* fs.writeFileString(path.join(mailConfig, "vector.yaml"), "mail"); - - yield* state.migrate("legacy"); - - expect(yield* fs.readFileString(path.join(mailConfig, "vector.yaml"))).toBe("mail"); - expect(yield* fs.exists(path.join(root, "legacy", "data", "vector-owned"))).toBe(false); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live("rejects a saved Vector id that escapes the stack without touching other stacks", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-escape-" }); - const state = yield* stateFor(root); - const sibling = path.join(root, "other", "data", "vector", "runtime", "vector"); - yield* fs.makeDirectory(sibling, { recursive: true }); - yield* fs.writeFileString(path.join(sibling, "vector.yaml"), "sibling"); - const escaping = "../../other/data/vector"; - const document = legacyDocument("legacy", root); - yield* fs.makeDirectory(path.join(root, "legacy", "data"), { recursive: true }); - yield* fs.writeFileString( - path.join(root, "legacy", "state.json"), - yield* Schema.encodeEffect(Json)({ - ...document, - instances: [ - ...document.instances.filter(({ creation }) => creation.service !== "vector"), - { id: escaping, creation: { service: "vector", config: {} } }, - ], - composition: { - members: [...document.composition.members, { id: escaping, activation: "eager" }], - dependencies: [], - }, - }), - ); - - yield* state.migrate("legacy"); - const failure = yield* state.read("legacy").pipe(Effect.flip); - - expect(failure.operation).toBe("decode"); - expect(yield* fs.readFileString(path.join(sibling, "vector.yaml"))).toBe("sibling"); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live("checks a state without Vector while another operation holds the registry lock", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-unlocked-" }); - const state = yield* stateFor(root); - yield* writeLegacyState(root, "legacy", root); - yield* state.migrate("legacy"); - const callerConfig = path.join(root, "legacy", "data", "vector", "runtime", "vector"); - yield* fs.makeDirectory(callerConfig, { recursive: true }); - yield* fs.writeFileString(path.join(callerConfig, "pipeline.yaml"), "caller"); - const held = yield* Deferred.make(); - const release = yield* Deferred.make(); - const holder = yield* state - .withLock(Deferred.succeed(held, undefined).pipe(Effect.andThen(Deferred.await(release)))) - .pipe(Effect.forkChild); - yield* Deferred.await(held); - - const checked = yield* state.migrate("legacy").pipe(Effect.timeout("1 second"), Effect.exit); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(holder); - - expect(Exit.isSuccess(checked)).toBe(true); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live("refuses to migrate a state document copied from another stack", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-copied-" }); - const state = yield* stateFor(root); - const file = yield* writeLegacyState(root, "copied", root); - yield* fs.writeFileString( - file, - yield* Schema.encodeEffect(Json)(legacyDocument("original", root)), - ); - const before = yield* fs.readFileString(file); - - const failure = yield* state.migrate("copied").pipe(Effect.flip); - - expect(failure.operation).toBe("identity"); - expect(yield* fs.readFileString(file)).toBe(before); - expect(yield* fs.exists(path.join(root, "original"))).toBe(false); - expect( - yield* fs.exists(path.join(root, "copied", "data", "vector", "runtime", "vector")), - ).toBe(true); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live("starts the owner of a legacy stack and drops Vector from its saved state", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-owner-" }); - const options = { - projectRoot: root, - stateRoot: `${root}/state`, - cacheRoot: `${root}/cache`, - runtime: "native", - } satisfies Parameters[0]; - const created = yield* create(options); - const file = yield* writeLegacyState(options.stateRoot, created.id, root); - - const stack = yield* open({ ...options, id: created.id, startOwner: true }); - yield* Effect.ensuring( - Effect.gen(function* () { - expect((yield* stack.services.list).map(({ id }) => id)).toEqual(["analytics", "mail"]); - expect(yield* stack.composition.describe).toEqual({ - members: [ - { id: "analytics", activation: "eager" }, - { id: "mail", activation: "eager" }, - ], - dependencies: [{ from: "mail", to: "analytics" }], - }); - const migrated = yield* fs.readFileString(file); - expect(yield* savedServices(migrated)).toEqual(["analytics", "mail"]); - - yield* stack.stop; - const reopened = yield* open({ ...options, id: created.id, startOwner: true }); - expect((yield* reopened.services.list).map(({ id }) => id)).toEqual([ - "analytics", - "mail", - ]); - expect(yield* fs.readFileString(file)).toBe(migrated); - }), - destroyTestStack(stack), - ); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - - it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( - "starts the owner of a legacy stack whose Vector migration fails", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-failed-" }); - const options = { - projectRoot: root, - stateRoot: `${root}/state`, - cacheRoot: `${root}/cache`, - runtime: "native", - } satisfies Parameters[0]; - const created = yield* create(options); - const file = yield* writeLegacyState(options.stateRoot, created.id, root); - const legacy = yield* fs.readFileString(file); - const data = path.join(options.stateRoot, created.id, "data"); - // Unlistable but writable: the migration cannot find Vector files, the owner still runs. - yield* fs.chmod(data, 0o300); - - const stack = yield* open({ ...options, id: created.id, startOwner: true }).pipe( - Effect.ensuring(fs.chmod(data, 0o700).pipe(Effect.orDie)), - ); - yield* Effect.ensuring( - Effect.gen(function* () { - expect((yield* stack.services.list).map(({ id }) => id)).toEqual(["analytics", "mail"]); - expect(yield* fs.readFileString(file)).toBe(legacy); - }), - destroyTestStack(stack), - ); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); -}); diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 183807fd10..9fbaef2dcf 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -9,17 +9,20 @@ import { Fiber, FileSystem, Layer, - Option, Path, - Schema, + Queue, Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- test-only native watcher for synchronous attachment, no Effect wrapper gives this guarantee. +import { watch as nodeWatch } from "node:fs"; import { launchHost } from "./HostProcess.ts"; -import { makeContainerRuntime } from "./runtime/Container.ts"; -import * as State from "./State.ts"; -import { reclaimStack, sweepOrphans } from "./Sweep.ts"; +import { makeContainerRuntime, type EngineTarget } from "./runtime/Container.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { reclaimDeletedStack, reclaimStack, sweepOrphans } from "./Sweep.ts"; import { engineStub, makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { awaitContainerRemoved } from "../tests/engine-events.ts"; +import { engineTarget, testEngine } from "../tests/engine-target.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; class SweepTestError extends Data.TaggedError("SweepTestError")<{ readonly message: string }> {} @@ -27,12 +30,16 @@ class SweepTestError extends Data.TaggedError("SweepTestError")<{ readonly messa const helperImage = "public.ecr.aws/docker/library/debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251"; -const docker = Effect.fn("SweepTest.docker")((args: ReadonlyArray) => +const engine = Effect.fn("SweepTest.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, [...engineTarget.argv, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -43,7 +50,7 @@ const docker = Effect.fn("SweepTest.docker")((args: ReadonlyArray) => { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`docker ${args.join(" ")} failed: ${stderr}`); + return yield* Effect.die(`${testEngine} ${args.join(" ")} failed: ${stderr}`); return stdout.trim(); }), ), @@ -55,7 +62,7 @@ const labels = (stackId: string, dataRoot: string) => [ ]; const containers = (stackId: string, dataRoot: string) => - docker([ + engine([ "ps", "--all", "--quiet", @@ -65,73 +72,56 @@ const containers = (stackId: string, dataRoot: string) => const createOwnedContainer = (stackId: string, dataRoot: string) => Effect.acquireRelease( - docker([ + engine([ "create", ...labels(stackId, dataRoot).flatMap((label) => ["--label", label.slice("label=".length)]), helperImage, ]), - (id) => docker(["rm", "--force", id]).pipe(Effect.ignore), + (id) => engine(["rm", "--force", id]).pipe(Effect.ignore), ); -/** Completes when Docker reports the container destroyed, including events before subscription. */ -const awaitDestroyed = (id: string, since: number) => - Effect.scoped( - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const events = yield* spawner.spawn( - ChildProcess.make( - "docker", - [ - "events", - "--since", - String(since), - "--filter", - `container=${id}`, - "--filter", - "event=destroy", - "--format", - "{{.Actor.ID}}", - ], - { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, - ), - ); - const destroyed = yield* events.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.filter((line) => line.trim() === id), - Stream.runHead, - ); - if (Option.isNone(destroyed)) - return yield* new SweepTestError({ message: "Docker event stream ended" }); - }), +/** + * Watches `directory` and waits for `entry`'s removal, scoped to the caller's own fork so the + * watcher closes as soon as it returns. `node:fs.watch` itself is synchronous, so acquiring it + * directly (rather than through `Stream`, which forks registration through `Stream.callback`, + * `Channel.callbackArray`, and `asyncQueue` before it actually attaches) guarantees the watcher is + * live before this returns: no canary handshake needed. Node reports non-recursive watch events by + * bare filename and sometimes coalesces them, so any event re-checks the real directory instead of + * trusting which path it names. The watcher's native resource is torn down whether or not removal + * ever arrives, so this never succeeds by a wait simply running out. + */ +const awaitRemoval = Effect.fn("SweepTest.awaitRemoval")(function* ( + directory: string, + entry: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const target = path.join(directory, entry); + const queue = yield* Queue.unbounded(); + yield* Effect.acquireRelease( + Effect.sync(() => + nodeWatch(directory, (_event, filename) => { + if (filename) Queue.offerUnsafe(queue, filename); + }), + ), + (watcher) => Effect.sync(() => watcher.close()), ); -const awaitRemoval = (directory: string, entry: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const target = path.join(directory, entry); - yield* fs.watch(directory).pipe( - Stream.filter((event) => event.path === entry || event.path === target), - Stream.mapEffect(() => fs.exists(target)), - Stream.takeUntil((exists) => !exists), - Stream.runDrain, - ); - }); + while (yield* fs.exists(target)) yield* Queue.take(queue); +}); const saved = ( id: string, projectRoot: string, - runtime: State.SavedStack["runtime"], - lifetime: State.StackLifetime, -): State.SavedStack => ({ + runtime: StackNamespace.SavedStack["runtime"], + lifetime: StackNamespace.StackLifetime, +): StackNamespace.SavedStack => ({ id, runtime, lifetime, identity: { projectRoot, branchContext: "sweep-test", stackName: id }, instances: [], composition: { members: [], dependencies: [] }, - ports: [], }); it.live.skipIf(process.platform === "win32")( @@ -152,11 +142,17 @@ it.live.skipIf(process.platform === "win32")( ); const rootA = path.dirname(path.dirname(dataA)); const cacheRoot = `${path.dirname(rootA)}/cache`; - const helper = yield* makeContainerRuntime({ engine: "docker", root: dataA }); + const helper = yield* makeContainerRuntime({ + target: engineTarget, + root: dataA, + }); yield* helper.prepare(helperImage); - const state = Context.get(yield* Layer.build(State.layer({ root: rootA })), State.Service); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: rootA })), + StackNamespace.Service, + ); - yield* state.save(saved(deadId, `${rootA}/dead`, "docker", "detached")); + yield* state.save(saved(deadId, `${rootA}/dead`, testEngine, "detached")); const dead = (yield* launchHost(state, { stateRoot: rootA, cacheRoot, stackId: deadId })) .endpoint; const orphan = yield* createOwnedContainer(deadId, dataA); @@ -175,15 +171,14 @@ it.live.skipIf(process.platform === "win32")( const sessionId = `session-${suffix}`; yield* state.save(saved(sessionId, `${rootA}/session`, "native", "session")); const nextId = `next-${suffix}`; - yield* state.save(saved(nextId, `${rootA}/next`, "native", "detached")); + yield* state.save(saved(nextId, `${rootA}/next`, testEngine, "detached")); const since = Math.floor((yield* Clock.currentTimeMillis) / 1000) - 1; const swept = yield* Effect.all( [ - awaitDestroyed(orphan, since), - awaitDestroyed(deleted, since), + awaitContainerRemoved(orphan, since), + awaitContainerRemoved(deleted, since), awaitRemoval(rootA, sessionId), - awaitRemoval(rootA, deletedId), awaitRemoval(path.join(rootA, deadId), "owner.json"), ], { concurrency: "unbounded" }, @@ -224,43 +219,86 @@ it.live.skipIf(process.platform === "win32")( { timeout: 180_000 }, ); -it.live("destroys a dead session stack whose saved Vector instance fails to migrate", () => +it.live( + "an orphan sweep removes destroyed stack directories once they are older than the restart window", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweep-prune-" }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root })), + StackNamespace.Service, + ); + const elevenMinutesAgo = Math.floor((yield* Clock.currentTimeMillis) / 1000) - 11 * 60; + const arrange = Effect.fn("SweepTest.arrange")(function* ( + id: string, + options: { readonly old: boolean; readonly file?: string; readonly dataFile?: string }, + ) { + const directory = path.join(root, id); + yield* fs.makeDirectory(path.join(directory, "data"), { recursive: true }); + if (options.file !== undefined) + yield* fs.writeFileString(path.join(directory, options.file), "x"); + if (options.dataFile !== undefined) + yield* fs.writeFileString(path.join(directory, "data", options.dataFile), "x"); + if (options.old) yield* fs.utimes(directory, elevenMinutesAgo, elevenMinutesAgo); + }); + yield* arrange("old-empty", { old: true }); + yield* arrange("fresh-empty", { old: false }); + yield* arrange("old-with-data", { old: true, dataFile: "PG_VERSION" }); + yield* arrange("old-with-lock", { old: true, file: "owner.lock" }); + yield* arrange("old-kept", { old: true }); + + yield* sweepOrphans({ + state, + stateRoot: root, + cacheRoot: path.join(root, "cache"), + ownerId: "old-kept", + }); + + expect(yield* fs.exists(path.join(root, "old-empty"))).toBe(false); + expect(yield* fs.exists(path.join(root, "fresh-empty", "data"))).toBe(true); + expect(yield* fs.exists(path.join(root, "old-with-data", "data", "PG_VERSION"))).toBe(true); + expect(yield* fs.exists(path.join(root, "old-with-lock", "owner.lock"))).toBe(true); + expect(yield* fs.exists(path.join(root, "old-kept", "data"))).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("reclaiming a session stack refuses a symlinked data directory and keeps its target", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const stateRoot = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweep-migrate-" }); - const real = Context.get(yield* Layer.build(State.layer({ root: stateRoot })), State.Service); - const state: State.Interface = { - ...real, - migrate: () => - Effect.fail(new State.StateError({ operation: "lock", message: "registry is busy" })), - }; - const id = "session-vector"; - yield* fs.makeDirectory(path.join(stateRoot, id), { recursive: true }); - yield* fs.writeFileString( - path.join(stateRoot, id, "state.json"), - yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ - ...saved(id, path.join(stateRoot, "project"), "native", "session"), - instances: [{ id: "vector", creation: { service: "vector", config: {} } }], - composition: { members: [{ id: "vector", activation: "eager" }], dependencies: [] }, - }), + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweep-symlinked-data-" }); + const stateRoot = path.join(root, "state"); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, ); + const stack = saved("symlinked-data", root, "native", "session"); + yield* state.save(stack); + const outside = path.join(root, "outside"); + yield* fs.makeDirectory(outside); + yield* fs.writeFileString(path.join(outside, "precious.txt"), "keep"); + yield* fs.symlink(outside, path.join(stateRoot, stack.id, "data")); - const reclaimed = yield* reclaimStack({ + const failure = yield* reclaimStack({ state, stateRoot, - cacheRoot: path.join(stateRoot, "cache"), - id, - }); + cacheRoot: path.join(root, "cache"), + id: stack.id, + }).pipe(Effect.flip); - expect(reclaimed).toBe("reclaimed"); - expect(yield* real.read(id)).toBeUndefined(); + expect(failure.message).toContain("symlink"); + expect(yield* fs.readFileString(path.join(outside, "precious.txt"))).toBe("keep"); + expect(yield* state.read(stack.id), "the registration stays for a retry").toBeDefined(); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); -it.live("sweeps deleted stacks of its own root only, on the engines its stacks use", () => +it.live("sweeps deleted stacks of its own root only, on the engine its owner pinned", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -268,23 +306,28 @@ it.live("sweeps deleted stacks of its own root only, on the engines its stacks u const stateRoot = yield* fs .makeTempDirectoryScoped({ prefix: "stack-sweep-deleted-" }) .pipe(Effect.flatMap(fs.realPath)); - const real = Context.get(yield* Layer.build(State.layer({ root: stateRoot })), State.Service); - const leased: Array = []; - const state: State.Interface = { + const real = Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, + ); + const swept: Array = []; + const state: StackNamespace.Interface = { ...real, - lease: (id) => + acquireLease: (id) => Effect.suspend(() => { - leased.push(id); - return real.lease(id); + swept.push(id); + return real.acquireLease(id); }), }; const deletedId = "d".repeat(64); + const heldId = "a".repeat(64); const foreignId = "f".repeat(64); const deletedData = path.join(stateRoot, deletedId, "data"); const engine = engineStub([ { id: "owner", stackId: "owner", root: path.join(stateRoot, "owner", "data") }, { id: "deleted", stackId: deletedId, root: deletedData }, { id: "deleted-sibling", stackId: deletedId, root: deletedData }, + { id: "held", stackId: heldId, root: path.join(stateRoot, heldId, "data") }, { id: "foreign", stackId: foreignId, @@ -293,26 +336,33 @@ it.live("sweeps deleted stacks of its own root only, on the engines its stacks u { id: "escape", stackId: "../escape", root: path.join(stateRoot, "..", "escape", "data") }, "malformed", ]); - const sweep = sweepOrphans({ + const pinned: EngineTarget = { + engine: "docker", + argv: ["--context", "pinned"], + daemonId: "pinned-daemon", + }; + const cacheRoot = path.join(stateRoot, "cache"); + yield* Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, + ).acquireLease(heldId); + + yield* sweepOrphans({ state, stateRoot, - cacheRoot: path.join(stateRoot, "cache"), + cacheRoot, ownerId: "owner", + engineTarget: pinned, }).pipe(Effect.provide(engine.layer)); - yield* state.save( - saved("owner", path.join(stateRoot, "owner-project"), "native", "detached"), - ); - yield* sweep; - expect(engine.commands, "a root of native stacks queries no engine").toEqual([]); - - yield* state.save( - saved("stopped", path.join(stateRoot, "stopped-project"), "docker", "detached"), - ); - yield* sweep; - expect(leased).toEqual(["stopped", deletedId]); + expect(swept, "only deleted stacks of this root are attempted").toEqual([deletedId, heldId]); + expect( + engine.remaining("docker").map(({ id }) => id), + "the held stack's containers stay", + ).toEqual(["owner", "held", "foreign", "escape"]); expect(engine.commands).toContainEqual([ "docker", + ...pinned.argv, "ps", "--all", "--quiet", @@ -322,23 +372,45 @@ it.live("sweeps deleted stacks of its own root only, on the engines its stacks u "--filter", `label=com.supabase.stack-root=${deletedData}`, ]); - expect(engine.remaining("docker").map(({ id }) => id)).toEqual([ - "owner", - "foreign", - "escape", - ]); + expect( + engine.commands.filter(([command]) => command === "podman"), + "only the owner's engine is queried", + ).toEqual([]); expect( yield* fs.exists(path.join(stateRoot, deletedId)), "the sweeper's lease leaves no directory behind", ).toBe(false); - const outcome = yield* reclaimStack({ + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("does not reclaim a stack that is registered as a deleted one", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateRoot = yield* fs + .makeTempDirectoryScoped({ prefix: "stack-reclaim-registered-" }) + .pipe(Effect.flatMap(fs.realPath)); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, + ); + const id = "b".repeat(64); + const engine = engineStub([ + { id: "live", stackId: id, root: path.join(stateRoot, id, "data") }, + ]); + yield* state.save(saved(id, path.join(stateRoot, "project"), "docker", "detached")); + + const outcome = yield* reclaimDeletedStack({ state, stateRoot, - cacheRoot: path.join(stateRoot, "cache"), - id: "stopped", - engine: "docker", + id, + engineTarget: { engine: "docker", argv: [], daemonId: "stub" }, }).pipe(Effect.provide(engine.layer)); - expect(outcome, "a stack registered again is not reclaimed as deleted").toBe("registered"); + + expect(outcome).toBe("registered"); + expect(engine.remaining("docker").map(({ id: removed }) => removed)).toEqual(["live"]); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 22204c7c40..d46f25183d 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -1,121 +1,167 @@ -import { Context, DateTime, Effect, FileSystem, Layer, Path } from "effect"; +import { Context, DateTime, Effect, Layer, Schema } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { sweepTimeout } from "./HostProcess.ts"; -import { isStackId } from "./identity/StackId.ts"; +import { StackIdSchema } from "./identity/StackId.ts"; import * as Owner from "./Owner.ts"; -import * as Container from "./runtime/Container.ts"; -import * as State from "./State.ts"; +import { listStackLabels, resolveEngineTarget, type EngineTarget } from "./runtime/Container.ts"; +import * as StackNamespace from "./StackNamespace.ts"; + +const isStackId = Schema.is(StackIdSchema); /** Runs a dead session stack's own destroy path in this process while holding its lease. */ const destroyStack = Effect.fn("Sweep.destroyStack")(function* ( - state: State.Interface, - saved: State.SavedStack, + state: StackNamespace.Interface, + saved: StackNamespace.SavedStack, dataRoot: string, cacheRoot: string, ) { - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(dataRoot, { recursive: true }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const engineTarget = + saved.runtime === "native" ? undefined : yield* resolveEngineTarget(spawner, saved.runtime); const context = yield* Layer.build( - Owner.layer({ saved, root: dataRoot, cacheRoot }).pipe( - Layer.provide(Layer.succeed(State.Service, state)), + Owner.layer({ saved, root: dataRoot, cacheRoot, engineTarget }).pipe( + Layer.provide(Layer.succeed(StackNamespace.Service, state)), ), ); yield* Context.get(context, Owner.Service).namespace.destroy; }); /** - * Cleans up a stack whose lease is free, holding that lease meanwhile and marking the hold so - * clients wait instead of mistaking it for a starting owner. Removes the stack's labelled - * containers, only those on `engine` once the stack is no longer registered, and destroys the - * stack when its lifetime is `session`. Returns `held` when another process holds the lease, - * `registered` when, with `engine`, the stack is registered again, and otherwise `reclaimed`. + * Holds `id`'s lease until the scope closes, marking the hold so clients wait instead of + * mistaking it for a starting owner. `false` when another process holds the lease. + */ +const holdLease = Effect.fn("Sweep.holdLease")(function* ( + state: StackNamespace.Interface, + id: string, +) { + const lease = yield* state + .acquireLease(id) + .pipe(Effect.catchTag("Namespace.LeaseHeldError", () => Effect.void)); + if (lease === undefined) return false; + yield* Effect.addFinalizer(() => lease.retractHolder.pipe(Effect.ignore)); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: DateTime.formatIso(yield* DateTime.now), + }); + return true; +}); + +/** + * Cleans up a stack whose lease is free, holding that lease meanwhile. Removes the stack's + * labelled containers, and destroys the stack when its lifetime is `session`. Returns `false` + * when another process holds the lease. */ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: { - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly stateRoot: string; readonly cacheRoot: string; readonly id: string; - readonly engine?: "docker" | "podman"; }) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const { state, id } = options; return yield* Effect.scoped( Effect.gen(function* () { - if (!(yield* state.lease(id))) return "held"; - yield* Effect.addFinalizer(() => state.retractHolder(id).pipe(Effect.ignore)); - yield* state.publishHolder(id, { - role: "sweeper", - pid: process.pid, - startedAt: DateTime.formatIso(yield* DateTime.now), - }); + if (!(yield* holdLease(state, id))) return false; const saved = yield* state.read(id); - if (options.engine !== undefined && saved !== undefined) return "registered"; - const stack = - saved ?? (options.engine === undefined ? undefined : { id, runtime: options.engine }); - if (stack === undefined) return "reclaimed"; - const dataRoot = path.join(yield* fs.realPath(options.stateRoot), id, "data"); - yield* Owner.sweepContainers(stack, dataRoot); - if (saved === undefined) return "reclaimed"; - // Reading drops saved Vector instances, so destroying does not depend on the migration. - yield* state - .migrate(id) - .pipe( - Effect.catch((error) => - Effect.logWarning(`Unable to migrate the saved state of stack ${id}`, error), - ), - ); + if (saved === undefined) return true; + const dataRoot = yield* StackNamespace.stackDataRootLabel(options.stateRoot, id); if (saved.lifetime === "session") yield* destroyStack(state, saved, dataRoot, options.cacheRoot); - return "reclaimed"; + else { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const engineTarget = + saved.runtime === "native" + ? undefined + : yield* resolveEngineTarget(spawner, saved.runtime); + yield* Owner.sweepContainers(saved, dataRoot, engineTarget); + } + return true; + }), + ).pipe(Effect.timeout(sweepTimeout)); +}); + +/** + * Removes the containers a stack with no registration left behind on `engineTarget`, holding its + * lease meanwhile; the stack's data in the shared database volume stays. Returns `held` when + * another process holds the lease, `registered` when the stack is registered, and otherwise + * `reclaimed`. + */ +export const reclaimDeletedStack = Effect.fn("Sweep.reclaimDeletedStack")(function* (options: { + readonly state: StackNamespace.Interface; + readonly stateRoot: string; + readonly id: string; + readonly engineTarget: EngineTarget; +}) { + const { state, id } = options; + const outcome = yield* Effect.scoped( + Effect.gen(function* () { + if (!(yield* holdLease(state, id))) return "held" as const; + if ((yield* state.read(id)) !== undefined) return "registered" as const; + const dataRoot = yield* StackNamespace.stackDataRootLabel(options.stateRoot, id); + yield* Owner.sweepContainers({ id }, dataRoot, options.engineTarget); + return "reclaimed" as const; }), ).pipe(Effect.timeout(sweepTimeout)); + yield* Effect.annotateCurrentSpan({ "sweep.outcome": outcome }); + return outcome; +}); + +/** The ids of stacks with containers on `target`'s engine labelled with a data root of `stateRoot`. */ +export const leftBehindStackIds = Effect.fn("Sweep.leftBehindStackIds")(function* ( + stateRoot: string, + target: EngineTarget, +) { + const ids = new Set(); + for (const { stackId, root } of yield* listStackLabels(target)) { + if (ids.has(stackId) || !isStackId(stackId)) continue; + if (root === (yield* StackNamespace.stackDataRootLabel(stateRoot, stackId))) ids.add(stackId); + } + return ids; }); /** * Keeps stack-labelled containers and session stacks alive only while their lease is held: every - * other stack of this state root whose lease is free is reclaimed, including a deleted stack whose - * containers remain on an engine the root's stacks use. + * other stack of this state root whose lease is free is reclaimed. With an `engineTarget`, so is + * every stack with no registration that left containers on that engine labelled with this state + * root. */ export const sweepOrphans = Effect.fn("Sweep.orphans")( function* (options: { - readonly state: State.Interface; + readonly state: StackNamespace.Interface; readonly stateRoot: string; readonly cacheRoot: string; readonly ownerId: string; + readonly engineTarget?: EngineTarget; }) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const reclaim = (id: string, engine?: "docker" | "podman") => - reclaimStack({ ...options, id, engine }).pipe( + const { state, stateRoot, cacheRoot, ownerId, engineTarget } = options; + const registered = new Set(); + for (const { id } of yield* state.list) { + registered.add(id); + if (id === ownerId) continue; + yield* reclaimStack({ state, stateRoot, cacheRoot, id }).pipe( Effect.catchCause((cause) => Effect.logWarning(`Orphan sweep of stack ${id} failed`, cause), ), ); - const stacks = yield* options.state.list; - for (const { id } of stacks) if (id !== options.ownerId) yield* reclaim(id); - const registered = new Set(stacks.map(({ id }) => id)); - const stateRoot = yield* fs.realPath(options.stateRoot); - const engines = new Set( - stacks.flatMap(({ runtime }) => (runtime === "native" ? [] : [runtime])), - ); - for (const engine of engines) { - const containers = yield* Container.listStackContainers(engine).pipe( - Effect.catch((error) => - Effect.logWarning(`Orphan sweep of ${engine} containers failed`, error).pipe( - Effect.as([]), + } + if (engineTarget !== undefined) { + const leftBehind = yield* leftBehindStackIds(stateRoot, engineTarget).pipe( + Effect.catchCause((cause) => + Effect.logWarning(`Orphan sweep of ${engineTarget.engine} containers failed`, cause).pipe( + Effect.as(new Set()), ), ), ); - const deleted = containers - .filter( - ({ stackId, root }) => - isStackId(stackId) && - !registered.has(stackId) && - root === path.join(stateRoot, stackId, "data"), - ) - .map(({ stackId }) => stackId); - for (const id of new Set(deleted)) yield* reclaim(id, engine); + for (const id of leftBehind) { + if (registered.has(id) || id === ownerId) continue; + yield* reclaimDeletedStack({ state, stateRoot, id, engineTarget }).pipe( + Effect.catchCause((cause) => + Effect.logWarning(`Orphan sweep of deleted stack ${id} failed`, cause), + ), + ); + } } + yield* state.pruneDestroyed(ownerId); }, Effect.catchCause((cause) => Effect.logWarning("Orphan sweep failed", cause)), ); diff --git a/packages/stack/src/composition/Supabase.native.integration.test.ts b/packages/stack/src/composition/Supabase.native.integration.test.ts index fffc8cc9b5..d99a7a695e 100644 --- a/packages/stack/src/composition/Supabase.native.integration.test.ts +++ b/packages/stack/src/composition/Supabase.native.integration.test.ts @@ -15,25 +15,25 @@ import { import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; -import { tmpdir } from "node:os"; -import * as State from "../State.ts"; -import type { SavedStack } from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import type { SavedStack } from "../StackNamespace.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; import { makeSupabaseComposition, SupabaseCompositionError, type SupabaseCompositionOperations, } from "./Supabase.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; // Below every OS ephemeral range, so another test's outbound socket cannot already hold them. const FIXED_STUDIO_PORT = 24_391; const FIXED_MAIL_PORT = 24_392; const stateFor = (root: string) => Effect.gen(function* () { - const context = yield* Layer.build(State.layer({ root })); - return Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root })); + return Context.get(context, StackNamespace.Service); }); const query = (url: string, statement: string) => @@ -60,7 +60,6 @@ const initial = (id: string): SavedStack => ({ instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); it.live("reports the failed binding and cleanup reason with the retained instance ID", () => diff --git a/packages/stack/src/composition/Supabase.ts b/packages/stack/src/composition/Supabase.ts index cef250d4fa..9a78d17237 100644 --- a/packages/stack/src/composition/Supabase.ts +++ b/packages/stack/src/composition/Supabase.ts @@ -1,11 +1,11 @@ import { Data, Effect, Exit, Redacted, Schema } from "effect"; import { postgresVersion } from "../Artifacts.ts"; -import { causeMessage, type CompositionConfig } from "../Orchestrator.ts"; +import { causeMessage, type CompositionConfig, isStoppedAndWakeDisabled } from "../Orchestrator.ts"; import type { Observation } from "../Rpc.ts"; import { ServiceCreation, type ServiceCreationInput } from "../services/Catalog.ts"; -import type { SavedStack, StackKeysInput } from "../State.ts"; +import type { SavedStack, StackKeysInput } from "../StackNamespace.ts"; import { credentialInputNames } from "../host/Credentials.ts"; -import { apiRoute, endpointNames, endpointPort } from "../host/Endpoints.ts"; +import { endpointNames, endpointPort, sharesApiEndpoint } from "../host/Endpoints.ts"; const DEFAULT_IDLE_MILLIS = 60_000; /** Studio idles slower than its peers: a background tab shouldn't cold-start it every minute. */ @@ -250,9 +250,6 @@ const differences = (left: unknown, right: unknown, path: string): ReadonlyArray return Object.is(left, right) ? [] : [path]; }; -const sharesApiEndpoint = (creation: ServiceCreationInput): boolean => - apiRoute(creation.service) !== undefined && endpointNames(creation).includes("http"); - /** Fixed ports requested for the shared API endpoint; composition accepts at most one. */ const fixedApiPorts = (creations: ReadonlyArray): ReadonlySet => new Set( @@ -425,7 +422,7 @@ export const makeSupabaseComposition = Effect.fn("Supabase.compose")( operations.status(id).pipe( Effect.mapError(compositionErrorFrom), Effect.flatMap((status) => - status.lifecycle === "stopped" && !status.wakeEnabled + isStoppedAndWakeDisabled(status) ? Effect.void : Effect.fail( compositionError( @@ -456,10 +453,7 @@ export const makeSupabaseComposition = Effect.fn("Supabase.compose")( const reusedByKind = new Map(reusedEntries.map((entry) => [entry.creation.service, entry])); const byKind = new Map(normalized.map((creation) => [creation.service, creation])); - const apiSource = normalized.find( - (creation) => - apiRoute(creation.service) !== undefined && endpointNames(creation).includes("http"), - ); + const apiSource = normalized.find(sharesApiEndpoint); if ( apiSource === undefined && normalized.some((creation) => ["auth", "functions", "studio"].includes(creation.service)) @@ -621,9 +615,7 @@ export const makeSupabaseComposition = Effect.fn("Supabase.compose")( creation.service !== "database" && endpointNames(creation).length > 0; return lazy - ? creation.service === "functions" - ? { id, activation: "lazy" as const } - : { id, activation: "lazy" as const, idleMillis: idleMillisFor(creation.service) } + ? { id, activation: "lazy" as const, idleMillis: idleMillisFor(creation.service) } : { id, activation: "eager" as const }; }); yield* operations.configure({ diff --git a/packages/stack/src/composition/Supabase.unit.test.ts b/packages/stack/src/composition/Supabase.unit.test.ts index 0036a1d9d4..f90497eebb 100644 --- a/packages/stack/src/composition/Supabase.unit.test.ts +++ b/packages/stack/src/composition/Supabase.unit.test.ts @@ -31,22 +31,24 @@ const makeInstance = ( }), removeData: () => Effect.void, }, - { id, config: {}, coordinate: orchestrator.admissionFor(id) }, + { id, config: {}, report: orchestrator.report }, ); - const instance: Orchestrator.RegisteredInstance = { + yield* orchestrator.register({ id, service: id, core, - startAt: (revision, inputs, wake, guard) => core.startAt(revision, inputs, wake, guard), - restart: (revision, inputs, config, guard) => core.restart(inputs, revision, guard), + launch: (generation, inputs) => core.launch(generation, inputs), + prepare: () => Effect.void, bind: Effect.void, close: Effect.void, + confirmRemoved: Effect.void, + release: Effect.void, + releasePorts: Effect.void, hasEndpoint: options.hasEndpoint ?? true, inputs: options.inputs ?? [], outputs: options.outputs ?? {}, - }; - yield* orchestrator.register(instance); - return instance; + }); + return { status: orchestrator.status(id), observation: orchestrator.changes(id) }; }); /** Merges optional config bindings onto a creation; the config union can't express this generically. */ @@ -55,8 +57,8 @@ const withValues = ( values: Record, ): C => ({ ...creation, config: { ...creation.config, ...values } }) as C; -const stopped = (instance: Orchestrator.RegisteredInstance) => - instance.core.observation.pipe( +const stopped = (instance: Effect.Success>) => + instance.observation.pipe( Stream.filter((state) => state.lifecycle === "stopped" && state.currentOperation === undefined), Stream.take(1), Stream.runDrain, @@ -142,7 +144,7 @@ it.live( inputs: ["databaseUrl"], outputs: { url: Effect.succeed("http://pgmeta") }, }); - const functions = yield* makeInstance(orchestrator, functionsId, { + yield* makeInstance(orchestrator, functionsId, { outputs: { url: Effect.succeed(`url::${functionsId}`) }, }); const studio = yield* makeInstance(orchestrator, studioId, { @@ -151,18 +153,18 @@ it.live( yield* orchestrator.configure(configuration); yield* orchestrator.startComposition; yield* orchestrator.restart(functionsId); - yield* functions.core.ready; - expect(yield* studio.core.get).toMatchObject({ lifecycle: "stopped", wakeEnabled: true }); + yield* orchestrator.ready(functionsId); + expect(yield* studio.status).toMatchObject({ lifecycle: "stopped", wakeEnabled: true }); const requestScope = yield* Scope.make(); yield* orchestrator.acquire(studioId).pipe(Scope.provide(requestScope)); yield* TestClock.adjust("1 second"); - expect((yield* database.core.get).lifecycle).toBe("running"); - expect((yield* pgmeta.core.get).lifecycle).toBe("running"); - expect((yield* studio.core.get).lifecycle).toBe("running"); + expect((yield* database.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("running"); yield* orchestrator.restart(functionsId); - yield* functions.core.ready; - expect((yield* studio.core.get).lifecycle).toBe("running"); + yield* orchestrator.ready(functionsId); + expect((yield* studio.status).lifecycle).toBe("running"); // The request finished; studio itself now idles on its own 5-minute timer. yield* Scope.close(requestScope, Exit.void); @@ -170,18 +172,22 @@ it.live( // Well past pgmeta's own 60s idle mark but within studio's 5-minute window: studio // isn't idle yet, so the dependent-blocks-sleep rule keeps pgmeta running too. yield* TestClock.adjust("90 seconds"); - expect((yield* studio.core.get).lifecycle).toBe("running"); - expect((yield* pgmeta.core.get).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); // One second before studio's 5-minute idle deadline, measured from the request. yield* TestClock.adjust("208 seconds"); - expect((yield* studio.core.get).lifecycle).toBe("running"); - expect((yield* pgmeta.core.get).lifecycle).toBe("running"); + expect((yield* studio.status).lifecycle).toBe("running"); + expect((yield* pgmeta.status).lifecycle).toBe("running"); const studioStopped = yield* stopped(studio).pipe(Effect.forkChild); const pgmetaStopped = yield* stopped(pgmeta).pipe(Effect.forkChild); yield* TestClock.adjust("2 seconds"); yield* Fiber.join(studioStopped); + expect((yield* pgmeta.status).lifecycle).toBe("running"); + + // Once studio's stop is confirmed, pgmeta idles on its own timer. + yield* TestClock.adjust("60 seconds"); yield* Fiber.join(pgmetaStopped); }), ).pipe(Effect.provide(TestClock.layer())), diff --git a/packages/stack/src/create.owner-rollback.integration.test.ts b/packages/stack/src/create.owner-rollback.integration.test.ts index 34ab4ff186..c309180f93 100644 --- a/packages/stack/src/create.owner-rollback.integration.test.ts +++ b/packages/stack/src/create.owner-rollback.integration.test.ts @@ -19,7 +19,7 @@ it.live( const dockerShim = path.join(binDir, "docker"); yield* fs.writeFileString( dockerShim, - "#!/bin/sh\necho 'docker daemon unreachable' >&2\nexit 1\n", + "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?' >&2\nexit 1\n", ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. @@ -43,11 +43,13 @@ it.live( const id = yield* deriveStackId(identity); const launchFailure = yield* Effect.flip(create({ ...options, startOwner: true })); - expect(launchFailure.message).toContain("docker daemon unreachable"); + expect(launchFailure.message).toContain("Cannot connect to the Docker daemon"); expect(yield* discover({ stateRoot: options.stateRoot })).toEqual([]); - const stackDirExit = yield* Effect.exit(fs.access(path.join(options.stateRoot, id))); - expect(Exit.isFailure(stackDirExit)).toBe(true); + const stateFileExit = yield* Effect.exit( + fs.access(path.join(options.stateRoot, id, "state.json")), + ); + expect(Exit.isFailure(stateFileExit)).toBe(true); const retried = yield* create({ ...options, runtime: "native" }); expect(retried.id).toBe(id); @@ -60,7 +62,7 @@ it.live("removes a stack it just registered when its owner launch is interrupted const path = yield* Path.Path; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-create-interrupt-" }); - // The shim signals through a FIFO once the owner's startup sweep is running, then blocks. + // The shim signals through a FIFO once the owner's startup reconcile is running, then blocks. const started = path.join(root, "sweep-started"); yield* Effect.scoped( spawner @@ -70,19 +72,34 @@ it.live("removes a stack it just registered when its owner launch is interrupted const binDir = path.join(root, "bin"); yield* fs.makeDirectory(binDir, { recursive: true }); const dockerShim = path.join(binDir, "docker"); + // An orphaned container from a never-fully-registered prior attempt, found by this owner's + // startup label sweep before it finishes registering. + const orphanId = "interrupt0000000000000000000000001"; yield* fs.writeFileString( dockerShim, - `#!/bin/sh\nif [ "$1" = "ps" ]; then echo started > '${started}'; exec sleep 60; fi\nexit 0\n`, + // The leading check strips the pinned `--host ` prefix every invocation now + // carries (fixed below via DOCKER_HOST), so "$1" below still sees the real command. + `#!/bin/sh\nif [ "$1" = "--host" ]; then shift 2; fi\nif [ "$1" = "ps" ]; then echo '${orphanId}'; exit 0; fi\nif [ "$1" = "rm" ]; then echo started > '${started}'; exec sleep 60; fi\nif [ "$1" = "info" ]; then echo fake-daemon-id; fi\nexit 0\n`, ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. const originalPath = process.env.PATH; // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. process.env.PATH = `${binDir}:${originalPath ?? ""}`; + // Fixes the pinned endpoint to a dummy socket, so resolving it never needs `context show`/ + // `context inspect`, which this minimal shim doesn't answer. + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + const originalDockerHost = process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env.DOCKER_HOST = "unix:///var/run/shimmed-docker.sock"; yield* Effect.addFinalizer(() => Effect.sync(() => { // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. process.env.PATH = originalPath; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + if (originalDockerHost === undefined) delete process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + else process.env.DOCKER_HOST = originalDockerHost; }), ); diff --git a/packages/stack/src/credentials.integration.test.ts b/packages/stack/src/credentials.integration.test.ts index c66ce6dfc6..2f704f035a 100644 --- a/packages/stack/src/credentials.integration.test.ts +++ b/packages/stack/src/credentials.integration.test.ts @@ -1,8 +1,8 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted } from "effect"; -import { tmpdir } from "node:os"; import { create, open } from "./effect.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -14,7 +14,7 @@ it.live("persists effective credentials across service creation, reopen, and res const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } as const; const stack = yield* create(options); @@ -53,7 +53,7 @@ it.live("resolves composition credentials before creating services", () => const stack = yield* create({ projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", }); yield* Effect.addFinalizer(() => stack.destroy.pipe(Effect.orDie)); diff --git a/packages/stack/src/defaults.integration.test.ts b/packages/stack/src/defaults.integration.test.ts index af71143ea4..1f0b3ab928 100644 --- a/packages/stack/src/defaults.integration.test.ts +++ b/packages/stack/src/defaults.integration.test.ts @@ -1,12 +1,12 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted } from "effect"; -import { tmpdir } from "node:os"; import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "./Defaults.ts"; import { create as createEffect, StackError } from "./effect.ts"; import { create as createPromise } from "./index.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; -const artifactCacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const artifactCacheRoot = testArtifactCacheRoot; const effectLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); const effectDatabaseConfig = { diff --git a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts index 595b75a6a5..8db341a4e5 100644 --- a/packages/stack/src/destroy.runtime-unavailable.integration.test.ts +++ b/packages/stack/src/destroy.runtime-unavailable.integration.test.ts @@ -1,37 +1,49 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Layer, Path } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { Effect, FileSystem, Layer, Path } from "effect"; import { create, discover } from "./effect.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -const shimDocker = Effect.fn("shimDocker")(function* ( - root: string, - script: string, - engine: "docker" | "podman" = "docker", -) { +// Strips the pinned `--host ` prefix every invocation now carries, so a shim's own +// `"$1"` checks see the real command unchanged; `DOCKER_HOST` below makes that prefix constant. +const stripPinPrefix = 'if [ "$1" = "--host" ]; then shift 2; fi\n'; + +const shimDocker = Effect.fn("shimDocker")(function* (root: string, script: string) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const binDir = path.join(root, "bin"); yield* fs.makeDirectory(binDir, { recursive: true }); - const dockerShim = path.join(binDir, engine); - yield* fs.writeFileString(dockerShim, script); + const dockerShim = path.join(binDir, "docker"); + yield* fs.writeFileString( + dockerShim, + script.replace("#!/bin/sh\n", `#!/bin/sh\n${stripPinPrefix}`), + ); yield* fs.chmod(dockerShim, 0o755); // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the detached host subprocess inherits PATH; this is not application config. const originalPath = process.env.PATH; // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. process.env.PATH = `${binDir}:${originalPath ?? ""}`; + // Fixes the pinned endpoint to a dummy socket, so resolving it never needs `context show`/ + // `context inspect`, which these minimal shims don't answer. + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + const originalDockerHost = process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env.DOCKER_HOST = "unix:///var/run/shimmed-docker.sock"; yield* Effect.addFinalizer(() => Effect.sync(() => { // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. process.env.PATH = originalPath; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + if (originalDockerHost === undefined) delete process.env.DOCKER_HOST; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + else process.env.DOCKER_HOST = originalDockerHost; }), ); }); it.live( - "removes a stack's registration and data when destroy finds no owner and its engine is unreachable", + "fails destroy with runtime-unavailable and keeps the stack registered when its engine is unreachable, then finishes the cleanup once a later owner can start", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -48,36 +60,23 @@ it.live( runtime: "docker", } satisfies Parameters[0]; const stack = yield* create(options); - const dataRoot = `${options.stateRoot}/${stack.id}/data`; - yield* fs.makeDirectory(`${dataRoot}/db-instance`, { recursive: true }); - yield* fs.writeFileString( - `${dataRoot}/db-instance/.supabase-database-storage.json`, - `{"backend":"docker","volume":"supabase-db-0123456789abcdef","namespace":"instance-${stack.id}-db-instance","cacheNamespace":"cache-${"0".repeat(32)}","daemonId":"daemon","initialized":true}`, - ); - const resolvedDataRoot = yield* fs.realPath(dataRoot); - const result = yield* stack.destroy; - expect(result).toEqual({ - runtimeCleanup: "skipped", - engine: "docker", - cleanupCommands: [ - `sh -c 'ids=$(docker ps --all --quiet --no-trunc --filter '\\''label=com.supabase.stack=${stack.id}'\\'' --filter '\\''label=com.supabase.stack-root=${resolvedDataRoot}'\\'') && { [ -z "$ids" ] || docker rm --force $ids; }'`, - expect.stringMatching( - new RegExp( - `^docker run --rm --mount 'type=volume,src=supabase-db-0123456789abcdef,dst=/store' '[^']+' /bin/sh -c 'rm -rf /store/instance-${stack.id}-db-instance'$`, - "u", - ), - ), - ], - }); + const failure = yield* Effect.flip(stack.destroy); + expect(failure.reason).toBe("runtime-unavailable"); + expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); + // The engine is back: destroy retried finishes the cleanup automatically. + yield* shimDocker( + root, + '#!/bin/sh\nif [ "$1" = "info" ]; then\n echo \'fake-daemon-id\'\nfi\nexit 0\n', + ); + yield* stack.destroy; expect(yield* discover({ stateRoot: options.stateRoot })).toEqual([]); - const stackDirExit = yield* Effect.exit(fs.access(`${options.stateRoot}/${stack.id}`)); - expect(Exit.isFailure(stackDirExit)).toBe(true); }).pipe(Effect.scoped, Effect.provide(layer)), + 30_000, ); -it.live("removes a stack offline when Windows reports its Docker daemon pipe is missing", () => +it.live("keeps a stack registered when Windows reports its Docker daemon pipe is missing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-windows-" }); @@ -93,14 +92,14 @@ it.live("removes a stack offline when Windows reports its Docker daemon pipe is runtime: "docker", }); - const result = yield* stack.destroy; + const failure = yield* Effect.flip(stack.destroy); - expect(result.runtimeCleanup).toBe("skipped"); - expect(yield* discover({ stateRoot })).toEqual([]); + expect(failure.reason).toBe("runtime-unavailable"); + expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("removes a stack offline when its engine CLI is not installed", () => +it.live("keeps a stack registered when its engine CLI is not installed", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-no-cli-" }); @@ -123,14 +122,14 @@ it.live("removes a stack offline when its engine CLI is not installed", () => }), ); - const result = yield* stack.destroy; + const failure = yield* Effect.flip(stack.destroy); - expect(result.runtimeCleanup).toBe("skipped"); - expect(yield* discover({ stateRoot })).toEqual([]); + expect(failure.reason).toBe("runtime-unavailable"); + expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("removes a stack offline when Docker reports its API socket is missing", () => +it.live("keeps a stack registered when Docker reports its API socket is missing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-missing-socket-" }); @@ -146,110 +145,13 @@ it.live("removes a stack offline when Docker reports its API socket is missing", runtime: "docker", }); - const result = yield* stack.destroy; - - expect(result.runtimeCleanup).toBe("skipped"); - expect(yield* discover({ stateRoot })).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(layer)), -); - -it.live( - "prints a container cleanup command that fails while the engine is down and removes every listed container once it is back", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-command-" }); - const unreachable = - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?' >&2\nexit 1\n"; - yield* shimDocker(root, unreachable); - const stack = yield* create({ - projectRoot: root, - stateRoot: `${root}/state`, - cacheRoot: `${root}/cache`, - runtime: "docker", - }); - const result = yield* stack.destroy; - if (result.runtimeCleanup !== "skipped") return yield* Effect.die("cleanup was not skipped"); - const [containerCommand] = result.cleanupCommands; - if (containerCommand === undefined) return yield* Effect.die("container command missing"); - const run = Effect.scoped( - spawner - .spawn(ChildProcess.make("/bin/sh", ["-c", containerCommand])) - .pipe(Effect.flatMap((child) => child.exitCode)), - ); - - expect(Number(yield* run)).not.toBe(0); - yield* fs.writeFileString( - `${root}/bin/docker`, - '#!/bin/sh\nif [ "$1" = "ps" ]; then exit 0; fi\necho "docker rm requires at least 1 argument" >&2\nexit 1\n', - ); - expect(Number(yield* run)).toBe(0); - // Removal succeeds only when both listed IDs arrive as separate arguments. - yield* fs.writeFileString( - `${root}/bin/docker`, - '#!/bin/sh\nif [ "$1" = "ps" ]; then printf "aaa111\\nbbb222\\n"; exit 0; fi\nif [ "$1" = "rm" ] && [ "$2" = "--force" ] && [ "$#" -eq 4 ] && [ "$3" = "aaa111" ] && [ "$4" = "bbb222" ]; then exit 0; fi\necho "unexpected arguments: $*" >&2\nexit 1\n', - ); - expect(Number(yield* run)).toBe(0); - }).pipe(Effect.scoped, Effect.provide(layer)), -); - -it.live("keeps a stack registered when Podman rejects its credentials", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-podman-auth-" }); - yield* shimDocker( - root, - "#!/bin/sh\necho 'Cannot connect to Podman. Please verify your connection to the Linux system using `podman system connection list`, or try `podman machine init` and `podman machine start` to manage a new Linux VM' >&2\necho 'Error: unable to connect to Podman socket: ssh: handshake failed: ssh: unable to authenticate, attempted methods [none publickey], no supported methods remain' >&2\nexit 125\n", - "podman", - ); - const stateRoot = `${root}/state`; - const stack = yield* create({ - projectRoot: root, - stateRoot, - cacheRoot: `${root}/cache`, - runtime: "podman", - }); - - const destroyFailure = yield* Effect.flip(stack.destroy); + const failure = yield* Effect.flip(stack.destroy); - expect(destroyFailure.message).toContain("unable to authenticate"); + expect(failure.reason).toBe("runtime-unavailable"); expect(yield* discover({ stateRoot })).toHaveLength(1); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live.skipIf(process.getuid?.() === 0)( - "keeps a stack and its data when offline destroy cannot remove container-owned host data", - () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-host-data-" }); - yield* shimDocker( - root, - "#!/bin/sh\necho 'Cannot connect to the Docker daemon at tcp://127.0.0.1:1. Is the docker daemon running?' >&2\nexit 1\n", - ); - const stateRoot = `${root}/state`; - const stack = yield* create({ - projectRoot: root, - stateRoot, - cacheRoot: `${root}/cache`, - runtime: "docker", - }); - // Stands in for PostgreSQL files a container wrote as its own user. - const instanceRoot = `${stateRoot}/${stack.id}/data/db-instance`; - yield* fs.makeDirectory(`${instanceRoot}/data`, { recursive: true }); - yield* fs.writeFileString(`${instanceRoot}/owned-file`, "owned data"); - yield* fs.chmod(`${instanceRoot}/data`, 0o000); - yield* Effect.addFinalizer(() => fs.chmod(`${instanceRoot}/data`, 0o755).pipe(Effect.ignore)); - - const destroyFailure = yield* Effect.flip(stack.destroy); - - expect(destroyFailure.message).toContain("start Docker and run destroy again"); - expect(yield* discover({ stateRoot })).toHaveLength(1); - expect(yield* fs.exists(`${instanceRoot}/owned-file`)).toBe(true); - }).pipe(Effect.scoped, Effect.provide(layer)), -); - it.live("keeps a stack registered when its container engine rejects the listing", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -268,6 +170,8 @@ it.live("keeps a stack registered when its container engine rejects the listing" const stack = yield* create(options); const destroyFailure = yield* Effect.flip(stack.destroy); + // The socket itself refuses the owner's own startup, resolving its pinned engine target: + // the owner never starts, so the stack stays registered for a later retry. expect(destroyFailure.message).toContain("permission denied"); expect(yield* discover({ stateRoot: options.stateRoot })).toHaveLength(1); @@ -278,18 +182,24 @@ it.live("keeps a stack registered when its container engine is reachable but cle Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-cleanup-failure-" }); + // The daemon is reachable and identifiable, and its label listing finds this stack's own + // container; only the targeted removal fails. + const containerId = "cleanupfail0000000000000000000001"; yield* shimDocker( root, [ "#!/bin/sh", 'if [ "$1" = "ps" ]; then', - ' echo "abc123def456"', + ` echo '${containerId}'`, " exit 0", "fi", 'if [ "$1" = "rm" ]; then', " echo 'docker: Error response from daemon: container removal failed' >&2", " exit 1", "fi", + 'if [ "$1" = "info" ]; then', + " echo 'fake-daemon-id'", + "fi", "exit 0", "", ].join("\n"), diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts index d6f65c1d2e..2a2011490d 100644 --- a/packages/stack/src/effect.integration.test.ts +++ b/packages/stack/src/effect.integration.test.ts @@ -9,13 +9,11 @@ import { Layer, Option, Redacted, - Schema, Scope, Stream, } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the test binds a dead owner's exact port. import * as Net from "node:net"; -import { tmpdir } from "node:os"; import { create, discover, @@ -27,21 +25,21 @@ import { } from "./effect.ts"; import { initialization, postgres } from "./Commands.ts"; import { fileURLToPath } from "node:url"; +import { testEngine } from "../tests/test-engine.ts"; +import type { ContainerEngine } from "./runtime/Container.ts"; import { launchHost } from "./HostProcess.ts"; import * as PromiseApi from "./index.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { assertOwnerExited, watchLeaseRelease } from "../tests/owner.ts"; import { foreignRelease } from "../tests/release-owner-fixture.ts"; import { engineStub } from "../tests/docker-fixture.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); // Below every OS ephemeral range, so another test's outbound socket cannot already hold it. const FIXED_API_PORT = 24_393; -const databaseOwnerMarker = Schema.fromJsonString( - Schema.Struct({ stackId: Schema.String, instanceId: Schema.String }), -); it.live("registers and discovers saved definitions without inventing live observations", () => Effect.gen(function* () { @@ -153,7 +151,7 @@ it.live("starts the owner on opt-in reopen without starting saved services", () ); it.live( - "stops the owner after destroy fails and retains saved data for retry", + "reports a failed destroy, exits the owner, and lets the next destroy reclaim the stack", () => Effect.scoped( Effect.gen(function* () { @@ -177,16 +175,16 @@ it.live( }, }); const dataRoot = `${options.stateRoot}/${stack.id}/data/${instance.id}`; - const marker = `${dataRoot}/.supabase-database-owner.json`; - yield* fs.makeDirectory(dataRoot, { recursive: true }); - const writeMarker = Schema.encodeEffect(databaseOwnerMarker); - yield* writeMarker({ stackId: "another-stack", instanceId: instance.id }).pipe( - Effect.flatMap((value) => fs.writeFileString(marker, value)), - ); + // Ownership is by location: a root that is a symlink (not a real owned directory, perhaps + // tampered with) is refused rather than traversed or removed. + yield* fs.remove(dataRoot, { recursive: true, force: true }); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "destroy-failure-outside-" }); + yield* fs.symlink(outside, dataRoot); const running = yield* discover(options); expect(running).toHaveLength(1); expect(running[0]?.host).toBeDefined(); + const released = yield* watchLeaseRelease(options.stateRoot, stack.id); const destroyExit = yield* stack.destroy.pipe(Effect.exit); expect(Exit.isFailure(destroyExit)).toBe(true); if (Exit.isFailure(destroyExit)) { @@ -198,19 +196,19 @@ it.live( { id: instance.id, succeeded: false, - error: expect.stringContaining("Database root belongs to another instance"), + error: expect.stringContaining("is a symlink"), }, ], }); } - expect(yield* fs.readFileString(marker)).toContain("another-stack"); + expect(yield* fs.readLink(dataRoot).pipe(Effect.isSuccess)).toBe(true); + yield* released; const retained = yield* discover(options); expect(retained).toHaveLength(1); expect(retained[0]?.host).toBeUndefined(); + yield* assertOwnerExited(running[0]?.host?.pid ?? 0); - yield* writeMarker({ stackId: stack.id, instanceId: instance.id }).pipe( - Effect.flatMap((value) => fs.writeFileString(marker, value)), - ); + yield* fs.remove(dataRoot, { force: true }); yield* stack.destroy; expect(yield* discover(options)).toHaveLength(0); }), @@ -218,14 +216,65 @@ it.live( { timeout: 30_000 }, ); -const resetDataStory = (runtime: "native" | "docker") => +it.live( + "stops a running database promptly while a client holds an idle TCP connection", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-stop-idle-tcp-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: testArtifactCacheRoot, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + yield* Effect.addFinalizer(() => destroyTestStack(stack)); + const database = yield* stack.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("idle-tcp-password"), + jwtSecret: Redacted.make("idle-tcp-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* database.start; + yield* database.ready; + const { databaseUrl } = yield* database.credentials({ from: "runtime" }); + if (databaseUrl === undefined) return yield* Effect.die("database credentials missing"); + const url = new URL(databaseUrl); + const client = yield* Effect.callback((resume) => { + const connection = Net.createConnection({ + host: url.hostname, + port: Number(url.port), + }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.once("error", (cause) => resume(Effect.fail(cause))); + return Effect.sync(() => connection.destroy()); + }); + + // An idle client must not hold the stop open. + yield* stack.stop.pipe(Effect.timeout("5 seconds")); + + client.destroy(); + expect(yield* discover(options)).toHaveLength(1); + expect((yield* discover(options))[0]?.host).toBeUndefined(); + }), + ).pipe(Effect.provide(layer)), + { timeout: 60_000 }, +); + +const resetDataStory = (runtime: "native" | ContainerEngine) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-reset-data-${runtime}-` }); const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime, } satisfies Parameters[0]; const stack = yield* create(options); @@ -399,7 +448,7 @@ it.live("resets native database data through the public RPC", () => resetDataSto timeout: 10 * 60_000, }); -it.live("resets Docker database data through the public RPC", () => resetDataStory("docker"), { +it.live("resets container database data through the public RPC", () => resetDataStory(testEngine), { timeout: 15 * 60_000, }); @@ -441,8 +490,8 @@ it.live("rejects an owner of another release while stop and destroy still reach runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const startForeignOwner = launchHost(state, { ...options, @@ -466,86 +515,109 @@ it.live("rejects an owner of another release while stop and destroy still reach }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("treats a sweeper's hold as no owner and starts one once the sweep ends", () => +it.live("discovers only the stacks whose id starts with the given prefix", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweeping-" }); - const options = { - projectRoot: root, - stateRoot: `${root}/state`, - cacheRoot: `${root}/cache`, - runtime: "native", - } satisfies Parameters[0]; - const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), - ); - yield* Effect.ensuring( - Effect.gen(function* () { - const sweep = yield* Scope.make(); - expect(yield* state.lease(stack.id).pipe(Scope.provide(sweep))).toBe(true); - yield* state.publishHolder(stack.id, { - role: "sweeper", - pid: process.pid, - startedAt: "2026-01-01T00:00:00.000Z", - }); - yield* stack.stop; - expect(yield* stack.composition.stop).toEqual([]); - expect((yield* discover(options))[0]?.host).toBeUndefined(); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-api-prefix-" }); + const stateRoot = `${root}/state`; + const locations = { projectRoot: root, stateRoot, cacheRoot: `${root}/cache` } as const; + const first = yield* create({ ...locations, name: "first", runtime: "native" }); + const second = yield* create({ ...locations, name: "second", runtime: "native" }); + const prefix = first.id.slice(0, 12); + expect(second.id.startsWith(prefix), "the prefix selects one of two stacks").toBe(false); + yield* fs.makeDirectory(`${stateRoot}/${prefix}-broken`); + yield* fs.writeFileString(`${stateRoot}/${prefix}-broken/state.json`, "{broken"); + yield* fs.makeDirectory(`${stateRoot}/other-broken`); + yield* fs.writeFileString(`${stateRoot}/other-broken/state.json`, "{broken"); - const starting = yield* stack.composition.start.pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* state.retractHolder(stack.id); - yield* Scope.close(sweep, Exit.void); - expect(yield* Fiber.join(starting)).toEqual([]); - expect((yield* discover(options))[0]?.host).toBeDefined(); - }), - destroyTestStack(stack), - ); + const reported: Array = []; + const selected = yield* discover({ + stateRoot, + idPrefix: prefix, + onInvalidState: (id) => Effect.sync(() => reported.push(id)), + }); + + expect(selected.map(({ definition }) => definition.id)).toEqual([first.id]); + expect(reported, "only a prefixed entry is reported as invalid").toEqual([`${prefix}-broken`]); }).pipe(Effect.scoped, Effect.provide(layer)), ); -it.live("creates a deleted stack again once a hold on its lease ends", () => +it.live( + "treats a sweeper's hold as no owner, refuses to stop, and starts one once the sweep ends", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweeping-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${root}/cache`, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), + ); + yield* Effect.ensuring( + Effect.gen(function* () { + const sweep = yield* Scope.make(); + const lease = yield* state.acquireLease(stack.id).pipe(Scope.provide(sweep)); + yield* lease.publishHolder({ + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + const refused = yield* Effect.flip(stack.stop); + expect(refused.message).toContain("holds the stack lease"); + expect(yield* stack.composition.stop).toEqual([]); + expect((yield* discover(options))[0]?.host).toBeUndefined(); + + const starting = yield* stack.composition.start.pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* lease.retractHolder; + yield* Scope.close(sweep, Exit.void); + expect(yield* Fiber.join(starting)).toEqual([]); + expect((yield* discover(options))[0]?.host).toBeDefined(); + }), + destroyTestStack(stack), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + +it.live("reclaims a registered stack's leftovers when stopping without an owner", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweeping-create-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ownerless-stop-" }); const options = { projectRoot: root, stateRoot: `${root}/state`, cacheRoot: `${root}/cache`, runtime: "native", + lifetime: "session", } satisfies Parameters[0]; - const { id } = yield* create(options); - yield* fs.remove(`${options.stateRoot}/${id}`, { recursive: true }); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), - ); - const hold = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(hold, Exit.void)); - expect(yield* state.lease(id).pipe(Scope.provide(hold))).toBe(true); - const log = state.ownerLog(id); - const contended = yield* fs.watch(`${options.stateRoot}/${id}`).pipe( - Stream.mapEffect(() => fs.readFileString(log).pipe(Effect.orElseSucceed(() => ""))), - Stream.takeUntil((text) => text.includes("holds the lease")), - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), + const identity = yield* resolveStackIdentity(options); + const id = yield* deriveStackId(identity); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); + yield* state.save({ + id, + identity, + lifetime: "session", + runtime: "native", + instances: [{ id: "leftover", creation: { service: "mail", config: {} } }], + composition: { members: [], dependencies: [] }, + }); + const leftover = `${options.stateRoot}/${id}/data/leftover`; + yield* fs.makeDirectory(leftover, { recursive: true }); + yield* fs.writeFileString(`${leftover}/state`, "left behind"); + const stack = yield* open({ ...options, id }); - const creating = yield* create({ ...options, startOwner: true }).pipe( - Effect.forkChild({ startImmediately: true }), - ); - yield* Fiber.join(contended); - yield* Scope.close(hold, Exit.void); - const stack = yield* Fiber.join(creating); + yield* stack.stop; - yield* Effect.ensuring( - Effect.gen(function* () { - expect(stack.id).toBe(id); - expect((yield* discover(options))[0]?.host).toBeDefined(); - }), - destroyTestStack(stack), - ); + expect(yield* state.read(id)).toBeUndefined(); + expect(yield* fs.exists(leftover)).toBe(false); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -592,10 +664,56 @@ it.live("finds a deleted stack only by the containers left in its own state root ); const file = `${root}/file`; yield* fs.writeFileString(file, ""); - expect( - Option.isNone(yield* found(deletedId, file)), - "an unusable registry is left to find", - ).toBe(true); + expect((yield* Effect.flip(found(deletedId, file))).operation, "an unusable state root").toBe( + "find", + ); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + +it.live("names the directory of a saved stack whose state cannot be decoded", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-unreadable-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${root}/cache`, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + const statePath = `${options.stateRoot}/${stack.id}/state.json`; + const saved = yield* fs.readFileString(statePath); + yield* fs.writeFileString( + statePath, + saved.replace('"instances":[]', '"instances":[{"id":"logs","service":"vector"}]'), + ); + + const failure = yield* find({ stateRoot: options.stateRoot, projectRoot: root }).pipe( + Effect.flip, + ); + + expect(failure.operation).toBe("find"); + expect(failure.message).toContain(`Stack ${stack.id} could not be read`); + expect(failure.message).toContain(`Remove its directory ${options.stateRoot}/${stack.id}`); + expect(yield* fs.exists(statePath)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + +it.live("removes nothing for a stack whose registration is unreadable", () => + Effect.gen(function* () { + const { fs, locations, containers } = yield* deletedStackRoot(); + yield* fs.makeDirectory(`${locations.stateRoot}/${deletedId}`); + yield* fs.writeFileString(`${locations.stateRoot}/${deletedId}/state.json`, "{broken"); + const engine = engineStub(containers); + + const failure = yield* findDeleted({ ...locations, id: deletedId }).pipe( + Effect.provide(engine.layer), + Effect.flip, + ); + + expect(failure.operation).toBe("find"); + expect(engine.remaining("docker")).toEqual(containers); + expect(engine.remaining("podman")).toEqual(containers); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -639,7 +757,7 @@ it.live("removes a deleted stack's containers from both engines, not another roo yield* findDeleted({ ...locations, id: deletedId }).pipe(Effect.provide(engine.layer)), ); - expect(yield* deleted.destroy).toEqual({ runtimeCleanup: "complete" }); + yield* deleted.destroy; const kept = containers.filter(({ id }) => id !== "deleted"); expect(engine.remaining("docker")).toEqual(kept); expect(engine.remaining("podman")).toEqual(kept); @@ -666,6 +784,24 @@ it.live("leaves a deleted stack's containers to a registration that returns duri }).pipe(Effect.scoped, Effect.provide(layer)), ); +it.live("leaves a deleted stack's containers alone while another process holds its lease", () => + Effect.gen(function* () { + const { locations, containers } = yield* deletedStackRoot(); + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: locations.stateRoot })), + ); + yield* state.acquireLease(deletedId); + const engine = engineStub(containers); + const deleted = Option.getOrThrow( + yield* findDeleted({ ...locations, id: deletedId }).pipe(Effect.provide(engine.layer)), + ); + + expect((yield* Effect.flip(deleted.destroy)).message).toContain("holds this stack's lease"); + expect(engine.remaining("docker")).toEqual(containers); + expect(engine.remaining("podman")).toEqual(containers); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + it.live("interrupts a call waiting for an owner while a sweeper holds the stack", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -677,14 +813,14 @@ it.live("interrupts a call waiting for an owner while a sweeper holds the stack" runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); yield* Effect.ensuring( Effect.gen(function* () { const sweep = yield* Scope.make(); - expect(yield* state.lease(stack.id).pipe(Scope.provide(sweep))).toBe(true); - yield* state.publishHolder(stack.id, { + const lease = yield* state.acquireLease(stack.id).pipe(Scope.provide(sweep)); + yield* lease.publishHolder({ role: "sweeper", pid: process.pid, startedAt: "2026-01-01T00:00:00.000Z", @@ -693,7 +829,7 @@ it.live("interrupts a call waiting for an owner while a sweeper holds the stack" const waited = yield* stack.composition.start.pipe(Effect.timeoutOption("200 millis")); expect(Option.isNone(waited), "the wait for the sweeper ends at the timeout").toBe(true); - yield* state.retractHolder(stack.id); + yield* lease.retractHolder; yield* Scope.close(sweep, Exit.void); expect(yield* stack.composition.start, "the handle still launches afterwards").toEqual([]); }), @@ -762,8 +898,8 @@ it.live("sends no call to a process that took a dead owner's port", () => const stack = yield* create(options); const mail = yield* stack.services.create({ service: "mail", config: {} }); yield* mail.status; - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const holder = yield* state.readHolder(stack.id); if (holder?.role !== "owner") return yield* Effect.die("Expected a live owner record"); @@ -793,8 +929,8 @@ it.live("sends no call to a process that took the port of a replaced owner", () const stack = yield* create(options); const mail = yield* stack.services.create({ service: "mail", config: {} }); yield* mail.status; - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const replaced = yield* state.readHolder(stack.id); if (replaced?.role !== "owner") return yield* Effect.die("Expected a live owner record"); @@ -822,7 +958,7 @@ it.live( const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } satisfies Parameters[0]; const handleScope = yield* Scope.make(); @@ -878,8 +1014,8 @@ it.live("confirms owner exit after shutdown even while a stray handle keeps its runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); const owner = yield* launchHost(state, { ...options, @@ -907,8 +1043,8 @@ it.live("replaces a dead session stack that holds the requested identity", () => } satisfies Parameters[0]; const identity = yield* resolveStackIdentity(options); const id = yield* deriveStackId(identity); - const state = yield* State.Service.pipe( - Effect.provide(State.layer({ root: options.stateRoot })), + const state = yield* StackNamespace.Service.pipe( + Effect.provide(StackNamespace.layer({ root: options.stateRoot })), ); yield* state.save({ id, @@ -917,7 +1053,6 @@ it.live("replaces a dead session stack that holds the requested identity", () => runtime: "native", instances: [{ id: "abandoned", creation: { service: "mail", config: {} } }], composition: { members: [], dependencies: [] }, - ports: [], }); yield* Effect.scoped( @@ -940,7 +1075,7 @@ it.live( const options = { projectRoot: root, stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", } satisfies Parameters[0]; const stack = yield* create(options); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index a980a17e7a..7cb89e2188 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -2,7 +2,6 @@ import { Cause, Context, Crypto, - DateTime, Deferred, Effect, Exit, @@ -40,17 +39,16 @@ import { type PlannedInstance, type SupabaseCompositionOptions, } from "./composition/Supabase.ts"; -import { listStackContainers, removeStackContainersCommand } from "./runtime/Container.ts"; -import { volumeDataCleanupCommands } from "./storage/DockerDatabaseStorage.ts"; import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; import { failureMessage } from "./internal/failure-message.ts"; -import * as State from "./State.ts"; -import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { engineUnreachable, resolveEngineTarget } from "./runtime/Container.ts"; +import { leftBehindStackIds, reclaimDeletedStack, reclaimStack } from "./Sweep.ts"; +import type { SavedStack, StackCredentials, StackKeysInput } from "./StackNamespace.ts"; import { StackError, type Definition, type Observation } from "./Rpc.ts"; import { sinceMillis, streamStackLogs as streamPersistedLogs } from "./host/LogStore.ts"; import { gatewayLog } from "./host/GatewayLog.ts"; import type { LogPosition, LogRecord, StackLogRecord } from "./host/LogRecord.ts"; -import { reclaimStack } from "./Sweep.ts"; import { ServiceCreationInput as ServiceCreationInputSchema, type ServiceCreation, @@ -67,7 +65,7 @@ import type { export { initialization, postgres } from "./Commands.ts"; export { resolveNativePostgresUser } from "./runtime/postgres-user.ts"; -export { apiRoute } from "./host/Endpoints.ts"; +export { apiRoute, sharesApiEndpoint } from "./host/Endpoints.ts"; export { gatewayLog }; export { StackError } from "./Rpc.ts"; export type { ServiceCreation } from "./services/Catalog.ts"; @@ -80,7 +78,7 @@ export type { SupabaseCompositionOptions, } from "./composition/Supabase.ts"; export { StackIdSchema as StackId } from "./identity/StackId.ts"; -export type { SavedStack } from "./State.ts"; +export type { SavedStack } from "./StackNamespace.ts"; export type { StackCredentials, StackKeysInput }; export type { Observation } from "./Rpc.ts"; export type { LogPosition, LogRecord, StackLogRecord }; @@ -91,7 +89,8 @@ export type { PostgresCommand, } from "./Commands.ts"; -const stateFor = (root: string) => State.Service.pipe(Effect.provide(State.layer({ root }))); +const stateFor = (root: string) => + StackNamespace.Service.pipe(Effect.provide(StackNamespace.layer({ root }))); /** Storage locations shared by clients and the detached stack owner. */ export interface StackLocations { @@ -107,7 +106,7 @@ export interface CreateOptions extends StackLocations { * A `session` stack starts its owner at creation and is destroyed when the creating handle's * scope closes or its process exits; a `detached` stack (the default) outlives its creator. */ - readonly lifetime?: State.StackLifetime; + readonly lifetime?: StackNamespace.StackLifetime; /** * Starts the owner at creation and lets it register the stack under its lease, so a failed or * interrupted launch leaves no registration behind. Session stacks always do this. @@ -202,19 +201,6 @@ export type ServiceInstances = { [K in Kind]: K extends "database" ? DatabaseInstance : ServiceInstance; }; type AnyInstance = ServiceInstances[Kind]; -/** - * The outcome of {@link Stack.destroy}. `skipped` means the stack's registration and host data - * were removed without its container engine, because the engine was unreachable; its containers - * and any database data in engine volumes remain, and `cleanupCommands` remove them once the - * engine is running. - */ -export type DestroyResult = - | { readonly runtimeCleanup: "complete" } - | { - readonly runtimeCleanup: "skipped"; - readonly engine: "docker" | "podman"; - readonly cleanupCommands: ReadonlyArray; - }; /** Options for streaming PostgreSQL command input and output. */ export interface PostgresCommandOptions { readonly args?: ReadonlyArray; @@ -284,7 +270,12 @@ export interface Stack { readonly readLogs: (options?: ReadLogsOptions) => Stream.Stream; }; readonly stop: Effect.Effect; - readonly destroy: Effect.Effect; + /** + * Removes the stack. When its container engine is unreachable, fails with reason + * `runtime-unavailable` and leaves the registration, data and claimed resources in place; + * calling it again once the engine is reachable finishes the cleanup. + */ + readonly destroy: Effect.Effect; readonly commands: { readonly run: CommandRunner; }; @@ -321,89 +312,11 @@ interface Connection { /** Replaced by a newer connection; closes once its last user finishes. */ retired: boolean; } -/** Finds a directory below `directory` that the current user cannot empty and delete. */ -const firstUnremovableDirectory = ( - fs: FileSystem.FileSystem, - path: Path.Path, - directory: string, -): Effect.Effect => - Effect.gen(function* () { - const entries = yield* fs.readDirectory(directory).pipe(Effect.option); - const writable = yield* fs.access(directory, { writable: true }).pipe(Effect.isSuccess); - if (Option.isNone(entries) || !writable) return directory; - for (const entry of entries.value) { - const child = path.join(directory, entry); - const info = yield* fs.stat(child).pipe(Effect.option); - if (Option.isNone(info) || info.value.type !== "Directory") continue; - if (yield* fs.readLink(child).pipe(Effect.isSuccess)) continue; - const blocked = yield* firstUnremovableDirectory(fs, path, child); - if (blocked !== undefined) return blocked; - } - return undefined; - }); - -/** - * Destroys a stack whose owner cannot start because its container engine is unreachable: under - * the stack's lease it removes the registration and host data, and returns the commands that - * remove the containers and engine-volume data left behind. - */ -const destroyWithoutEngine = Effect.fn("Stack.destroyWithoutEngine")(function* ( - state: State.Interface, - saved: SavedStack, - locations: StackLocations, - engine: "docker" | "podman", -) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const id = saved.id; - const dataRoot = path.join(locations.stateRoot, id, "data"); - return yield* Effect.scoped( - Effect.gen(function* () { - if (!(yield* state.lease(id))) - return yield* failure( - "destroy", - "An owner for this stack started during destroy; run destroy again", - ); - yield* Effect.addFinalizer(() => state.retractHolder(id).pipe(Effect.ignore)); - yield* state.publishHolder(id, { - role: "sweeper", - pid: process.pid, - startedAt: DateTime.formatIso(yield* DateTime.now), - }); - const current = yield* state.read(id); - // Container-written host data, such as database files below Docker 26, can belong to the - // container user; only the engine can delete it, so refuse before deleting anything. - const blocked = - current !== undefined && (yield* fs.exists(dataRoot)) - ? yield* firstUnremovableDirectory(fs, path, dataRoot) - : undefined; - if (blocked !== undefined) { - const engineName = engine === "docker" ? "Docker" : "Podman"; - return yield* failure( - "destroy", - `Stack data at ${blocked} can only be removed by ${engineName}; start ${engineName} and run destroy again`, - ); - } - // Containers are labelled with the resolved data root the owner ran with. - const root = yield* fs.realPath(dataRoot).pipe(Effect.orElseSucceed(() => dataRoot)); - const cleanupCommands = [ - removeStackContainersCommand({ engine, stackId: id, root }), - ...(yield* volumeDataCleanupCommands({ engine, root, fs, path })), - ]; - if (current !== undefined) { - yield* fs.remove(dataRoot, { recursive: true, force: true }); - yield* state.withLock(state.remove(id)); - } - return { runtimeCleanup: "skipped", engine, cleanupCommands } as const; - }), - ).pipe(Effect.mapError((cause) => failure("destroy", cause))); -}); - /** `launch` starts an owner when none is live; `attach` requires a live one. */ type Reach = "launch" | "attach"; const makeHandle = Effect.fn("Stack.makeHandle")(function* ( - state: State.Interface, + state: StackNamespace.Interface, saved: SavedStack, locations: StackLocations, seed: { readonly access?: HostAccess; readonly creator?: boolean } = {}, @@ -411,7 +324,12 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( // Calls and streams release what they borrow in their own scope, not in the handle's. const services = Context.omit(Scope.Scope)( yield* Effect.context< - HttpClient.HttpClient | FileSystem.FileSystem | Path.Path | Crypto.Crypto | Scope.Scope + | HttpClient.HttpClient + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | Scope.Scope >(), ); const crypto = yield* Crypto.Crypto; @@ -558,29 +476,42 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( Effect.provideContext(services), ), ).pipe(Stream.mapError((cause) => failure(operation, cause))); + /** Stop with no owner: the leftovers of a failed or killed owner are reclaimed under the lease. */ + const reclaimWithoutOwner = reclaimStack({ + state, + stateRoot: locations.stateRoot, + cacheRoot: locations.cacheRoot, + id: saved.id, + }).pipe( + Effect.flatMap((reclaimed) => + reclaimed + ? Effect.void + : Effect.fail( + new StackError({ + operation: "shutdown", + message: "Another process holds the stack lease; run stop again", + }), + ), + ), + ); const shutdown = Effect.fn("Stack.shutdown")(function* (destroy: boolean) { const operation = destroy ? "destroy" : "shutdown"; yield* invalidate(); - const engine = saved.runtime === "native" ? undefined : saved.runtime; // Shutdown uses the release-stable endpoint, so it reaches owners of any release. - const { endpoint, refusal, engineUnavailable } = yield* Effect.scoped( + const { endpoint, refusal } = yield* Effect.scoped( Effect.gen(function* () { - const idle = { endpoint: undefined, refusal: Exit.void, engineUnavailable: false }; + const idle = { endpoint: undefined, refusal: Exit.void }; const live = yield* connectHost(state, saved.id, { anyRelease: true }).pipe( Effect.map(Option.some), Effect.catchIf(ownerAbsent, () => - destroy - ? launchHost(state, launchOptions).pipe(Effect.map(Option.some)) - : Effect.succeed(Option.none()), + Effect.gen(function* () { + if (destroy) return Option.some(yield* launchHost(state, launchOptions)); + yield* reclaimWithoutOwner; + return Option.none(); + }), ), Effect.catchIf(stackGone, () => Effect.succeed(Option.none())), - // The owner's startup sweep reports an unreachable engine before any owner serves. - Effect.catchIf( - (cause) => engine !== undefined && hasReason("runtime-unavailable")(cause), - () => Effect.succeed("engine-unavailable" as const), - ), ); - if (live === "engine-unavailable") return { ...idle, engineUnavailable: true }; if (Option.isNone(live)) return idle; const access = live.value; const endpoint = access.endpoint; @@ -600,46 +531,21 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( ), Effect.exit, ); - return { endpoint, refusal, engineUnavailable: false }; + return { endpoint, refusal }; }), ).pipe( Effect.provideContext(services), Effect.mapError((cause) => failure(operation, cause)), ); - if (engineUnavailable && engine !== undefined) - return yield* destroyWithoutEngine(state, saved, locations, engine).pipe( - Effect.provideContext(services), - ); - if (endpoint === undefined) return { runtimeCleanup: "complete" } as const; - if (Exit.isFailure(refusal)) { - const shutdownFailure = Option.match(Cause.findErrorOption(refusal.cause), { + if (endpoint === undefined) return; + if (Exit.isFailure(refusal)) + return yield* Option.match(Cause.findErrorOption(refusal.cause), { onNone: () => failure(operation, Cause.pretty(refusal.cause)), onSome: (cause) => failure(operation, cause), }); - if (!destroy) return yield* shutdownFailure; - const exitResult = yield* waitForOwnerExit(endpoint.pid, ownerExitProbe(fs)).pipe( - Effect.mapError((cause) => failure("shutdown-exit", cause)), - Effect.exit, - ); - if (Exit.isFailure(exitResult)) { - const exitFailure = Option.match(Cause.findErrorOption(exitResult.cause), { - onNone: () => failure("shutdown-exit", Cause.pretty(exitResult.cause)), - onSome: (cause) => failure("shutdown-exit", cause), - }); - const exitStatus = exitFailure.message.includes("still running") - ? "Owner is still running after failed destroy" - : "Owner exit was not confirmed after failed destroy"; - return yield* new StackError({ - ...shutdownFailure, - message: `${shutdownFailure.message}; ${exitStatus}; exit probe: ${exitFailure.message}`, - }); - } - return yield* shutdownFailure; - } yield* waitForOwnerExit(endpoint.pid, ownerExitProbe(fs)).pipe( Effect.mapError((cause) => failure("shutdown-exit", cause)), ); - return { runtimeCleanup: "complete" } as const; }); const snapshotScope = (options: DatabaseSnapshotOptions | undefined) => @@ -945,7 +851,6 @@ export const create = Effect.fn("Stack.create")( runtime: options.runtime, instances: [], composition: { members: [], dependencies: [] }, - ports: [], }; const locations = { stateRoot: options.stateRoot, cacheRoot: options.cacheRoot }; const existing = yield* state.read(id); @@ -1008,15 +913,18 @@ export const discover = Effect.fn("Stack.discover")( function* ( options: Pick & { readonly idPrefix?: string; - readonly onInvalidState?: (id: string, error: State.StateError) => Effect.Effect; + readonly onInvalidState?: ( + id: string, + error: StackNamespace.NamespaceError, + ) => Effect.Effect; }, ) { const selected = (id: string) => options.idPrefix === undefined || id.startsWith(options.idPrefix); const onInvalidState = options.onInvalidState; - const state = yield* State.Service.pipe( + const state = yield* StackNamespace.Service.pipe( Effect.provide( - State.layer({ + StackNamespace.layer({ root: options.stateRoot, onInvalidState: onInvalidState === undefined @@ -1052,58 +960,65 @@ export const find = Effect.fn("Stack.find")( const state = yield* stateFor(options.stateRoot); const id = "id" in options ? options.id : yield* deriveStackId(yield* resolveStackIdentity(options)); - const definition = yield* state.read(id); + const path = yield* Path.Path; + const definition = yield* state.read(id).pipe( + Effect.catchIf( + (error) => error.operation === "decode", + (error) => + failure( + "find", + `Stack ${id} could not be read: ${error.message}. Remove its directory ${path.join(options.stateRoot, id)} to discard it. Containers it left behind are removed when a container stack next starts under this state root or when the stack is destroyed by id, and its port reservations are released when another stack needs them; its data in the shared database volume is not removed.`, + ), + ), + ); if (definition === undefined) return Option.none(); return Option.some({ definition, host: yield* observeHost(state, definition) }); }, Effect.mapError((cause) => failure("find", cause)), ); -/** Selects a stack that is no longer registered while containers labelled with its data root remain. */ +/** A stack with no registration that left containers behind; destroying it removes them. */ +export type DeletedStack = Pick; + +const engineLabel = (engine: "docker" | "podman") => (engine === "docker" ? "Docker" : "Podman"); + +/** + * Selects a stack that is no longer registered while containers labelled with its data root in + * `stateRoot` remain on Docker or Podman. Destroying it removes only those containers, never the + * stack's data in the shared database volume. + */ export const findDeleted = Effect.fn("Stack.findDeleted")( function* (options: StackLocations & { readonly id: string }) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const services = yield* Effect.context< - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | HttpClient.HttpClient - | ChildProcessSpawner.ChildProcessSpawner + FileSystem.FileSystem | Path.Path | ChildProcessSpawner.ChildProcessSpawner >(); - // A registry that cannot be opened or read is left to `find`, which reports it. - const registry = yield* Effect.gen(function* () { - const state = yield* stateFor(options.stateRoot); - const saved = yield* state.read(options.id); - return { state, saved, stateRoot: yield* fs.realPath(options.stateRoot) }; - }).pipe(Effect.option); - if (Option.isNone(registry) || registry.value.saved !== undefined) - return Option.none(); - const { state, stateRoot } = registry.value; - const root = path.join(stateRoot, options.id, "data"); + const spawner = Context.get(services, ChildProcessSpawner.ChildProcessSpawner); + const state = yield* stateFor(options.stateRoot); + if ((yield* state.read(options.id)) !== undefined) return Option.none(); const probes = yield* Effect.forEach( ["docker", "podman"] as const, (engine) => - listStackContainers(engine).pipe( - Effect.map((containers) => - containers.some( - (container) => container.stackId === options.id && container.root === root, + resolveEngineTarget(spawner, engine).pipe( + Effect.flatMap((target) => + leftBehindStackIds(options.stateRoot, target).pipe( + Effect.map((ids) => Option.some({ target, found: ids.has(options.id) })), ), ), - Effect.catchIf( - (error) => error.reason === "engine-unavailable", - () => Effect.succeed(false), - ), + Effect.catchIf(engineUnreachable, () => Effect.succeedNone), Effect.result, - Effect.map((listed) => ({ engine, listed })), + Effect.map((outcome) => ({ engine, outcome })), ), { concurrency: "unbounded" }, ); - const matched = probes.filter(({ listed }) => Result.isSuccess(listed) && listed.success); - const [unlisted] = probes.flatMap(({ engine, listed }) => - Result.isFailure(listed) + const matched = probes.flatMap(({ engine, outcome }) => + Result.isSuccess(outcome) && Option.isSome(outcome.success) && outcome.success.value.found + ? [{ engine, target: outcome.success.value.target }] + : [], + ); + const [unlisted] = probes.flatMap(({ engine, outcome }) => + Result.isFailure(outcome) ? [ - `Unable to list ${engine === "docker" ? "Docker" : "Podman"} containers while looking for stack ${options.id}'s leftovers: ${listed.failure.message}`, + `Unable to list ${engineLabel(engine)} containers while looking for stack ${options.id}'s leftovers: ${failureMessage(outcome.failure)}`, ] : [], ); @@ -1112,8 +1027,15 @@ export const findDeleted = Effect.fn("Stack.findDeleted")( return yield* failure("find", unlisted); } const destroy = Effect.gen(function* () { - for (const { engine } of matched) { - const refusal = Match.value(yield* reclaimStack({ ...options, state, engine })).pipe( + for (const { target } of matched) { + const refusal = Match.value( + yield* reclaimDeletedStack({ + state, + stateRoot: options.stateRoot, + id: options.id, + engineTarget: target, + }), + ).pipe( Match.when("reclaimed", () => undefined), Match.when("held", () => "Another process holds this stack's lease; run destroy again"), Match.when("registered", () => `Stack ${options.id} was registered again during destroy`), @@ -1124,9 +1046,8 @@ export const findDeleted = Effect.fn("Stack.findDeleted")( if (unlisted !== undefined) return yield* failure( "destroy", - `Removed the ${matched.map(({ engine }) => (engine === "docker" ? "Docker" : "Podman")).join(" and ")} containers stack ${options.id} left behind. ${unlisted}`, + `Removed the ${matched.map(({ target }) => engineLabel(target.engine)).join(" and ")} containers stack ${options.id} left behind. ${unlisted}`, ); - return { runtimeCleanup: "complete" } as const; }).pipe( Effect.mapError((cause) => failure("destroy", cause)), Effect.provideContext(services), @@ -1136,8 +1057,6 @@ export const findDeleted = Effect.fn("Stack.findDeleted")( Effect.mapError((cause) => failure("find", cause)), ); -export type DeletedStack = Pick; - /** Selects the persisted logs of a stack; its owner does not need to run. */ export interface StreamStackLogsOptions extends Pick { readonly stackId: string; @@ -1154,7 +1073,12 @@ export const streamStackLogs = ( Stream.unwrap( Effect.gen(function* () { const path = yield* Path.Path; - const root = yield* State.stackLogsRoot(path, options.stateRoot, options.stackId); + if (!Schema.is(StackNamespace.SavedStack.fields.id)(options.stackId)) + return yield* new StackNamespace.NamespaceError({ + operation: "identity", + message: `Invalid state id: ${options.stackId}`, + }); + const root = StackNamespace.stackLogsRoot(path, options.stateRoot, options.stackId); return streamPersistedLogs({ root, ...(options.instances === undefined ? {} : { instances: options.instances }), diff --git a/packages/stack/src/functions/FunctionsBootstrap.ts b/packages/stack/src/functions/FunctionsBootstrap.ts index 340deed73b..b4b72c5d7b 100644 --- a/packages/stack/src/functions/FunctionsBootstrap.ts +++ b/packages/stack/src/functions/FunctionsBootstrap.ts @@ -1,7 +1,8 @@ -import { Crypto, Data, Effect, FileSystem, Path, PlatformError, Schema } from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. -import { rmdir } from "node:fs/promises"; +import { Crypto, Data, Effect, FileSystem, Path, type PlatformError, Schema } from "effect"; +import { contentDigestHex } from "../internal/content-digest.ts"; +import { publishGeneration } from "../internal/generation-publish.ts"; import { StackIdSchema, type StackId } from "../identity/StackId.ts"; +import { isSafeId } from "../identity/SafeId.ts"; export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstrapError")<{ readonly message: string; @@ -10,12 +11,12 @@ export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstra }> {} export interface FunctionsBootstrapOwner { + /** The owned directory a caller mounts statically; generations publish underneath it. */ + readonly root: string; /** Publishes the stack-owned Edge Runtime main service for the current session. */ readonly write: (input: { readonly content: string; }) => Effect.Effect; - /** Removes only this stack's functions bootstrap root. */ - readonly cleanupAll: Effect.Effect; } export interface FunctionsBootstrapOwnerOptions { @@ -27,6 +28,8 @@ export interface FunctionsBootstrapOwnerOptions { const failure = (message: string, fields: Readonly> = {}) => new FunctionsBootstrapError({ message, ...fields }); +const generationPrefix = "generation-"; + const mapFs = ( path: string, operation: string, @@ -43,95 +46,32 @@ export const makeFunctionsBootstrapOwner = Effect.fn("FunctionsBootstrap.makeOwn yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( Effect.mapError((cause) => failure("Invalid stack identity", { cause })), ); - if (!/^[a-zA-Z0-9_-]+$/u.test(options.instanceId)) - return yield* failure("Invalid Functions instance identity"); + if (!isSafeId(options.instanceId)) return yield* failure("Invalid Functions instance identity"); const root = path.join(options.root, options.instanceId, "runtime", "functions"); - const removeEmptyDirectory = (directory: string) => - Effect.tryPromise({ - try: () => rmdir(directory), - catch: (cause) => - failure("Unable to clean Functions parent directory", { path: directory, cause }), - }).pipe( - Effect.catch((cause) => { - const code = - typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause - ? cause.cause.code - : undefined; - return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" - ? Effect.void - : Effect.fail(cause); - }), - ); const write = Effect.fn("FunctionsBootstrap.write")(function* (input: { readonly content: string; }) { if (input.content.includes("\u0000")) return yield* failure("Functions bootstrap contains an invalid character"); - const target = path.join(root, "index.ts"); - const configFile = path.join(root, "deno.json"); - return yield* Effect.gen(function* () { - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - failure("Unable to allocate functions bootstrap file", { cause }), - ), - ); - const temporary = path.join(root, `.index.ts.${token}.tmp`); - return yield* Effect.gen(function* () { - yield* mapFs( - root, - "create functions bootstrap directory", - fs.makeDirectory(root, { recursive: true, mode: 0o700 }), - ); - yield* mapFs(root, "secure functions bootstrap directory", fs.chmod(root, 0o700)); - // An empty workspace root stops Deno config discovery before any ancestor package.json or - // workspace; a plain `{}` still joins an ancestor Deno workspace and fails membership. - yield* mapFs( - configFile, - "write functions bootstrap config", - fs.writeFileString(configFile, '{"workspace":[]}\n', { mode: 0o600 }), - ); - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* mapFs( - temporary, - "create functions bootstrap file", - fs.open(temporary, { flag: "w", mode: 0o600 }), - ); - yield* mapFs( - temporary, - "write functions bootstrap file", - file.writeAll(new TextEncoder().encode(input.content)), - ); - yield* mapFs(temporary, "sync functions bootstrap file", file.sync); - }), - ); - yield* mapFs(temporary, "secure functions bootstrap file", fs.chmod(temporary, 0o600)); - yield* mapFs(target, "publish functions bootstrap file", fs.rename(temporary, target)); - yield* mapFs(target, "secure published functions bootstrap file", fs.chmod(target, 0o600)); - return yield* mapFs( - target, - "resolve published functions bootstrap file", - fs.realPath(target), - ); - }).pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - }); + const hash = yield* mapFs( + root, + "hash functions bootstrap content", + contentDigestHex(crypto, input.content), + ); + const generation = yield* mapFs( + root, + "publish functions bootstrap generation", + // An empty workspace root stops Deno config discovery before any ancestor package.json or + // workspace; a plain `{}` still joins an ancestor Deno workspace and fails membership. + publishGeneration(fs, path, root, `${generationPrefix}${hash}`, [ + { name: "deno.json", content: '{"workspace":[]}\n', mode: 0o600 }, + { name: "index.ts", content: input.content, mode: 0o600 }, + ]), + ); + const target = path.join(generation, "index.ts"); + return yield* mapFs(target, "resolve published functions bootstrap file", fs.realPath(target)); }); - const cleanupAll = mapFs( - root, - "clean functions bootstrap files", - fs.remove(root, { recursive: true, force: true }), - ).pipe( - Effect.andThen(removeEmptyDirectory(path.dirname(root))), - Effect.andThen(removeEmptyDirectory(path.dirname(path.dirname(root)))), - Effect.withSpan("FunctionsBootstrap.cleanupAll"), - ); - return { write, cleanupAll }; + return { root, write }; }); diff --git a/packages/stack/src/functions/functions-bootstrap.integration.test.ts b/packages/stack/src/functions/functions-bootstrap.integration.test.ts index cb1e008534..257420668e 100644 --- a/packages/stack/src/functions/functions-bootstrap.integration.test.ts +++ b/packages/stack/src/functions/functions-bootstrap.integration.test.ts @@ -1,6 +1,6 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Path } from "effect"; +import { Deferred, Effect, FileSystem, Fiber, Path } from "effect"; import { StackIdSchema } from "../identity/StackId.ts"; import { makeFunctionsBootstrapOwner } from "./FunctionsBootstrap.ts"; @@ -17,65 +17,32 @@ const setupBootstrapOwner = (prefix: string) => return { fs, root, stackId, owner }; }); +/** `ino` for every file a generation publishes together, keyed by name. */ +const generationInodes = (fs: FileSystem.FileSystem, path: Path.Path, target: string) => + Effect.gen(function* () { + const generation = path.dirname(target); + return { + index: (yield* fs.stat(path.join(generation, "index.ts"))).ino, + config: (yield* fs.stat(path.join(generation, "deno.json"))).ino, + }; + }); + describe("functions bootstrap owner", () => { it.live("publishes a private Functions bootstrap file with restrictive modes", () => Effect.gen(function* () { const path = yield* Path.Path; const { fs, root, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-"); const target = yield* owner.write({ content: "export default 1" }); - expect(target).toContain( - path.join(root, "functions-one", "runtime", "functions", "index.ts"), + expect(path.dirname(path.dirname(target))).toContain( + path.join(root, "functions-one", "runtime", "functions"), ); + expect(path.basename(target)).toBe("index.ts"); expect(((yield* fs.stat(path.dirname(target))).mode ?? 0) & 0o777).toBe(0o700); expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); expect(yield* fs.readFileString(target)).toBe("export default 1"); }).pipe(Effect.provide(NodeServices.layer)), ); - it.live("cleans one Functions bootstrap while preserving another instance", () => - Effect.gen(function* () { - const path = yield* Path.Path; - const { fs, root, stackId, owner } = yield* setupBootstrapOwner( - "stack-functions-bootstrap-cleanup-", - ); - const target = yield* owner.write({ content: "export default 1" }); - const other = yield* makeFunctionsBootstrapOwner({ - root, - stackId, - instanceId: "functions-two", - }); - const otherTarget = yield* other.write({ content: "export default 2" }); - - expect(yield* fs.exists(target)).toBe(true); - - yield* owner.cleanupAll; - - expect(yield* fs.exists(target)).toBe(false); - expect(yield* fs.readFileString(otherTarget)).toBe("export default 2"); - expect(yield* fs.exists(path.join(root, "functions-one", "runtime", "functions"))).toBe( - false, - ); - expect(yield* fs.exists(path.join(root, "functions-one"))).toBe(false); - expect(yield* fs.exists(path.join(root, "functions-two"))).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("recreates readable Functions bootstrap content after cleanup", () => - Effect.gen(function* () { - const { fs, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-recreate-"); - const first = yield* owner.write({ content: "export default 1" }); - - expect(yield* fs.exists(first)).toBe(true); - - yield* owner.cleanupAll; - - expect(yield* fs.exists(first)).toBe(false); - const recreated = yield* owner.write({ content: "export default 2" }); - - expect(yield* fs.readFileString(recreated)).toBe("export default 2"); - }).pipe(Effect.provide(NodeServices.layer)), - ); - it.live("returns the canonical published path for a symlinked state root", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -100,16 +67,47 @@ describe("functions bootstrap owner", () => { "functions-one", "runtime", "functions", - "index.ts", ); - expect(target).toBe(expected); + expect(path.dirname(path.dirname(target))).toBe(expected); + expect(path.basename(target)).toBe("index.ts"); expect(yield* fs.readFileString(target)).toBe("export default 2"); expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("keeps every published file's identity when identical content is written twice", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const { fs, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-idempotent-"); + const first = yield* owner.write({ content: "export default 1" }); + const before = yield* generationInodes(fs, path, first); + + const second = yield* owner.write({ content: "export default 1" }); + + expect(second).toBe(first); + expect(yield* generationInodes(fs, path, second)).toEqual(before); + expect(yield* fs.readFileString(first)).toBe("export default 1"); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("converges concurrent writers of identical content on one complete generation", () => + Effect.gen(function* () { + const { fs, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-race-"); + const gate = yield* Deferred.make(); + const fibers = yield* Effect.forEach( + Array.from({ length: 20 }, () => undefined), + () => + Deferred.await(gate).pipe( + Effect.andThen(owner.write({ content: "export default race" })), + Effect.forkScoped, + ), + ); + yield* Deferred.succeed(gate, undefined); + const targets = yield* Effect.forEach(fibers, (fiber) => Fiber.join(fiber)); - yield* owner.cleanupAll; - expect( - yield* fs.exists(path.join(canonicalRoot, "functions-one", "runtime", "functions")), - ).toBe(false); + expect(new Set(targets).size).toBe(1); + const target = targets[0]!; + expect(yield* fs.readFileString(target)).toBe("export default race"); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts index 269d9307c2..840ca45fcd 100644 --- a/packages/stack/src/host/CommandRunner.initialization.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.initialization.integration.test.ts @@ -10,7 +10,7 @@ import { type ArtifactSource, } from "../preparation/ArtifactStore.ts"; import { PreparationError } from "../preparation/Errors.ts"; -import type { StackCredentials } from "../State.ts"; +import type { StackCredentials } from "../StackNamespace.ts"; import * as CommandRunner from "./CommandRunner.ts"; const target = @@ -33,14 +33,18 @@ if (authVersion === undefined) { const prepareAuthArtifact = Effect.fn(function* (cacheRoot: string, script: string) { if (target === undefined) return yield* Effect.fail("Unsupported test platform"); const fs = yield* FileSystem.FileSystem; - const { releaseVersion } = yield* resolveArtifact({ service: "auth" }); + const { releaseVersion, natives } = yield* resolveArtifact({ service: "auth" }); const request: ArtifactRequest = { key: `slim-services/auth/${releaseVersion}/${target}`, requiredRuntimePaths: ["bin/auth"], executablePath: "bin/auth", }; + // The generation directory is content-addressed by this digest: it must match what + // `CommandRunner`'s own real artifact source resolves for this platform, or the fixture + // published here is invisible to it and it downloads the real catalog-pinned binary instead. + const digest = natives[target].archive; const source: ArtifactSource = { - checksum: () => Effect.succeed("0".repeat(64)), + checksum: () => Effect.succeed(digest), materialize: (_request, destination) => Effect.gen(function* () { yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); diff --git a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts index 240d72a977..e806b20aec 100644 --- a/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts +++ b/packages/stack/src/host/CommandRunner.native-cleanup.integration.test.ts @@ -3,10 +3,69 @@ import { expect, it } from "@effect/vitest"; import { Context, Effect, FileSystem, Layer, Path, Stream } from "effect"; import { systemError } from "effect/PlatformError"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { resolveArtifact } from "../Artifacts.ts"; import { postgres } from "../Commands.ts"; +import { + makeArtifactStore, + type ArtifactRequest, + type ArtifactSource, +} from "../preparation/ArtifactStore.ts"; +import { PreparationError } from "../preparation/Errors.ts"; import * as CommandRunner from "./CommandRunner.ts"; -it.live.skipIf(process.platform === "win32")( +const target = + process.platform === "darwin" && process.arch === "arm64" + ? "darwin-arm64" + : process.platform === "linux" && process.arch === "x64" + ? "linux-amd64" + : process.platform === "linux" && process.arch === "arm64" + ? "linux-arm64" + : undefined; + +/** + * Publishes a trivial fixture in place of the real postgres artifact, at the exact cache key and + * digest `CommandRunner`'s own native launch resolves: its later `useNativeArtifact` call then + * hits this cached generation instead of downloading and running the real, much larger postgres + * distribution, whose first execution after a fresh extraction costs several additional seconds + * on top of the download (one-time code-signature validation of a freshly written binary). + */ +const prepareDatabaseArtifact = Effect.fn(function* (cacheRoot: string) { + if (target === undefined) return yield* Effect.fail("Unsupported test platform"); + const fs = yield* FileSystem.FileSystem; + const { releaseVersion, natives, requiredRuntimePaths, executablePath } = yield* resolveArtifact({ + service: "database", + }); + const request: ArtifactRequest = { + key: `slim-services/postgres/${releaseVersion}/${target}`, + requiredRuntimePaths, + executablePath, + }; + const digest = natives[target].archive; + const source: ArtifactSource = { + checksum: () => Effect.succeed(digest), + materialize: (_request, destination) => + Effect.gen(function* () { + for (const relative of requiredRuntimePaths) { + const file = `${destination}/${relative}`; + yield* fs.makeDirectory(file.slice(0, file.lastIndexOf("/")), { recursive: true }); + yield* fs.writeFileString(file, `#!${process.execPath}\nprocess.exit(0);\n`); + yield* fs.chmod(file, 0o755); + } + }).pipe( + Effect.mapError( + (cause) => + new PreparationError({ + message: `Unable to write database command fixture: ${cause.message}`, + cause, + }), + ), + ), + }; + const store = yield* makeArtifactStore({ cacheRoot, source }); + yield* store.prepare(request); +}); + +it.live.skipIf(target === undefined || process.platform === "win32")( "retries failed native workload cleanup when the stack runner is cleaned up", () => Effect.scoped( @@ -15,6 +74,7 @@ it.live.skipIf(process.platform === "win32")( const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "native-runner-cleanup-" }); const cacheRoot = path.join(root, "cache"); + yield* prepareDatabaseArtifact(cacheRoot); let isRunningCalls = 0; const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const spawner = ChildProcessSpawner.make((command) => diff --git a/packages/stack/src/host/CommandRunner.ts b/packages/stack/src/host/CommandRunner.ts index 1a318c2892..216b1d9842 100644 --- a/packages/stack/src/host/CommandRunner.ts +++ b/packages/stack/src/host/CommandRunner.ts @@ -17,15 +17,18 @@ import { ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; import { slimImageMirrors, - prepareNativeArtifact, postgresVersion, resolveArtifact, + useNativeArtifact, } from "../Artifacts.ts"; -import { makeContainerRuntime, type HostGateway } from "../runtime/Container.ts"; +import { isSafeId } from "../identity/SafeId.ts"; +import * as Environment from "../namespace/Environment.ts"; +import { borrow } from "../namespace/Paths.ts"; +import { makeContainerRuntime, type EngineTarget, type HostGateway } from "../runtime/Container.ts"; import { spawnNativeProcess } from "../runtime/NativeProcess.ts"; import { awaitCommandOutput, type CommandOutputResult } from "../runtime/CommandOutput.ts"; import type { CommandInvocation as CommandInvocationType } from "../Commands.ts"; -import type { StackCredentials } from "../State.ts"; +import type { StackCredentials } from "../StackNamespace.ts"; import { resolveInitializationCommand } from "../services/Initialization.ts"; export class CommandError extends Data.TaggedError("CommandError")<{ @@ -77,6 +80,8 @@ const makeCommandRunner = (options: { readonly runtime: "native" | "docker" | "podman"; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -84,16 +89,18 @@ const makeCommandRunner = (options: { const crypto = yield* Crypto.Crypto; const http = yield* HttpClient.HttpClient; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - if (!/^[a-zA-Z0-9_-]+$/u.test(options.stackId)) return yield* failure("Invalid stack identity"); + if (!isSafeId(options.stackId)) return yield* failure("Invalid stack identity"); const container = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeContainerRuntime({ - engine: options.runtime, + target: options.engineTarget, root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), }); + const borrowCallerPath = (candidate: string) => + borrow(fs, path, candidate, options.root, (_operation, cause) => failure(cause)); const jobsRoot = path.join(options.root, "jobs"); yield* fs .makeDirectory(jobsRoot, { recursive: true, mode: 0o700 }) @@ -165,11 +172,19 @@ const makeCommandRunner = (options: { postgresCommand.pgProve !== undefined ) return yield* failure("pgProve options require the pg_prove command"); + for (const mount of [ + ...(initialization?.mounts ?? []), + ...(postgresCommand?.pgProve?.mounts ?? []), + ]) + yield* borrowCallerPath(mount.source); const version = initialization?.version ?? postgresVersion(String(postgresCommand?.command.major)); const process = yield* Effect.gen(function* () { if (container === undefined) { - const artifact = yield* prepareNativeArtifact( + // Pins the generation for the life of this job's scope, before the spawn below: the + // pin is registered ahead of the process, so it releases only after the process's + // own cleanup finalizer (added later) has run. + const artifact = yield* useNativeArtifact( { service: initialization?.service ?? "database", version }, options.cacheRoot, ).pipe( @@ -179,6 +194,11 @@ const makeCommandRunner = (options: { Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(HttpClient.HttpClient, http), ); + // The job's own scoped temp directory, removed with it, so the confined environment + // needs no cleanup of its own. + const environment = yield* Environment.confine(fs, path, directory).pipe( + Effect.mapError(failure), + ); const child = yield* spawnNativeProcess( { executable: path.join( @@ -188,6 +208,8 @@ const makeCommandRunner = (options: { ), args: postgresCommand?.args ?? initialization?.args ?? [], env: postgresCommand?.env ?? initialization?.env ?? {}, + environment, + artifactLockPath: artifact.lockPath, cwd: postgresCommand?.pgProve?.cwd ?? initialization?.cwd ?? @@ -311,4 +333,6 @@ export const layer = (options: { readonly runtime: "native" | "docker" | "podman"; /** Shares one host-gateway probe with the host's other container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; }) => Layer.effect(Service, makeCommandRunner(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/host/Credentials.ts b/packages/stack/src/host/Credentials.ts index 55d95eed28..391deb4785 100644 --- a/packages/stack/src/host/Credentials.ts +++ b/packages/stack/src/host/Credentials.ts @@ -6,7 +6,7 @@ import { } from "../Defaults.ts"; import { ServiceCreation, type ServiceCreationInput } from "../services/Catalog.ts"; import { resolveStackKeys } from "../services/ServiceConfig.ts"; -import type { SavedStack, StackCredentials, StackKeysInput } from "../State.ts"; +import type { SavedStack, StackCredentials, StackKeysInput } from "../StackNamespace.ts"; export class CredentialError extends Data.TaggedError("CredentialError")<{ readonly message: string; @@ -190,15 +190,11 @@ const credentialInputs: { export const credentialInputNames = (service: ServiceCreation["service"]): ReadonlyArray => Object.keys(credentialInputs[service]); -/** - * Completes a creation with the stack credentials it consumes. The JWT secret and database - * credentials always come from the record; other inputs keep an explicit value. - */ -export const withCredentials = ( +const fillCredentials = ( creation: ServiceCreationInput, + config: Record, credentials: StackCredentials, ): Effect.Effect => { - const config: Record = { ...creation.config }; for (const [input, source] of Object.entries(credentialInputs[creation.service])) { const value = credentials[source]; if (creation.service === "database") config[input] = Redacted.make(value); @@ -206,3 +202,28 @@ export const withCredentials = ( } return Schema.decodeUnknownEffect(ServiceCreation)({ ...creation, config }); }; + +/** + * Completes a creation with the stack credentials it consumes. The JWT secret and database + * credentials always come from the record; other inputs keep an explicit value. + */ +export const withCredentials = ( + creation: ServiceCreationInput, + credentials: StackCredentials, +): Effect.Effect => + fillCredentials(creation, { ...creation.config }, credentials); + +/** + * Moves a saved creation from the previous credential record to the next. An input that differs + * from the previous record is explicit and kept; one equal to it follows the next record. + */ +export const refreshCredentials = ( + creation: ServiceCreation, + previous: StackCredentials, + next: StackCredentials, +): Effect.Effect => { + const config: Record = { ...creation.config }; + for (const [input, source] of Object.entries(credentialInputs[creation.service])) + if (config[input] === previous[source]) delete config[input]; + return fillCredentials(creation, config, next); +}; diff --git a/packages/stack/src/host/Endpoints.ts b/packages/stack/src/host/Endpoints.ts index 036f245874..b3684815c6 100644 --- a/packages/stack/src/host/Endpoints.ts +++ b/packages/stack/src/host/Endpoints.ts @@ -2,7 +2,7 @@ import { allowedEndpointNames, type ServiceCreation } from "../services/Catalog. import type { ServiceEndpoint } from "../services/Recipe.ts"; import { apiPath as storageApiPath } from "../services/Storage.ts"; import type { NetworkEndpoint } from "../Network.ts"; -import { ProxyError, type BackendAddress } from "../Proxy.ts"; +import type { BackendAddress } from "../Proxy.ts"; import { Data, Effect, Redacted } from "effect"; export class EndpointError extends Data.TaggedError("EndpointError")<{ @@ -48,6 +48,10 @@ export const apiRoute = (service: ServiceCreation["service"]): string | undefine } }; +/** Whether the service's configured HTTP endpoint is served on the shared API listener. */ +export const sharesApiEndpoint = (creation: EndpointIntents): boolean => + apiRoute(creation.service) !== undefined && endpointNames(creation).includes("http"); + type RouteKeys = NonNullable[number]["keyRewrite"]>["keys"]; /** Routes a service's HTTP endpoint on the shared API listener, or `undefined` for a dedicated one. */ @@ -96,15 +100,10 @@ export const joinRoutes = (creation: ServiceCreation, name: string): NetworkEndp : undefined; /** Translates a launched runtime's endpoint into the proxy's backend address. */ -export const backendAddress = ( - endpoint: ServiceEndpoint, -): Effect.Effect => { - if (endpoint.kind === "unix") - return endpoint.path === undefined - ? Effect.fail(new ProxyError({ message: "Unix endpoint has no path" })) - : Effect.succeed({ path: `${endpoint.path}/.s.PGSQL.${endpoint.port}` }); - return Effect.succeed({ host: endpoint.host ?? "127.0.0.1", port: endpoint.port }); -}; +export const backendAddress = (endpoint: ServiceEndpoint): BackendAddress => + endpoint.kind === "unix" + ? { path: endpoint.path } + : { host: endpoint.host ?? "127.0.0.1", port: endpoint.port }; export const publicUrl = (host: string, port: number) => `http://${host}:${port}`; diff --git a/packages/stack/src/host/GatewayLog.ts b/packages/stack/src/host/GatewayLog.ts index 4df69aaf56..2a6ffb4fcb 100644 --- a/packages/stack/src/host/GatewayLog.ts +++ b/packages/stack/src/host/GatewayLog.ts @@ -36,12 +36,12 @@ export const formatAccess = (access: HttpAccess) => const encoder = new TextEncoder(); -/** The gateway stream's output, its access sink, and an observation of one launch per owner run. */ +/** The gateway stream's output, its access sink, and the launches of one owner run. */ export interface GatewayLog { readonly logs: CatalogLogs; /** Records an access once the owner run's launch began; earlier ones have no reader yet. */ readonly record: HttpAccessSink; - readonly observation: Stream.Stream<{ readonly launchId: number }>; + readonly launches: Stream.Stream; /** Begins this owner run's launch, numbered after the retained ones. */ readonly begin: (launchId: number) => Effect.Effect; } @@ -60,10 +60,7 @@ export const make = Effect.gen(function* () { : publish("stdout", encoder.encode(`${formatAccess(access)}\n`)), ), ), - observation: Stream.fromEffect(Deferred.await(launch)).pipe( - Stream.map((launchId) => ({ launchId })), - Stream.concat(Stream.never), - ), + launches: Stream.fromEffect(Deferred.await(launch)).pipe(Stream.concat(Stream.never)), begin: (launchId) => launchOutputPublisher(output, launchId).pipe( Effect.flatMap(({ part }) => part), diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 8f58d6c5d1..fd664e88cc 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -26,7 +26,6 @@ import { type Received, } from "../../tests/fake-logflare.ts"; import { makeManualClock, type ManualClock } from "../../tests/manual-clock.ts"; -import type { ServiceObservation } from "../Service.ts"; import type { LaunchOutput } from "../runtime/Session.ts"; import { CatalogError } from "../services/Recipe.ts"; import * as GatewayLog from "./GatewayLog.ts"; @@ -73,27 +72,11 @@ const drive = ( return posts; }); -const observationOf = ( - id: string, - overrides: Partial>, -): ServiceObservation => ({ - id, - config: undefined, - lifecycle: "stopped", - health: undefined, - error: undefined, - cleanupError: undefined, - exit: undefined, - currentOperation: undefined, - launchId: undefined, - intentRevision: 0, - wakeEnabled: true, - registered: true, - ...overrides, -}); +type Serving = LogForwarder.ForwardedInstance["serving"] extends Stream.Stream ? A : never; + +const serving = (launchId = 1): Serving => ({ serving: true, launchId }); -const serving = (id: string) => - observationOf(id, { lifecycle: "running", health: "healthy", launchId: 1 }); +const notServing = (launchId?: number): Serving => ({ serving: false, launchId }); const encoder = new TextEncoder(); @@ -104,14 +87,12 @@ const serviceSource = ( ) => Effect.gen(function* () { const logs = yield* PubSub.unbounded(); - const observation = yield* SubscriptionRef.make(serving(service)); + const observation = yield* SubscriptionRef.make(serving()); yield* store.attach({ service, instanceId: service, logs: PubSub.subscribe(logs), - observation: SubscriptionRef.changes(observation).pipe( - Stream.map(({ launchId }) => ({ launchId })), - ), + launches: SubscriptionRef.changes(observation).pipe(Stream.map(({ launchId }) => launchId)), }); let seq = 0; const written = yield* Queue.unbounded(); @@ -141,25 +122,21 @@ const serviceSource = ( service, endpoint: () => Effect.fail(new CatalogError({ operation: "endpoint", message: "unused" })), creation: Effect.die("Only the Analytics creation is read"), - observation: SubscriptionRef.changes(observation), + serving: SubscriptionRef.changes(observation), } satisfies LogForwarder.ForwardedInstance, }; }); const analyticsTarget = (port: Effect.Effect) => Effect.gen(function* () { - const observation = yield* SubscriptionRef.make(observationOf("analytics", {})); + const observation = yield* SubscriptionRef.make(notServing()); return { observation, - set: (awake: boolean) => - SubscriptionRef.set( - observation, - awake ? serving("analytics") : observationOf("analytics", { lifecycle: "stopped" }), - ), + set: (awake: boolean) => SubscriptionRef.set(observation, awake ? serving() : notServing()), /** Restarts Analytics as a new launch, which loses whatever the previous one had queued. */ relaunch: (launchId: number) => - SubscriptionRef.set(observation, observationOf("analytics", { lifecycle: "stopped" })).pipe( - Effect.andThen(SubscriptionRef.set(observation, { ...serving("analytics"), launchId })), + SubscriptionRef.set(observation, notServing()).pipe( + Effect.andThen(SubscriptionRef.set(observation, serving(launchId))), ), instance: { id: "analytics", @@ -173,7 +150,7 @@ const analyticsTarget = (port: Effect.Effect) => })), ), creation: Effect.succeed({ service: "analytics" as const, config: { apiKey: "test-key" } }), - observation: SubscriptionRef.changes(observation), + serving: SubscriptionRef.changes(observation), } satisfies LogForwarder.ForwardedInstance, }; }); @@ -292,24 +269,14 @@ describe("LogForwarder", () => { yield* database.log("while awake"); const beforeSleep = yield* logflare.next; - yield* SubscriptionRef.set( - analytics.observation, - observationOf("analytics", { lifecycle: "stopping", launchId: 1 }), - ); + yield* SubscriptionRef.set(analytics.observation, notServing(1)); yield* forwarder.awaitShipping(false); - yield* SubscriptionRef.set(analytics.observation, observationOf("analytics", {})); + yield* SubscriptionRef.set(analytics.observation, notServing()); yield* database.log("while asleep"); yield* Ref.set(port, logflare.port); yield* logflare.respond(503); - yield* SubscriptionRef.set( - analytics.observation, - observationOf("analytics", { lifecycle: "starting", health: "starting", launchId: 2 }), - ); - yield* SubscriptionRef.set( - analytics.observation, - observationOf("analytics", { lifecycle: "running", health: "starting", launchId: 2 }), - ); - yield* SubscriptionRef.set(analytics.observation, { ...serving("analytics"), launchId: 2 }); + yield* SubscriptionRef.set(analytics.observation, notServing(2)); + yield* SubscriptionRef.set(analytics.observation, serving(2)); yield* forwarder.awaitShipping(true); yield* logflare.apply(); yield* release; @@ -431,27 +398,6 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("removes an instance's stale cursor write directories when it attaches", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const { root, store, logflare, database, analytics } = yield* fixture(); - const directory = path.join(root, "database", "database"); - const stale = path.join(directory, ".state-write-stale"); - const recent = path.join(directory, ".state-write-recent"); - yield* fs.makeDirectory(stale, { recursive: true }); - yield* fs.makeDirectory(recent, { recursive: true }); - // A numeric file time is in seconds. - const twoDaysAgo = (yield* Clock.currentTimeMillis) / 1000 - 2 * 24 * 60 * 60; - yield* fs.utimes(stale, twoDaysAgo, twoDaysAgo); - - yield* startForwarder(store, logflare, [analytics.instance, database.instance]); - - expect(yield* fs.exists(stale)).toBe(false); - expect(yield* fs.exists(recent)).toBe(true); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - it.live("detaches an instance only once its cursor write landed, so its logs can go", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -1197,7 +1143,7 @@ describe("LogForwarder", () => { yield* store.attach({ ...GatewayLog.gatewayLog, logs: gateway.logs, - observation: gateway.observation, + launches: gateway.launches, }); yield* gateway.begin(1); yield* analytics.set(true); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 395d11a85d..3b006a4a5d 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -20,11 +20,10 @@ import { } from "effect"; import { HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http"; import type { CompositionConfig } from "../Orchestrator.ts"; -import type { ServiceObservation } from "../Service.ts"; import type { ServiceCreation } from "../services/Catalog.ts"; import type { CatalogError, ServiceEndpoint } from "../services/Recipe.ts"; import { errorCode } from "../internal/sharing-violation.ts"; -import { reapStaleWrites, writeFileAtomically } from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; import { isShippedService, logflareEvent, @@ -42,7 +41,11 @@ export interface ForwardedInstance { readonly endpoint: (name: string) => Effect.Effect; /** The current saved creation; Analytics' API key is read from it per body. */ readonly creation: Effect.Effect; - readonly observation: Stream.Stream>; + /** Whether the instance is running and healthy, and which launch it is serving. */ + readonly serving: Stream.Stream<{ + readonly serving: boolean; + readonly launchId: number | undefined; + }>; } /** An owner log stream without a service instance; it ships while the owner runs. */ @@ -384,15 +387,10 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar }).pipe(Semaphore.withPermits(rebinding, 1)); const trackTarget = (instance: ForwardedInstance) => - instance.observation.pipe( - Stream.takeUntil((observation) => !observation.registered), - Stream.runForEach((observation) => + instance.serving.pipe( + Stream.runForEach(({ serving, launchId }) => Ref.update(candidates, (current) => - new Map(current).set(instance.id, { - instance, - serving: observation.lifecycle === "running" && observation.health === "healthy", - launchId: observation.launchId, - }), + new Map(current).set(instance.id, { instance, serving, launchId }), ).pipe(Effect.andThen(rebind)), ), Effect.ensuring( @@ -449,7 +447,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const target = path.join(directory, cursorFile); if (cursor === undefined) return yield* fs.remove(target, { force: true }); const content = yield* encodeCursor(cursor); - yield* writeFileAtomically(fs, path, { directory, target, content }); + yield* StackNamespace.publish(fs, path, { target, content }); }); /** Saves progress before acting on it, so the in-memory position never passes the file. */ @@ -501,9 +499,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const remaining = deadline - (yield* Clock.currentTimeMillis); yield* client .execute( - HttpClientRequest.post( - `http://${endpoint.host ?? "127.0.0.1"}:${endpoint.port}/api/logs`, - ).pipe( + HttpClientRequest.post(`http://127.0.0.1:${endpoint.port}/api/logs`).pipe( HttpClientRequest.setUrlParam("source_name", source), HttpClientRequest.setHeader("x-api-key", apiKey), HttpClientRequest.bodyText(body, "application/json"), @@ -854,13 +850,6 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar }), ).pipe(current.users.withPermits(1)); - const unregistered = (instance: ForwardedInstance) => - instance.observation.pipe( - Stream.filter((observation) => !observation.registered), - Stream.take(1), - Stream.runDrain, - ); - /** Waits until the instance joins the composition, or leaves it. */ const membership = (instanceId: string, member: boolean) => SubscriptionRef.changes(memberIds).pipe( @@ -870,16 +859,11 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ); /** - * Ships until the instance unregisters (`until`) or the store detaches its logs, which ends a - * session. A service instance ships only while it is a composition member; the gateway stream - * belongs to the owner. + * Ships until the instance is detached or the store detaches its logs, which ends a session. A + * service instance ships only while it is a composition member; the gateway stream belongs to + * the owner. */ - const forward = ( - id: string, - service: ShippedService, - until: Effect.Effect, - composed: boolean, - ) => + const forward = (id: string, service: ShippedService, composed: boolean) => Effect.gen(function* () { while (true) { if (composed) yield* membership(id, true); @@ -911,7 +895,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ); if (detached) return yield* Effect.logDebug(`Log shipping of ${id} stopped with its logs`); } - }).pipe(Effect.raceFirst(until)); + }); const followers = yield* FiberMap.make(); const attach = Effect.fn("LogForwarder.attach")(function* ( @@ -920,20 +904,10 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar if (instance.service === "analytics") yield* FiberMap.run(followers, instance.id, trackTarget(instance)); else if (isShippedService(instance.service)) { - // A cursor write cut short by a crash leaves its temporary directory behind. - yield* options.logs.directory(instance.id).pipe( - Effect.flatMap((directory) => reapStaleWrites(fs, path, directory)), - Effect.ignore, - ); yield* FiberMap.run( followers, instance.id, - forward( - instance.id, - instance.service, - "observation" in instance ? unregistered(instance) : Effect.never, - "observation" in instance, - ), + forward(instance.id, instance.service, "serving" in instance), ); } }); diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index 6afa5b2a60..c8cdaae296 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -8,6 +8,7 @@ import { Fiber, FileSystem, Latch, + Option, Path, PlatformError, PubSub, @@ -37,9 +38,7 @@ const fakeInstance = ( options.capacity === undefined ? yield* PubSub.unbounded() : yield* PubSub.sliding(options.capacity); - const launch = yield* SubscriptionRef.make<{ readonly launchId: number | undefined }>({ - launchId: undefined, - }); + const launch = yield* SubscriptionRef.make(undefined); const sequences = new Map(); const chunk = (launchId: number, text: string, stream: "stdout" | "stderr" = "stdout") => { const seq = sequences.get(launchId) ?? 0; @@ -58,7 +57,7 @@ const fakeInstance = ( service: options.service ?? "auth", instanceId, logs: PubSub.subscribe(logs), - observation: SubscriptionRef.changes(launch), + launches: SubscriptionRef.changes(launch), publish: (...chunks: ReadonlyArray) => Clock.currentTimeMillis.pipe( Effect.flatMap((time) => @@ -69,7 +68,7 @@ const fakeInstance = ( ), ), chunk, - setLaunch: (launchId: number | undefined) => SubscriptionRef.set(launch, { launchId }), + setLaunch: (launchId: number | undefined) => SubscriptionRef.set(launch, launchId), }; }); @@ -213,7 +212,7 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("writes a launch's partial line when its observation leaves the launch", () => + it.effect("writes a launch's partial line when the instance leaves the launch", () => Effect.gen(function* () { const root = yield* tempRoot("log-store-partial-"); const { store } = yield* openStore(root); @@ -252,14 +251,12 @@ describe("LogStore", () => { const left = yield* Deferred.make(); const fake = yield* fakeInstance("queued-end"); // The store pulls past the transition only after it has handled it. - const observation = Stream.make({ launchId: 1 }).pipe( - Stream.concat( - Stream.fromEffect(Deferred.await(leave).pipe(Effect.as({ launchId: undefined }))), - ), + const launches: Stream.Stream = Stream.make(1).pipe( + Stream.concat(Stream.fromEffect(Deferred.await(leave).pipe(Effect.as(undefined)))), Stream.concat(Stream.fromEffectDrain(Deferred.succeed(left, undefined))), Stream.concat(Stream.never), ); - yield* store.attach({ ...fake, observation }); + yield* store.attach({ ...fake, launches }); const reader = yield* collect(store.read("queued-end", { from: "oldest", follow: true })); const euro = encoder.encode("€"); const withBytes = (chunk: LaunchOutput, bytes: ReadonlyArray): LaunchOutput => ({ @@ -944,6 +941,30 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("numbers launches above retained history when the newest segment is empty", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-empty-newest-"); + const opened = yield* openStore(root); + const instance = yield* fakeInstance("crashed"); + yield* opened.store.attach(instance); + const reader = yield* collect( + opened.store.read("crashed", { from: "oldest", tail: 0, follow: true }), + ); + yield* instance.publish(instance.chunk(7, "launch seven\n")); + yield* reader.take(2); + yield* opened.close; + yield* fs.writeFileString(path.join(root, "auth", "crashed", segmentName(2)), ""); + + const restarted = yield* openStore(root); + + expect( + yield* restarted.store.latestLaunchId({ service: "auth", instanceId: "crashed" }), + ).toEqual(Option.some(7)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("follows only records written after an empty tailed read subscribes", () => Effect.gen(function* () { const root = yield* tempRoot("log-store-end-"); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index db419b38de..39b2397fe9 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -77,7 +77,8 @@ export interface AttachedInstance { readonly service: string; readonly instanceId: string; readonly logs: CatalogLogs; - readonly observation: Stream.Stream<{ readonly launchId: number | undefined }>; + /** The launch the instance is in, or `undefined` once that launch has ended. */ + readonly launches: Stream.Stream; } export interface Interface { @@ -259,31 +260,40 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { Effect.mapError(storeError("read")), ); - /** - * Reads only the last chunk of each segment, newest first, until one holds a complete record; - * a crash right after rotation can leave the newest segment empty or cut. - */ + /** Highest launch id in the last chunk of one segment, skipping its first, possibly cut, line. */ + const segmentLaunchId = (directory: string, generation: number) => { + const file = path.join(directory, segmentName(generation)); + return fs.stat(file).pipe( + Effect.mapError(storeError("read")), + Effect.flatMap((info) => { + const offset = Math.max(0, Number(info.size) - readChunkBytes); + return readChunk(file, offset).pipe( + Effect.map( + Option.match({ + onNone: () => Option.none(), + onSome: (bytes) => { + let latest: number | undefined; + for (const line of decoder + .decode(bytes) + .split("\n") + .slice(offset > 0 ? 1 : 0, -1)) { + const launchId = parseRecord(line, { generation, byteOffset: 0 })?.launchId; + if (launchId !== undefined) latest = Math.max(latest ?? 0, launchId); + } + return Option.fromNullishOr(latest); + }, + }), + ), + ); + }), + ); + }; + + /** Newest launch id on disk; a crash can leave the newest segment empty, so older ones are walked. */ const latestLaunchId = Effect.fnUntraced(function* (directory: string) { for (const generation of (yield* generations(directory)).toReversed()) { - const file = path.join(directory, segmentName(generation)); - const size = yield* fs.stat(file).pipe( - Effect.map((info) => Option.some(Number(info.size))), - Effect.catchIf(isNotFound, () => Effect.succeed(Option.none())), - Effect.mapError(storeError("read")), - ); - if (Option.isNone(size)) continue; - const offset = Math.max(0, size.value - readChunkBytes); - const bytes = yield* readChunk(file, offset); - if (Option.isNone(bytes)) continue; - let latest: number | undefined; - for (const line of decoder - .decode(bytes.value) - .split("\n") - .slice(offset > 0 ? 1 : 0, -1)) { - const launchId = parseRecord(line, { generation, byteOffset: 0 })?.launchId; - if (launchId !== undefined) latest = Math.max(latest ?? 0, launchId); - } - if (latest !== undefined) return Option.some(latest); + const launchId = yield* segmentLaunchId(directory, generation); + if (Option.isSome(launchId)) return launchId; } return Option.none(); }); @@ -960,8 +970,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio const flushLaunches = Effect.gen(function* () { const previous = yield* Ref.make(undefined); - yield* instance.observation.pipe( - Stream.map(({ launchId }) => launchId), + yield* instance.launches.pipe( Stream.changes, Stream.runForEach((launchId) => Ref.getAndSet(previous, launchId).pipe( diff --git a/packages/stack/src/host/LogflareStorage.integration.test.ts b/packages/stack/src/host/LogflareStorage.integration.test.ts index e379409061..9afb851aba 100644 --- a/packages/stack/src/host/LogflareStorage.integration.test.ts +++ b/packages/stack/src/host/LogflareStorage.integration.test.ts @@ -3,8 +3,8 @@ import { PgClient } from "@effect/sql-pg"; import { describe, expect, it } from "@effect/vitest"; import { Context, Effect, Exit, FileSystem, Layer, Redacted, Ref, Scope } from "effect"; import { connect, createServer, type Socket } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. -import { tmpdir } from "node:os"; -import { makeService } from "../Service.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type DatabaseConfig } from "../services/Database.ts"; import * as LogflareStorage from "./LogflareStorage.ts"; @@ -31,12 +31,12 @@ const analyticsDatabase = Effect.gen(function* () { stackId: "logflare-storage", instanceId: "database", root, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, runtime: "native", }); // The service runs its operations in its own scope, which must stay open while it is destroyed. const serviceScope = yield* Scope.make(); - const service = yield* makeService(recipe.definition, { id: "database", config }).pipe( + const service = yield* makeStandaloneService(recipe.definition, { id: "database", config }).pipe( Scope.provide(serviceScope), ); yield* Effect.addFinalizer(() => @@ -116,7 +116,6 @@ describe("LogflareStorage", { timeout: 180_000 }, () => { const stored = yield* storage.storedIds("postgres.logs", [storedId, missingId]); const uncreated = yield* storage.storedIds("auth.logs", [storedId]); const unknown = yield* storage.storedIds("storage.logs", [storedId]).pipe(Effect.flip); - expect([...stored]).toEqual([storedId]); expect([...uncreated]).toEqual([]); expect(unknown.message).toBe("Analytics has no storage.logs sources"); diff --git a/packages/stack/src/host/LogflareStorage.ts b/packages/stack/src/host/LogflareStorage.ts index 62c5eacf24..c434e74607 100644 --- a/packages/stack/src/host/LogflareStorage.ts +++ b/packages/stack/src/host/LogflareStorage.ts @@ -48,7 +48,7 @@ export const analyticsDatabase = Effect.fnUntraced(function* ( ); for (const name of replaced) url.searchParams.delete(name); // A socket directory is passed as the `host` parameter, which takes precedence over the hostname. - if (bound.kind === "unix" && bound.path !== undefined) url.searchParams.set("host", bound.path); + if (bound.kind === "unix") url.searchParams.set("host", bound.path); else url.hostname = bound.host ?? "127.0.0.1"; url.port = String(bound.port); return { url: url.toString() } satisfies AnalyticsDatabase; diff --git a/packages/stack/src/identity/SafeId.ts b/packages/stack/src/identity/SafeId.ts new file mode 100644 index 0000000000..ad2e1b32ba --- /dev/null +++ b/packages/stack/src/identity/SafeId.ts @@ -0,0 +1,11 @@ +import { Schema } from "effect"; + +/** An identifier safe to use as a single path segment: letters, digits, `_` and `-`. */ +export const SafeId = Schema.String.pipe( + Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { + identifier: "SafeId", + message: "Expected a safe id", + }), +); + +export const isSafeId = Schema.is(SafeId); diff --git a/packages/stack/src/identity/StackId.ts b/packages/stack/src/identity/StackId.ts index f13f2f9dc5..04d460536c 100644 --- a/packages/stack/src/identity/StackId.ts +++ b/packages/stack/src/identity/StackId.ts @@ -7,7 +7,7 @@ export type StackId = string & { readonly [StackIdTypeId]: "StackId" }; const SHA256_HEX = /^[0-9a-f]{64}$/; -export const isStackId = (value: string): value is StackId => SHA256_HEX.test(value); +const isStackId = (value: string): value is StackId => SHA256_HEX.test(value); export const StackIdSchema = Schema.String.pipe( Schema.refine((value): value is StackId => isStackId(value), { diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index d504bb5fa0..5b7774723e 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -12,7 +12,7 @@ export type { PostgresCommandOptions, Stack, } from "./PromiseClient.ts"; -export type { StackCredentials, StackKeysInput } from "./State.ts"; +export type { StackCredentials, StackKeysInput } from "./StackNamespace.ts"; export { initialization, postgres } from "./Commands.ts"; export { StackError } from "./Rpc.ts"; export type { CompositionConfig } from "./Orchestrator.ts"; @@ -21,7 +21,6 @@ export type { CreateOptions, CreationChange, DatabaseSnapshotOptions, - DestroyResult, FindOptions, FoundStack, LogPosition, diff --git a/packages/stack/src/internal/artifacts.ts b/packages/stack/src/internal/artifacts.ts index 5d9e1fb81b..79c79b468c 100644 --- a/packages/stack/src/internal/artifacts.ts +++ b/packages/stack/src/internal/artifacts.ts @@ -7,6 +7,7 @@ export { postgresVersion, prepareNativeArtifact, resolveArtifact, + useNativeArtifact, type ArtifactKind, type ArtifactPin, type NativePin, diff --git a/packages/stack/src/internal/composition.ts b/packages/stack/src/internal/composition.ts new file mode 100644 index 0000000000..830238b291 --- /dev/null +++ b/packages/stack/src/internal/composition.ts @@ -0,0 +1,2 @@ +/** Composition planner shared with the CLI's test fakes to reproduce saved-stack comparison. */ +export { planSupabaseComposition } from "../composition/Supabase.ts"; diff --git a/packages/stack/src/internal/content-digest.ts b/packages/stack/src/internal/content-digest.ts new file mode 100644 index 0000000000..5be27d5e65 --- /dev/null +++ b/packages/stack/src/internal/content-digest.ts @@ -0,0 +1,13 @@ +import { Effect, type Crypto, type PlatformError } from "effect"; + +const hex = (bytes: Uint8Array): string => + Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + +/** A 16-character hex SHA-256 prefix, used to name content-addressed files and directories. */ +export const contentDigestHex = ( + crypto: Crypto.Crypto, + content: string, +): Effect.Effect => + crypto + .digest("SHA-256", new TextEncoder().encode(content)) + .pipe(Effect.map((digest) => hex(digest).slice(0, 16))); diff --git a/packages/stack/src/internal/dispatch.integration.test.ts b/packages/stack/src/internal/dispatch.integration.test.ts index 7a547d4ef4..c306ab2434 100644 --- a/packages/stack/src/internal/dispatch.integration.test.ts +++ b/packages/stack/src/internal/dispatch.integration.test.ts @@ -4,7 +4,7 @@ import { expect, it } from "@effect/vitest"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { fileURLToPath } from "node:url"; import { HostEndpoint } from "../HostProcess.ts"; -import * as State from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; const fixturePath = fileURLToPath( new URL("../../tests/compiled-dispatch-fixture.ts", import.meta.url), @@ -56,8 +56,8 @@ it.live("dispatches compiled owner and native launchers through the production b yield* compileFixture(executable); const stateRoot = path.join(root, "state"); const cacheRoot = path.join(root, "cache"); - const stateContext = yield* Layer.build(State.layer({ root: stateRoot })); - const state = Context.get(stateContext, State.Service); + const stateContext = yield* Layer.build(StackNamespace.layer({ root: stateRoot })); + const state = Context.get(stateContext, StackNamespace.Service); const stackId = "compiled-dispatch"; yield* state.save({ id: stackId, @@ -66,7 +66,6 @@ it.live("dispatches compiled owner and native launchers through the production b instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const runOwner = (mode: "owner" | "stop") => diff --git a/packages/stack/src/internal/engine.ts b/packages/stack/src/internal/engine.ts new file mode 100644 index 0000000000..1bdc19a370 --- /dev/null +++ b/packages/stack/src/internal/engine.ts @@ -0,0 +1,2 @@ +/** Container engine target resolution shared with the CLI's runtime selection. */ +export { resolveEngineTarget, type ContainerEngine } from "../runtime/Container.ts"; diff --git a/packages/stack/src/internal/generation-publish.ts b/packages/stack/src/internal/generation-publish.ts new file mode 100644 index 0000000000..4d0b4c7f4d --- /dev/null +++ b/packages/stack/src/internal/generation-publish.ts @@ -0,0 +1,54 @@ +import { Effect, type FileSystem, type Path, type PlatformError } from "effect"; + +export interface GenerationFile { + readonly name: string; + readonly content: string; + readonly mode: number; +} + +let counter = 0; + +/** + * Publishes `files` together as `root/name` by renaming one staged directory into place, so readers + * never see a partial generation. A rename can't replace a non-empty directory, so a collision with + * an already-complete generation (same content-derived name) is success. + */ +export const publishGeneration = Effect.fn("Internal.publishGeneration")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + name: string, + files: ReadonlyArray, +) { + const generation = path.join(root, name); + const isComplete = (directory: string): Effect.Effect => + Effect.forEach(files, (file) => fs.exists(path.join(directory, file.name))).pipe( + Effect.map((results) => results.every(Boolean)), + ); + if (yield* isComplete(generation)) return generation; + yield* fs.makeDirectory(root, { recursive: true, mode: 0o700 }); + const stage = path.join(root, `.${name}.${process.pid}-${++counter}.tmp`); + // Uninterruptible: an abandoned in-flight rename could otherwise publish a stage that cleanup has + // already partly removed. + yield* Effect.gen(function* () { + yield* fs.makeDirectory(stage, { recursive: true, mode: 0o700 }); + for (const file of files) + yield* fs.writeFileString(path.join(stage, file.name), file.content, { mode: file.mode }); + yield* fs.rename(stage, generation).pipe( + Effect.catch((cause) => + isComplete(generation).pipe( + Effect.orElseSucceed(() => false), + Effect.flatMap((complete) => (complete ? Effect.void : Effect.fail(cause))), + ), + ), + ); + }).pipe( + Effect.ensuring( + fs + .remove(stage, { recursive: true, force: true }) + .pipe(Effect.catchTag("PlatformError", () => Effect.void)), + ), + Effect.uninterruptible, + ); + return generation; +}); diff --git a/packages/stack/src/internal/host-process.ts b/packages/stack/src/internal/host-process.ts index eb459d0c37..395218ffea 100644 --- a/packages/stack/src/internal/host-process.ts +++ b/packages/stack/src/internal/host-process.ts @@ -1,8 +1,17 @@ -import { Cause, Effect, Exit, Option, Schema } from "effect"; +import { Cause, Duration, Effect, Exit, identity, Option, Schema } from "effect"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. import { closeSync, writeSync } from "node:fs"; -import { SavedStack } from "../State.ts"; -import { runStackHost, StackHostError, type StackHostOptions } from "../StackHost.ts"; +import { SavedStack } from "../StackNamespace.ts"; +import { + RegistrationCheckInterval, + runStackHost, + StackHostError, + type StackHostOptions, +} from "../StackHost.ts"; + +interface HostProcessOverrides extends Pick { + readonly registrationCheckInterval?: Duration.Input; +} const writeLine = (value: unknown) => Effect.gen(function* () { @@ -62,7 +71,10 @@ const options = ( }; }); -const program = (args: ReadonlyArray, overrides: Pick) => +const program = ( + args: ReadonlyArray, + { registrationCheckInterval, ...overrides }: HostProcessOverrides, +) => Effect.gen(function* () { let reported = false; const report = (value: unknown) => @@ -82,7 +94,11 @@ const program = (args: ReadonlyArray, overrides: Pick Effect.callback((resume) => { @@ -96,7 +112,7 @@ const flushed = (stream: NodeJS.WriteStream) => */ export const runHostProcess = ( args: ReadonlyArray, - overrides: Pick = {}, + overrides: HostProcessOverrides = {}, ): Promise => Effect.runPromise( program(args, overrides).pipe( diff --git a/packages/stack/src/internal/test-run-label.ts b/packages/stack/src/internal/test-run-label.ts index 8cdcdefb24..c0302f0839 100644 --- a/packages/stack/src/internal/test-run-label.ts +++ b/packages/stack/src/internal/test-run-label.ts @@ -1,6 +1,6 @@ import { Config, Effect, Option } from "effect"; -/** Environment variable a test run sets so stack-created Docker/Podman resources can be labelled. */ +/** Environment variable a test run sets so stack-created Docker resources can be labelled. */ export const testRunEnvVar = "SUPABASE_STACK_TEST_RUN"; /** Label key stamped on volumes and containers a test run creates. */ diff --git a/packages/stack/src/lifetime.integration.test.ts b/packages/stack/src/lifetime.integration.test.ts index da1a17a058..fb73e3afdf 100644 --- a/packages/stack/src/lifetime.integration.test.ts +++ b/packages/stack/src/lifetime.integration.test.ts @@ -1,37 +1,36 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { - Context, - Data, - Effect, - Fiber, - FileSystem, - Layer, - Option, - Path, - Schema, - Stream, -} from "effect"; +import { Context, Data, Effect, Fiber, FileSystem, Layer, Option, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { create, open } from "./effect.ts"; -import * as State from "./State.ts"; -import { assertOwnerExited, captureOwnerPid, watchLeaseRelease } from "../tests/owner.ts"; +import { launchHost, ownerClient, ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; +import * as StackNamespace from "./StackNamespace.ts"; +import { + assertOwnerExited, + captureOwnerPid, + shutdownOwner, + watchLeaseRelease, +} from "../tests/owner.ts"; +import { watchEntry } from "../tests/watch-entry.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; class LifetimeTestError extends Data.TaggedError("LifetimeTestError")<{ readonly message: string; }> {} const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; const sessionFixture = fileURLToPath( new URL("../tests/session-client-fixture.ts", import.meta.url), ); +const shortRegistrationPollFixture = fileURLToPath( + new URL("../tests/short-registration-poll-fixture.ts", import.meta.url), +); const stateFor = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const collect = (child: ChildProcessSpawner.ChildProcessHandle) => @@ -78,20 +77,6 @@ const alive = (pid: number) => { } }; -/** Completes once `entry` disappears from `directory`; subscribe before triggering the removal. */ -const awaitRemoval = (directory: string, entry: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const target = path.join(directory, entry); - yield* fs.watch(directory).pipe( - Stream.filter((event) => event.path === entry || event.path === target), - Stream.mapEffect(() => fs.exists(target)), - Stream.takeUntil((exists) => !exists), - Stream.runDrain, - ); - }); - it.live.skipIf(process.platform === "win32")( "destroys a session stack and its native processes when the creating process is killed", () => @@ -139,19 +124,14 @@ it.live.skipIf(process.platform === "win32")( )(ready); const state = yield* stateFor(stateRoot); expect(yield* state.leased(stackId)).toBe(true); - expect((yield* state.claims).find(({ id }) => id === stackId)?.ports.length).toBeGreaterThan( - 0, - ); const ownerPid = yield* captureOwnerPid({ stateRoot, cacheRoot }, stackId); const owned = yield* descendantsOf(ownerPid); expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); - const removed = yield* awaitRemoval(stateRoot, stackId).pipe( - Effect.forkChild({ startImmediately: true }), - ); + const removed = yield* watchEntry(`${stateRoot}/${stackId}`, "state.json", false); const released = yield* watchLeaseRelease(stateRoot, stackId); yield* creator.kill({ killSignal: "SIGKILL" }); - yield* Fiber.join(removed).pipe( + yield* removed.pipe( Effect.timeoutOrElse({ duration: "1 minute", orElse: () => @@ -162,8 +142,8 @@ it.live.skipIf(process.platform === "win32")( // Releasing the lease is the owner's last act, after its native processes have exited. yield* released; expect(owned.filter(alive), "native processes die with their owner").toEqual([]); - expect(yield* fs.exists(`${stateRoot}/${stackId}`)).toBe(false); - expect((yield* state.claims).some(({ id }) => id === stackId)).toBe(false); + expect(yield* fs.exists(`${stateRoot}/${stackId}/state.json`)).toBe(false); + expect(yield* state.read(stackId)).toBeUndefined(); }).pipe(Effect.scoped, Effect.provide(layer)), { timeout: 180_000 }, ); @@ -188,7 +168,7 @@ it.live("destroys a session stack when its creating handle closes", () => yield* assertOwnerExited(ownerPid); const state = yield* stateFor(locations.stateRoot); expect(yield* state.read(id)).toBeUndefined(); - expect(yield* fs.exists(`${locations.stateRoot}/${id}`)).toBe(false); + expect(yield* fs.exists(`${locations.stateRoot}/${id}/state.json`)).toBe(false); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -222,3 +202,99 @@ it.live("lets only the creating handle start a session stack's owner", () => expect(yield* state.leased(id)).toBe(false); }).pipe(Effect.scoped, Effect.provide(layer)), ); + +it.live.skipIf(process.platform === "win32")( + "exits and stops its native workload, freeing its port, when its registration is confirmed gone", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registration-lost-native-" }); + const stateRoot = `${root}/state`; + const state = yield* stateFor(stateRoot); + const stackId = "registration-lost-native"; + yield* state.save({ + id: stackId, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: stackId }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + // The shortened poll interval comes only from this dedicated test entrypoint, through + // the internal `Context.Reference`; production startup never reads an env var or `Config`. + const access = yield* launchHost(state, { + stateRoot, + cacheRoot, + stackId, + entrypoint: shortRegistrationPollFixture, + }); + const client = yield* ownerClient(access); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + const status = yield* client.status({ id: mail.id }); + const port = status.endpoints.find((endpoint) => endpoint.name === "http")?.port; + if (port === undefined) return yield* Effect.die("Missing mail http endpoint"); + const ownerPid = access.endpoint.pid; + const owned = yield* descendantsOf(ownerPid); + expect(owned.length, "the owner runs the native mail service").toBeGreaterThan(0); + + // Subscribes to the owner's own exit signal (its lease release, the owner's last act) before + // triggering the deletion, rather than polling for it afterward. + const leaseReleased = yield* watchLeaseRelease(stateRoot, stackId); + yield* fs.remove(`${stateRoot}/${stackId}/state.json`); + yield* leaseReleased; + // The lease is already confirmed released, so the process is already exiting or exited; + // `waitForOwnerExit`'s own internal cap no longer bounds detection, drain and cleanup — only + // this final, now-fast confirmation. + yield* waitForOwnerExit(ownerPid, ownerExitProbe(fs)).pipe(Effect.timeout("10 seconds")); + + expect(owned.filter(alive), "native processes die with their owner").toEqual([]); + expect( + yield* fs.exists(`${stateRoot}/${stackId}/state.json`), + "no registration is republished", + ).toBe(false); + + // The stopped workload frees its port: a fresh stack can claim the exact same port. + const reclaimedRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "stack-registration-lost-native-reclaim-", + }); + const reclaimedStateRoot = `${reclaimedRoot}/state`; + const reclaimedState = yield* stateFor(reclaimedStateRoot); + const reclaimedId = "registration-lost-native-reclaim"; + yield* reclaimedState.save({ + id: reclaimedId, + runtime: "native", + identity: { projectRoot: reclaimedRoot, branchContext: "main", stackName: reclaimedId }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + const reclaimedAccess = yield* launchHost(reclaimedState, { + stateRoot: reclaimedStateRoot, + cacheRoot, + stackId: reclaimedId, + }); + const reclaimedClient = yield* ownerClient(reclaimedAccess); + const reclaimedMail = yield* reclaimedClient.createService({ + service: "mail", + config: {}, + endpoints: { http: { port } }, + }); + yield* reclaimedClient.startService({ id: reclaimedMail.id }); + yield* reclaimedClient.readyService({ id: reclaimedMail.id }); + const reclaimedStatus = yield* reclaimedClient.status({ id: reclaimedMail.id }); + expect(reclaimedStatus.endpoints.find((endpoint) => endpoint.name === "http")?.port).toBe( + port, + ); + yield* shutdownOwner(reclaimedAccess, true); + yield* waitForOwnerExit(reclaimedAccess.endpoint.pid, ownerExitProbe(fs)).pipe( + Effect.timeout("15 seconds"), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), + { timeout: 60_000 }, +); diff --git a/packages/stack/src/namespace/Capabilities.ts b/packages/stack/src/namespace/Capabilities.ts new file mode 100644 index 0000000000..8849318dd4 --- /dev/null +++ b/packages/stack/src/namespace/Capabilities.ts @@ -0,0 +1,18 @@ +import { Schema } from "effect"; + +/** A failure from any namespace operation; `operation` names the step that failed. */ +export class NamespaceError extends Schema.TaggedError()( + "Namespace.NamespaceError", + { + operation: Schema.String, + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + }, +) {} + +export const namespaceError = (operation: string, cause: unknown): NamespaceError => + new NamespaceError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); diff --git a/packages/stack/src/namespace/Environment.integration.test.ts b/packages/stack/src/namespace/Environment.integration.test.ts new file mode 100644 index 0000000000..d1c349f6e4 --- /dev/null +++ b/packages/stack/src/namespace/Environment.integration.test.ts @@ -0,0 +1,52 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Path, Schema } from "effect"; +import { confine, EnvironmentSymlinkError } from "./Environment.ts"; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("native environment confinement", () => { + it.live("creates every confined directory under a fresh root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "environment-confine-" }); + const environment = yield* confine(fs, path, root); + expect(yield* fs.exists(environment.values.XDG_CACHE_HOME)).toBe(true); + }), + ), + ); + + it.live( + "refuses to confine on restart when a reused component is a symlink planted by the step-down user, and creates nothing outside", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "environment-confine-symlink-", + }); + // First launch creates the confined layout normally. + yield* confine(fs, path, root); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "environment-confine-outside-", + }); + const sentinel = path.join(outside, "sentinel"); + yield* fs.writeFileString(sentinel, "do not create here"); + // After handover, the step-down user that now owns `root` swaps `.cache` for a symlink + // to outside before the stack restarts. + const cache = path.join(root, ".cache"); + yield* fs.remove(cache, { recursive: true }); + yield* fs.symlink(outside, cache); + const failure = yield* confine(fs, path, root).pipe(Effect.flip); + if (!Schema.is(EnvironmentSymlinkError)(failure)) + return yield* Effect.die(`Expected an EnvironmentSymlinkError, got ${failure}`); + expect(failure.path).toBe(cache); + expect(yield* fs.readDirectory(outside)).toEqual(["sentinel"]); + }), + ), + ); +}); diff --git a/packages/stack/src/namespace/Environment.ts b/packages/stack/src/namespace/Environment.ts new file mode 100644 index 0000000000..32c8e601b4 --- /dev/null +++ b/packages/stack/src/namespace/Environment.ts @@ -0,0 +1,104 @@ +import { Effect, FileSystem, Path, Schema } from "effect"; +import type { PlatformError } from "effect/PlatformError"; +import { lstatPath } from "./drivers/FileSystem.ts"; + +/** + * Variables a native workload's environment is confined to. Every one of them is rooted under the + * owned directory {@link confine} is given, so a caller cannot redirect a workload's home, temp, + * cache, config, data, state or Deno directory outside it. + */ +const CONFINED_KEYS = [ + "HOME", + "TMPDIR", + "TMP", + "TEMP", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + "DENO_DIR", +] as const; +type ConfinedKey = (typeof CONFINED_KEYS)[number]; + +/** The owned environment a native workload's spec must carry; constructed only by {@link confine}. */ +export interface NativeEnvironment { + readonly values: Readonly>; +} + +/** A caller tried to set an environment variable the namespace confines; it names every one. */ +export class EnvironmentOverrideError extends Schema.TaggedError()( + "Namespace.EnvironmentOverrideError", + { keys: Schema.Array(Schema.String) }, +) {} + +/** + * A reused environment component (the owned root or one nested under it) is a symlink or junction, + * which its current owner (a step-down user, for example) could have planted between launches. + */ +export class EnvironmentSymlinkError extends Schema.TaggedError()( + "Namespace.EnvironmentSymlinkError", + { path: Schema.String }, +) {} + +/** Every path component from `root` down to (and including) `target`, root first. */ +const componentsFrom = (path: Path.Path, root: string, target: string): ReadonlyArray => { + const relative = path.relative(root, target); + const segments = relative.split(path.sep).filter((segment) => segment.length > 0); + const components: Array = [root]; + let current = root; + for (const segment of segments) { + current = path.join(current, segment); + components.push(current); + } + return components; +}; + +/** + * Builds the confined environment for a native workload rooted at `root`: `root` itself is HOME, + * with a temp, cache, config, data, state and Deno directory nested under it. On reuse, a prior + * launch's confined directories could have been replaced with a symlink or junction by their + * current owner; every component from `root` down is rejected if so, before anything is created + * through it. Creates every directory so the workload finds them ready on first use. + */ +export const confine = Effect.fn("Namespace.Environment.confine")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, +): Effect.fn.Return { + const home = root; + const tmp = path.join(home, "tmp"); + const values: Record = { + HOME: home, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_DATA_HOME: path.join(home, ".local", "share"), + XDG_STATE_HOME: path.join(home, ".local", "state"), + DENO_DIR: path.join(home, ".cache", "deno"), + }; + const directories = new Set(Object.values(values)); + const components = new Set([root]); + for (const directory of directories) + for (const component of componentsFrom(path, root, directory)) components.add(component); + for (const component of components) + if ((yield* lstatPath(component))?.type === "SymbolicLink") + return yield* new EnvironmentSymlinkError({ path: component }); + for (const directory of directories) + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + return { values }; +}); + +/** + * Merges `environment`'s confined values into `callerEnv`, failing instead of letting `callerEnv` + * override any of them. + */ +export const apply = ( + environment: NativeEnvironment, + callerEnv: Readonly> = {}, +): Effect.Effect, EnvironmentOverrideError> => { + const overridden = CONFINED_KEYS.filter((key) => key in callerEnv); + if (overridden.length > 0) return Effect.fail(new EnvironmentOverrideError({ keys: overridden })); + return Effect.succeed({ ...callerEnv, ...environment.values }); +}; diff --git a/packages/stack/src/namespace/Lease.ts b/packages/stack/src/namespace/Lease.ts new file mode 100644 index 0000000000..411192a9c2 --- /dev/null +++ b/packages/stack/src/namespace/Lease.ts @@ -0,0 +1,279 @@ +import { Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope } from "effect"; +import type { DatabaseSync } from "node:sqlite"; +import { isSafeId } from "../identity/SafeId.ts"; +import { retrySharingViolation } from "../internal/sharing-violation.ts"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import * as Publication from "./Publication.ts"; +import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; +import { acquireLock, errcode, isBusy, isMissing, takeLock } from "./drivers/Sqlite.ts"; + +/** What the current lease holder publishes: an owner's control endpoint, or a sweeper's marker. */ +export const LeaseHolder = Schema.Union([ + Schema.Struct({ + role: Schema.Literal("owner"), + port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), + pid: Schema.Int, + release: Schema.String, + lifetime: Schema.Literals(["session", "detached"]), + startedAt: Schema.String, + secret: Schema.String, + }), + Schema.Struct({ role: Schema.Literal("sweeper"), pid: Schema.Int, startedAt: Schema.String }), +]); +export type LeaseHolder = Schema.Schema.Type; + +/** A live owner already holds this stack's lease. */ +export class LeaseHeldError extends Schema.TaggedError()( + "Namespace.LeaseHeldError", + { + stackId: Schema.String, + message: Schema.String, + }, +) {} + +/** + * Obtained only by acquiring a stack's owner lease, for the life of that acquisition's scope. + * `publishHolder` and `retractHolder` are the only way to write or clear this process's holder + * record, so neither can run without the lease that makes the record meaningful. + */ +interface LeaseHandle { + readonly stackId: string; + readonly ownerLog: string; + readonly publishHolder: (record: LeaseHolder) => Effect.Effect; + /** Removes the holder record while the lease path still names the file this lease locked. */ + readonly retractHolder: Effect.Effect; + /** False once the lease path no longer names the file this lease locked (confirmed missing or replaced). */ + readonly pathHeld: Effect.Effect; +} + +export interface Interface { + readonly acquireLease: ( + id: string, + ) => Effect.Effect; + /** Reports whether any process currently holds the stack's owner lease. */ + readonly leased: (id: string) => Effect.Effect; + /** Only meaningful while the lease is held; a record left by a dead holder is stale. */ + readonly readHolder: (id: string) => Effect.Effect; + /** The file that receives the stack owner's stdout and stderr. */ + readonly ownerLog: (id: string) => string; +} + +export const OWNER_FILE = "owner.json"; +export const OWNER_LOG_FILE = "owner.log"; + +const checkId = (id: string): Effect.Effect => + isSafeId(id) ? Effect.void : Effect.fail(namespaceError("identity", `Invalid state id: ${id}`)); + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; + /** Whether the registry still has a saved document for this stack id. */ + readonly isRegistered: (id: string) => Effect.Effect; + /** Observes a lease request that found the lease held and is waiting for it. */ + readonly onLeaseContended?: (id: string) => Effect.Effect; +} + +export const make = ( + options: Options, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stackRoot = (id: string) => path.join(options.root, id); + const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); + const ownerPath = (id: string) => path.join(stackRoot(id), OWNER_FILE); + const ownerLog = (id: string) => path.join(stackRoot(id), OWNER_LOG_FILE); + const retryRead = retrySharingViolation(options.platform); + + /** An open file that was unlinked: SQLite IOERR_VNODE on macOS, IOERR_FSTAT on Linux. */ + const isMoved = (error: NamespaceError) => + errcode(error.cause, 6922) || errcode(error.cause, 1802); + const identityOf = (info: FileSystem.File.Info) => + `${info.dev}:${Option.getOrElse(info.ino, () => "")}`; + const fileIdentity = (file: string) => + fs.stat(file).pipe(Effect.map(identityOf), Effect.option); + /** + * Deletes an unregistered stack's lease file while it is still the file locked as `identity`; + * `false` means it is still in place. + */ + const removeLeaseFile = (id: string, identity: string) => + Effect.all([options.isRegistered(id), fileIdentity(leasePath(id))]).pipe( + Effect.flatMap(([registered, current]) => + registered || !Option.contains(current, identity) + ? Effect.void + : fs.remove(leasePath(id), { force: true }), + ), + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + + const acquireLease = Effect.fn("Namespace.Lease.acquire")(function* (id: string) { + yield* checkId(id); + const target = leasePath(id); + const scope = yield* Scope.Scope; + let held: string | undefined; + let unlinked = false; + yield* Effect.addFinalizer(() => + held !== undefined + ? (unlinked ? Effect.void : removeLeaseFile(id, held)).pipe( + Effect.andThen(removeEmptyDirectory(stackRoot(id))), + Effect.ignore, + ) + : Effect.void, + ); + let contended = false; + const noteContention = Effect.suspend(() => + contended + ? Effect.void + : Effect.sync(() => (contended = true)).pipe( + Effect.andThen(options.onLeaseContended?.(id) ?? Effect.void), + ), + ); + + /** Closes `attemptScope`, noting contention first so busy is reported at most once. */ + const abandon = ( + attemptScope: Scope.Closeable, + error: NamespaceError, + ): Effect.Effect => + (isBusy(error) ? noteContention : Effect.void).pipe( + Effect.andThen(Scope.close(attemptScope, Exit.void)), + Effect.andThen(Effect.fail(error)), + ); + + /** + * A waiter can open the lease file just before its holder unlinks it, and then lock the + * unlinked file; comparing the path's file before opening and after locking rejects that. + * Up to 8 quick retries cover that race; a busy failure propagates instead, so the retry + * around this whole attempt (below) can wait it out on a slower schedule. + */ + const tryOnce: Effect.Effect<{ readonly ownerLog: string }, NamespaceError, Scope.Scope> = + Effect.gen(function* () { + for (let attempt = 0; attempt < 8; attempt++) { + // A releasing holder may remove the empty stack directory at any moment. + yield* fs + .makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => namespaceError("lease", cause))); + const before = yield* fileIdentity(target); + const attemptScope = yield* Scope.fork(scope, "sequential"); + const connection = yield* acquireLock(target, "create").pipe( + Scope.provide(attemptScope), + Effect.map(Option.some), + Effect.catchIf(isMissing, () => Effect.succeed(Option.none())), + Effect.catch((error) => abandon(attemptScope, error)), + ); + if (Option.isNone(connection)) { + yield* Scope.close(attemptScope, Exit.void); + continue; + } + const locked = yield* takeLock(connection.value).pipe( + Effect.as(true), + Effect.catchIf(isMoved, () => Effect.succeed(false)), + Effect.catch((error) => abandon(attemptScope, error)), + ); + const after = locked ? yield* fileIdentity(target) : Option.none(); + if (Option.isSome(before) && Option.isSome(after) && before.value === after.value) { + held = after.value; + yield* Scope.addFinalizer( + attemptScope, + removeLeaseFile(id, after.value).pipe( + Effect.map((removed) => (unlinked = removed)), + ), + ); + return { ownerLog: ownerLog(id) }; + } + yield* Scope.close(attemptScope, Exit.void); + } + return yield* namespaceError("lease", `The lease file of stack ${id} keeps changing`); + }); + + const outcome = yield* tryOnce.pipe( + Effect.retry({ + schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "500 millis" })), + while: isBusy, + }), + Effect.catchIf( + isBusy, + () => + new LeaseHeldError({ + stackId: id, + message: `Another owner holds the lease of stack ${id}`, + }), + ), + ); + const pathHeld = fs.stat(target).pipe( + Effect.map((info) => identityOf(info) === held), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(false), + ), + Effect.mapError((cause) => namespaceError("lease", cause)), + ); + return { + stackId: id, + ownerLog: outcome.ownerLog, + pathHeld, + publishHolder: Effect.fn("Namespace.Lease.publishHolder")(function* (record: LeaseHolder) { + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(LeaseHolder))( + record, + ).pipe(Effect.mapError((cause) => namespaceError("encode", cause))); + // The lease already guarantees a single writer, so an atomic rename over any existing + // record (a live one, or a crashed holder's stale one) is enough: a reader never sees a + // gap where the file briefly doesn't exist, unlike a prior remove followed by a publish. + yield* Publication.publish(fs, path, { + target: ownerPath(id), + content: serialized, + platform: options.platform, + }); + }), + retractHolder: pathHeld.pipe( + Effect.flatMap((held) => + held + ? fs + .remove(ownerPath(id), { force: true }) + .pipe(Effect.mapError((cause) => namespaceError("remove", cause))) + : Effect.void, + ), + Effect.withSpan("Namespace.Lease.retractHolder"), + ), + } satisfies LeaseHandle; + }); + + const leased = Effect.fn("Namespace.Lease.leased")(function* (id: string) { + yield* checkId(id); + return yield* Effect.scoped( + acquireLock(leasePath(id), "existing").pipe( + Effect.flatMap((connection) => takeLock(connection).pipe(Effect.as(false))), + Effect.catchIf(isBusy, () => Effect.succeed(true)), + ), + ).pipe( + // A lease file that is missing, or that its holder unlinked meanwhile, is not held. + Effect.catchIf( + (error) => isMissing(error) || isMoved(error), + () => Effect.succeed(false), + ), + ); + }); + const decodeHolder = Schema.decodeEffect(Schema.fromJsonString(LeaseHolder)); + const readHolder = Effect.fn("Namespace.Lease.readHolder")(function* (id: string) { + yield* checkId(id); + const target = ownerPath(id); + // The holder may retract its record at any moment, so a missing record is not an error. + const text = yield* fs.readFileString(target).pipe( + retryRead, + Effect.map(Option.some), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + Effect.mapError((cause) => namespaceError("read", cause)), + ); + if (Option.isNone(text)) return undefined; + return yield* decodeHolder(text.value).pipe( + Effect.mapError((cause) => + namespaceError("decode", `Unable to decode ${target}: ${cause}`), + ), + ); + }); + + return { acquireLease, leased, readHolder, ownerLog }; + }); diff --git a/packages/stack/src/namespace/Namespace.e2e.test.ts b/packages/stack/src/namespace/Namespace.e2e.test.ts new file mode 100644 index 0000000000..910719dd37 --- /dev/null +++ b/packages/stack/src/namespace/Namespace.e2e.test.ts @@ -0,0 +1,417 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + Context, + Data, + Deferred, + Effect, + Fiber, + FileSystem, + Layer, + Path, + Ref, + Schema, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; +import { + connectHost, + launchHost, + ownerExitProbe, + shutdownHost, + waitForOwnerExit, +} from "../HostProcess.ts"; +import { create, open } from "../effect.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import { captureOwnerPid, watchLeaseRelease } from "../../tests/owner.ts"; +import { testEngine } from "../../tests/test-engine.ts"; + +class PublishBarrierError extends Data.TaggedError("PublishBarrierError")<{ + readonly message: string; +}> {} +class LockFixtureError extends Data.TaggedError("LockFixtureError")<{ + readonly message: string; +}> {} + +const makeTestState = (root: string) => + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), + ); + +const ownerFixture = fileURLToPath(new URL("../../tests/host-process-fixture.ts", import.meta.url)); +const publishLoopFixture = fileURLToPath( + new URL("../../tests/publish-loop-fixture.ts", import.meta.url), +); +const lockFixture = fileURLToPath( + new URL("../../tests/namespace-lock-fixture.ts", import.meta.url), +); + +const testLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +const savedStack = (root: string, id: string): StackNamespace.SavedStack => ({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "local" }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, +}); + +it.live( + "a new owner acquires a SIGKILLed owner's lease, with no manual cleanup", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + // Scoped outside the owners: their scope below must fully close (confirming the second + // owner has actually exited, not just acknowledged shutdown) before this directory's own + // removal runs, or Windows reports EBUSY on a file the still-exiting owner has open. + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-sigkill-" }); + const state = yield* makeTestState(root); + const id = "stack"; + yield* state.save(savedStack(root, id)); + const options = { stateRoot: root, cacheRoot: root, stackId: id, entrypoint: ownerFixture }; + + yield* Effect.scoped( + Effect.gen(function* () { + const first = yield* launchHost(state, options); + expect(yield* state.leased(id)).toBe(true); + + const released = yield* watchLeaseRelease(root, id); + yield* Effect.sync(() => process.kill(first.endpoint.pid, "SIGKILL")); + yield* released; + expect(yield* state.leased(id)).toBe(false); + + // No file is removed or repaired here; the kernel alone released the dead owner's lease. + const second = yield* Effect.acquireRelease(launchHost(state, options), (access) => + shutdownHost(access, true).pipe( + Effect.ignore, + Effect.andThen( + waitForOwnerExit(access.endpoint.pid, ownerExitProbe(fs)).pipe(Effect.ignore), + ), + ), + ); + expect(second.endpoint.pid).not.toBe(first.endpoint.pid); + expect(yield* connectHost(state, id)).toEqual(second); + }), + ); + }), + ).pipe(Effect.provide(testLayer)), + 20_000, +); + +it.live( + "killing a writer exactly at the staging-to-publish boundary leaves the prior document intact and lets the next publish recover", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-publish-" }); + const state = yield* makeTestState(root); + const id = "stack"; + yield* state.save(savedStack(root, id)); + + const writer = yield* spawner.spawn( + ChildProcess.make(process.execPath, [publishLoopFixture, root, id], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ); + const ready = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* writer.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* writer.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.tap((line) => + line === "about-to-publish" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + Stream.runDrain, + Effect.forkScoped, + ); + const writerFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new PublishBarrierError({ message: `${reason}: ${text}` })), + ), + ); + // The fixture announces this boundary itself, right before the real rename call; killing + // here lands in the staging-to-publish gap on every run, not "most of the time". + yield* Deferred.await(ready).pipe( + Effect.raceFirst( + writer.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + writerFailure(`Writer exited before readiness: ${String(cause)}`), + onSuccess: (code) => writerFailure(`Writer exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => writerFailure("Writer did not reach the publish boundary"), + }), + ); + yield* Effect.sync(() => process.kill(Number(writer.pid), "SIGKILL")); + yield* writer.exitCode.pipe(Effect.ignore); + yield* Fiber.join(output); + yield* Fiber.join(diagnostics); + + const target = path.join(root, id, "state.json"); + const raw = yield* fs.readFileString(target); + const decoded = yield* Schema.decodeEffect( + Schema.fromJsonString(StackNamespace.SavedStack), + )(raw); + // The killed writer never reached the rename: the prior document is untouched, and only + // its staging file, never a partial target, is left behind. + expect(decoded).toEqual(savedStack(root, id)); + expect( + (yield* fs.readDirectory(path.join(root, id))).filter((entry) => entry.endsWith(".tmp")), + ).toHaveLength(1); + + yield* state.save({ + ...decoded, + identity: { ...decoded.identity, stackName: "recovered" }, + }); + expect(yield* state.read(id)).toEqual({ + ...decoded, + identity: { ...decoded.identity, stackName: "recovered" }, + }); + }), + ).pipe(Effect.provide(testLayer)), + 20_000, +); + +it.live( + "serializes real writer processes and releases a killed registry holder while its child survives", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-lock-process-" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawn = (args: ReadonlyArray) => + spawner.spawn( + ChildProcess.make(process.execPath, [lockFixture, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + forceKillAfter: "1 second", + }), + ); + const readOutput = (child: ChildProcessSpawner.ChildProcessHandle) => + Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + + const state = yield* makeTestState(root); + const saved = savedStack(root, "stack-main"); + yield* state.save(saved); + + yield* Effect.forEach( + ["first", "second", "third"], + (id) => + Effect.scoped( + Effect.gen(function* () { + const writer = yield* spawn(["write", root, id]); + const [, stderr, code] = yield* readOutput(writer); + expect(code, stderr).toBe(0); + }), + ), + { concurrency: "unbounded" }, + ); + expect((yield* state.read(saved.id))?.instances.map(({ id }) => id).sort()).toEqual([ + "first", + "second", + "third", + ]); + + const holder = yield* spawn(["hold", root]); + const locked = yield* Deferred.make(); + const childReady = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* holder.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* holder.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => + line === "locked" + ? Deferred.succeed(locked, undefined).pipe(Effect.asVoid) + : line.startsWith("child:") + ? Deferred.succeed(childReady, line.slice(6)).pipe(Effect.asVoid) + : Effect.void, + ), + Effect.forkScoped, + ); + const holderFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new LockFixtureError({ message: `${reason}: ${text}` })), + ), + ); + yield* Effect.all([Deferred.await(locked), Deferred.await(childReady)]).pipe( + Effect.raceFirst( + holder.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + holderFailure(`Holder exited before readiness: ${String(cause)}`), + onSuccess: (code) => holderFailure(`Holder exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => holderFailure("Holder readiness timed out"), + }), + ); + const childPid = yield* Schema.decodeEffect(Schema.FiniteFromString)( + yield* Deferred.await(childReady), + ); + const kill = (pid: number) => + Effect.try({ + try: () => process.kill(pid, "SIGKILL"), + catch: (cause) => new LockFixtureError({ message: String(cause) }), + }); + yield* Effect.addFinalizer(() => + kill(childPid).pipe( + Effect.ignore, + Effect.andThen(Fiber.join(output)), + Effect.andThen(Fiber.join(diagnostics)), + Effect.andThen(waitForOwnerExit(childPid, ownerExitProbe(fs)).pipe(Effect.ignore)), + Effect.timeout("10 seconds"), + Effect.orDie, + ), + ); + yield* kill(Number(holder.pid)); + yield* holder.exitCode.pipe(Effect.exit); + yield* state.withLock( + state.save({ ...saved, identity: { ...saved.identity, stackName: "after-crash" } }), + ); + expect((yield* state.read(saved.id))?.identity.stackName).toBe("after-crash"); + yield* Effect.try({ + try: () => process.kill(childPid, 0), + catch: (cause) => + new LockFixtureError({ + message: `Child did not survive the killed holder: ${String(cause)}`, + }), + }); + }), + ).pipe(Effect.provide(testLayer)), + 20_000, +); + +const engine = Effect.fn("Namespace.e2e.engine")((args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make(testEngine, args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + Stream.mkString(Stream.decodeText(child.stdout)), + Stream.mkString(Stream.decodeText(child.stderr)), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* Effect.die(`${testEngine} ${args.join(" ")} failed: ${stderr}`); + return stdout.trim(); + }), + ), +); + +// Needs a real container engine, which only the Linux CI runner has. +it.live.skipIf(process.platform === "win32" || process.platform === "darwin")( + "removes exactly the containers a SIGKILLed owner left behind, with no manual cleanup", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-e2e-container-" }); + const stateRoot = `${root}/state`; + const cacheRoot = `${root}/cache`; + const stack = yield* create({ + projectRoot: root, + stateRoot, + cacheRoot, + runtime: testEngine, + startOwner: true, + }); + yield* Effect.addFinalizer(() => stack.destroy.pipe(Effect.ignore)); + const mail = yield* stack.services.create({ service: "mail", config: {} }); + yield* mail.start; + yield* mail.ready; + + const containerIds = (yield* engine([ + "ps", + "--all", + "--quiet", + "--no-trunc", + "--filter", + `label=com.supabase.stack=${stack.id}`, + "--filter", + `label=com.supabase.instance=${mail.id}`, + ])) + .split("\n") + .filter((id) => id.length > 0); + expect(containerIds).toHaveLength(1); + const [containerId] = containerIds; + + const ownerPid = yield* captureOwnerPid({ stateRoot, cacheRoot }, stack.id); + const released = yield* watchLeaseRelease(stateRoot, stack.id); + yield* Effect.sync(() => process.kill(ownerPid, "SIGKILL")); + yield* released; + + // Killing the owner leaves the container running; nothing in this test removes it. + expect( + yield* engine(["inspect", "--format", "{{.State.Status}}", String(containerId)]), + ).not.toBe(""); + + const reopened = yield* open({ + id: stack.id, + stateRoot, + cacheRoot, + startOwner: true, + }); + yield* Effect.addFinalizer(() => reopened.destroy.pipe(Effect.ignore)); + + const remaining = yield* engine([ + "ps", + "--all", + "--quiet", + "--no-trunc", + "--filter", + `label=com.supabase.stack=${stack.id}`, + ]); + expect(remaining).toBe(""); + const inspectExit = yield* engine([ + "inspect", + "--format", + "{{.State.Status}}", + String(containerId), + ]).pipe(Effect.exit); + expect(inspectExit._tag).toBe("Failure"); + }), + ).pipe(Effect.provide(testLayer)), + 120_000, +); diff --git a/packages/stack/src/State.integration.test.ts b/packages/stack/src/namespace/Namespace.integration.test.ts similarity index 52% rename from packages/stack/src/State.integration.test.ts rename to packages/stack/src/namespace/Namespace.integration.test.ts index 2c8b7e6a7d..645663b1e6 100644 --- a/packages/stack/src/State.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.integration.test.ts @@ -1,30 +1,36 @@ -import { NodeServices } from "@effect/platform-node"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { TestClock } from "effect/testing"; import { Context, - DateTime, Deferred, Effect, Exit, Fiber, FileSystem, Layer, + Option, Path, PlatformError, Ref, Schema, + Scheduler, Scope, } from "effect"; -import * as State from "./State.ts"; -import type { SavedStack } from "./State.ts"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- synchronous existence check between manual scheduler steps. +import { existsSync } from "node:fs"; +import { create, discover } from "../effect.ts"; +import { destroyTestStack } from "../../tests/stack-cleanup.ts"; +import { errorCode } from "../internal/sharing-violation.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import * as Paths from "./Paths.ts"; const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); -const initial: SavedStack = { +const initial: StackNamespace.SavedStack = { id: "stack-main", identity: { projectRoot: "/tmp/project", @@ -35,19 +41,20 @@ const initial: SavedStack = { instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; const run = (effect: Effect.Effect) => Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); -describe("durable stack state", () => { +const stackLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +describe("durable registry", () => { it.live("reopens saved state and serializes concurrent read-modify-write operations", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-" }); const first = yield* makeTestState(root); const second = yield* makeTestState(root); yield* first.save(initial); @@ -84,7 +91,7 @@ describe("durable stack state", () => { run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-invalid-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-invalid-" }); const store = yield* makeTestState(root); yield* store.save(initial); const unsafe = yield* store.read("../outside").pipe(Effect.exit); @@ -99,204 +106,7 @@ describe("durable stack state", () => { expect(String(malformed.cause)).toContain("state.json"); } const valid = yield* store.read(initial.id); - expect(Schema.is(State.SavedStack)(valid)).toBe(true); - }), - ), - ); - - it.effect("reports replacement errno and destination while preserving the prior state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-state-rename-", - }); - const renameCalls = yield* Ref.make(0); - const failRename = yield* Ref.make(false); - const firstFailure = yield* Deferred.make(); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Ref.updateAndGet(renameCalls, (calls) => calls + 1).pipe( - Effect.flatMap(() => - Effect.map(Ref.get(failRename), (shouldFail) => ({ - shouldFail, - })), - ), - Effect.flatMap(({ shouldFail }) => - shouldFail && from.endsWith("state.json") && to.endsWith("state.json") - ? Deferred.succeed(firstFailure, undefined).pipe( - Effect.andThen( - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - description: "injected replacement failure", - cause: Object.assign(new Error("sharing violation"), { - code: "EPERM", - }), - }), - ), - ), - ) - : fs.rename(from, to), - ), - ), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* Ref.set(renameCalls, 0); - yield* Ref.set(failRename, true); - const saving = yield* store - .save({ - ...initial, - identity: { ...initial.identity, stackName: "next" }, - }) - .pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - yield* TestClock.adjust("950 millis"); - const error = yield* Fiber.join(saving).pipe(Effect.flip); - expect(error.operation).toBe("publish"); - expect(error.message).toContain("EPERM"); - expect(error.message).toContain(path.join(root, "stack-main", "state.json")); - expect(yield* Ref.get(renameCalls)).toBe(13); - expect(yield* store.read(initial.id)).toEqual(initial); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - }), - ), - ); - - it.effect("recovers from each Windows replacement errno using the same temporary state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - for (const code of ["EPERM", "EACCES", "EBUSY"]) { - const root = yield* fs.makeTempDirectoryScoped({ - prefix: `stack-state-retry-${code}-`, - }); - const calls = yield* Ref.make(0); - const temporaryPaths = yield* Ref.make>([]); - const failOnce = yield* Ref.make(false); - const firstFailure = yield* Deferred.make(); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Effect.gen(function* () { - yield* Ref.update(calls, (count) => count + 1); - if (yield* Ref.get(failOnce)) { - yield* Ref.set(failOnce, false); - yield* Ref.update(temporaryPaths, (paths) => [...paths, from]); - yield* Deferred.succeed(firstFailure, undefined); - return yield* PlatformError.systemError({ - _tag: - code === "EACCES" - ? "PermissionDenied" - : code === "EBUSY" - ? "Busy" - : "Unknown", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - cause: Object.assign(new Error("sharing violation"), { - code, - }), - }); - } - yield* Ref.update(temporaryPaths, (paths) => [...paths, from]); - return yield* fs.rename(from, to); - }), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* Ref.set(calls, 0); - yield* Ref.set(temporaryPaths, []); - yield* Ref.set(failOnce, true); - const publishing = yield* store - .save({ - ...initial, - identity: { ...initial.identity, stackName: code }, - }) - .pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - expect(yield* store.read(initial.id)).toEqual(initial); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(true); - yield* TestClock.adjust("10 millis"); - yield* Fiber.join(publishing); - expect(yield* Ref.get(calls)).toBe(2); - const retriedPaths = yield* Ref.get(temporaryPaths); - expect(retriedPaths).toHaveLength(2); - expect(retriedPaths[0]).toContain(".state-write-"); - expect(retriedPaths[1]).toBe(retriedPaths[0]); - expect((yield* store.read(initial.id))?.identity.stackName).toBe(code); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - } - }), - ), - ); - - it.effect("does not retry non-Windows or unrelated replacement errors", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - for (const [platform, code] of [ - ["darwin", "EPERM"], - ["win32", "EINVAL"], - ] as const) { - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-state-no-retry-", - }); - const calls = yield* Ref.make(0); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Ref.update(calls, (count) => count + 1).pipe( - Effect.andThen( - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - cause: Object.assign(new Error("injected failure"), { - code, - }), - }), - ), - ), - ), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - const error = yield* store.save(initial).pipe(Effect.flip); - expect(error.operation).toBe("publish"); - expect(yield* Ref.get(calls)).toBe(1); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - } + expect(Schema.is(StackNamespace.SavedStack)(valid)).toBe(true); }), ), ); @@ -332,12 +142,12 @@ describe("durable stack state", () => { }), }); const store = yield* Layer.build( - State.layer({ + StackNamespace.layer({ root: options.root, platform: options.platform, onInvalidState: (id) => Ref.update(reported, (ids) => [...ids, id]), }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); return { store, firstFailure, reported }; }); @@ -345,7 +155,9 @@ describe("durable stack state", () => { run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-retry-" }); + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-list-retry-", + }); yield* (yield* makeTestState(root)).save(initial); const { store, firstFailure, reported } = yield* listingWithReadFailures({ root, @@ -371,7 +183,9 @@ describe("durable stack state", () => { ["win32", "EBUSY"], ["linux", "EACCES"], ] as const) { - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-skip-" }); + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-list-skip-", + }); yield* (yield* makeTestState(root)).save(initial); const { store, firstFailure, reported } = yield* listingWithReadFailures({ root, @@ -391,17 +205,20 @@ describe("durable stack state", () => { ); it.live.skipIf(process.platform === "win32" || process.getuid?.() === 0)( - "lists and claims readable stacks past a sibling directory it cannot access", + "lists readable stacks past a sibling directory it cannot access", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-eacces-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-eacces-" }); const reported: Array = []; const store = yield* Layer.build( - State.layer({ root, onInvalidState: (id) => Effect.sync(() => reported.push(id)) }), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); + StackNamespace.layer({ + root, + onInvalidState: (id) => Effect.sync(() => reported.push(id)), + }), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); yield* store.save(initial); const locked = path.join(root, "locked"); yield* fs.makeDirectory(locked, { mode: 0o700 }); @@ -412,7 +229,6 @@ describe("durable stack state", () => { expect((yield* store.list).map(({ id }) => id)).toEqual([initial.id]); expect(reported).toEqual(["locked"]); - expect((yield* store.claims).map(({ id }) => id)).toEqual([initial.id]); }), ), ); @@ -422,21 +238,23 @@ describe("durable stack state", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-list-invalid-" }); + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-list-invalid-", + }); const reported: Array = []; const store = yield* Layer.build( - State.layer({ + StackNamespace.layer({ root, onInvalidState: (id) => Effect.sync(() => reported.push(id)), }), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); yield* store.save(initial); yield* fs.makeDirectory(path.join(root, "malformed")); yield* fs.writeFileString(path.join(root, "malformed", "state.json"), "{broken"); yield* fs.makeDirectory(path.join(root, "mismatched")); yield* fs.writeFileString( path.join(root, "mismatched", "state.json"), - yield* Schema.encodeEffect(Schema.fromJsonString(State.SavedStack))(initial), + yield* Schema.encodeEffect(Schema.fromJsonString(StackNamespace.SavedStack))(initial), ); yield* fs.makeDirectory(path.join(root, "directory", "state.json"), { recursive: true }); yield* fs.writeFileString(path.join(root, "stray-file"), ""); @@ -457,14 +275,14 @@ describe("durable stack state", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-typed-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-typed-" }); const reported: Array = []; const store = yield* Layer.build( - State.layer({ + StackNamespace.layer({ root, onInvalidState: (id) => Effect.sync(() => reported.push(id)), }), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); yield* store.save(initial); const mail = { service: "mail", config: {} }; const invalid = { @@ -503,104 +321,18 @@ describe("durable stack state", () => { ), ); - it.effect("cleans up a cancelled Windows replacement and releases the state lock", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "stack-state-cancel-rename-", - }); - const calls = yield* Ref.make(0); - const failRename = yield* Ref.make(false); - const firstFailure = yield* Deferred.make(); - const injectedFs = Layer.effect( - FileSystem.FileSystem, - Effect.succeed({ - ...fs, - rename: (from: string, to: string) => - Effect.gen(function* () { - if (from.endsWith("state.json") && to.endsWith("state.json")) { - yield* Ref.update(calls, (count) => count + 1); - if (yield* Ref.get(failRename)) { - yield* Deferred.succeed(firstFailure, undefined); - return yield* PlatformError.systemError({ - _tag: "Busy", - module: "FileSystem", - method: "rename", - pathOrDescriptor: to, - cause: Object.assign(new Error("sharing violation"), { - code: "EBUSY", - }), - }); - } - } - return yield* fs.rename(from, to); - }), - }), - ); - const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); - yield* store.save(initial); - yield* Ref.set(calls, 0); - yield* Ref.set(failRename, true); - const next = { - ...initial, - identity: { ...initial.identity, stackName: "next" }, - }; - const saving = yield* store.withLock(store.save(next)).pipe(Effect.forkScoped); - yield* Deferred.await(firstFailure); - yield* TestClock.adjust("1 millis"); - expect(yield* Ref.get(calls)).toBe(1); - yield* Fiber.interrupt(saving); - expect(yield* store.read(initial.id)).toEqual(initial); - expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); - - yield* Ref.set(failRename, false); - yield* store.withLock(store.save(next)); - expect((yield* store.read(initial.id))?.identity.stackName).toBe("next"); - }), - ), - ); - - it.live("reaps write leftovers of dead writers and keeps recent ones on open", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-reap-" }); - const abandoned = path.join(root, ".state-write-abandoned"); - const inProgress = path.join(root, ".state-write-in-progress"); - for (const directory of [abandoned, inProgress]) { - yield* fs.makeDirectory(directory); - yield* fs.writeFileString(path.join(directory, "state.json"), "{}"); - } - const twoDaysAgo = DateTime.toDateUtc(DateTime.subtract(yield* DateTime.now, { days: 2 })); - yield* fs.utimes(abandoned, twoDaysAgo, twoDaysAgo); - - const store = yield* makeTestState(root); - - expect(yield* fs.exists(abandoned)).toBe(false); - expect(yield* fs.exists(path.join(inProgress, "state.json"))).toBe(true); - yield* store.save(initial); - expect(yield* store.read(initial.id)).toEqual(initial); - }), - ), - ); - - it.live("removes empty stack parents without deleting unowned data", () => + it.live("unregisters a stack and empties its data without deleting unowned data", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-registry-remove-" }); const store = yield* makeTestState(root); yield* store.save(initial); yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); yield* store.remove(initial.id); - expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + expect(yield* fs.exists(path.join(root, initial.id, "state.json"))).toBe(false); + expect(yield* fs.readDirectory(path.join(root, initial.id, "data"))).toEqual([]); yield* store.save(initial); yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); @@ -616,7 +348,7 @@ describe("durable stack state", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-logs-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-remove-logs-" }); const held = yield* Ref.make(2); const injectedFs = Layer.succeed(FileSystem.FileSystem, { ...fs, @@ -637,16 +369,18 @@ describe("durable stack state", () => { }), }); const store = yield* Layer.build( - State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); + StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); yield* store.save(initial); - const segment = path.join(store.logsRoot(initial.id), "auth", "instance", "0000000001.log"); + const logs = Paths.stackLogsRoot(path, root, initial.id); + const segment = path.join(logs, "auth", "instance", "0000000001.log"); yield* fs.makeDirectory(path.dirname(segment), { recursive: true }); yield* fs.writeFileString(segment, "record\n"); yield* store.remove(initial.id); - expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + expect(yield* fs.exists(logs)).toBe(false); + expect(yield* store.read(initial.id)).toBeUndefined(); }), ), ); @@ -656,7 +390,7 @@ describe("durable stack state", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-failed-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-remove-failed-" }); const denied = yield* Ref.make(true); const injectedFs = Layer.succeed(FileSystem.FileSystem, { ...fs, @@ -673,10 +407,11 @@ describe("durable stack state", () => { }), }); const store = yield* Layer.build( - State.layer({ root }).pipe(Layer.provide(injectedFs)), - ).pipe(Effect.map((context) => Context.get(context, State.Service))); + StackNamespace.layer({ root }).pipe(Layer.provide(injectedFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); yield* store.save(initial); - const segment = path.join(store.logsRoot(initial.id), "auth", "instance", "0000000001.log"); + const logs = Paths.stackLogsRoot(path, root, initial.id); + const segment = path.join(logs, "auth", "instance", "0000000001.log"); yield* fs.makeDirectory(path.dirname(segment), { recursive: true }); yield* fs.writeFileString(segment, "record\n"); @@ -690,7 +425,7 @@ describe("durable stack state", () => { expect(listed).toEqual([initial.id]); expect(logsKept).toBe(true); expect(yield* store.read(initial.id)).toBeUndefined(); - expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + expect(yield* fs.exists(logs)).toBe(false); }), ), ); @@ -701,19 +436,23 @@ describe("durable stack state", () => { run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-traverse-" }); + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "namespace-registry-traverse-", + }); yield* fs.chmod(root, 0o755); yield* makeTestState(root); expect((yield* fs.stat(root)).mode & 0o777).toBe(0o701); }), ), ); +}); +describe("registry lock", () => { it.effect("keeps a lock owner protected when a waiting fiber is cancelled", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-lock-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-cancel-" }); const first = yield* makeTestState(root); const second = yield* makeTestState(root); const entered = yield* Deferred.make(); @@ -742,11 +481,12 @@ describe("durable stack state", () => { }), ), ); + it.effect("times out contention without releasing the holder's lock", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-timeout-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-timeout-" }); const first = yield* makeTestState(root); const second = yield* makeTestState(root); const entered = yield* Deferred.make(); @@ -773,9 +513,12 @@ describe("durable stack state", () => { run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-failure-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-failure-" }); const state = yield* makeTestState(root); - const failure = new State.StateError({ operation: "test", message: "body failed" }); + const failure = new StackNamespace.NamespaceError({ + operation: "test", + message: "body failed", + }); expect(yield* state.withLock(Effect.fail(failure)).pipe(Effect.flip)).toBe(failure); const defect = yield* state.withLock(Effect.die("body defect")).pipe(Effect.exit); expect(Exit.hasDies(defect)).toBe(true); @@ -789,20 +532,81 @@ describe("durable stack state", () => { }), ), ); + it.effect("fails invalid lock files without retrying or changing saved state", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-invalid-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-invalid-" }); const state = yield* makeTestState(root); yield* fs.writeFileString(`${root}/.registry-lock.sqlite`, "not a database"); const error = yield* state.withLock(state.save(initial)).pipe(Effect.flip); - expect(error).toBeInstanceOf(State.StateError); + expect(error).toBeInstanceOf(StackNamespace.NamespaceError); expect(error.operation).toBe("lock"); expect(yield* state.read(initial.id)).toBeUndefined(); }), ), ); + + it.live( + "releases the lock when the holder is interrupted right after taking it, before the caller body runs", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lock-interrupt-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + const lockPath = `${root}/.registry-lock.sqlite`; + const started = yield* Deferred.make(); + + // Yielding after every single fiber step, and only resuming a step once flushed, gives + // deterministic single-step control over the forked fiber below, with no production seam. + const tasks: Array<() => void> = []; + const dispatcher: Scheduler.SchedulerDispatcher = { + scheduleTask: (task) => tasks.push(task), + flush: () => { + for (const task of tasks.splice(0, tasks.length)) task(); + }, + }; + const stepScheduler: Scheduler.Scheduler = { + executionMode: "async", + shouldYield: (fiber) => fiber.currentOpCount >= fiber.maxOpsBeforeYield, + makeDispatcher: () => dispatcher, + }; + + const holder = yield* first + .withLock(Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never))) + .pipe( + Effect.provideService(Scheduler.Scheduler, stepScheduler), + Effect.provideService(Scheduler.MaxOpsBeforeYield, 10), + Effect.forkScoped, + ); + // The fork itself is scheduled on the default scheduler; let it actually start (handing + // control to `stepScheduler`) before single-stepping it below. + yield* Effect.yieldNow; + yield* Effect.yieldNow; + + // Single-step the acquisition until the lock file exists on disk, then stop: this always + // lands in the gap between "lock taken" and "the caller body starts" that the fix closes. + for (let step = 0; step < 20_000 && !existsSync(lockPath); step++) dispatcher.flush(); + expect(existsSync(lockPath)).toBe(true); + expect(holder.pollUnsafe()).toBeUndefined(); + expect(Option.isNone(yield* Deferred.poll(started))).toBe(true); + + // Interrupting and resuming through the same controlled dispatcher keeps the whole + // sequence deterministic: no real scheduler or timer is involved on either side. + holder.interruptUnsafe(); + for (let step = 0; step < 20_000 && holder.pollUnsafe() === undefined; step++) + dispatcher.flush(); + expect(holder.pollUnsafe()).not.toBeUndefined(); + + // No contention wait: a leaked lock would make this block for up to 5 seconds. + yield* second.withLock(second.save(initial)).pipe(Effect.timeout("200 millis")); + expect(yield* second.read(initial.id)).toEqual(initial); + }), + ), + ); }); describe("stack owner lease", () => { @@ -810,30 +614,31 @@ describe("stack owner lease", () => { run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lease-handoff-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-handoff-" }); const holder = yield* makeTestState(root); const contended = yield* Deferred.make(); const waiter = Context.get( yield* Layer.build( - State.layer({ + StackNamespace.layer({ root, onLeaseContended: () => Deferred.succeed(contended, undefined).pipe(Effect.asVoid), }), ), - State.Service, + StackNamespace.Service, ); const observer = yield* makeTestState(root); const holderScope = yield* Scope.make(); - expect(yield* holder.lease("gone").pipe(Scope.provide(holderScope))).toBe(true); + const holderLease = yield* holder.acquireLease("gone").pipe(Scope.provide(holderScope)); + expect(holderLease.stackId).toBe("gone"); const waiterScope = yield* Scope.make(); const waiting = yield* waiter - .lease("gone") + .acquireLease("gone") .pipe(Scope.provide(waiterScope), Effect.forkChild({ startImmediately: true })); yield* Deferred.await(contended); yield* Scope.close(holderScope, Exit.void); - expect(yield* Fiber.join(waiting)).toBe(true); + expect((yield* Fiber.join(waiting)).stackId).toBe("gone"); expect(yield* observer.leased("gone"), "the path names the file the waiter locked").toBe( true, ); @@ -846,6 +651,27 @@ describe("stack owner lease", () => { ), ); + it.live("fails a second owner with a typed error while the first still holds the lease", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-held-" }); + const holder = yield* makeTestState(root); + const contender = yield* makeTestState(root); + const holderScope = yield* Scope.make(); + yield* holder.acquireLease("busy").pipe(Scope.provide(holderScope)); + + const rejection = yield* contender.acquireLease("busy").pipe(Effect.scoped, Effect.flip); + expect(Schema.is(StackNamespace.LeaseHeldError)(rejection)).toBe(true); + if (!Schema.is(StackNamespace.LeaseHeldError)(rejection)) + return yield* Effect.die(rejection); + expect(rejection.stackId).toBe("busy"); + + yield* Scope.close(holderScope, Exit.void); + }), + ), + ); + // Windows refuses to delete a stack directory while its lease file is open. it.live.skipIf(process.platform === "win32")( "keeps a successor's lease file when a holder of a removed stack releases", @@ -853,25 +679,61 @@ describe("stack owner lease", () => { run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lease-successor-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-successor-" }); const state = yield* makeTestState(root); const first = yield* Scope.make(); yield* Effect.addFinalizer(() => Scope.close(first, Exit.void)); - expect(yield* state.lease("gone").pipe(Scope.provide(first))).toBe(true); + yield* state.acquireLease("gone").pipe(Scope.provide(first)); yield* fs.remove(`${root}/gone`, { recursive: true }); - expect(yield* state.lease("gone")).toBe(true); + const successor = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(successor, Exit.void)); + yield* state.acquireLease("gone").pipe(Scope.provide(successor)); yield* Scope.close(first, Exit.void); + expect(yield* state.leased("gone")).toBe(true); }), ), ); + // Windows refuses to delete a stack directory while its lease file is open. + it.live.skipIf(process.platform === "win32")( + "keeps a successor's discovery record when the holder of a removed stack retracts its own", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-record-" }); + const state = yield* makeTestState(root); + const record = (pid: number): StackNamespace.LeaseHolder => ({ + role: "sweeper", + pid, + startedAt: "2026-01-01T00:00:00.000Z", + }); + const first = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(first, Exit.void)); + const removed = yield* state.acquireLease("gone").pipe(Scope.provide(first)); + yield* removed.publishHolder(record(1)); + yield* fs.remove(`${root}/gone`, { recursive: true }); + + const successor = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(successor, Exit.void)); + const current = yield* state.acquireLease("gone").pipe(Scope.provide(successor)); + yield* current.publishHolder(record(2)); + yield* removed.retractHolder; + + expect(yield* state.readHolder("gone")).toEqual(record(2)); + yield* current.retractHolder; + expect(yield* state.readHolder("gone")).toBeUndefined(); + }), + ), + ); + it.live("reports a free lease without creating a lease file", () => run( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lease-probe-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-probe-" }); const state = yield* makeTestState(root); yield* state.save(initial); expect(yield* state.leased(initial.id)).toBe(false); @@ -882,4 +744,300 @@ describe("stack owner lease", () => { }), ), ); + + it.effect("retries a transient holder-read failure and then succeeds", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-holder-read-retry-" }); + const target = path.join(root, initial.id, "owner.json"); + const firstFailure = yield* Deferred.make(); + const failed = yield* Ref.make(false); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + readFileString: (file: string, encoding?: string) => + Effect.gen(function* () { + if (file === target && !(yield* Ref.get(failed))) { + yield* Ref.set(failed, true); + yield* Deferred.succeed(firstFailure, undefined); + return yield* PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "readFile", + pathOrDescriptor: file, + cause: Object.assign(new Error("injected transient read failure"), { + code: "EBUSY", + }), + }); + } + return yield* fs.readFileString(file, encoding); + }), + }); + const state = Context.get( + yield* Layer.build( + StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(flakyFs)), + ), + StackNamespace.Service, + ); + yield* state.save(initial); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(initial.id).pipe(Scope.provide(scope)); + const record: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(record); + + const reading = yield* state.readHolder(initial.id).pipe(Effect.forkScoped); + yield* Deferred.await(firstFailure); + yield* TestClock.adjust("10 millis"); + expect(yield* Fiber.join(reading)).toEqual(record); + yield* Scope.close(scope, Exit.void); + }), + ), + ); + + it.live("publishes successfully despite a transient rename failure", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "flaky-cleanup"; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-publish-cleanup-" }); + const stackDirectory = path.join(root, id); + const ownerJson = path.join(stackDirectory, "owner.json"); + // A single ignored attempt (the pre-fix behavior) only ever makes one call per site; + // three consecutive failures need a real retry loop to still converge. + const remaining = yield* Ref.make(3); + const flakyFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + rename: (oldPath: string, newPath: string) => + Effect.gen(function* () { + if (newPath === ownerJson && (yield* Ref.get(remaining)) > 0) { + yield* Ref.update(remaining, (count) => count - 1); + return yield* PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "rename", + pathOrDescriptor: newPath, + cause: Object.assign(new Error("injected transient rename failure"), { + code: "EBUSY", + }), + }); + } + return yield* fs.rename(oldPath, newPath); + }), + }); + const state = Context.get( + yield* Layer.build( + StackNamespace.layer({ root, platform: "win32" }).pipe(Layer.provide(flakyFs)), + ), + StackNamespace.Service, + ); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(id).pipe(Scope.provide(scope)); + const record: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(record); + + expect(yield* Ref.get(remaining), "all injected failures were consumed").toBe(0); + expect(yield* state.readHolder(id)).toEqual(record); + expect( + (yield* fs.readDirectory(stackDirectory)).filter((entry) => entry.endsWith(".tmp")), + ).toEqual([]); + yield* Scope.close(scope, Exit.void); + }), + ), + ); + + it.live("never reports a holder record missing while replacing it with a fresh one", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const id = "atomic-replace"; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-lease-atomic-" }); + const ownerJson = path.join(root, id, "owner.json"); + const gate = yield* Deferred.make(); + const entered = yield* Deferred.make(); + // Only the second publish (the replacement) is gated; the first establishes R1 normally. + let publishCount = 0; + const gatedFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + rename: (oldPath: string, newPath: string) => { + if (newPath !== ownerJson) return fs.rename(oldPath, newPath); + publishCount++; + return publishCount === 2 + ? Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(gate)), + Effect.andThen(fs.rename(oldPath, newPath)), + ) + : fs.rename(oldPath, newPath); + }, + }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root }).pipe(Layer.provide(gatedFs))), + StackNamespace.Service, + ); + const scope = yield* Scope.make(); + const lease = yield* state.acquireLease(id).pipe(Scope.provide(scope)); + const r1: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid, + startedAt: "2026-01-01T00:00:00.000Z", + }; + yield* lease.publishHolder(r1); + expect(yield* state.readHolder(id)).toEqual(r1); + + const r2: StackNamespace.LeaseHolder = { + role: "sweeper", + pid: process.pid + 1, + startedAt: "2026-01-01T00:00:01.000Z", + }; + const publishing = yield* lease.publishHolder(r2).pipe(Effect.forkScoped); + // Runs before the publisher's interruption (reverse order), so a failed assertion can't + // leave the uninterruptible publish blocked on the gate during teardown. + yield* Effect.addFinalizer(() => Deferred.succeed(gate, undefined)); + yield* Deferred.await(entered); + + // At the publication gate (the rename not yet landed), a reader must still see the prior + // record, never a gap: a remove-then-publish sequence would briefly have no file at all. + expect(yield* state.readHolder(id)).toEqual(r1); + + yield* Deferred.succeed(gate, undefined); + yield* Fiber.join(publishing); + expect(yield* state.readHolder(id)).toEqual(r2); + yield* Scope.close(scope, Exit.void); + }), + ), + ); +}); + +describe("discovery", () => { + it.live("lists identical stack identities under two state roots independently", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const base = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-discover-" }); + const projectRoot = `${base}/project`; + yield* fs.makeDirectory(projectRoot, { recursive: true }); + const locations = (root: string) => + ({ + projectRoot, + stateRoot: `${base}/${root}/state`, + cacheRoot: `${base}/${root}/cache`, + runtime: "native", + }) satisfies Parameters[0]; + const a = yield* create(locations("a")); + const b = yield* create(locations("b")); + expect(a.id).toBe(b.id); + + const listedA = yield* discover({ stateRoot: `${base}/a/state` }); + const listedB = yield* discover({ stateRoot: `${base}/b/state` }); + expect(listedA.map(({ definition }) => definition.id)).toEqual([a.id]); + expect(listedB.map(({ definition }) => definition.id)).toEqual([b.id]); + + yield* destroyTestStack(a); + yield* destroyTestStack(b); + }).pipe(Effect.scoped, Effect.provide(stackLayer)), + ); +}); + +describe("publication", () => { + // Windows has no directory fsync to fail. + it.effect.skipIf(process.platform === "win32")( + "fails publication when the directory fsync reports EIO", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const eio = (method: string, file: string) => + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method, + pathOrDescriptor: file, + cause: Object.assign(new Error("injected directory fsync failure"), { code: "EIO" }), + }); + // Covers both ways opening the directory for its own fsync can report EIO: the `open` + // call itself failing, and `open` succeeding but the handle's own `sync` failing. The + // second case is backed by a real regular-file handle (scoped like any other), not the + // real directory: opening a directory for `sync` hits Windows's own supported `EISDIR` + // path instead of ever reaching the injected failure. + for (const mode of ["open", "sync"] as const) { + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-fsync-eio-" }); + const directory = path.join(root, initial.id); + const syncProbe = path.join(root, ".sync-probe"); + yield* fs.writeFileString(syncProbe, ""); + const failingFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + open: (file: string, options?: Parameters[1]) => { + if (file !== directory || options?.flag !== "r") return fs.open(file, options); + if (mode === "open") return Effect.fail(eio("open", file)); + return fs + .open(syncProbe, options) + .pipe( + Effect.map((handle) => ({ ...handle, sync: Effect.fail(eio("fsync", file)) })), + ); + }, + }); + const store = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(failingFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + const failure = yield* store.save(initial).pipe(Effect.flip); + expect(failure, mode).toBeInstanceOf(StackNamespace.NamespaceError); + expect(failure.operation, mode).toBe("publish"); + expect(errorCode(failure.cause), mode).toBe("EIO"); + } + }), + ), + ); + + it.effect("leaves no staging file after a partial write fails with ENOSPC", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-publish-enospc-" }); + const directory = path.join(root, initial.id); + const stagingPrefix = path.join(directory, ".state.json."); + const failingFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + // A real disk ENOSPC lands after some bytes already reached the file; writing a + // truncated prefix for real before failing reproduces that instead of never creating it. + writeFileString: (file: string, content: string, options?: { readonly mode?: number }) => + file.startsWith(stagingPrefix) + ? fs.writeFileString(file, content.slice(0, 1), options).pipe( + Effect.andThen( + Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "writeFile", + pathOrDescriptor: file, + cause: Object.assign(new Error("injected partial write failure"), { + code: "ENOSPC", + }), + }), + ), + ), + ) + : fs.writeFileString(file, content, options), + }); + const store = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(failingFs)), + ).pipe(Effect.map((context) => Context.get(context, StackNamespace.Service))); + const failure = yield* store.save(initial).pipe(Effect.flip); + expect(failure).toBeInstanceOf(StackNamespace.NamespaceError); + expect( + (yield* fs.readDirectory(directory)).filter((entry) => entry.endsWith(".tmp")), + ).toEqual([]); + }), + ), + ); }); diff --git a/packages/stack/src/State.windows.integration.test.ts b/packages/stack/src/namespace/Namespace.windows.e2e.test.ts similarity index 90% rename from packages/stack/src/State.windows.integration.test.ts rename to packages/stack/src/namespace/Namespace.windows.e2e.test.ts index cc43a6ca78..acedc44d4a 100644 --- a/packages/stack/src/State.windows.integration.test.ts +++ b/packages/stack/src/namespace/Namespace.windows.e2e.test.ts @@ -13,19 +13,18 @@ import { Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "./State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; -const saved: State.SavedStack = { +const saved: StackNamespace.SavedStack = { id: "stack-main", identity: { projectRoot: "C:\\project", branchContext: "test", stackName: "windows" }, runtime: "native", instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; -class HolderError extends Data.TaggedError("StateWindowsHolderError")<{ +class HolderError extends Data.TaggedError("NamespaceWindowsHolderError")<{ readonly message: string; }> {} @@ -42,7 +41,7 @@ const testSharingViolation = () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-windows-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "namespace-windows-" }); const path = yield* Path.Path; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const target = path.join(root, saved.id, "state.json"); @@ -74,8 +73,10 @@ const testSharingViolation = () => ), }), ); - const context = yield* Layer.build(State.layer({ root }).pipe(Layer.provide(injectedFs))); - const state = Context.get(context, State.Service); + const context = yield* Layer.build( + StackNamespace.layer({ root }).pipe(Layer.provide(injectedFs)), + ); + const state = Context.get(context, StackNamespace.Service); yield* state.save(saved); yield* fs.writeFileString(path.join(root, "hold-state.ps1"), sharingHolder); @@ -167,9 +168,12 @@ const testSharingViolation = () => ); expect((yield* state.read(saved.id))?.identity.stackName).toBe("recovered"); expect(yield* Ref.get(failures).pipe(Effect.map((seen) => seen.length))).toBeGreaterThan(0); + // Only the staging document the recovered publish produced is left under the stack directory. expect( - (yield* fs.readDirectory(root)).some((entry) => entry.startsWith(".state-write-")), - ).toBe(false); + (yield* fs.readDirectory(path.join(root, saved.id))).filter((entry) => + entry.endsWith(".tmp"), + ), + ).toEqual([]); yield* Fiber.join(output); yield* Fiber.join(diagnostics); }), diff --git a/packages/stack/src/namespace/Paths.integration.test.ts b/packages/stack/src/namespace/Paths.integration.test.ts new file mode 100644 index 0000000000..b9fe0f9fea --- /dev/null +++ b/packages/stack/src/namespace/Paths.integration.test.ts @@ -0,0 +1,286 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Data, Effect, FileSystem, Path } from "effect"; +import { borrow, destroyOwnedRoot, resolveStackDataRoot } from "./Paths.ts"; + +class TestPathsError extends Data.TaggedError("TestPathsError")<{ + readonly operation: string; + readonly cause: unknown; +}> {} + +const onError = (operation: string, cause: unknown) => new TestPathsError({ operation, cause }); + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("ownership is by location", () => { + it.live("rejects a borrowed path that resolves inside the stack's data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-reject-" }); + const ownedRoot = path.join(dataRoot, "instance"); + yield* fs.makeDirectory(ownedRoot, { recursive: true }); + const inside = path.join(ownedRoot, "config.yaml"); + yield* fs.writeFileString(inside, "borrowed"); + const failure = yield* borrow(fs, path, inside, dataRoot, onError).pipe(Effect.flip); + expect(failure.operation).toBe("configure"); + }), + ), + ); + + it.live("rejects a borrowed path equal to the stack's data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-equal-" }); + const failure = yield* borrow(fs, path, dataRoot, dataRoot, onError).pipe(Effect.flip); + expect(failure.operation).toBe("configure"); + }), + ), + ); + + it.live("rejects a borrowed path inside a sibling instance under the same data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-sibling-" }); + const siblingInstance = path.join(dataRoot, "mail-instance"); + yield* fs.makeDirectory(siblingInstance, { recursive: true }); + const inside = path.join(siblingInstance, "pipeline.yaml"); + yield* fs.writeFileString(inside, "borrowed"); + const failure = yield* borrow(fs, path, inside, dataRoot, onError).pipe(Effect.flip); + expect(failure.operation).toBe("configure"); + }), + ), + ); + + it.live("accepts a borrowed path outside the stack's data root", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-accept-" }); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-caller-" }).pipe( + Effect.flatMap((root) => { + const file = path.join(root, "caller-config.yaml"); + return fs.writeFileString(file, "borrowed").pipe(Effect.as(file)); + }), + ); + const borrowed = yield* borrow(fs, path, outside, dataRoot, onError); + expect(borrowed.path).toBe(outside); + }), + ), + ); + + it.live("accepts a borrowed path that does not exist yet, under a real ancestor", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-pending-" }); + const outsideRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "owned-root-pending-out-", + }); + const notYetCreated = path.join(outsideRoot, "not-created-yet.yaml"); + yield* borrow(fs, path, notYetCreated, dataRoot, onError); + }), + ), + ); + + it.live("recursively removes a real owned root after running removeData", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-destroy-" }); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.makeDirectory(path.join(ownedRoot, "nested"), { recursive: true }); + yield* fs.writeFileString(path.join(ownedRoot, "nested", "file"), "data"); + let removeDataRan = false; + yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + Effect.sync(() => { + removeDataRan = true; + }), + onError, + ); + expect(removeDataRan).toBe(true); + expect(yield* fs.exists(ownedRoot)).toBe(false); + }), + ), + ); + + it.live("runs removeData even when the owned root was never created", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-absent-" }); + const ownedRoot = path.join(parentRoot, "instance"); + let removeDataRan = false; + yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + Effect.sync(() => { + removeDataRan = true; + }), + onError, + ); + expect(removeDataRan).toBe(true); + }), + ), + ); + + it.live("keeps listed entries after removeData, removing the rest", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-keep-" }); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.makeDirectory(ownedRoot, { recursive: true }); + yield* fs.writeFileString(path.join(ownedRoot, "snapshots"), "keep-me"); + yield* fs.writeFileString(path.join(ownedRoot, "data"), "remove-me"); + yield* destroyOwnedRoot(fs, path, ownedRoot, parentRoot, Effect.void, onError, [ + "snapshots", + ]); + expect(yield* fs.readDirectory(ownedRoot)).toEqual(["snapshots"]); + }), + ), + ); + + it.live( + "refuses to destroy a root that is a symlink to an outside directory, which survives", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-symlink-" }); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "owned-root-outside-" }); + const sentinel = path.join(outside, "sentinel"); + yield* fs.writeFileString(sentinel, "do not remove me"); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.symlink(outside, ownedRoot); + // Destructive: if confirmRealOwnedDirectory ran after this (or not at all), the sentinel + // outside the owned root would actually be deleted, not just silently skipped. + const failure = yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + fs + .remove(sentinel, { force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))), + onError, + ).pipe(Effect.flip); + expect(failure.operation).toBe("destroy"); + expect(yield* fs.exists(sentinel)).toBe(true); + }), + ), + ); + + it.live("refuses to destroy a symlinked root even when keep entries are requested", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const parentRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "owned-root-symlink-keep-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "owned-root-outside-keep-", + }); + const sentinel = path.join(outside, "sentinel"); + yield* fs.writeFileString(sentinel, "do not remove me"); + const ownedRoot = path.join(parentRoot, "instance"); + yield* fs.symlink(outside, ownedRoot); + const failure = yield* destroyOwnedRoot( + fs, + path, + ownedRoot, + parentRoot, + fs + .remove(sentinel, { force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))), + onError, + ["sentinel"], + ).pipe(Effect.flip); + expect(failure.operation).toBe("destroy"); + expect(yield* fs.exists(sentinel)).toBe(true); + }), + ), + ); +}); + +describe("the stack data root", () => { + it.live("is refused at launch when it is a symlink", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-data-root-symlink-" }); + const outside = path.join(root, "outside"); + yield* fs.makeDirectory(outside); + yield* fs.makeDirectory(path.join(root, "state", "stack"), { recursive: true }); + yield* fs.symlink(outside, path.join(root, "state", "stack", "data")); + + const failure = yield* resolveStackDataRoot(path.join(root, "state"), "stack").pipe( + Effect.flip, + ); + + expect(failure.message).toContain("symlink"); + }), + ), + ); + + it.live("is refused when the stack directory is a symlink to an outside directory", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-dir-symlink-" }); + const outside = path.join(root, "outside"); + yield* fs.makeDirectory(path.join(outside, "data"), { recursive: true }); + const survivor = path.join(outside, "data", "keep"); + yield* fs.writeFileString(survivor, "do not remove me"); + yield* fs.makeDirectory(path.join(root, "state")); + yield* fs.symlink(outside, path.join(root, "state", "stack")); + + const failure = yield* resolveStackDataRoot(path.join(root, "state"), "stack").pipe( + Effect.flip, + ); + + expect(failure.message).toContain("symlink"); + expect(yield* fs.exists(survivor)).toBe(true); + }), + ), + ); + + it.live("is refused at launch when it is a dangling symlink", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-data-root-dangling-" }); + yield* fs.makeDirectory(path.join(root, "state", "stack"), { recursive: true }); + yield* fs.symlink(path.join(root, "missing"), path.join(root, "state", "stack", "data")); + + const failure = yield* resolveStackDataRoot(path.join(root, "state"), "stack").pipe( + Effect.flip, + ); + + expect(failure.message).toContain("symlink"); + }), + ), + ); +}); diff --git a/packages/stack/src/namespace/Paths.ts b/packages/stack/src/namespace/Paths.ts new file mode 100644 index 0000000000..35cfe439e0 --- /dev/null +++ b/packages/stack/src/namespace/Paths.ts @@ -0,0 +1,207 @@ +import { Effect, FileSystem, Path } from "effect"; +import { namespaceError } from "./Capabilities.ts"; +import { lstatPath } from "./drivers/FileSystem.ts"; + +const BorrowedPathBrand: unique symbol = Symbol("Namespace.BorrowedPath"); + +/** + * A caller-supplied path the namespace only reads or mounts, never deletes. The brand makes + * {@link borrow} the only way to construct one: a service that needs a caller path in `prepare`, + * `mounts`, `env`, or `args` must hold a `BorrowedPath`, so a path that was never validated against + * the stack's data root cannot be used in its place. + */ +export interface BorrowedPath { + readonly path: string; + readonly [BorrowedPathBrand]: true; +} + +/** + * Resolves `target`'s real path, walking up to its nearest existing ancestor and rejoining the + * remaining segments when it (or an ancestor) does not exist yet, so a symlinked ancestor cannot + * smuggle a not-yet-created path inside an owned root. + */ +const resolveReal = ( + fs: FileSystem.FileSystem, + path: Path.Path, + target: string, + onError: (operation: string, cause: unknown) => E, +): Effect.Effect => + Effect.gen(function* () { + const exists = yield* fs + .exists(target) + .pipe(Effect.mapError((cause) => onError("configure", cause))); + if (exists) + return yield* fs + .realPath(target) + .pipe(Effect.mapError((cause) => onError("configure", cause))); + const parent = path.dirname(target); + if (parent === target) return path.resolve(target); + const realParent = yield* resolveReal(fs, path, parent, onError); + return path.join(realParent, path.basename(target)); + }); + +/** + * The only way to construct a {@link BorrowedPath}. Ownership is by location: everything under the + * stack's data root belongs to the stack, so a borrowed (read-only) caller path must resolve + * neither to it nor inside it — checked by real path, equality included, after resolving through + * any ancestor symlink. Call before preparation or launch uses `candidate`. + */ +export const borrow = Effect.fn("Namespace.Paths.borrow")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + candidate: string, + dataRoot: string, + onError: (operation: string, cause: unknown) => E, + ): Effect.Effect => + Effect.gen(function* () { + const [realCandidate, realDataRoot] = yield* Effect.all([ + resolveReal(fs, path, candidate, onError), + resolveReal(fs, path, dataRoot, onError), + ]); + if (realCandidate === realDataRoot || realCandidate.startsWith(`${realDataRoot}${path.sep}`)) + return yield* Effect.fail( + onError("configure", `${candidate} resolves inside the owned data root ${dataRoot}`), + ); + return { path: candidate, [BorrowedPathBrand]: true }; + }), +); + +/** + * The data-root label its containers carry, derived without creating the directory, for a stack + * whose directory may be gone; it equals {@link resolveStackDataRoot} while the directory exists. + */ +export const stackDataRootLabel = Effect.fn("Namespace.Paths.stackDataRootLabel")(function* ( + stateRoot: string, + stackId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + return path.join(yield* fs.realPath(stateRoot), stackId, "data"); +}); + +/** The directory that holds a stack's persisted service logs, beside its data root. */ +export const stackLogsRoot = (path: Path.Path, stateRoot: string, stackId: string): string => + path.join(path.normalize(stateRoot), stackId, "logs"); + +/** The container mount target for an instance-scoped directory owned on the host. */ +export const containerInstancePath = "/instance"; + +/** Owned per-launch environment-file directory name under a stack's data root. */ +export const CONTAINER_ENV_DIRNAME = ".container-env"; + +/** + * Confirms `root` is a real directory (not a symlink or junction) whose real path resolves inside + * `parentRoot`'s real path. {@link destroyOwnedRoot} runs this before any destructive step, + * including the `keep` branch, so a symlink planted at `root` (or replacing it between launches) + * is refused rather than traversed or removed. + */ +const confirmRealOwnedDirectory = ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + parentRoot: string, + onError: (operation: string, cause: unknown) => E, +): Effect.Effect => + Effect.gen(function* () { + const info = yield* lstatPath(root).pipe(Effect.mapError((cause) => onError("destroy", cause))); + if (info?.type === "SymbolicLink") + return yield* Effect.fail(onError("destroy", `${root} is a symlink, not an owned directory`)); + if (info === undefined || info.type !== "Directory") + return yield* Effect.fail(onError("destroy", `${root} is not a directory`)); + const realRoot = yield* fs + .realPath(root) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + const realParent = yield* fs + .realPath(parentRoot) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + if (realRoot !== realParent && !realRoot.startsWith(`${realParent}${path.sep}`)) + return yield* Effect.fail(onError("destroy", `${root} resolves outside ${parentRoot}`)); + }); + +/** + * Destroys the owned root `root` (a child of `parentRoot`), confirming it first in every branch, + * including `keep`, and before `removeData` runs. `removeData` always runs, even when `root` was + * never created on disk, since a recipe's data may live entirely outside the host filesystem (an + * engine volume, for example). One race is accepted and left unguarded: an ancestor directory + * replaced between the confirmation below and the removal that follows it — a stack's own owner is + * the only writer of its roots. + */ +export const destroyOwnedRoot = Effect.fn("Namespace.Paths.destroyOwnedRoot")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + parentRoot: string, + removeData: Effect.Effect, + onError: (operation: string, cause: unknown) => E, + /** Root entries kept after removeData; when empty the root itself is removed. */ + keep: ReadonlyArray = [], + ): Effect.Effect => + Effect.gen(function* () { + const exists = yield* fs + .exists(root) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + if (exists) yield* confirmRealOwnedDirectory(fs, path, root, parentRoot, onError); + yield* removeData; + if (!exists) return; + if (keep.length === 0) { + yield* fs + .remove(root, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + return; + } + const names = yield* fs + .readDirectory(root) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + for (const name of names) + if (!keep.includes(name)) + yield* fs + .remove(path.join(root, name), { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => onError("destroy", cause))); + }), +); + +/** + * Refuses a stack directory that is a symlink or resolves outside the state root, and a stack's + * `data` directory when it is a symlink (dangling or not), not a directory, or resolves outside the + * stack directory, so nothing is created in or removed from a target the stack does not own. A + * missing directory passes: only launch creates it. + */ +export const confirmStackDataRoot = Effect.fn("Namespace.Paths.confirmStackDataRoot")(function* ( + stateRoot: string, + stackId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stackRoot = path.join(stateRoot, stackId); + const dataRoot = path.join(stackRoot, "data"); + const stackInfo = yield* lstatPath(stackRoot).pipe( + Effect.mapError((cause) => namespaceError("destroy", cause)), + ); + if (stackInfo !== undefined) + yield* confirmRealOwnedDirectory(fs, path, stackRoot, stateRoot, namespaceError); + const info = yield* lstatPath(dataRoot).pipe( + Effect.mapError((cause) => namespaceError("destroy", cause)), + ); + if (info !== undefined) + yield* confirmRealOwnedDirectory(fs, path, dataRoot, stackRoot, namespaceError); +}); + +/** + * The canonical data root of a stack: its `data` directory under `stateRoot`, created if missing + * and resolved through symlinks. Launch and every cleanup path derive container labels and native + * socket directories from this one value, so each reaches the same resources however the state + * root is spelled. A `data` directory that is itself a symlink is refused. + */ +export const resolveStackDataRoot = Effect.fn("Namespace.Paths.resolveStackDataRoot")(function* ( + stateRoot: string, + stackId: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dataRoot = path.join(stateRoot, stackId, "data"); + yield* confirmStackDataRoot(stateRoot, stackId); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + return yield* fs.realPath(dataRoot); +}); diff --git a/packages/stack/src/namespace/Pin.integration.test.ts b/packages/stack/src/namespace/Pin.integration.test.ts new file mode 100644 index 0000000000..116f4e0c6d --- /dev/null +++ b/packages/stack/src/namespace/Pin.integration.test.ts @@ -0,0 +1,71 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Scope, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { pin } from "./Pin.ts"; + +/** Reports whether a separate process can take an EXCLUSIVE lock on `lockPath` right now. */ +const exclusiveLockOutcome = (lockPath: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const script = ` + import { DatabaseSync } from "node:sqlite"; + const database = new DatabaseSync(process.argv[1]); + database.exec("PRAGMA busy_timeout = 0"); + try { + database.exec("BEGIN EXCLUSIVE"); + console.log("acquired"); + } catch (error) { + if (error.errcode !== 5) { + console.error(error); + process.exit(1); + } + console.log("busy"); + } + `; + const child = yield* spawner.spawn( + ChildProcess.make(process.execPath, ["-e", script, lockPath], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + }), + ); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(Number(exitCode), stderr).toBe(0); + return stdout.trim(); + }), + ); + +describe("pins", () => { + it.live( + "another process stays excluded after a second in-process pin on the same file is released", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-pin-" }); + const lockPath = `${directory}/generation.lock`; + + const holderScope = yield* Scope.make(); + yield* pin(lockPath).pipe(Scope.provide(holderScope)); + const competitorScope = yield* Scope.make(); + yield* pin(lockPath).pipe(Scope.provide(competitorScope)); + yield* Scope.close(competitorScope, Exit.void); + + expect(yield* exclusiveLockOutcome(lockPath)).toBe("busy"); + + yield* Scope.close(holderScope, Exit.void); + expect(yield* exclusiveLockOutcome(lockPath)).toBe("acquired"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + 20_000, + ); +}); diff --git a/packages/stack/src/namespace/Pin.ts b/packages/stack/src/namespace/Pin.ts new file mode 100644 index 0000000000..cb8af6472e --- /dev/null +++ b/packages/stack/src/namespace/Pin.ts @@ -0,0 +1,29 @@ +import { DateTime, Effect, FileSystem, Schedule, Scope } from "effect"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import { acquireLock, isBusy, takeSharedLock } from "./drivers/Sqlite.ts"; + +/** Rides out a retirement sweep's brief EXCLUSIVE hold instead of failing a concurrent pin. */ +const pinRetrySchedule = Schedule.spaced("10 millis").pipe( + Schedule.upTo({ duration: "2 seconds" }), +); + +/** + * Holds a SHARED lock on `lockPath` (one generation's digest lock file) for the life of the + * enclosing scope: every consumer of a prepared generation's paths must stay inside this scope, and + * must not resolve or create `lockPath`'s generation directory before this resolves. Touches the + * lock file's mtime on acquisition, which is retirement's only liveness signal for a holder in + * another process. The caller must ensure `lockPath`'s parent directory already exists. + */ +export const pin = Effect.fn("Pin.pin")(function* ( + lockPath: string, +): Effect.fn.Return { + const fs = yield* FileSystem.FileSystem; + const connection = yield* acquireLock(lockPath, "create"); + yield* takeSharedLock(connection).pipe( + Effect.retry({ schedule: pinRetrySchedule, while: isBusy }), + ); + const now = DateTime.toDateUtc(yield* DateTime.now); + yield* fs + .utimes(lockPath, now, now) + .pipe(Effect.mapError((cause) => namespaceError("pin-touch", cause))); +}); diff --git a/packages/stack/src/namespace/PortReservations.ts b/packages/stack/src/namespace/PortReservations.ts new file mode 100644 index 0000000000..e8eb458360 --- /dev/null +++ b/packages/stack/src/namespace/PortReservations.ts @@ -0,0 +1,361 @@ +import { userInfo } from "node:os"; +import type { DatabaseSync } from "node:sqlite"; +import { NodeServices } from "@effect/platform-node"; +import { Context, Effect, FileSystem, Layer, Path, Predicate, Scope, Stream } from "effect"; +import { ChildProcess } from "effect/unstable/process"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import { restrictDirectoryToOwner } from "../runtime/postgres-user.ts"; +import { errcode, openDatabase } from "./drivers/Sqlite.ts"; +import { lstatPath } from "./drivers/FileSystem.ts"; + +/** Identifies the stack that owns a port reservation. */ +export interface Holder { + readonly stackId: string; + readonly stateRoot: string; +} + +const SQLITE_CONSTRAINT_PRIMARYKEY = 1555; + +export interface Interface { + /** The port this stack already reserves for `endpoint`, if it holds one. */ + readonly find: ( + stateRoot: string, + stackId: string, + endpoint: string, + ) => Effect.Effect; + /** + * Reserves `port` for `(stateRoot, stackId, endpoint)`. Resolves to `undefined` once the row is + * committed, or the live {@link Holder} already occupying `port` when it is taken. + */ + readonly reserve: ( + stateRoot: string, + stackId: string, + endpoint: string, + port: number, + ) => Effect.Effect; + /** + * Atomically replaces `port`'s row with `(stateRoot, stackId, endpoint)`, but only while it still + * belongs to `expected`. Resolves to `false` without changing anything once another reservation + * has already claimed, reclaimed, or released it. + */ + readonly reclaim: ( + port: number, + expected: Holder, + reservation: { + readonly stateRoot: string; + readonly stackId: string; + readonly endpoint: string; + }, + ) => Effect.Effect; + /** Releases one endpoint's reservation; a no-op when it holds none. */ + readonly release: ( + stateRoot: string, + stackId: string, + endpoint: string, + ) => Effect.Effect; + /** Releases every reservation a stack holds, at its state root. */ + readonly releaseStack: ( + stateRoot: string, + stackId: string, + ) => Effect.Effect; +} + +export class Service extends Context.Service()( + "@supabase/stack/PortReservations", +) {} + +const schema = ` +CREATE TABLE IF NOT EXISTS reservation ( + port INTEGER PRIMARY KEY, + state_root TEXT NOT NULL, + stack_id TEXT NOT NULL, + endpoint TEXT NOT NULL, + UNIQUE (state_root, stack_id, endpoint) +)`; + +interface HolderRow { + readonly state_root: string; + readonly stack_id: string; +} + +const isPortRow = (value: unknown): value is { readonly port: number } => + Predicate.hasProperty(value, "port") && typeof value.port === "number"; + +const isHolderRow = (value: unknown): value is HolderRow => + Predicate.hasProperty(value, "state_root") && + typeof value.state_root === "string" && + Predicate.hasProperty(value, "stack_id") && + typeof value.stack_id === "string"; + +const connectionFor = (file: string): Effect.Effect => + openDatabase(file, (connection) => { + // Short and bounded: real contention between two of this user's processes is brief, and + // a blocking wait here is simpler than an async retry loop for a database this small. Set + // before the schema statement, so even first-time table creation waits out a racing peer + // instead of surfacing a spurious busy error. + connection.exec("PRAGMA busy_timeout = 2000"); + connection.exec(schema); + }); + +/** + * Confirms `directory` (created 0700 if missing) is a real directory this uid owns, with the + * no-follow {@link lstatPath} so a symlink planted ahead of us is refused rather than traversed. + * Validates before changing any mode, so a symlink is never followed by `chmod` either. The + * directory is an ancestor of native data, so an existing traverse-only grant is kept. + */ +const securePrivateDirectory = Effect.fn("PortReservations.securePrivateDirectory")(function* ( + fs: FileSystem.FileSystem, + directory: string, +) { + yield* fs + .makeDirectory(directory, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => namespaceError("root", cause))); + const info = yield* lstatPath(directory).pipe( + Effect.mapError((cause) => namespaceError("root", cause)), + ); + if (info === undefined || info.type !== "Directory") + return yield* namespaceError("root", `${directory} is not a directory`); + const uid = process.getuid?.(); + if (uid !== undefined && info.uid !== uid) + return yield* namespaceError( + "root", + `${directory} belongs to uid ${String(info.uid)}, not the current user`, + ); + yield* restrictDirectoryToOwner(fs, directory).pipe( + Effect.mapError((cause) => namespaceError("root", cause)), + ); +}); + +/** `getent passwd ` prints one `name:passwd:uid:gid:gecos:dir:shell` line; field 6 is home. */ +const parseGetent = (output: string): string | undefined => output.trim().split(":")[5]; + +/** `dscacheutil -q user -a uid ` prints `key: value` lines; `dir:` is home. */ +const parseDscacheutil = (output: string): string | undefined => { + for (const line of output.split("\n")) { + const match = /^dir:\s*(.+)$/u.exec(line); + if (match?.[1] !== undefined) return match[1]; + } + return undefined; +}; + +/** + * Resolves the current user's home directly from the OS, never `$HOME`: on POSIX, `os.userInfo()` + * itself prefers `$HOME` once a process starts with it set, which would let a sandboxed or + * overridden environment move the registry. `getent`/`dscacheutil` read the system database + * instead, so a caller's environment cannot steer this. Windows has neither a POSIX uid nor those + * tools, so there `os.userInfo().homedir` reads the user profile directly instead. + */ +const resolvePasswdHome = Effect.fn("PortReservations.resolvePasswdHome")(function* () { + if (process.platform === "win32") { + const home = userInfo().homedir; + if (home.length === 0) + return yield* namespaceError("home", "Unable to resolve the Windows user profile home"); + return home; + } + const uid = process.getuid?.(); + if (uid === undefined) + return yield* namespaceError( + "home", + "No uid to resolve a passwd home for (not a POSIX process)", + ); + const [executable, args, parse] = + process.platform === "darwin" + ? (["dscacheutil", ["-q", "user", "-a", "uid", String(uid)], parseDscacheutil] as const) + : (["getent", ["passwd", String(uid)], parseGetent] as const); + yield* Effect.annotateCurrentSpan({ + "process.executable.name": executable, + "process.arg_count": args.length, + }); + const output = yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* ChildProcess.make(executable, args, { + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + const [text, exitCode] = yield* Effect.all( + [child.stdout.pipe(Stream.decodeText, Stream.mkString), child.exitCode], + { concurrency: 2 }, + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(exitCode)); + return Number(exitCode) === 0 ? text : ""; + }), + ).pipe( + Effect.timeout("5 seconds"), + Effect.mapError((cause) => namespaceError("home", cause)), + ); + const home = parse(output); + if (home === undefined || home.length === 0) + return yield* namespaceError("home", `Unable to resolve the passwd home for uid ${uid}`); + return home; +}); + +/** + * Computed once per process, replayed for every later caller: the OS passwd home for this uid, + * always through the real Node spawner. Tests regularly swap `ChildProcessSpawner` to sandbox + * container orchestration; this low-level OS fact is not part of that contract, and must resolve + * the same way regardless of what the ambient context provides. + */ +const resolvedHome: Effect.Effect = Effect.runSync( + Effect.cached(resolvePasswdHome().pipe(Effect.provide(NodeServices.layer))), +); + +/** + * One SQLite database per OS user at `/.supabase/ports.sqlite`, the only authority on + * which stack owns a public port across every state root on this machine. + */ +const make = Effect.fn("PortReservations.make")(function* (): Effect.fn.Return< + Interface, + NamespaceError, + FileSystem.FileSystem | Path.Path | Scope.Scope +> { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const directory = path.join(yield* resolvedHome, ".supabase"); + yield* securePrivateDirectory(fs, directory); + const connection = yield* connectionFor(path.join(directory, "ports.sqlite")); + + const toHolder = (row: HolderRow): Holder => ({ + stackId: row.stack_id, + stateRoot: row.state_root, + }); + + const find = Effect.fn("PortReservations.find")(function* ( + stateRoot: string, + stackId: string, + endpoint: string, + ) { + const row = yield* Effect.try({ + try: () => + connection + .prepare( + "SELECT port FROM reservation WHERE state_root = ? AND stack_id = ? AND endpoint = ?", + ) + .get(stateRoot, stackId, endpoint), + catch: (cause) => namespaceError("find", cause), + }); + return isPortRow(row) ? row.port : undefined; + }); + + const holderOf = (port: number): HolderRow | undefined => { + const row = connection + .prepare("SELECT state_root, stack_id FROM reservation WHERE port = ?") + .get(port); + return isHolderRow(row) ? row : undefined; + }; + + // Untraced: `Ports.ts`'s auto scan calls `reserve` and `reclaim` once per candidate port, and its + // own span already records the attempt and failure counts for the whole scan. + const reserve = Effect.fnUntraced(function* ( + stateRoot: string, + stackId: string, + endpoint: string, + port: number, + ) { + return yield* Effect.try({ + try: () => { + const insertRow = () => + connection + .prepare( + "INSERT INTO reservation (port, state_root, stack_id, endpoint) VALUES (?, ?, ?, ?)", + ) + .run(port, stateRoot, stackId, endpoint); + connection.exec("BEGIN IMMEDIATE"); + try { + try { + insertRow(); + } catch (error) { + if (!errcode(error, SQLITE_CONSTRAINT_PRIMARYKEY)) throw error; + // Still holding the write lock taken above: the conflicting row cannot change until we + // commit or roll back, so this read is a consistent view of who really holds `port`. + const row = holderOf(port); + if (row !== undefined) { + connection.exec("ROLLBACK"); + return toHolder(row); + } + // The row that caused the conflict is already gone within this same transaction's own + // view (for example a concurrent release just ahead of us); nothing else can touch it + // while we hold the lock, so retrying the insert now must succeed. + insertRow(); + } + connection.exec("COMMIT"); + return undefined; + } catch (error) { + connection.exec("ROLLBACK"); + throw error; + } + }, + catch: (cause) => namespaceError("reserve", cause), + }); + }); + + const reclaim = Effect.fnUntraced(function* ( + port: number, + expected: Holder, + reservation: { + readonly stateRoot: string; + readonly stackId: string; + readonly endpoint: string; + }, + ) { + return yield* Effect.try({ + try: () => { + connection.exec("BEGIN IMMEDIATE"); + try { + const current = holderOf(port); + if ( + current === undefined || + current.state_root !== expected.stateRoot || + current.stack_id !== expected.stackId + ) { + connection.exec("ROLLBACK"); + return false; + } + connection.prepare("DELETE FROM reservation WHERE port = ?").run(port); + connection + .prepare( + "INSERT INTO reservation (port, state_root, stack_id, endpoint) VALUES (?, ?, ?, ?)", + ) + .run(port, reservation.stateRoot, reservation.stackId, reservation.endpoint); + connection.exec("COMMIT"); + return true; + } catch (error) { + connection.exec("ROLLBACK"); + throw error; + } + }, + catch: (cause) => namespaceError("reclaim", cause), + }); + }); + + const release = Effect.fn("PortReservations.release")(function* ( + stateRoot: string, + stackId: string, + endpoint: string, + ) { + yield* Effect.try({ + try: () => + connection + .prepare("DELETE FROM reservation WHERE state_root = ? AND stack_id = ? AND endpoint = ?") + .run(stateRoot, stackId, endpoint), + catch: (cause) => namespaceError("release", cause), + }); + }); + + const releaseStack = Effect.fn("PortReservations.releaseStack")(function* ( + stateRoot: string, + stackId: string, + ) { + yield* Effect.try({ + try: () => + connection + .prepare("DELETE FROM reservation WHERE state_root = ? AND stack_id = ?") + .run(stateRoot, stackId), + catch: (cause) => namespaceError("release", cause), + }); + }); + + return { find, reserve, reclaim, release, releaseStack }; +}); + +export const layer = Layer.effect(Service, make().pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/namespace/Publication.ts b/packages/stack/src/namespace/Publication.ts new file mode 100644 index 0000000000..6b3c2558ec --- /dev/null +++ b/packages/stack/src/namespace/Publication.ts @@ -0,0 +1,45 @@ +import { Effect, FileSystem, Path } from "effect"; +import { retrySharingViolation } from "../internal/sharing-violation.ts"; +import { namespaceError } from "./Capabilities.ts"; +import * as Drivers from "./drivers/FileSystem.ts"; + +export interface PublishOptions { + readonly target: string; + readonly content: string; + readonly platform?: NodeJS.Platform; +} + +let counter = 0; + +/** + * Writes `content` as `target`, so a reader only ever sees the prior document or the complete new + * one. Stages the full content in `target`'s own directory and fsyncs it, publishes it with an + * atomic rename, then fsyncs the directory. The staging path and its cleanup are reserved before + * anything is written to it, so a write that fails partway (for example `ENOSPC`) still leaves no + * file behind; a process killed mid publish leaves only the staging file, never a partially + * written target. + */ +export const publish = Effect.fn("Namespace.publish")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + options: PublishOptions, +) { + const directory = path.dirname(options.target); + const staging = path.join( + directory, + `.${path.basename(options.target)}.${process.pid}-${++counter}.tmp`, + ); + const retry = retrySharingViolation(options.platform); + yield* Effect.acquireUseRelease( + Effect.succeed(staging), + () => + Effect.gen(function* () { + yield* fs.writeFileString(staging, options.content, { mode: 0o600 }); + yield* Drivers.fsyncFile(fs, staging); + yield* retry(fs.rename(staging, options.target)); + yield* Drivers.fsyncDirectory(fs, directory); + }), + // The rename already clears the staging name on success; only a failed attempt leaves it. + () => retry(fs.remove(staging, { force: true })).pipe(Effect.ignore), + ).pipe(Effect.mapError((cause) => namespaceError("publish", cause))); +}); diff --git a/packages/stack/src/namespace/Registry.integration.test.ts b/packages/stack/src/namespace/Registry.integration.test.ts new file mode 100644 index 0000000000..2afdec4030 --- /dev/null +++ b/packages/stack/src/namespace/Registry.integration.test.ts @@ -0,0 +1,53 @@ +import { NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Effect, Fiber, FileSystem, Layer, Path } from "effect"; +import { TestClock } from "effect/testing"; +import * as StackNamespace from "../StackNamespace.ts"; + +const RETENTION_SECONDS = 11 * 60; + +it.effect( + "pruning a destroyed shell cannot remove directories while its id is being registered", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-registry-prune-" }); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root })), + StackNamespace.Service, + ); + const id = "recreated"; + const directory = path.join(root, id); + yield* fs.makeDirectory(path.join(directory, "data"), { recursive: true }); + const destroyedAtSeconds = 1_767_225_600; + yield* fs.utimes(directory, destroyedAtSeconds, destroyedAtSeconds); + yield* TestClock.setTime((destroyedAtSeconds + RETENTION_SECONDS) * 1000); + + const scope = yield* Effect.scope; + yield* state.withLock( + Effect.gen(function* () { + const pruning = yield* state.pruneDestroyed("other").pipe(Effect.forkIn(scope)); + // The controlled clock moves the prune's lock retries along until it finishes, whether + // by pruning or by giving up on the busy lock. + yield* Fiber.await(pruning).pipe( + Effect.race(TestClock.adjust("50 millis").pipe(Effect.forever)), + ); + expect(yield* fs.exists(path.join(directory, "data"))).toBe(true); + yield* state.save({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "test", stackName: id }, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + }); + }), + ); + + expect(yield* fs.exists(path.join(directory, "data"))).toBe(true); + expect((yield* state.read(id))?.id).toBe(id); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/namespace/Registry.ts b/packages/stack/src/namespace/Registry.ts new file mode 100644 index 0000000000..38aa0b5768 --- /dev/null +++ b/packages/stack/src/namespace/Registry.ts @@ -0,0 +1,288 @@ +import { Clock, Effect, Exit, FileSystem, Option, Path, Schedule, Schema, Scope } from "effect"; +import { CompositionConfig } from "../Orchestrator.ts"; +import { ServiceCreation } from "../services/Catalog.ts"; +import { SafeId } from "../identity/SafeId.ts"; +import { retrySharingViolation } from "../internal/sharing-violation.ts"; +import { namespaceError, type NamespaceError } from "./Capabilities.ts"; +import { OWNER_FILE, OWNER_LOG_FILE } from "./Lease.ts"; +import { stackLogsRoot } from "./Paths.ts"; +import * as Publication from "./Publication.ts"; +import { removeEmptyDirectory } from "./drivers/FileSystem.ts"; +import { acquireLock, isBusy, takeLock } from "./drivers/Sqlite.ts"; + +/** How long a destroyed stack's empty directories stay, so a same-id restart finds them. */ +const DESTROYED_RETENTION_MILLIS = 10 * 60 * 1000; + +const SavedInstance = Schema.Struct({ + id: SafeId, + creation: Schema.toCodecJson(ServiceCreation), +}); +interface SavedInstance extends Schema.Schema.Type {} + +/** API and JWT signing keys that override the keys a stack derives by default. */ +export const StackKeysInput = Schema.Struct({ + publishableKey: Schema.optionalKey(Schema.String), + secretKey: Schema.optionalKey(Schema.String), + anonKey: Schema.optionalKey(Schema.String), + anonKeyIsOverride: Schema.optionalKey(Schema.Boolean), + serviceRoleKey: Schema.optionalKey(Schema.String), + serviceRoleKeyIsOverride: Schema.optionalKey(Schema.Boolean), + gotrueJwtKeys: Schema.optionalKey(Schema.String), + publicSigningKeys: Schema.optionalKey(Schema.String), + remoteJwks: Schema.optionalKey(Schema.String), +}); +export interface StackKeysInput extends Schema.Schema.Type {} + +export const StackCredentials = Schema.Struct({ + jwtSecret: Schema.String, + postgresRootKey: Schema.String, + databasePassword: Schema.String, + publishableKey: Schema.String, + secretKey: Schema.String, + anonKey: Schema.String, + serviceRoleKey: Schema.String, + jwks: Schema.String, + gotrueJwtKeys: Schema.String, + remoteJwks: Schema.String, + anonKeyIsOverride: Schema.Boolean, + serviceRoleKeyIsOverride: Schema.Boolean, +}); +export interface StackCredentials extends Schema.Schema.Type {} + +/** A session stack is destroyed when its creator exits; a detached stack outlives it. */ +export const StackLifetime = Schema.Literals(["session", "detached"]); +export type StackLifetime = Schema.Schema.Type; + +export const SavedStack = Schema.Struct({ + id: SafeId, + lifetime: StackLifetime, + identity: Schema.Struct({ + projectRoot: Schema.String, + branchContext: Schema.String, + stackName: Schema.String, + }), + runtime: Schema.Literals(["native", "docker", "podman"]), + instances: Schema.Array(SavedInstance).check( + Schema.makeFilter((instances) => + new Set(instances.map(({ id }) => id)).size === instances.length + ? undefined + : "Expected unique instance ids", + ), + ), + composition: CompositionConfig, + credentials: Schema.optionalKey(StackCredentials), +}); +export interface SavedStack extends Schema.Schema.Type {} + +export interface Interface { + readonly read: (id: string) => Effect.Effect; + /** Skips each stack entry that stays unreadable after transient retries, reporting it to `onInvalidState`. */ + readonly list: Effect.Effect, NamespaceError>; + readonly save: (state: SavedStack) => Effect.Effect; + readonly remove: (id: string) => Effect.Effect; + /** Removes the empty directories `remove` left for stacks destroyed over ten minutes ago, other than `keep`. */ + readonly pruneDestroyed: (keep: string) => Effect.Effect; + /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ + readonly withLock: ( + effect: Effect.Effect, + ) => Effect.Effect; +} + +const checkId = (id: string): Effect.Effect => + Schema.is(SafeId)(id) + ? Effect.void + : Effect.fail(namespaceError("identity", `Invalid state id: ${id}`)); + +const decodeState = ( + text: string, + id: string, + target: string, +): Effect.Effect => + Schema.decodeEffect(Schema.fromJsonString(SavedStack))(text).pipe( + Effect.mapError((cause) => + namespaceError( + "decode", + `Unable to decode state ${target} for stack ${id}: ${cause.message}`, + ), + ), + ); + +export interface Options { + readonly root: string; + readonly platform?: NodeJS.Platform; + readonly onInvalidState?: (id: string, error: NamespaceError) => Effect.Effect; +} + +export const make = ( + options: Options, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = options.root; + const lock = path.join(root, ".registry-lock.sqlite"); + + const stackRoot = (id: string) => path.join(root, id); + const statePath = (id: string) => path.join(stackRoot(id), "state.json"); + const retryRead = (effect: Effect.Effect) => + effect.pipe( + retrySharingViolation(options.platform), + Effect.mapError((cause) => namespaceError("read", cause)), + ); + const read = Effect.fn("Namespace.Registry.read")(function* (id: string) { + yield* checkId(id); + const target = statePath(id); + const exists = yield* fs.exists(target).pipe(retryRead); + if (!exists) return undefined; + const text = yield* fs.readFileString(target).pipe(retryRead); + const state = yield* decodeState(text, id, target); + if (state.id !== id) + return yield* namespaceError("identity", "State document identity does not match its path"); + return state; + }); + const stackIds = fs.readDirectory(root).pipe( + Effect.map((entries) => entries.filter(Schema.is(SafeId))), + Effect.mapError((cause) => namespaceError("list", cause)), + ); + const readEntries = ( + readEntry: (id: string) => Effect.Effect, + onSkipped: (id: string, error: NamespaceError) => Effect.Effect, + ) => + Effect.gen(function* () { + const entries: Array = []; + for (const id of yield* stackIds) { + const value = yield* readEntry(id).pipe( + Effect.catch((error) => onSkipped(id, error).pipe(Effect.as(undefined))), + ); + if (value !== undefined) entries.push(value); + } + return entries; + }); + const list = Effect.fn("Namespace.Registry.list")(() => + readEntries(read, (id, error) => options.onInvalidState?.(id, error) ?? Effect.void), + ); + const save = Effect.fn("Namespace.Registry.save")(function* (state: SavedStack) { + yield* checkId(state.id); + yield* fs + .makeDirectory(stackRoot(state.id), { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => namespaceError("write", cause))); + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(SavedStack))(state).pipe( + Effect.mapError((cause) => namespaceError("encode", cause)), + ); + yield* Publication.publish(fs, path, { + target: statePath(state.id), + content: serialized, + platform: options.platform, + }); + }); + const remove = Effect.fn("Namespace.Registry.remove")(function* (id: string) { + yield* checkId(id); + // Logs go before the state file, so a stack whose logs remain stays listed for another removal. + yield* fs.remove(stackLogsRoot(path, root, id), { recursive: true, force: true }).pipe( + retrySharingViolation(options.platform), + Effect.mapError((cause) => namespaceError("remove", cause)), + ); + for (const file of [ + statePath(id), + path.join(stackRoot(id), OWNER_FILE), + path.join(stackRoot(id), OWNER_LOG_FILE), + ]) + yield* fs + .remove(file, { force: true }) + .pipe(Effect.mapError((cause) => namespaceError("remove", cause))); + // The stack and data directories stay: deleting and recreating them for a restart under the + // same id lets the container engine's file share report fresh bind sources as missing. + }); + const isDestroyedShell = Effect.fnUntraced(function* (id: string, now: number) { + const directory = stackRoot(id); + if ((yield* fs.stat(directory)).type !== "Directory") return false; + const entries = yield* fs.readDirectory(directory); + if (entries.length !== 1 || entries[0] !== "data") return false; + const data = path.join(directory, "data"); + if ((yield* fs.stat(data)).type !== "Directory") return false; + if ((yield* fs.readDirectory(data)).length > 0) return false; + // Read after the contents, so a destroy that just emptied the directory counts as recent. + const modified = Option.getOrUndefined((yield* fs.stat(directory)).mtime)?.getTime(); + return modified !== undefined && now - modified >= DESTROYED_RETENTION_MILLIS; + }); + /** + * One attempt: forks `attemptScope` under `guardScope` before opening anything, so there is + * always a scope to close if the open or the `BEGIN IMMEDIATE` below fails or defects. Runs + * uninterruptibly as a single unit; the retry waiting on busy, and `effect` once this + * succeeds, both stay interruptible around it. + */ + const acquireAttempt = ( + guardScope: Scope.Scope, + ): Effect.Effect => + Effect.uninterruptible( + Effect.gen(function* () { + const attemptScope = yield* Scope.fork(guardScope, "sequential"); + yield* acquireLock(lock, "create").pipe( + Scope.provide(attemptScope), + Effect.flatMap(takeLock), + Effect.onExit((exit) => + Exit.isFailure(exit) ? Scope.close(attemptScope, exit) : Effect.void, + ), + ); + return attemptScope; + }), + ); + /** + * Holds the lock in its own standalone scope, so `effect`'s ambient scope (for example a port + * listener it binds) stays whatever the caller already had, not one this lock closes early. + */ + const withLock = Effect.fn("Namespace.Registry.withLock")( + (effect: Effect.Effect) => + Effect.acquireUseRelease( + Scope.make(), + (guardScope) => + acquireAttempt(guardScope).pipe( + Effect.retry({ + schedule: Schedule.spaced("50 millis").pipe( + Schedule.upTo({ duration: "5 seconds" }), + ), + while: isBusy, + }), + Effect.catchIf(isBusy, () => + namespaceError( + "lock", + "Stack registry is locked by another operation; retry shortly", + ), + ), + Effect.andThen(effect), + ), + (guardScope, exit) => Scope.close(guardScope, exit), + ), + ); + + const pruneDestroyed = Effect.fn("Namespace.Registry.pruneDestroyed")(function* (keep: string) { + const now = yield* Clock.currentTimeMillis; + let removed = 0; + for (const id of yield* stackIds) { + if (id === keep) continue; + // Rechecked under the registry lock: a same-id recreate either saved its state before the + // check or waits until the directories are gone. + const pruned = yield* Effect.gen(function* () { + if (!(yield* isDestroyedShell(id, now))) return false; + return yield* withLock( + Effect.gen(function* () { + if (!(yield* isDestroyedShell(id, now))) return false; + yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); + yield* removeEmptyDirectory(stackRoot(id)); + return true; + }), + ); + }).pipe( + Effect.catch((error) => + Effect.logWarning(`Could not prune destroyed stack directory ${id}`, error).pipe( + Effect.as(false), + ), + ), + ); + if (pruned) removed += 1; + } + yield* Effect.annotateCurrentSpan({ removed }); + }); + + return { read, list: list(), save, remove, pruneDestroyed, withLock }; + }); diff --git a/packages/stack/src/namespace/drivers/FileSystem.ts b/packages/stack/src/namespace/drivers/FileSystem.ts new file mode 100644 index 0000000000..962c0185e9 --- /dev/null +++ b/packages/stack/src/namespace/drivers/FileSystem.ts @@ -0,0 +1,77 @@ +import { Effect, FileSystem, Predicate } from "effect"; +import { systemError, type PlatformError } from "effect/PlatformError"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has neither a no-follow stat nor a non-recursive directory removal. +import { lstat, rmdir } from "node:fs/promises"; +import { namespaceError, type NamespaceError } from "../Capabilities.ts"; + +export interface LstatInfo { + readonly type: "Directory" | "SymbolicLink" | "File" | "Other"; + readonly uid: number; + readonly mode: number; +} + +const toLstatInfo = (stats: Awaited>): LstatInfo => ({ + type: stats.isSymbolicLink() + ? "SymbolicLink" + : stats.isDirectory() + ? "Directory" + : stats.isFile() + ? "File" + : "Other", + uid: Number(stats.uid), + mode: Number(stats.mode), +}); + +const toLstatError = (path: string, cause: unknown): PlatformError => + systemError({ + _tag: Predicate.hasProperty(cause, "code") && cause.code === "ENOENT" ? "NotFound" : "Unknown", + module: "FileSystem", + method: "lstat", + pathOrDescriptor: path, + cause, + }); + +/** + * No-follow metadata for `path`: a single `lstat` syscall that never resolves a symlink, closing + * the time-of-check gap between a probe and a later symlink-following stat. Resolves to `undefined` + * when `path` has no entry at all. + */ +export const lstatPath = (path: string): Effect.Effect => + Effect.tryPromise({ try: () => lstat(path), catch: (cause) => toLstatError(path, cause) }).pipe( + Effect.map(toLstatInfo), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + // oxlint-disable-next-line effecttsgo/effect-succeed-with-void -- the success type is `undefined`, not `void`. + () => Effect.succeed(undefined), + ), + ); + +/** Removes a directory only if it is already empty; any other failure propagates. */ +export const removeEmptyDirectory = (directory: string): Effect.Effect => + Effect.tryPromise({ + try: () => rmdir(directory), + catch: (cause) => namespaceError("cleanup", cause), + }).pipe( + Effect.catchIf( + (error) => + Predicate.hasProperty(error.cause, "code") && + ["ENOENT", "ENOTEMPTY", "EEXIST"].includes(String(error.cause.code)), + () => Effect.void, + ), + ); + +/** Flushes a file's contents to durable storage. */ +export const fsyncFile = ( + fs: FileSystem.FileSystem, + target: string, +): Effect.Effect => + Effect.scoped(fs.open(target, { flag: "r+" }).pipe(Effect.flatMap((file) => file.sync))); + +/** Flushes a directory's own metadata; a no-op on Windows, which can't open or fsync directories. */ +export const fsyncDirectory = ( + fs: FileSystem.FileSystem, + directory: string, +): Effect.Effect => + process.platform === "win32" + ? Effect.void + : Effect.scoped(fs.open(directory, { flag: "r" }).pipe(Effect.flatMap((file) => file.sync))); diff --git a/packages/stack/src/namespace/drivers/Sqlite.ts b/packages/stack/src/namespace/drivers/Sqlite.ts new file mode 100644 index 0000000000..2c062b0281 --- /dev/null +++ b/packages/stack/src/namespace/drivers/Sqlite.ts @@ -0,0 +1,126 @@ +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no OS-owned cross-process lock primitive. +import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; +import { Effect, Predicate, Scope } from "effect"; +import { namespaceError, type NamespaceError } from "../Capabilities.ts"; +import { takeSharedLockSync } from "./sqlite-pin.ts"; + +const openConnection = (path: string, mode: "create" | "existing") => + Effect.try({ + try: () => + mode === "create" + ? new DatabaseSync(path) + : new DatabaseSync(new URL(`${pathToFileURL(path).href}?mode=rw`)), + catch: (cause) => namespaceError("lock-open", cause), + }); +const closeConnection = (connection: DatabaseSync) => + Effect.try({ + // Rolls back first: closing a connection mid-transaction must release its lock immediately, + // and must not depend on `close()` alone to do so. A connection with no open transaction (for + // example `acquireLock("create")` with no lock ever taken) has nothing to roll back. + try: () => { + try { + connection.exec("ROLLBACK"); + } catch { + // No open transaction; nothing to roll back. + } + connection.close(); + }, + catch: (cause) => namespaceError("lock-close", cause), + }); + +/** + * Opens `path` for the enclosing scope and runs `init` once on the new connection; a failing + * `init` closes the connection before the error propagates. + */ +export const openDatabase = ( + path: string, + init: (connection: DatabaseSync) => void, +): Effect.Effect => + Effect.acquireRelease( + Effect.try({ + try: () => { + const connection = new DatabaseSync(path); + try { + init(connection); + } catch (error) { + connection.close(); + throw error; + } + return connection; + }, + catch: (cause) => namespaceError("open", cause), + }), + (connection) => + closeConnection(connection).pipe( + Effect.catch((error) => Effect.logWarning(`Unable to close ${path}`, error)), + ), + ); + +/** Matches a `NamespaceError`'s cause against one of `node:sqlite`'s numeric `errcode`s. */ +export const errcode = (cause: unknown, code: number) => + Predicate.hasProperty(cause, "errcode") && cause.errcode === code; +/** SQLite reports lock contention under `busy_timeout = 0` with this errcode. */ +export const isBusy = (error: NamespaceError) => errcode(error.cause, 5); +/** `acquireLock("existing")` reports this errcode when the database file does not exist. */ +export const isMissing = (error: NamespaceError) => errcode(error.cause, 14); +/** + * Opens `path`'s connection for the enclosing scope. Every connection this process opens on a + * path shares one SQLite library instance, which tracks locks per inode across connections and + * keeps them held until the last one closes, so a second in-process connection contends with the + * first exactly as another process would. + */ +export const acquireLock = ( + path: string, + mode: "create" | "existing", +): Effect.Effect => + Effect.acquireRelease(openConnection(path, mode), (connection) => + closeConnection(connection).pipe( + Effect.catch((error) => Effect.logWarning(`Unable to close ${path}`, error)), + ), + ); + +/** + * Takes the connection's RESERVED write lock once; fails busy under any contention instead of + * waiting. `BEGIN IMMEDIATE` alone excludes other writers, but not an existing reader's SHARED + * lock (it only escalates to EXCLUSIVE, conflicting with readers, when a write is attempted), so + * this is enough for writer-versus-writer exclusion only, such as one staging slot's own lock. + */ +export const takeLock = (connection: DatabaseSync): Effect.Effect => + Effect.gen(function* () { + yield* Effect.try({ + try: () => connection.exec("PRAGMA busy_timeout = 0"), + catch: (cause) => namespaceError("lock", cause), + }); + yield* Effect.try({ + try: () => connection.exec("BEGIN IMMEDIATE"), + catch: (cause) => namespaceError("lock", cause), + }); + }); + +/** + * Takes the connection's EXCLUSIVE lock once, immediately, in rollback-journal mode: unlike + * `BEGIN IMMEDIATE`, this conflicts with an existing reader's SHARED lock right away, which is + * what retirement needs to safely exclude a live pin. Fails busy under any contention. + */ +export const takeExclusiveLock = (connection: DatabaseSync): Effect.Effect => + Effect.gen(function* () { + yield* Effect.try({ + try: () => connection.exec("PRAGMA busy_timeout = 0"), + catch: (cause) => namespaceError("lock", cause), + }); + yield* Effect.try({ + try: () => connection.exec("BEGIN EXCLUSIVE"), + catch: (cause) => namespaceError("lock", cause), + }); + }); + +/** + * Takes the connection's SHARED read lock once: a real `SELECT` against a lazily created one-row + * table, held until the connection commits or closes. Busy under a concurrent EXCLUSIVE holder. + */ +export const takeSharedLock = (connection: DatabaseSync): Effect.Effect => + Effect.try({ + try: () => takeSharedLockSync(connection), + catch: (cause) => namespaceError("lock", cause), + }); diff --git a/packages/stack/src/namespace/drivers/sqlite-pin.ts b/packages/stack/src/namespace/drivers/sqlite-pin.ts new file mode 100644 index 0000000000..f7d496bede --- /dev/null +++ b/packages/stack/src/namespace/drivers/sqlite-pin.ts @@ -0,0 +1,37 @@ +// Framework-agnostic: shared by the Effect-wrapped driver (`Sqlite.ts`, used in-process) and the +// standalone native launcher, which runs outside any Effect runtime and calls these directly. +import type { DatabaseSync } from "node:sqlite"; + +/** + * The one-row table a pin's `SELECT` reads. It writes only when missing, so later pins never take a + * write-intent lock that would conflict with a concurrent reader. + */ +const ensurePinTable = (connection: DatabaseSync): void => { + const hasTable = connection + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'pin'") + .get(); + if (hasTable === undefined) + connection.exec("CREATE TABLE IF NOT EXISTS pin (id INTEGER PRIMARY KEY)"); + const hasRow = connection.prepare("SELECT 1 FROM pin LIMIT 1").get(); + if (hasRow === undefined) connection.exec("INSERT OR IGNORE INTO pin (id) VALUES (1)"); +}; + +/** + * Takes a SHARED read lock: a deferred `BEGIN` plus a real `SELECT`, held until the connection + * commits or closes. Multiple SHARED holders coexist; only a concurrent EXCLUSIVE holder (a + * retirement sweep) contends, surfacing as SQLite's busy errcode under `busy_timeout = 0`. + */ +export const takeSharedLockSync = (connection: DatabaseSync): void => { + connection.exec("PRAGMA busy_timeout = 0"); + ensurePinTable(connection); + connection.exec("BEGIN"); + try { + connection.prepare("SELECT id FROM pin LIMIT 1").get(); + } catch (error) { + // A deferred `BEGIN` takes no lock by itself; only this `SELECT` does, so a failure here + // still leaves the transaction open. Roll back before propagating, or a retry on this same + // connection would hit "cannot start a transaction within a transaction" instead of busy. + connection.exec("ROLLBACK"); + throw error; + } +}; diff --git a/packages/stack/src/port-reservation.e2e.test.ts b/packages/stack/src/port-reservation.e2e.test.ts new file mode 100644 index 0000000000..1d55cd9476 --- /dev/null +++ b/packages/stack/src/port-reservation.e2e.test.ts @@ -0,0 +1,404 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Cause, Context, Effect, Exit, FileSystem, Layer, Option } from "effect"; +import * as Net from "node:net"; +import { create, open, type Observation } from "./effect.ts"; +import { ownerExitProbe, waitForOwnerExit } from "./HostProcess.ts"; +import * as PortReservations from "./namespace/PortReservations.ts"; +import { captureOwnerPid } from "../tests/owner.ts"; +import { destroyTestStack } from "../tests/stack-cleanup.ts"; +import { testEngine } from "../tests/test-engine.ts"; +import type { ContainerEngine } from "./runtime/Container.ts"; + +/** + * The per-user port reservation registry: a stopped stack keeps its automatic ports across state + * roots unless another stack's configured port takes one over, a restart recovers the same ports + * or fails with a structured `PortConflict`, and destroy (but never stop or a kill) releases them. + */ + +const servicesLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(servicesLayer)); + +const mailOn = (port: number | "auto") => + ({ service: "mail" as const, config: {}, endpoints: { http: { port } } }) as const; + +const portOf = (observation: Observation): number => { + const endpoint = observation.endpoints.find((candidate) => candidate.name === "http"); + if (endpoint === undefined) throw new Error(`Missing http endpoint in ${observation.id}`); + return endpoint.port; +}; + +/** A plain external listener, not a stack, holding `host:port` outside this package entirely. */ +const foreignListener = (host: string, port: number) => + Effect.acquireRelease( + Effect.callback((resume) => { + const server = Net.createServer(); + server.once("error", (cause) => resume(Effect.fail(cause))); + server.listen(port, host, () => resume(Effect.succeed(server))); + }), + (server) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }), + ); + +const makeStack = Effect.fn("PortReservation.makeStack")(function* ( + fs: FileSystem.FileSystem, + cacheRoot: string, + runtime: "native" | ContainerEngine = "native", +) { + const root = yield* fs.makeTempDirectoryScoped({ prefix: `port-reservation-${runtime}-` }); + const stack = yield* create({ + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot, + runtime, + }); + yield* Effect.addFinalizer(() => destroyTestStack(stack)); + return { stack, stateRoot: `${root}/state` }; +}); + +const conflictHolderStackId = (exit: Exit.Exit) => { + if (Exit.isSuccess(exit)) return undefined; + const failure = Cause.findErrorOption(exit.cause); + if (Option.isNone(failure)) return undefined; + const error = failure.value as { readonly conflict?: { readonly holder?: unknown } }; + const holder = error.conflict?.holder; + return typeof holder === "object" && holder !== null && "stackId" in holder + ? (holder as { readonly stackId: string }).stackId + : undefined; +}; + +const conflictIsForeign = (exit: Exit.Exit) => { + if (Exit.isSuccess(exit)) return false; + const failure = Cause.findErrorOption(exit.cause); + if (Option.isNone(failure)) return false; + const error = failure.value as { readonly conflict?: { readonly holder?: unknown } }; + return error.conflict?.holder === "foreign"; +}; + +/** + * Runs `use` with `HOME` pointed at `home`; a spawned owner inherits it at spawn time. Restored + * afterward, so this only ever affects the one owner launched inside `use`. + */ +const withFakeHome = (home: string, use: Effect.Effect) => + Effect.acquireUseRelease( + Effect.sync(() => { + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- a one-shot, restored test fixture, not application config. + const previous = process.env.HOME; + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- see above. + process.env.HOME = home; + return previous; + }), + () => use, + (previous) => + Effect.sync(() => { + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- restores the mutation made above. + if (previous === undefined) delete process.env.HOME; + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- restores the mutation made above. + else process.env.HOME = previous; + }), + ); + +it.live( + "a configured port takes over a stopped stack's automatic port in another state root, and the stopped stack restarts on a new automatic port", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "port-reservation-cache-" }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.stop; + + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + yield* mailB.start; + yield* mailB.ready; + expect(portOf(yield* mailB.status)).toBe(port); + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).not.toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a SIGKILLed owner's automatic reservation survives, the stack restarts on the same port, and destroy then releases it", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-kill-cache-", + }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + const pid = yield* captureOwnerPid({ stateRoot: stateRootA, cacheRoot }, stackA.id); + yield* Effect.sync(() => process.kill(pid, "SIGKILL")); + // The owner's own exit already confirms the kernel has reclaimed its listeners. + yield* waitForOwnerExit(pid, ownerExitProbe(fs)); + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + + yield* reopenedA.destroy; + const portReservations = Context.get( + yield* Layer.build(PortReservations.layer), + PortReservations.Service, + ); + expect( + yield* portReservations.find( + yield* fs.realPath(stateRootA), + stackA.id, + `${mailA.id}:http`, + ), + ).toBeUndefined(); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a foreign listener occupying a stopped stack's saved port fails its restart as a PortConflict(foreign), never reassigning it", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-foreign-cache-", + }); + const { stack: stackA, stateRoot: stateRootA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + yield* stackA.stop; + + const reopenedA = yield* open({ id: stackA.id, stateRoot: stateRootA, cacheRoot }); + const reopenedMailA = yield* reopenedA.services.get(mailA.id); + + // A process outside the registry occupies A's saved port for exactly this inner scope. + yield* Effect.scoped( + Effect.gen(function* () { + yield* foreignListener("127.0.0.1", port); + const restartA = yield* Effect.exit(reopenedMailA.start); + expect(Exit.isFailure(restartA)).toBe(true); + expect(conflictIsForeign(restartA)).toBe(true); + }), + ); + + // Never reassigned: once the foreign listener goes away, the retry recovers the same port. + yield* reopenedMailA.start; + yield* reopenedMailA.ready; + expect(portOf(yield* reopenedMailA.status)).toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "deleting a stopped stack's state root makes its reserved ports reclaimable by another stack", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-reclaim-cache-", + }); + // Plain (unscoped): this test deletes `root` itself, so no scope finalizer should also + // try to remove it afterward. + const root = yield* fs.makeTempDirectory({ prefix: "port-reservation-reclaim-a-" }); + const stateRootA = `${root}/state`; + const stackA = yield* create({ + projectRoot: root, + stateRoot: stateRootA, + cacheRoot, + runtime: "native", + }); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.stop; + + // The whole state root disappears without going through destroy (an unmounted volume, or a + // deleted checkout): a confirmed ENOENT on its registration is what makes its row stale. + yield* fs.remove(root, { recursive: true, force: true }); + + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + yield* mailB.start; + yield* mailB.ready; + expect(portOf(yield* mailB.status)).toBe(port); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "a native and a Docker stack racing for the same reserved port never both succeed", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-backend-cache-", + }); + // Destroyed, not merely stopped, so the port is genuinely free for B and C to race over; + // a stopped stack would keep the row and both racers would lose to it instead. + const { stack: stackA } = yield* makeStack(fs, cacheRoot, "native"); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + yield* stackA.destroy; + + const { stack: stackB } = yield* makeStack(fs, cacheRoot, "native"); + const { stack: stackC } = yield* makeStack(fs, cacheRoot, testEngine); + const mailB = yield* stackB.services.create(mailOn(port)); + const mailC = yield* stackC.services.create(mailOn(port)); + const [exitB, exitC] = yield* Effect.all( + [Effect.exit(mailB.start), Effect.exit(mailC.start)], + { concurrency: "unbounded" }, + ); + const successes = [exitB, exitC].filter(Exit.isSuccess); + expect(successes).toHaveLength(1); + const loser = Exit.isFailure(exitB) ? exitB : exitC; + expect(conflictHolderStackId(loser)).toBeDefined(); + }), + ), + { timeout: 180_000 }, +); + +it.live( + "an owner spawned with HOME pointed elsewhere still reserves in the real per-user registry", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-home-cache-", + }); + const { stack: stackA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + const fakeHome = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-fake-home-", + }); + // The owner is spawned lazily, on this stack's first RPC call; the fake HOME must still be + // set when that happens, so the whole setup runs inside the same window. + const startB = yield* withFakeHome( + fakeHome, + Effect.gen(function* () { + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const mailB = yield* stackB.services.create(mailOn(port)); + return yield* Effect.exit(mailB.start); + }), + ); + expect(Exit.isFailure(startB)).toBe(true); + expect(conflictHolderStackId(startB)).toBe(stackA.id); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "composing a member on an already-claimed port surfaces a structured conflict naming the holder", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-composition-cache-", + }); + const { stack: stackA } = yield* makeStack(fs, cacheRoot); + const mailA = yield* stackA.services.create(mailOn("auto")); + yield* mailA.start; + yield* mailA.ready; + const port = portOf(yield* mailA.status); + + // The composition registers every member's public port up front, including a lazy + // one like mail, so the conflict surfaces here rather than at a later `.start()`. + const { stack: stackB } = yield* makeStack(fs, cacheRoot); + const composeB = yield* Effect.exit(stackB.composition.supabase([mailOn(port)])); + expect(Exit.isFailure(composeB)).toBe(true); + expect(conflictHolderStackId(composeB)).toBe(stackA.id); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "restarting a composition surfaces a structured conflict from one member's own failing outcome", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-composition-outcome-cache-", + }); + const { stack } = yield* makeStack(fs, cacheRoot); + const members = yield* stack.composition.supabase([mailOn("auto")]); + yield* stack.composition.start; + const statuses = yield* Effect.forEach(members, (member) => member.status); + const mail = statuses.find((status) => status.config.service === "mail"); + if (mail === undefined) return yield* Effect.die("mail missing from the composition"); + const port = portOf(mail); + yield* stack.composition.stop; + + // Composing and starting cleanly the first time means the top-level failure below cannot + // itself carry the conflict; only `startComposition`'s own per-member outcome does. + yield* foreignListener("127.0.0.1", port); + const restart = yield* Effect.exit(stack.composition.start); + expect(Exit.isFailure(restart)).toBe(true); + expect(conflictIsForeign(restart)).toBe(true); + }), + ), + { timeout: 120_000 }, +); + +it.live( + "pinning a public port inside the native range fails naming the range and reserves nothing", + () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "port-reservation-native-range-cache-", + }); + const { stack, stateRoot } = yield* makeStack(fs, cacheRoot); + const mail = yield* stack.services.create(mailOn(15_432)); + + const failure = yield* Effect.flip(mail.start); + + expect(failure.message).toContain("10000-19999"); + expect(failure.message).toContain("reserved for native service backends"); + const reservationContext = yield* Layer.build(PortReservations.layer); + const portReservations = Context.get(reservationContext, PortReservations.Service); + expect( + yield* portReservations.find(yield* fs.realPath(stateRoot), stack.id, `${mail.id}:http`), + ).toBeUndefined(); + }), + ), + { timeout: 120_000 }, +); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 1b21d3c92e..5ed94c3beb 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -7,13 +7,21 @@ import { Path, PlatformError, Predicate, - Schema, + Scope, } from "effect"; import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; import { validateRelativePath, validateSha256 } from "./Integrity.ts"; import { restrictDirectoryToOwner } from "../runtime/postgres-user.ts"; +import { fsyncDirectory } from "../namespace/drivers/FileSystem.ts"; +import { + acquireLock, + isMissing, + takeExclusiveLock, + takeLock, +} from "../namespace/drivers/Sqlite.ts"; +import * as Pin from "../namespace/Pin.ts"; /** A concrete artifact identity. `key` may contain subdirectories but never an absolute or traversing path. */ export interface ArtifactRequest { @@ -56,10 +64,12 @@ export interface ArtifactStoreOptions { readonly source: ArtifactSource; } -interface PreparedArtifact { +export interface PreparedArtifact { readonly key: string; - /** Installed artifact directory. Required runtime paths are relative to this directory. */ + /** Installed, content-addressed generation directory. Required runtime paths are relative to it. */ readonly path: string; + /** The generation's digest lock file; pinned by `use` for the life of its returned scope. */ + readonly lockPath: string; readonly sha256: string; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath?: string; @@ -68,14 +78,14 @@ interface PreparedArtifact { type ArtifactStoreError = PreparationError | ArtifactIntegrityError; -const ARTIFACT_FORMAT = "supabase-stack-artifact-v3"; -const METADATA_NAME = ".artifact.json"; -const EXECUTABLE_MODE = 0o755; -/** - * Child of the staging directory that sources materialize into: GNU tar resets its extraction - * target's mtime from the archive, which must not make a staging directory look like a leftover. - */ +/** Child of a staging directory that sources materialize into, renamed whole to publish. */ const STAGING_CONTENT_NAME = "content"; +/** Per-key directory holding every in-progress preparer's and retirer's staging slot. */ +const STAGING_DIR_NAME = ".staging"; +const DIRECTORY_MODE = 0o755; +/** A generation is eligible for retirement once its digest lock file's mtime is this old. */ +const RETIREMENT_AGE_MILLIS = 30 * 24 * 60 * 60 * 1000; +const DIGEST_NAME = /^[0-9a-f]{64}$/u; type ArtifactPathKind = "file" | "directory" | "symlink"; @@ -91,19 +101,6 @@ const artifactError = (message: string, fields: Readonly const metadataError = (message: string, fields: Readonly> = {}) => new ArtifactIntegrityError({ ...fields, message }); -const ArtifactMetadataSchema = Schema.Struct({ - format: Schema.Literal(ARTIFACT_FORMAT), - key: Schema.String, - sha256: Schema.String, - requiredRuntimePaths: Schema.Array(Schema.String), - requiredRuntimeKinds: Schema.Record( - Schema.String, - Schema.Literals(["file", "directory", "symlink"]), - ), - executablePath: Schema.optional(Schema.String), -}); -type ArtifactMetadata = Schema.Schema.Type; - const mapFs = ( path: string, operation: string, @@ -163,47 +160,11 @@ const validateRequest = (request: ArtifactRequest): Effect.Effect>, -): ArtifactMetadata => ({ - format: ARTIFACT_FORMAT, - key: request.key, - sha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - requiredRuntimeKinds, - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), -}); - -const encodeMetadata = (metadata: ArtifactMetadata): Effect.Effect => - Schema.encodeEffect(Schema.fromJsonString(ArtifactMetadataSchema))(metadata).pipe( - Effect.mapError((cause) => - artifactError(`Unable to encode artifact metadata: ${String(cause)}`), - ), - ); - -const readMetadata = ( - fs: FileSystem.FileSystem, - metadataPath: string, -): Effect.Effect, ArtifactIntegrityError> => - Effect.matchEffect(fs.readFileString(metadataPath), { - onFailure: (cause) => - isNotFound(cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - metadataError("Cached artifact metadata is unreadable", { - path: metadataPath, - cause, - }), - ), - onSuccess: (text) => - Schema.decodeEffect(Schema.fromJsonString(ArtifactMetadataSchema))(text).pipe( - Effect.map(Option.some), - Effect.orElseSucceed(() => Option.none()), - ), - }); - +/** + * Creates `directory` (and any missing ancestors up to `canonicalRoot`) and restricts every + * segment along the way to its owner, preserving a traverse-only grant: every container level + * under the cache root, not just the leaf, must stay owner-restricted as the tree grows. + */ const ensureDirectory = ( fs: FileSystem.FileSystem, path: Path.Path, @@ -220,7 +181,14 @@ const ensureDirectory = ( for (const segment of relative.split(path.sep).filter((value) => value.length > 0)) { current = path.join(current, segment); const exists = yield* mapFs(current, "inspect artifact directory", fs.exists(current)); - if (!exists) break; + if (!exists) + // `recursive: true` tolerates a concurrent preparer that just created this very segment + // between the `exists` check above and this call; it is otherwise a single-directory make. + yield* mapFs( + current, + "create artifact directory", + fs.makeDirectory(current, { recursive: true, mode: 0o700 }), + ); const real = yield* fs.realPath(current).pipe( Effect.mapError((cause) => artifactError(`Unable to resolve artifact directory: ${cause.message}`, { @@ -233,25 +201,8 @@ const ensureDirectory = ( return yield* artifactError("Artifact directory contains a symlink", { path: current }); if (!pathAtOrBelow(root, real, path.sep)) return yield* artifactError("Artifact directory escapes cache root", { path: current }); + yield* mapFs(current, "secure artifact directory", restrictDirectoryToOwner(fs, current)); } - yield* mapFs( - resolved, - "create artifact directory", - fs.makeDirectory(resolved, { recursive: true, mode: 0o700 }), - ); - const real = yield* fs.realPath(resolved).pipe( - Effect.mapError((cause) => - artifactError(`Unable to resolve artifact directory: ${cause.message}`, { - path: resolved, - cause, - }), - ), - ); - if (real !== resolved) - return yield* artifactError("Artifact directory contains a symlink", { path: resolved }); - if (!pathAtOrBelow(root, real, path.sep)) - return yield* artifactError("Artifact directory escapes cache root", { path: resolved }); - yield* mapFs(resolved, "secure artifact directory", restrictDirectoryToOwner(fs, resolved)); }); const ensureSafeRoot = ( @@ -447,12 +398,12 @@ const validateFreshRuntimePaths = ( return Object.fromEntries(entries); }); -const ensureSafePaths = ( +/** Shallow containment+kind check for an already-published, trusted generation: no recursion. */ +const inspectRequiredPaths = ( fs: FileSystem.FileSystem, path: Path.Path, root: string, realRoot: string, - metadata: ArtifactMetadata, relativePaths: ReadonlyArray, ): Effect.Effect>, ArtifactIntegrityError> => Effect.gen(function* () { @@ -471,27 +422,22 @@ const ensureSafePaths = ( ), ); if (!exists) - return yield* metadataError("Cached artifact is missing a required runtime path", { - path: relative, - }); + return yield* metadataError( + `Cached artifact is missing ${candidate}; remove ${root} to reinstall it`, + { path: relative }, + ); const inspected = yield* inspectFreshPath(fs, path, candidate, realRoot); - const expectedKind = metadata.requiredRuntimeKinds[relative]; - if (expectedKind === undefined || inspected.kind !== expectedKind) - return yield* metadataError("Cached artifact runtime path changed basic kind", { - path: relative, - expected: expectedKind, - actual: inspected.kind, - }); entries.push([relative, inspected]); } return Object.fromEntries(entries); }); +/** Kind-only executable check, used before a fresh publish normalizes its final mode. */ const ensureExecutableFile = ( fs: FileSystem.FileSystem, executable: string, -): Effect.Effect => { - return fs.stat(executable).pipe( +): Effect.Effect => + fs.stat(executable).pipe( Effect.mapError((cause) => metadataError("Cached artifact executable cannot be inspected", { path: executable, @@ -509,73 +455,32 @@ const ensureExecutableFile = ( ), ), ); -}; -const identityMismatch = (request: ArtifactRequest, metadata: ArtifactMetadata): boolean => - metadata.key !== request.key || metadata.executablePath !== request.executablePath; - -const unrecordedRequiredPaths = ( - request: ArtifactRequest, - metadata: ArtifactMetadata, -): ReadonlyArray => - request.requiredRuntimePaths.filter( - (relative) => metadata.requiredRuntimeKinds[relative] === undefined, - ); - -const sha256Of = ( - request: ArtifactRequest, - metadata: ArtifactMetadata, -): Effect.Effect => - validateSha256(metadata.sha256).pipe( +/** A published generation is never chmodded again, so a cache hit must check the bit directly. */ +const ensureExecutableMode = ( + fs: FileSystem.FileSystem, + generation: string, + executable: string, +): Effect.Effect => + fs.stat(executable).pipe( Effect.mapError((cause) => - metadataError("Cached artifact metadata contains an invalid SHA-256", { - key: request.key, + metadataError("Cached artifact executable cannot be inspected", { + path: executable, cause, }), ), + Effect.flatMap((info) => + (info.mode & 0o100) !== 0 + ? Effect.void + : Effect.fail( + metadataError( + `Cached artifact executable ${executable} is missing its executable bit; remove ${generation} to reinstall it`, + { path: executable }, + ), + ), + ), ); -const writeBytesSync = ( - fs: FileSystem.FileSystem, - path: string, - bytes: Uint8Array, -): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(path, { flag: "wx", mode: 0o600 }).pipe( - Effect.mapError((cause) => - artifactError(`Unable to create artifact source file: ${cause.message}`, { - path, - cause, - }), - ), - ); - yield* file.writeAll(bytes).pipe( - Effect.mapError((cause) => - artifactError(`Unable to write artifact source file: ${cause.message}`, { - path, - cause, - }), - ), - ); - yield* file.sync.pipe( - Effect.mapError((cause) => - artifactError(`Unable to sync artifact source file: ${cause.message}`, { path, cause }), - ), - ); - }), - ); - -const writeMetadataSync = ( - fs: FileSystem.FileSystem, - path: string, - metadata: ArtifactMetadata, -): Effect.Effect => - Effect.gen(function* () { - const encoded = yield* encodeMetadata(metadata); - yield* writeBytesSync(fs, path, new TextEncoder().encode(encoded)); - }); - const cleanup = (fs: FileSystem.FileSystem, path: string): Effect.Effect => fs .remove(path, { recursive: true, force: true }) @@ -585,85 +490,290 @@ const cleanup = (fs: FileSystem.FileSystem, path: string): Effect.Effect => - Effect.gen(function* () { - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - artifactError(`Unable to allocate artifact metadata temporary name: ${cause.message}`, { - path: metadataPath, - cause, - }), + cacheRoot: string, + source: ArtifactSource, + request: ArtifactRequest, +): Effect.fn.Return { + const keyRoot = path.resolve(cacheRoot, request.key); + if (!pathWithin(cacheRoot, keyRoot, path.sep)) + return yield* artifactError("Artifact key escapes cache root", { key: request.key }); + const expectedSha256 = yield* source.checksum(request).pipe( + Effect.flatMap((sha256) => + validateSha256(sha256).pipe( + Effect.mapError((cause) => + metadataError("Artifact source returned an invalid SHA-256", { + key: request.key, + cause, + }), + ), ), - ); - const temporary = path.join(path.dirname(metadataPath), `${METADATA_NAME}.${token}.tmp`); - yield* Effect.gen(function* () { - yield* writeMetadataSync(fs, temporary, metadata); - yield* mapFs( - metadataPath, - "update cached artifact metadata", - fs.rename(temporary, metadataPath), - ); - }).pipe(Effect.onExit(() => cleanup(fs, temporary))); - }); + ), + ); + return { + keyRoot, + stagingRoot: path.join(keyRoot, STAGING_DIR_NAME), + expectedSha256, + generationPath: path.join(keyRoot, expectedSha256), + lockPath: path.join(keyRoot, `${expectedSha256}.lock`), + }; +}); -const orphanMaxAgeMillis = 24 * 60 * 60 * 1000; +const stagingDirFor = (path: Path.Path, stagingRoot: string, token: string) => + path.join(stagingRoot, token); +const stagingLockFor = (path: Path.Path, stagingRoot: string, token: string) => + path.join(stagingRoot, `${token}.lock`); -const leftoverToken = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u; +/** A staging lock's own file, or the rollback journal SQLite leaves while a transaction is open. */ +const STAGING_LOCK_SUFFIX = /\.lock(-journal)?$/u; -/** Matches the temporary and quarantine names the store creates beside `targetName`. */ -const isLeftoverOf = (targetName: string, name: string): boolean => { - const prefix = `.${targetName}.`; - const suffix = [".tmp", ".invalid"].find((candidate) => name.endsWith(candidate)); - return ( - suffix !== undefined && - name.startsWith(prefix) && - leftoverToken.test(name.slice(prefix.length, name.length - suffix.length)) +/** + * Every `.staging` token with a directory. A bare lock file is skipped: its owner creates the + * lock before taking it and the directory only afterwards, so reaping it could unlink a lock + * about to be taken. + */ +const listStagingTokens = ( + fs: FileSystem.FileSystem, + stagingRoot: string, +): Effect.Effect> => + fs.readDirectory(stagingRoot).pipe( + Effect.map((names) => names.filter((name) => !STAGING_LOCK_SUFFIX.test(name))), + Effect.orElseSucceed(() => []), ); -}; /** - * Best-effort removal of one target's temp/quarantine siblings that a hard kill left behind. - * Only entries older than `orphanMaxAgeMillis` are removed, so a concurrent in-progress download - * by another process is never touched. + * Removes a `.staging/` directory and its lock file only when the lock can be taken + * without blocking: a live preparer or retirer holding it keeps this from ever touching + * in-progress work. Best-effort: any failure (including contention) is silently skipped. */ -const reapLeftoversOf = ( +const reapStagingToken = ( + fs: FileSystem.FileSystem, + path: Path.Path, + stagingRoot: string, + token: string, +): Effect.Effect => { + const lockPath = stagingLockFor(path, stagingRoot, token); + const dir = stagingDirFor(path, stagingRoot, token); + return Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing").pipe( + Effect.catchIf(isMissing, () => acquireLock(lockPath, "create")), + ); + yield* takeLock(connection); + yield* fs.remove(dir, { recursive: true, force: true }); + yield* fs.remove(lockPath, { force: true }); + yield* fs.remove(`${lockPath}-journal`, { force: true }); + }), + ).pipe(Effect.ignore); +}; + +const reapStaleStaging = ( fs: FileSystem.FileSystem, path: Path.Path, - target: string, + stagingRoot: string, ): Effect.Effect => + listStagingTokens(fs, stagingRoot).pipe( + Effect.flatMap((tokens) => + Effect.forEach(tokens, (token) => reapStagingToken(fs, path, stagingRoot, token), { + discard: true, + concurrency: 1, + }), + ), + ); + +/** A symlink is never a cache entry: the cache neither walks nor writes through one. */ +const isSymlink = (fs: FileSystem.FileSystem, target: string): Effect.Effect => + // readLink succeeds only on a symlink. + fs.readLink(target).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + +interface CacheWalk { + readonly generations: ReadonlyArray<{ readonly keyRoot: string; readonly digest: string }>; + readonly stagingRoots: ReadonlyArray; +} + +/** + * Recursively walks the cache root and returns every generation (a digest-named directory) and + * every `.staging` root, without ever descending into either: a key embeds its release version + * (see `Artifacts.ts`'s `artifactKey`), so a CLI upgrade leaves a previous release's generations + * behind a now-unreachable key that only a root-wide walk like this ever revisits. A plain + * directory-name listing, never a generation's own (potentially large) published contents, keeps + * this cheap. + */ +const walkCache = ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, +): Effect.Effect => Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - const parent = path.dirname(target); - const targetName = path.basename(target); - const names = yield* fs.readDirectory(parent); - yield* Effect.forEach( - names.filter((name) => isLeftoverOf(targetName, name)), - (name) => { - const candidate = path.join(parent, name); - return fs.stat(candidate).pipe( - Effect.flatMap((info) => - Option.match(info.mtime, { - onNone: () => Effect.void, - onSome: (mtime) => - now - mtime.getTime() > orphanMaxAgeMillis - ? fs.remove(candidate, { recursive: true, force: true }) - : Effect.void, - }), - ), - Effect.ignore, - ); - }, - { discard: true }, - ); - }).pipe(Effect.ignore); + const names = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); + const generations: Array = []; + const stagingRoots: Array = []; + for (const name of names) { + if (yield* isSymlink(fs, path.join(root, name))) continue; + if (name === STAGING_DIR_NAME) { + stagingRoots.push(path.join(root, name)); + continue; + } + if (DIGEST_NAME.test(name)) { + generations.push({ keyRoot: root, digest: name }); + continue; + } + const nested = yield* walkCache(fs, path, path.join(root, name)); + generations.push(...nested.generations); + stagingRoots.push(...nested.stagingRoots); + } + return { generations, stagingRoots }; + }); -const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( +/** + * Retires one generation. The digest lock is held only for the age re-check and the rename into + * a lock-guarded staging slot, so a concurrent pin is never blocked behind the recursive delete; + * the slot lock keeps the delete from being reaped while it runs. A crash after the rename leaves + * an unlocked staging directory that the next sweep reaps. + */ +const retireGeneration = Effect.fn("ArtifactStore.retireGeneration")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + keyRoot: string, + digest: string, +): Effect.fn.Return { + const generationPath = path.join(keyRoot, digest); + const lockPath = path.join(keyRoot, `${digest}.lock`); + const stagingRoot = path.join(keyRoot, STAGING_DIR_NAME); + const stagingLockPath = yield* Effect.scoped( + Effect.gen(function* () { + const slotScope = yield* Scope.Scope; + const slot = yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(lockPath, "existing"); + yield* takeExclusiveLock(connection); + const info = yield* fs.stat(lockPath); + const mtime = Option.getOrUndefined(info.mtime); + const now = yield* Clock.currentTimeMillis; + if (mtime === undefined || now - mtime.getTime() <= RETIREMENT_AGE_MILLIS) + return undefined; + if (yield* isSymlink(fs, stagingRoot)) return undefined; + yield* fs.makeDirectory(stagingRoot, { recursive: true, mode: 0o700 }); + const token = yield* crypto.randomUUIDv4; + const slotLockPath = stagingLockFor(path, stagingRoot, token); + const slotDir = stagingDirFor(path, stagingRoot, token); + const slotConnection = yield* acquireLock(slotLockPath, "create").pipe( + Scope.provide(slotScope), + ); + yield* takeLock(slotConnection); + yield* fs.rename(generationPath, slotDir); + return { slotDir, slotLockPath }; + }), + ); + if (slot === undefined) return undefined; + yield* fs.remove(slot.slotDir, { recursive: true, force: true }); + return slot.slotLockPath; + }), + ).pipe(Effect.orElseSucceed(() => undefined)); + if (stagingLockPath === undefined) return false; + yield* fs + .remove(stagingLockPath, { force: true }) + .pipe(Effect.andThen(fs.remove(`${stagingLockPath}-journal`, { force: true })), Effect.ignore); + return true; +}); + +/** + * One best-effort sweep over the whole cache root. It reaps every unlocked `.staging` entry under + * any key root (a crashed preparer's leftovers), then retires each published generation whose + * digest lock is uncontended and whose lock-file mtime, rechecked under that lock, is older than + * the retention window. Digest lock files are never deleted (a stable inode); only the generation + * directory they guard is. A per-entry failure (including contention from a live pin or another + * retirer) is skipped, never propagated: this sweep must never block the `prepare`/`use` call it + * runs alongside. A pin held by this process contends with the sweep's lock exactly as a pin held + * by another process does. + */ +const sweepCache = Effect.fn("ArtifactStore.sweep")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + cacheRoot: string, +): Effect.fn.Return { + const { generations, stagingRoots } = yield* walkCache(fs, path, cacheRoot); + for (const stagingRoot of stagingRoots) yield* reapStaleStaging(fs, path, stagingRoot); + let retired = 0; + for (const { keyRoot, digest } of generations) { + if (yield* retireGeneration(fs, path, crypto, keyRoot, digest)) retired++; + } + yield* Effect.annotateCurrentSpan({ "artifact.retired_count": retired }); + return retired; +}); + +/** A cache hit: the generation directory exists, so every listed path is checked, never rehashed. */ +const checkHit = Effect.fn("ArtifactStore.checkHit")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + cacheRoot: string, + request: ArtifactRequest, + resolved: ResolvedGeneration, +): Effect.fn.Return, ArtifactStoreError> { + const exists = yield* mapFs( + resolved.generationPath, + "inspect cached artifact", + fs.exists(resolved.generationPath), + ); + if (!exists) return Option.none(); + const realRoot = yield* ensureSafeRoot(fs, path, resolved.generationPath, cacheRoot).pipe( + Effect.catch((error) => (isMissingArtifactRoot(error) ? Effect.void : Effect.fail(error))), + ); + if (realRoot === undefined) return Option.none(); + const inspected = yield* inspectRequiredPaths( + fs, + path, + resolved.generationPath, + realRoot, + request.requiredRuntimePaths, + ); + if (request.executablePath !== undefined) { + const executable = inspected[request.executablePath]; + if (executable === undefined) + return yield* metadataError("Cached artifact executable path is not recorded", { + path: request.executablePath, + }); + yield* ensureExecutableFile(fs, executable.realPath); + yield* ensureExecutableMode(fs, resolved.generationPath, executable.realPath); + } + return Option.some({ + key: request.key, + path: resolved.generationPath, + lockPath: resolved.lockPath, + sha256: resolved.expectedSha256, + requiredRuntimePaths: [...request.requiredRuntimePaths], + ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), + outcome: "cached" as const, + }); +}); + +const isPublishTargetTaken = (cause: unknown): boolean => + cause instanceof PlatformError.PlatformError && + cause.reason instanceof PlatformError.SystemError && + hasErrnoCode(cause.reason.cause) && + ["EEXIST", "ENOTEMPTY"].includes(String(cause.reason.cause.code)); + +/** + * Downloads, validates and publishes a fresh generation. Staged under a lock-guarded + * `.staging//content`, published by renaming the completed content to `/` and + * fsyncing the parent. A losing preparer (publish target already taken) accepts the winner's + * generation instead of failing, since both staged the same content-addressed digest. + */ +const publishGeneration = Effect.fn("ArtifactStore.publish")(function* ( fs: FileSystem.FileSystem, path: Path.Path, crypto: Crypto.Crypto, @@ -671,175 +781,46 @@ const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( cacheRoot: string, source: ArtifactSource, request: ArtifactRequest, + resolved: ResolvedGeneration, onProgress?: (state: "downloading" | "preparing") => void, -) { - return yield* Effect.gen(function* () { - const target = path.resolve(cacheRoot, request.key); - const targetParent = path.dirname(target); - yield* ensureDirectory(fs, path, targetParent, cacheRoot); - yield* reapLeftoversOf(fs, path, target); - const metadataPath = path.join(target, METADATA_NAME); - const checkCached: Effect.Effect< - Option.Option, - ArtifactStoreError - > = Effect.gen(function* () { - const realRoot = yield* ensureSafeRoot(fs, path, target, cacheRoot).pipe( - Effect.map(Option.some), - Effect.catch((error) => - isMissingArtifactRoot(error) ? Effect.succeed(Option.none()) : Effect.fail(error), +): Effect.fn.Return { + yield* mapFs( + resolved.stagingRoot, + "create artifact staging root", + fs.makeDirectory(resolved.stagingRoot, { recursive: true, mode: 0o700 }), + ); + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => + artifactError(`Unable to allocate artifact staging name: ${cause.message}`, { + key: request.key, + cause, + }), + ), + ); + const stagingDir = stagingDirFor(path, resolved.stagingRoot, token); + const stagingLockPath = stagingLockFor(path, resolved.stagingRoot, token); + const content = path.join(stagingDir, STAGING_CONTENT_NAME); + yield* Effect.scoped( + Effect.gen(function* () { + const connection = yield* acquireLock(stagingLockPath, "create").pipe( + Effect.mapError((cause) => + artifactError(`Unable to lock artifact staging directory: ${cause.message}`, { + key: request.key, + cause, + }), ), ); - if (Option.isNone(realRoot)) return Option.none(); - const cachedMetadata = yield* readMetadata(fs, metadataPath); - if (Option.isNone(cachedMetadata)) return Option.none(); - const metadata = cachedMetadata.value; - // A key names immutable content and sources unpack whole archives, so only a different key - // or executable is a miss; paths beyond the recorded ones are checked in the published tree. - if (identityMismatch(request, metadata)) return Option.none(); - const sha256 = yield* sha256Of(request, metadata); - const newPaths = unrecordedRequiredPaths(request, metadata); - const newPathSet = new Set(newPaths); - const recordedPaths = request.requiredRuntimePaths.filter( - (relative) => !newPathSet.has(relative), - ); - // Published content is not rehashed on cache hits: metadata and cheap structural checks - // protect the cache boundary; content tampering may execute or fail later when the - // workload starts. - const recordedSafePaths = yield* ensureSafePaths( - fs, - path, - target, - realRoot.value, - metadata, - recordedPaths, - ); - // A path the recorded metadata has not seen yet gets the same containment and safety - // validation a fresh publish applies, before its kind is trusted and recorded. - const newSafePaths = - newPaths.length > 0 - ? yield* validateFreshRuntimePaths(fs, path, target, realRoot.value, newPaths).pipe( - Effect.mapError((cause) => - metadataError( - `Cached artifact ${request.key} cannot serve newly required runtime paths (${cause.message}); remove ${target} to download it again`, - { key: request.key, path: target, cause }, - ), - ), - ) - : {}; - const safePaths = { ...recordedSafePaths, ...newSafePaths }; - if (newPaths.length > 0) { - const newKinds = Object.fromEntries( - Object.entries(newSafePaths).map(([relative, inspected]) => [relative, inspected.kind]), - ); - const appendedPaths = newPaths.filter( - (relative) => !metadata.requiredRuntimePaths.includes(relative), - ); - // The paths above are already validated; a failed write only loses the recording, so - // it must not fail preparation. A later request re-validates and retries the write. - yield* updateMetadataAtomic(fs, path, crypto, metadataPath, { - ...metadata, - requiredRuntimePaths: [...metadata.requiredRuntimePaths, ...appendedPaths], - requiredRuntimeKinds: { ...metadata.requiredRuntimeKinds, ...newKinds }, - }).pipe( - Effect.catch((cause) => - Effect.logWarning("Unable to record newly validated artifact runtime paths", cause), - ), - ); - } - if (request.executablePath !== undefined) { - const executable = safePaths[request.executablePath]; - if (executable === undefined) - return yield* metadataError("Cached artifact executable path is not recorded", { - path: request.executablePath, - }); - yield* ensureExecutableFile(fs, executable.realPath); - } - return Option.some({ - key: request.key, - path: target, - sha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), - outcome: "cached" as const, - }); - }); - const inspectCache = (): Effect.Effect, ArtifactStoreError> => - Effect.gen(function* () { - const exists = yield* mapFs(target, "inspect cached artifact", fs.exists(target)); - if (!exists) return Option.none(); - const cached = yield* checkCached; - if (Option.isSome(cached)) return cached; - const stillExists = yield* mapFs(target, "inspect cached artifact", fs.exists(target)); - if (!stillExists) return Option.none(); - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - artifactError(`Unable to allocate artifact replacement name: ${cause.message}`, { - key: request.key, - cause, - }), - ), - ); - const replacement = path.join(targetParent, `.${path.basename(target)}.${token}.invalid`); - const moved = yield* Effect.acquireUseRelease( - fs.rename(target, replacement).pipe( - Effect.as(Option.some(replacement)), - Effect.catch((cause) => - isNotFound(cause) - ? Effect.succeed(Option.none()) - : Effect.fail( - artifactError(`Unable to replace invalid cached artifact: ${String(cause)}`, { - path: target, - cause, - }), - ), - ), - ), - (renamed) => Effect.succeed(Option.isSome(renamed)), - (renamed) => (Option.isSome(renamed) ? cleanup(fs, renamed.value) : Effect.void), - ); - if (moved) { - return Option.none(); - } - const raced = yield* mapFs(target, "inspect cached artifact", fs.exists(target)); - if (!raced) return Option.none(); - const concurrent = yield* checkCached; - if (Option.isSome(concurrent)) return concurrent; - return yield* metadataError("Concurrent cached artifact metadata is invalid", { - path: metadataPath, - }); - }); - const initial = yield* inspectCache(); - if (Option.isSome(initial)) return initial.value; - - // Artifact keys identify immutable published versions. Once a valid entry exists, its - // persisted digest is authoritative; only a cache miss consults the upstream checksum. - const expectedSha256 = yield* source.checksum(request).pipe( - Effect.flatMap((sha256) => - validateSha256(sha256).pipe( - Effect.mapError((cause) => - metadataError("Artifact source returned an invalid SHA-256", { - key: request.key, - cause, - }), - ), + yield* takeLock(connection).pipe( + Effect.mapError((cause) => + artifactError(`Unable to lock artifact staging directory: ${cause.message}`, { + key: request.key, + cause, + }), ), - ), - ); - - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - artifactError(`Unable to allocate artifact temporary name: ${cause.message}`, { - key: request.key, - cause, - }), - ), - ); - const staging = path.join(targetParent, `.${path.basename(target)}.${token}.tmp`); - const content = path.join(staging, STAGING_CONTENT_NAME); - const published = yield* Effect.gen(function* () { + ); yield* ensureDirectory(fs, path, content, cacheRoot); const contentRoot = yield* ensureSafeRoot(fs, path, content, cacheRoot); - yield* source.materialize(request, content, expectedSha256, onProgress).pipe( + yield* source.materialize(request, content, resolved.expectedSha256, onProgress).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(Crypto.Crypto, crypto), @@ -847,59 +828,107 @@ const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( // already-owned process service captured by its constructor. Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner), ); - const runtimePaths = yield* validateFreshRuntimePaths( + const inspected = yield* validateFreshRuntimePaths( fs, path, content, contentRoot, request.requiredRuntimePaths, ); - const runtimeKinds = Object.fromEntries( - Object.entries(runtimePaths).map(([relative, inspected]) => [relative, inspected.kind]), - ); - yield* writeMetadataSync( - fs, - path.join(content, METADATA_NAME), - metadataFor(request, expectedSha256, runtimeKinds), - ); if (request.executablePath !== undefined) { - const executable = runtimePaths[request.executablePath]; + const executable = inspected[request.executablePath]; if (executable === undefined) return yield* metadataError("Fresh artifact executable path is not recorded", { path: request.executablePath, }); yield* ensureExecutableFile(fs, executable.realPath); - yield* mapFs( - executable.realPath, - "set executable artifact mode", - fs.chmod(executable.realPath, EXECUTABLE_MODE), - ); } - const beforePublish = yield* inspectCache(); - if (Option.isSome(beforePublish)) return beforePublish.value; - const rename = mapFs(content, "publish artifact", fs.rename(content, target)).pipe( - Effect.as(undefined), + yield* mapFs(content, "normalize artifact mode", fs.chmod(content, DIRECTORY_MODE)); + // Protects the narrow window right after publish: the digest lock file's mtime is touched + // here regardless of any pin the caller already holds, so a generation republished after a + // past retirement never inherits its lock file's stale, pre-retirement mtime. + return yield* Effect.scoped( + Pin.pin(resolved.lockPath).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.mapError((cause) => + artifactError(`Unable to pin artifact generation before publishing: ${cause.message}`, { + key: request.key, + cause, + }), + ), + Effect.andThen( + fs.rename(content, resolved.generationPath).pipe( + Effect.as(true as const), + Effect.catch((cause) => + isPublishTargetTaken(cause) + ? Effect.succeed(false as const) + : Effect.fail( + artifactError(`Unable to publish artifact: ${String(cause)}`, { + path: resolved.generationPath, + cause, + }), + ), + ), + Effect.flatMap((won) => + won + ? mapFs( + resolved.keyRoot, + "publish artifact", + fsyncDirectory(fs, resolved.keyRoot), + ) + : Effect.void, + ), + ), + ), + ), ); - const recoverPublish = (): Effect.Effect => - Effect.gen(function* () { - const probe = yield* inspectCache(); - if (Option.isSome(probe)) return probe.value; - return yield* rename; - }); - const recovered: Effect.Effect = - rename.pipe(Effect.catch(recoverPublish)); - return yield* recovered; - }).pipe(Effect.onExit(() => cleanup(fs, staging))); - if (published !== undefined) return published; - return { - key: request.key, - path: target, - sha256: expectedSha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), - outcome: "downloaded" as const, - }; - }); + }), + ).pipe( + Effect.onExit(() => + // Runs after the staging lock connection above has already closed (`Effect.scoped`'s own + // finalizer fires first), so removing its file here is always safe, on every outcome. + cleanup(fs, stagingDir).pipe( + Effect.andThen(fs.remove(stagingLockPath, { force: true })), + Effect.andThen(fs.remove(`${stagingLockPath}-journal`, { force: true })), + Effect.ignore, + ), + ), + ); + const hit = yield* checkHit(fs, path, cacheRoot, request, resolved); + if (Option.isNone(hit)) + return yield* artifactError("Unable to publish artifact: the published generation vanished", { + path: resolved.generationPath, + }); + return { ...hit.value, outcome: "downloaded" as const }; +}); + +const prepareOrResolve = Effect.fn("ArtifactStore.prepareOrResolve")(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + crypto: Crypto.Crypto, + childProcessSpawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + cacheRoot: string, + source: ArtifactSource, + request: ArtifactRequest, + resolved: ResolvedGeneration, + onProgress?: (state: "downloading" | "preparing") => void, +): Effect.fn.Return { + yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); + const hit = yield* checkHit(fs, path, cacheRoot, request, resolved); + if (Option.isSome(hit)) return hit.value; + const published = yield* publishGeneration( + fs, + path, + crypto, + childProcessSpawner, + cacheRoot, + source, + request, + resolved, + onProgress, + ); + yield* sweepCache(fs, path, crypto, cacheRoot); + return published; }); export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* ( @@ -936,15 +965,72 @@ export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* if (rootInfo.type !== "Directory") return yield* artifactError("Artifact cache root must be a directory", { path: cacheRoot }); yield* mapFs(cacheRoot, "secure artifact cache root", restrictDirectoryToOwner(fs, cacheRoot)); + const prepare = Effect.fn("ArtifactStore.prepare")(function* ( request: ArtifactRequest, onProgress?: (state: "downloading" | "preparing") => void, ) { yield* validateRequest(request); - const target = path.resolve(cacheRoot, request.key); - if (!pathWithin(cacheRoot, target, path.sep)) - return yield* artifactError("Artifact key escapes cache root", { key: request.key }); - const prepared = yield* makeArtifactOperation( + const resolved = yield* resolveGeneration(path, cacheRoot, options.source, request); + // Ahead-of-time preparation pins nothing for its caller, but a warm generation's own + // inspection and a fresh one's publication both still need protection from a concurrent + // retirement sweep elsewhere; a SHARED pin held only for this call's own duration, released + // before returning, gives that without leaving anything pinned for the caller. + yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); + const prepared = yield* Effect.scoped( + Pin.pin(resolved.lockPath).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.mapError((cause) => + artifactError(`Unable to pin artifact generation: ${cause.message}`, { + key: request.key, + cause, + }), + ), + Effect.andThen( + prepareOrResolve( + fs, + path, + crypto, + childProcessSpawner, + cacheRoot, + options.source, + request, + resolved, + onProgress, + ), + ), + ), + ); + yield* Effect.annotateCurrentSpan({ + "artifact.outcome": prepared.outcome, + "artifact.digest": resolved.expectedSha256, + }); + return prepared; + }); + + /** + * The one scoped launch-time operation: pins the generation, resolves or prepares it, and + * returns its paths. Every consumer must use those paths only inside this scope. + */ + const use = Effect.fn("ArtifactStore.use")(function* ( + request: ArtifactRequest, + onProgress?: (state: "downloading" | "preparing") => void, + ) { + yield* validateRequest(request); + const resolved = yield* resolveGeneration(path, cacheRoot, options.source, request); + // The digest lock file's parent must exist before it can be created; `prepareOrResolve` below + // ensures it again, idempotently, before resolving or preparing the generation itself. + yield* ensureDirectory(fs, path, resolved.keyRoot, cacheRoot); + yield* Pin.pin(resolved.lockPath).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.mapError((cause) => + artifactError(`Unable to pin artifact generation: ${cause.message}`, { + key: request.key, + cause, + }), + ), + ); + const prepared = yield* prepareOrResolve( fs, path, crypto, @@ -952,10 +1038,15 @@ export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* cacheRoot, options.source, request, + resolved, onProgress, ); - yield* Effect.annotateCurrentSpan({ "artifact.outcome": prepared.outcome }); + yield* Effect.annotateCurrentSpan({ + "artifact.outcome": prepared.outcome, + "artifact.digest": resolved.expectedSha256, + }); return prepared; }); - return { prepare }; + + return { prepare, use }; }); diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index 8f637d0a42..94ef0482ac 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -3,17 +3,27 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Crypto, + Data, DateTime, Deferred, + Duration, Effect, Exit, Fiber, FileSystem, Layer, Option, - PlatformError, - Schema, + Path, + Schedule, + Scope, + Stream, } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import * as Environment from "../namespace/Environment.ts"; +import { defaultNativeProcessLauncher, spawnNativeProcess } from "../runtime/NativeProcess.ts"; +import { acquireLock, isBusy, takeExclusiveLock } from "../namespace/drivers/Sqlite.ts"; import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; import { makeArtifactStore, type ArtifactRequest, type ArtifactSource } from "./ArtifactStore.ts"; import { verifySha256 } from "./Integrity.ts"; @@ -33,6 +43,18 @@ const errorOf = (exit: Exit.Exit): E | undefined => const withPlatform = (effect: Effect.Effect) => Effect.scoped(effect).pipe(Effect.provide(layer)); +const mapMaterializeError = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + cause instanceof PreparationError || cause instanceof ArtifactIntegrityError + ? cause + : new PreparationError({ + message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + ), + ); + const sourceWriting = (bytes: Uint8Array = archive): ArtifactSource => ({ checksum: () => Effect.succeed(archiveSha256), materialize: (_request, destination, expectedSha256) => @@ -40,45 +62,104 @@ const sourceWriting = (bytes: Uint8Array = archive): ArtifactSource => ({ const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { mode: 0o755 }); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { mode: 0o644 }); const crypto = yield* Crypto.Crypto; yield* verifySha256(bytes, expectedSha256).pipe(Effect.provideService(Crypto.Crypto, crypto)); - }).pipe( - Effect.mapError((cause) => - cause instanceof ArtifactIntegrityError - ? cause - : new PreparationError({ - message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }); -const replacesMalformedMetadata = (label: string, metadata: string | undefined) => - withPlatform( +/** A second digest, published by `triggerSweep` under a fresh key per call. */ +const archiveAlt = new TextEncoder().encode("archive-alt"); +const archiveAltSha256 = createHash("sha256").update(archiveAlt).digest("hex"); +const sourceWritingAlt = (): ArtifactSource => ({ + checksum: () => Effect.succeed(archiveAltSha256), + materialize: (_request, destination, expectedSha256) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: `supabase-stack-artifact-${label}-metadata-`, + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres alt", { + mode: 0o755, }); - const target = `${root}/${request.key}`; - yield* fs.makeDirectory(`${target}/bin`, { recursive: true }); - yield* fs.writeFileString(`${target}/bin/postgres`, "unverified postgres"); - if (metadata !== undefined) yield* fs.writeFileString(`${target}/.artifact.json`, metadata); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config alt", { mode: 0o644 }); + const crypto = yield* Crypto.Crypto; + yield* verifySha256(archiveAlt, expectedSha256).pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + }).pipe(mapMaterializeError), +}); + +/** A cache miss under a fresh key: the only event that triggers a retirement sweep. */ +const triggerSweep = (root: string) => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const key = `database/postgres-sweep-${yield* crypto.randomUUIDv4}`; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWritingAlt() }); + return yield* store.prepare({ ...request, key }); + }); + +/** Sets `path`'s mtime 31 days in the past: a legitimate test-only lever on retirement's age fence. */ +const ageLockFile = (fs: FileSystem.FileSystem, path: string) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const aged = DateTime.toDate(DateTime.subtract(now, { days: 31 })); + yield* fs.utimes(path, aged, aged); + }); + +/** Matches a staging token's own lock file, or the rollback journal SQLite leaves transiently. */ +const isStagingLockFile = (name: string) => /\.lock(-journal)?$/u.test(name); + +const digestDirectories = (fs: FileSystem.FileSystem, keyRoot: string) => + fs.readDirectory(keyRoot).pipe( + Effect.orElseSucceed(() => []), + Effect.map((names) => names.filter((name) => /^[0-9a-f]{64}$/u.test(name))), + ); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const prepared = yield* store.prepare(request); +const fixturePath = fileURLToPath( + new URL("../../tests/artifact-store-fixture.ts", import.meta.url), +); - expect(prepared.outcome).toBe("downloaded"); - expect(yield* fs.readFileString(`${target}/bin/postgres`)).toBe("native postgres"); - expect(yield* fs.exists(`${target}/.artifact.json`)).toBe(true); - }), +const spawnFixture = (args: ReadonlyArray) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner.spawn( + ChildProcess.make(process.execPath, [fixturePath, ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + forceKillAfter: "1 second", + }), + ); + }); + +const runToCompletion = (handle: ChildProcessSpawner.ChildProcessHandle) => + Effect.all( + [ + handle.stdout.pipe(Stream.decodeText, Stream.mkString), + handle.stderr.pipe(Stream.decodeText, Stream.mkString), + handle.exitCode, + ], + { concurrency: "unbounded" }, + ); + +/** Reads the fixture's first stdout line without waiting for it to exit: `use`/`stall` block forever. */ +const firstLine = (handle: ChildProcessSpawner.ChildProcessHandle) => + handle.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runHead, + Effect.map(Option.getOrThrow), + ); + +const killAndAwaitExit = (handle: ChildProcessSpawner.ChildProcessHandle) => + Effect.sync(() => process.kill(Number(handle.pid), "SIGKILL")).pipe( + Effect.andThen(handle.exitCode.pipe(Effect.ignore)), ); describe("verified native artifact preparation", () => { - it.live("downloads and atomically publishes an executable artifact tree", () => + it.live("downloads and publishes a content-addressed generation", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -86,8 +167,12 @@ describe("verified native artifact preparation", () => { const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); const prepared = yield* store.prepare(request); expect(prepared.outcome).toBe("downloaded"); + expect(prepared.path).toBe(`${yield* fs.realPath(root)}/${request.key}/${archiveSha256}`); expect(yield* fs.exists(`${prepared.path}/bin/postgres`)).toBe(true); - expect((yield* fs.stat(`${prepared.path}/bin/postgres`)).mode & 0o111).not.toBe(0); + expect((yield* fs.stat(prepared.path)).mode & 0o777).toBe(0o755); + expect((yield* fs.stat(`${prepared.path}/bin/postgres`)).mode & 0o777).toBe(0o755); + expect((yield* fs.stat(`${prepared.path}/etc/postgres.conf`)).mode & 0o777).toBe(0o644); + expect(yield* fs.exists(prepared.lockPath)).toBe(true); }), ), ); @@ -142,14 +227,15 @@ describe("verified native artifact preparation", () => { const second = yield* store.prepare(request); expect(first.outcome).toBe("downloaded"); expect(second.outcome).toBe("cached"); - expect(checksumCalls).toBe(1); + expect(second.path).toBe(first.path); + expect(checksumCalls).toBe(2); expect(materializeCalls).toBe(1); }), ), ); it.live( - "keeps the published tree and reuses it when required runtime paths expand on the same key", + "serves a newly required path already present in the published tree without redownloading", () => withPlatform( Effect.gen(function* () { @@ -157,14 +243,9 @@ describe("verified native artifact preparation", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-artifact-paths-expand-", }); - let checksumCalls = 0; let materializeCalls = 0; const source: ArtifactSource = { - checksum: () => - Effect.sync(() => { - checksumCalls += 1; - return archiveSha256; - }), + checksum: () => Effect.succeed(archiveSha256), materialize: (entry, destination) => Effect.sync(() => { materializeCalls += 1; @@ -184,232 +265,12 @@ describe("verified native artifact preparation", () => { expect(second.path).toBe(first.path); expect(stillPublishedIno).toEqual(publishedIno); expect(second.requiredRuntimePaths).toEqual([...request.requiredRuntimePaths]); - expect(checksumCalls).toBe(1); expect(materializeCalls).toBe(1); - expect(yield* fs.readFileString(`${second.path}/bin/postgres`)).toBe("native postgres"); - const metadata = yield* Schema.decodeEffect( - Schema.fromJsonString( - Schema.Struct({ - requiredRuntimePaths: Schema.Array(Schema.String), - requiredRuntimeKinds: Schema.Record(Schema.String, Schema.String), - }), - ), - )(yield* fs.readFileString(`${second.path}/.artifact.json`)); - expect(metadata.requiredRuntimePaths).toContain("etc/postgres.conf"); - expect(metadata.requiredRuntimeKinds["etc/postgres.conf"]).toBe("file"); + expect(yield* fs.readFileString(`${second.path}/etc/postgres.conf`)).toBe("config"); }), ), ); - it.live("replaces a cached tree with unknown artifact metadata", () => - replacesMalformedMetadata("unknown", '{"format":"supabase-stack-artifact-v0"}'), - ); - - it.live("replaces a cached tree with missing artifact metadata", () => - replacesMalformedMetadata("missing", undefined), - ); - - it.live("reuses a cache hit after required file contents are modified", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-tampered-runtime-", - }); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const published = yield* store.prepare(request); - yield* fs.writeFileString(`${published.path}/bin/postgres`, "tampered executable"); - const cached = yield* store.prepare(request); - expect(cached.outcome).toBe("cached"); - }), - ), - ); - - it.live("reuses cached directories without hashing their contents", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-directory-runtime-", - }); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-directory-runtime", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/share/runtime/config`, "config"); - yield* fs.symlink("config", `${destination}/share/runtime/config-link`); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const prepared = yield* store.prepare(directoryRequest); - expect((yield* store.prepare(directoryRequest)).outcome).toBe("cached"); - - yield* fs.writeFileString(`${prepared.path}/share/runtime/config`, "tampered config"); - expect((yield* store.prepare(directoryRequest)).outcome).toBe("cached"); - }), - ), - ); - - it.live("rejects a non-EINVAL readLink failure on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-readlink-error-", - }); - const publisher = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - yield* publisher.prepare(request); - const failingFs: FileSystem.FileSystem = { - ...fs, - readLink: (candidate) => - candidate.endsWith("/bin/postgres") - ? Effect.fail( - PlatformError.systemError({ - _tag: "PermissionDenied", - module: "FileSystem", - method: "readLink", - pathOrDescriptor: candidate, - cause: { code: "EACCES" }, - }), - ) - : fs.readLink(candidate), - }; - const checker = yield* makeArtifactStore({ - cacheRoot: root, - source: sourceWriting(), - }).pipe(Effect.provideService(FileSystem.FileSystem, failingFs)); - - const exit = yield* checker.prepare(request).pipe(Effect.exit); - const error = errorOf(exit); - - expect(error).toBeInstanceOf(ArtifactIntegrityError); - expect(error?.message).toContain("Unable to inspect required runtime path"); - }), - ), - ); - - it.live("does not recursively enumerate required directories on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-no-recursion-", - }); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-cache-no-recursion", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/share/runtime/config`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const publisher = yield* makeArtifactStore({ cacheRoot: root, source }); - yield* publisher.prepare(directoryRequest); - const guardedFs: FileSystem.FileSystem = { - ...fs, - readDirectory: (candidate, options) => - candidate.endsWith("/share/runtime") - ? Effect.fail( - PlatformError.systemError({ - _tag: "Busy", - module: "FileSystem", - method: "readDirectory", - pathOrDescriptor: candidate, - cause: { code: "EBUSY" }, - }), - ) - : fs.readDirectory(candidate, options), - }; - const checker = yield* makeArtifactStore({ - cacheRoot: root, - source, - }).pipe(Effect.provideService(FileSystem.FileSystem, guardedFs)); - - const cached = yield* checker.prepare(directoryRequest); - - expect(cached.outcome).toBe("cached"); - }), - ), - ); - - it.live("rejects a missing required path on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-missing-", - }); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const published = yield* store.prepare(request); - yield* fs.remove(`${published.path}/etc/postgres.conf`); - - const exit = yield* store.prepare(request).pipe(Effect.exit); - - expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - }), - ), - ); - - it.live("rejects a cache hit whose newly required path is missing from the published tree", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-new-path-missing-", - }); - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - const narrow: ArtifactRequest = { ...request, requiredRuntimePaths: ["bin/postgres"] }; - const published = yield* store.prepare(narrow); - const publishedIno = (yield* fs.stat(published.path)).ino; - - const wider: ArtifactRequest = { - ...request, - requiredRuntimePaths: ["bin/postgres", "share/missing"], - }; - const exit = yield* store.prepare(wider).pipe(Effect.exit); - - const error = errorOf(exit); - expect(error).toBeInstanceOf(ArtifactIntegrityError); - expect(error?.message).toContain(`remove ${published.path} to download it again`); - expect(yield* fs.exists(published.path)).toBe(true); - expect((yield* fs.stat(published.path)).ino).toEqual(publishedIno); - }), - ), - ); - it.live("rejects a required path whose basic kind changes on a cache hit", () => withPlatform( Effect.gen(function* () { @@ -441,31 +302,8 @@ describe("verified native artifact preparation", () => { }); const outsideExecutable = `${outside}/postgres`; yield* fs.writeFileString(outsideExecutable, "outside"); - const symlinkRequest: ArtifactRequest = { - ...request, - key: "database/postgres-cache-link", - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres.real`, "native postgres"); - yield* fs.symlink("postgres.real", `${destination}/bin/postgres`); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); + const symlinkRequest: ArtifactRequest = { ...request, key: "database/postgres-cache-link" }; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); const published = yield* store.prepare(symlinkRequest); yield* fs.remove(`${published.path}/bin/postgres`); yield* fs.symlink(outsideExecutable, `${published.path}/bin/postgres`); @@ -477,7 +315,7 @@ describe("verified native artifact preparation", () => { ), ); - it.live("rejects a checksum mismatch and removes only its temporary tree", () => + it.live("rejects a checksum mismatch and leaves no generation or staging behind", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -490,9 +328,11 @@ describe("verified native artifact preparation", () => { }); const exit = yield* store.prepare(request).pipe(Effect.exit); expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); + const staging = yield* fs + .readDirectory(`${root}/${request.key}/.staging`) + .pipe(Effect.orElseSucceed(() => [])); + expect(staging).toEqual([]); }), ), ); @@ -508,27 +348,17 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin/postgres`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare(request).pipe(Effect.exit); expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); }), ), ); @@ -544,21 +374,17 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres.real`, "native postgres"); + yield* fs.writeFileString(`${destination}/bin/postgres.real`, "native postgres", { + mode: 0o755, + }); yield* fs.symlink("postgres.real", `${destination}/bin/postgres`); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { + mode: 0o644, + }); + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const prepared = yield* store.prepare(request); @@ -586,123 +412,19 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); yield* fs.symlink(outsideExecutable, `${destination}/bin/postgres`); yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare(request).pipe(Effect.exit); expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); expect(yield* fs.exists(outsideExecutable)).toBe(true); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); - }), - ), - ); - - it.live("rejects an escaping symlink nested in a fresh required directory", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-nested-link-escape-", - }); - const outside = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-nested-link-outside-", - }); - const outsideConfig = `${outside}/config`; - yield* fs.writeFileString(outsideConfig, "outside"); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-nested-link", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.symlink(outsideConfig, `${destination}/share/runtime/config`); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const exit = yield* store.prepare(directoryRequest).pipe(Effect.exit); - - expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - expect(yield* fs.exists(`${root}/database/postgres-nested-link`)).toBe(false); - }), - ), - ); - - it.live("rejects an escaping symlink nested in a newly required directory on a cache hit", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-new-nested-link-escape-", - }); - const outside = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-cache-new-nested-link-outside-", - }); - const outsideConfig = `${outside}/config`; - yield* fs.writeFileString(outsideConfig, "outside"); - const directoryRequest: ArtifactRequest = { - ...request, - key: "database/postgres-cache-new-nested-link", - requiredRuntimePaths: ["bin/postgres", "share/runtime"], - }; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination) => - Effect.gen(function* () { - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/share/runtime`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.symlink(outsideConfig, `${destination}/share/runtime/config`); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const narrow: ArtifactRequest = { - ...directoryRequest, - requiredRuntimePaths: ["bin/postgres"], - }; - const published = yield* store.prepare(narrow); - - const exit = yield* store.prepare(directoryRequest).pipe(Effect.exit); - - expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); - expect(yield* fs.exists(published.path)).toBe(true); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); }), ), ); @@ -719,31 +441,22 @@ describe("verified native artifact preparation", () => { checksum: () => Effect.succeed(archiveSha256), materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; yield* fs.writeFileString(`${destination}/partial`, "partial"); yield* Deferred.succeed(started, undefined); return yield* Effect.never; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const preparation = yield* Effect.forkChild(store.prepare(request)); yield* Deferred.await(started); yield* Fiber.interrupt(preparation); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* fs.exists(`${root}/database/postgres`)).toBe(false); + expect(yield* digestDirectories(fs, `${root}/${request.key}`)).toEqual([]); }), ), ); - it.live("allows separate stores to duplicate work and converge safely", () => + it.live("allows separate stores to duplicate work and converge on one published generation", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -756,27 +469,23 @@ describe("verified native artifact preparation", () => { let calls = 0; const source: ArtifactSource = { checksum: () => Effect.succeed(archiveSha256), - materialize: (entry, destination) => + materialize: (_entry, destination) => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; calls += 1; yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { + mode: 0o755, + }); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { + mode: 0o644, + }); yield* calls === 1 ? Deferred.succeed(firstStarted, undefined) : Deferred.succeed(secondStarted, undefined); yield* Deferred.await(release); - return; - }).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: `materialization failed: ${cause.message}`, - cause, - }), - ), - ), + }).pipe(mapMaterializeError), }; const firstStore = yield* makeArtifactStore({ cacheRoot: root, source }); const secondStore = yield* makeArtifactStore({ cacheRoot: root, source }); @@ -789,6 +498,10 @@ describe("verified native artifact preparation", () => { expect(calls).toBe(2); expect(results[0].path).toBe(results[1].path); expect(yield* fs.exists(`${results[0].path}/bin/postgres`)).toBe(true); + const staging = yield* fs + .readDirectory(`${root}/${request.key}/.staging`) + .pipe(Effect.orElseSucceed(() => [])); + expect(staging).toEqual([]); }), ), ); @@ -796,16 +509,13 @@ describe("verified native artifact preparation", () => { it.live("rejects traversal in keys and required runtime paths before touching the source", () => withPlatform( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-artifact-path-" }); + const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-path-", + }); let called = false; const source: ArtifactSource = { checksum: () => Effect.succeed(archiveSha256), - materialize: () => - Effect.sync(() => { - called = true; - return; - }), + materialize: () => Effect.sync(() => (called = true)), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const keyExit = yield* store.prepare({ ...request, key: "../escape" }).pipe(Effect.exit); @@ -831,11 +541,7 @@ describe("verified native artifact preparation", () => { let called = false; const source: ArtifactSource = { checksum: () => Effect.succeed(archiveSha256), - materialize: () => - Effect.sync(() => { - called = true; - return; - }), + materialize: () => Effect.sync(() => (called = true)), }; const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare({ ...request, key: "nested/postgres" }).pipe(Effect.exit); @@ -848,110 +554,802 @@ describe("verified native artifact preparation", () => { ); }); -describe("orphaned temp/quarantine sweep", () => { - it.live("removes a prepared key's old leftovers and keeps fresh and unrelated ones", () => +describe("a required path missing from the archive", () => { + const narrowSource = (): ArtifactSource => ({ + checksum: () => Effect.succeed(archiveSha256), + materialize: (_request, destination, expectedSha256) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { + mode: 0o755, + }); + const crypto = yield* Crypto.Crypto; + yield* verifySha256(archive, expectedSha256).pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + }).pipe(mapMaterializeError), + }); + + it.live("fails with a typed error on a cold cache, and leaves the cache unchanged", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-sweep-", + prefix: "supabase-stack-artifact-missing-cold-", }); - const now = yield* DateTime.now; - const old = DateTime.toDate(DateTime.subtract(now, { hours: 25 })); - const recent = DateTime.toDate(DateTime.subtract(now, { hours: 1 })); - const parent = `${root}/database`; - const oldTemp = `${parent}/.postgres.00000000-0000-4000-8000-000000000001.tmp`; - const oldQuarantine = `${parent}/.postgres.00000000-0000-4000-8000-000000000002.invalid`; - const freshTemp = `${parent}/.postgres.00000000-0000-4000-8000-000000000003.tmp`; - const freshQuarantine = `${parent}/.postgres.00000000-0000-4000-8000-000000000004.invalid`; - const unrelated = `${parent}/nested/.x.tmp`; - const leftovers = [oldTemp, oldQuarantine, freshTemp, freshQuarantine, unrelated]; - for (const leftover of leftovers) { - yield* fs.makeDirectory(leftover, { recursive: true }); - yield* fs.writeFileString(`${leftover}/marker`, "leftover"); - } - for (const leftover of [oldTemp, oldQuarantine, unrelated]) - yield* fs.utimes(leftover, old, old); - for (const leftover of [freshTemp, freshQuarantine]) - yield* fs.utimes(leftover, recent, recent); - - const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); - expect(yield* fs.exists(oldTemp), "construction leaves the cache alone").toBe(true); - expect((yield* store.prepare(request)).outcome).toBe("downloaded"); - - expect(yield* fs.exists(oldTemp)).toBe(false); - expect(yield* fs.exists(oldQuarantine)).toBe(false); - expect(yield* fs.exists(freshTemp)).toBe(true); - expect(yield* fs.exists(freshQuarantine)).toBe(true); - expect(yield* fs.exists(unrelated)).toBe(true); + const key = "database/postgres-missing-cold"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: narrowSource() }); + const wideRequest: ArtifactRequest = { ...request, key }; + const exit = yield* store.prepare(wideRequest).pipe(Effect.exit); + expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); + expect(yield* digestDirectories(fs, `${root}/${key}`)).toEqual([]); }), ), ); - it.live("never reaps an in-progress extraction whose source backdated its own destination", () => + it.live("fails with a typed error on a warm cache, and leaves the cache unchanged", () => withPlatform( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const crypto = yield* Crypto.Crypto; const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-artifact-reaper-race-", + prefix: "supabase-stack-artifact-missing-warm-", }); - const backdated = yield* Deferred.make(); - const secondReachedMaterialize = yield* Deferred.make(); - let materializeCalls = 0; - const source: ArtifactSource = { - checksum: () => Effect.succeed(archiveSha256), - materialize: (_entry, destination, expectedSha256) => - Effect.gen(function* () { - materializeCalls += 1; - if (materializeCalls > 1) { - // The second preparation arrives here after running the reaper; it fails instead - // of racing the first to publish, which keeps the interleaving deterministic. - yield* Deferred.succeed(secondReachedMaterialize, undefined); - return yield* new PreparationError({ - message: "the second preparation stops before publishing", - }); - } - yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); - yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); - yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres"); - yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config"); - yield* verifySha256(archive, expectedSha256).pipe( - Effect.provideService(Crypto.Crypto, crypto), - ); - // GNU tar sets an extraction destination's own mtime from the archive's `./` - // entry; the slim-services archives carry epoch mtimes there. - yield* fs.utimes(destination, 0, 0); - yield* Deferred.succeed(backdated, undefined); - yield* Deferred.await(secondReachedMaterialize); - }).pipe( - Effect.mapError((cause) => - cause instanceof PreparationError || cause instanceof ArtifactIntegrityError - ? cause - : new PreparationError({ - message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - ), + const key = "database/postgres-missing-warm"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: narrowSource() }); + const narrowRequest: ArtifactRequest = { + ...request, + key, + requiredRuntimePaths: ["bin/postgres"], }; - const store = yield* makeArtifactStore({ cacheRoot: root, source }); - - const first = yield* Effect.forkChild(store.prepare(request)); - yield* Deferred.await(backdated); - - // Runs the same leftover reaper the first operation's staging directory is exposed to; - // it refuses its own materialization once it gets there, so it can never win a race to - // publish and mask a reap that already happened. - const second = yield* store.prepare(request).pipe(Effect.exit); - expect(Exit.isFailure(second)).toBe(true); - - const published = yield* Fiber.join(first); + const published = yield* store.prepare(narrowRequest); expect(published.outcome).toBe("downloaded"); - expect(yield* fs.exists(`${published.path}/bin/postgres`)).toBe(true); - expect(yield* fs.readFileString(`${published.path}/bin/postgres`)).toBe("native postgres"); - expect(yield* fs.readFileString(`${published.path}/etc/postgres.conf`)).toBe("config"); - }), - ), + + const wideRequest: ArtifactRequest = { ...request, key }; + const exit = yield* store.prepare(wideRequest).pipe(Effect.exit); + expect(errorOf(exit)).toBeInstanceOf(ArtifactIntegrityError); + expect(yield* digestDirectories(fs, `${root}/${key}`)).toEqual([archiveSha256]); + expect(yield* fs.exists(published.path)).toBe(true); + }), + ), + ); +}); + +describe("concurrent preparers across processes", () => { + it.live( + "both succeed, exactly one generation is published, and no staging is left", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-cross-process-", + }); + const key = "database/postgres-cross-process"; + const handles = yield* Effect.all( + [spawnFixture(["prepare", root, key]), spawnFixture(["prepare", root, key])], + { concurrency: "unbounded" }, + ); + const results = yield* Effect.all(handles.map(runToCompletion), { + concurrency: "unbounded", + }); + const parsed = results.map(([stdout, stderr, exitCode]) => { + expect(Number(exitCode), stderr).toBe(0); + const match = /prepared:(\w+):(.+):([0-9a-f]+)/.exec(stdout.trim()); + if (match === null) throw new Error(`Unparseable fixture output: ${stdout}`); + return { outcome: match[1], path: match[2], sha256: match[3] }; + }); + expect(parsed[0]?.path).toBe(parsed[1]?.path); + expect(parsed[0]?.sha256).toBe(parsed[1]?.sha256); + const staging = yield* fs + .readDirectory(`${root}/${key}/.staging`) + .pipe(Effect.orElseSucceed(() => [])); + expect(staging).toEqual([]); + }), + ), + 20_000, + ); +}); + +describe("lock-guarded staging", () => { + it.live( + "reaps a SIGKILLed preparer's staging directory on the next preparer, which publishes normally", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sigkill-reap-", + }); + const key = "database/postgres-sigkill-reap"; + const handle = yield* spawnFixture(["stall", root, key]); + yield* firstLine(handle); + const stagingRoot = `${root}/${key}/.staging`; + const tokensBefore = (yield* fs.readDirectory(stagingRoot)).filter( + (name) => !isStagingLockFile(name), + ); + expect(tokensBefore).toHaveLength(1); + + yield* killAndAwaitExit(handle); + + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + expect(prepared.outcome).toBe("downloaded"); + const tokensAfter = (yield* fs.readDirectory(stagingRoot)).filter( + (name) => !isStagingLockFile(name), + ); + expect(tokensAfter).toHaveLength(0); + }), + ), + 20_000, + ); + + it.live( + "reaps a crashed preparer's orphan staging under a published key when a miss runs under another key", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-orphan-reap-", + }); + const key = "database/postgres-orphan-reap"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const published = yield* store.prepare({ ...request, key }); + const orphan = `${root}/${key}/.staging/crashed-token`; + yield* fs.makeDirectory(`${orphan}/content`, { recursive: true }); + yield* fs.writeFileString(`${orphan}/content/partial`, "partial download"); + + const hit = yield* store.prepare({ ...request, key }); + expect(hit.outcome).toBe("cached"); + expect(yield* fs.exists(orphan)).toBe(true); + + yield* triggerSweep(root); + + expect(yield* fs.exists(orphan)).toBe(false); + expect(yield* fs.exists(published.path)).toBe(true); + }), + ), + 20_000, + ); + + it.live.skipIf(process.platform === "win32")( + "never follows a symlinked staging root or cache subdirectory out of the cache", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-link-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-outside-", + }); + const key = "database/postgres-sweep-link"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + yield* store.prepare({ ...request, key }); + yield* fs.makeDirectory(`${outside}/token`, { recursive: true }); + yield* fs.writeFileString(`${outside}/token/precious`, "keep"); + yield* fs.writeFileString(`${outside}/precious`, "keep"); + yield* fs.remove(`${root}/${key}/.staging`, { recursive: true, force: true }); + yield* fs.symlink(outside, `${root}/${key}/.staging`); + yield* fs.symlink(outside, `${root}/linked-subdir`); + + yield* triggerSweep(root); + + expect(yield* fs.exists(`${outside}/token/precious`)).toBe(true); + expect(yield* fs.exists(`${outside}/precious`)).toBe(true); + }), + ), + 20_000, + ); + + it.live.skipIf(process.platform === "win32")( + "retires nothing through a symlinked staging root under an aged unpinned generation", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-retire-link-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-retire-outside-", + }); + const key = "database/postgres-retire-link"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + yield* fs.writeFileString(`${outside}/precious`, "keep"); + yield* fs.remove(`${root}/${key}/.staging`, { recursive: true, force: true }); + yield* fs.symlink(outside, `${root}/${key}/.staging`); + yield* ageLockFile(fs, prepared.lockPath); + + yield* triggerSweep(root); + + expect(yield* fs.readDirectory(outside)).toEqual(["precious"]); + expect(yield* fs.exists(prepared.path)).toBe(true); + }), + ), + 20_000, + ); + + it.live( + "leaves a staging lock file that has no directory yet for the preparer that is about to take it", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-opening-lock-", + }); + const key = "database/postgres-opening-lock"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + yield* store.prepare({ ...request, key }); + const openingLock = `${root}/${key}/.staging/opening-token.lock`; + yield* fs.writeFile(openingLock, new Uint8Array()); + + yield* triggerSweep(root); + + expect(yield* fs.exists(openingLock)).toBe(true); + }), + ), + 20_000, + ); + + it.live( + "never touches a live preparer's staging directory while another preparer runs", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-live-stage-", + }); + const key = "database/postgres-live-stage"; + const handle = yield* spawnFixture(["stall", root, key]); + yield* firstLine(handle); + const stagingRoot = `${root}/${key}/.staging`; + const tokensBefore = (yield* fs.readDirectory(stagingRoot)).filter( + (name) => !isStagingLockFile(name), + ); + expect(tokensBefore).toHaveLength(1); + const liveToken = tokensBefore[0]; + if (liveToken === undefined) throw new Error("No live staging token found"); + + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + expect(prepared.outcome).toBe("downloaded"); + + const tokensAfter = yield* fs.readDirectory(stagingRoot); + expect(tokensAfter).toEqual(expect.arrayContaining([liveToken, `${liveToken}.lock`])); + expect(yield* fs.exists(`${stagingRoot}/${liveToken}/content`)).toBe(true); + + yield* killAndAwaitExit(handle); + }), + ), + 20_000, + ); +}); + +describe("pins", () => { + it.live( + "keeps a generation pinned for one in-process consumer while another releases, and retires it only once both are gone", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-pin-inprocess-", + }); + const key = "database/postgres-pin-inprocess"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + + const firstScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); + const firstUse = yield* store.use({ ...request, key }).pipe(Scope.provide(firstScope)); + const secondScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); + const secondUse = yield* store.use({ ...request, key }).pipe(Scope.provide(secondScope)); + expect(firstUse.path).toBe(secondUse.path); + + yield* ageLockFile(fs, firstUse.lockPath); + + // Sweeping while both consumers hold the pin: busy against this process's own open + // connection, so the stale generation survives untouched. + yield* triggerSweep(root); + expect(yield* fs.exists(firstUse.path)).toBe(true); + + yield* Scope.close(firstScope, Exit.void); + yield* triggerSweep(root); + expect(yield* fs.exists(firstUse.path)).toBe(true); + + yield* Scope.close(secondScope, Exit.void); + yield* triggerSweep(root); + expect(yield* fs.exists(firstUse.path)).toBe(false); + expect(yield* fs.exists(firstUse.lockPath)).toBe(true); + }), + ), + ); + + it.live( + "a running native workload's generation survives a concurrent retirement sweep, across processes, until it releases", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-pin-cross-process-", + }); + const key = "database/postgres-pin-cross-process"; + const handle = yield* spawnFixture(["use", root, key]); + const line = yield* firstLine(handle); + const match = /^used:(\w+):(.+):(.+)$/.exec(line); + if (match === null) throw new Error(`Unparseable fixture output: ${line}`); + const generationPath = match[2]; + const lockPath = match[3]; + if (generationPath === undefined || lockPath === undefined) + throw new Error(`Unparseable fixture output: ${line}`); + + yield* ageLockFile(fs, lockPath); + yield* triggerSweep(root); + expect(yield* fs.exists(generationPath)).toBe(true); + + yield* killAndAwaitExit(handle); + yield* ageLockFile(fs, lockPath); + yield* triggerSweep(root); + expect(yield* fs.exists(generationPath)).toBe(false); + expect(yield* fs.exists(lockPath)).toBe(true); + }), + ), + 20_000, + ); + + it.live("a retirement sweep that runs between prepare and use cannot break the launch", () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-between-", + }); + const key = "database/postgres-sweep-between"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + expect(prepared.outcome).toBe("downloaded"); + + // Ages the generation past the retention window: ahead-of-time `prepare` only pins + // itself for its own duration (see `ArtifactStore.prepare`), so by now it is unpinned + // and genuinely eligible for retirement, unlike the pinned scenarios covered above. + yield* ageLockFile(fs, prepared.lockPath); + + // A cache miss under another key forces a sweep as a side effect. + yield* triggerSweep(root); + expect(yield* fs.exists(prepared.path)).toBe(false); + expect(yield* fs.exists(prepared.lockPath)).toBe(true); + + // `use` transparently re-prepares the now-retired generation instead of failing the + // launch, publishing it fresh again at the same content-addressed path. + const used = yield* Effect.scoped(store.use({ ...request, key })); + expect(used.outcome).toBe("downloaded"); + expect(used.path).toBe(prepared.path); + expect(yield* fs.exists(`${used.path}/bin/postgres`)).toBe(true); + }), + ), + ); + + it.live("a consumer pins a generation while its retirement is still deleting the old copy", () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-retire-delete-", + }); + const key = "database/postgres-retire-delete"; + const setupStore = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* setupStore.prepare({ ...request, key }); + yield* ageLockFile(fs, prepared.lockPath); + const keyRoot = path.dirname(prepared.path); + + const deleteStarted = yield* Deferred.make(); + const releaseDelete = yield* Deferred.make(); + const sweepingStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + remove: (...args: Parameters) => + args[0].startsWith(`${keyRoot}/.staging/`) && !isStagingLockFile(args[0]) + ? Deferred.succeed(deleteStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseDelete)), + Effect.andThen(fs.remove(...args)), + ) + : fs.remove(...args), + }), + ); + yield* Effect.addFinalizer(() => Deferred.succeed(releaseDelete, undefined)); + + const sweepFiber = yield* sweepingStore + .prepare({ ...request, key: "database/postgres-retire-delete-trigger" }) + .pipe(Effect.forkScoped); + yield* Deferred.await(deleteStarted); + expect(yield* digestDirectories(fs, keyRoot)).toEqual([]); + + const consumerStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWriting(), + }); + const used = yield* Effect.scoped(consumerStore.use({ ...request, key })); + expect(used.path).toBe(prepared.path); + expect(yield* fs.exists(`${used.path}/bin/postgres`)).toBe(true); + + yield* Deferred.succeed(releaseDelete, undefined); + yield* Fiber.join(sweepFiber); + const leftovers = yield* fs + .readDirectory(`${keyRoot}/.staging`) + .pipe(Effect.map((names) => names.filter((name) => !isStagingLockFile(name)))); + expect(leftovers).toEqual([]); + }), + ), + ); + + it.live( + "a sweep retires an aged unpinned generation and spares an aged one this process pins", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-sweep-pinned-", + }); + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const pinned = yield* Effect.scoped( + Effect.gen(function* () { + const pinnedUse = yield* store.use({ + ...request, + key: "database/postgres-aged-pinned", + }); + const unpinned = yield* store.prepare({ + ...request, + key: "database/postgres-aged-unpinned", + }); + yield* ageLockFile(fs, pinnedUse.lockPath); + yield* ageLockFile(fs, unpinned.lockPath); + + yield* triggerSweep(root); + + expect(yield* fs.exists(unpinned.path)).toBe(false); + expect(yield* fs.exists(pinnedUse.path)).toBe(true); + return pinnedUse; + }), + ); + expect(yield* fs.exists(pinned.path)).toBe(true); + }), + ), + ); + + it.live("a cache hit leaves aged unpinned generations alone until the next miss", () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-hit-no-sweep-", + }); + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const warm = yield* store.prepare({ ...request, key: "database/postgres-warm" }); + const stale = yield* store.prepare({ ...request, key: "database/postgres-stale" }); + yield* ageLockFile(fs, stale.lockPath); + + const hit = yield* store.prepare({ ...request, key: "database/postgres-warm" }); + expect(hit.outcome).toBe("cached"); + expect(hit.path).toBe(warm.path); + expect(yield* fs.exists(stale.path)).toBe(true); + + yield* triggerSweep(root); + expect(yield* fs.exists(stale.path)).toBe(false); + }), + ), + ); + + it.live( + "a fresh pin never fails busy against a same-process retirement sweep triggered through the real store", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-pin-sweep-race-", + }); + const key = "database/postgres-pin-sweep-race"; + + const setupStore = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* setupStore.prepare({ ...request, key }); + // Idle and past the retention window: ahead-of-time `prepare` only pins itself for its + // own duration (see `ArtifactStore.prepare`), so by now nothing in this process still + // pins it, making it a legitimate, un-pinned sweep target. + yield* ageLockFile(fs, prepared.lockPath); + const keyRoot = path.dirname(prepared.path); + + // Gates the sweep right after it opens its own connection on the target's lock path + // (inside `acquireLock`/`takeExclusiveLock`, just before `retireGeneration`' + // `fs.stat` call), holding it open across a real async gap under full test control. + const sweepIsHolding = yield* Deferred.make(); + const releaseSweep = yield* Deferred.make(); + const sweepStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWritingAlt(), + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + stat: (...args: Parameters) => + args[0] === prepared.lockPath + ? Deferred.succeed(sweepIsHolding, undefined).pipe( + Effect.andThen(Deferred.await(releaseSweep)), + Effect.andThen(fs.stat(...args)), + ) + : fs.stat(...args), + }), + ); + + // Gates the fresh `use` right before it calls `Pin.pin`, once it has resolved the same + // generation and confirmed the key directory (the step immediately preceding the pin). + const freshPinApproaching = yield* Deferred.make(); + const letFreshPinProceed = yield* Deferred.make(); + const freshPinStore = yield* makeArtifactStore({ + cacheRoot: root, + source: sourceWriting(), + }).pipe( + Effect.provideService(FileSystem.FileSystem, { + ...fs, + realPath: (...args: Parameters) => + args[0] === keyRoot + ? Deferred.succeed(freshPinApproaching, undefined).pipe( + Effect.andThen(Deferred.await(letFreshPinProceed)), + Effect.andThen(fs.realPath(...args)), + ) + : fs.realPath(...args), + }), + ); + + // A failed assertion below must not leave either fiber parked on its gate forever. + yield* Effect.addFinalizer(() => + Effect.all([ + Deferred.succeed(releaseSweep, undefined), + Deferred.succeed(letFreshPinProceed, undefined), + ]), + ); + + const sweepFiber = yield* sweepStore + .prepare({ ...request, key }) + .pipe(Effect.exit, Effect.forkScoped); + yield* Deferred.await(sweepIsHolding); + + const freshPinFiber = yield* Effect.scoped(freshPinStore.use({ ...request, key })).pipe( + Effect.exit, + Effect.forkScoped, + ); + yield* Deferred.await(freshPinApproaching); + + // Releases the fresh pin into its own `Pin.pin` call first, while the sweep still holds + // the identical path open, then lets the sweep finish: this is the exact collision an + // unsynchronized sweep would lose. + yield* Deferred.succeed(letFreshPinProceed, undefined); + yield* Deferred.succeed(releaseSweep, undefined); + + const sweepExit = yield* Fiber.join(sweepFiber); + const freshPinExit = yield* Fiber.join(freshPinFiber); + const message = (exit: Exit.Exit) => + Exit.isFailure(exit) ? Cause.pretty(exit.cause) : ""; + expect(Exit.isSuccess(sweepExit), message(sweepExit)).toBe(true); + expect(Exit.isSuccess(freshPinExit), message(freshPinExit)).toBe(true); + }), + ), + ); + + it.live( + "the native launcher holds its own pin independent of the owner, releasing it only once it reports the workload's exit", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-launcher-pin-", + }); + const key = "database/postgres-launcher-pin"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + + const environment = yield* Environment.confine(fs, path, `${root}/home`); + const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); + yield* spawnNativeProcess( + { + executable: "/bin/sleep", + args: ["30"], + environment, + artifactLockPath: prepared.lockPath, + gracefulStopSignal: "SIGTERM", + gracefulStopTimeout: "1 second", + }, + defaultNativeProcessLauncher(), + ).pipe(Scope.provide(processScope)); + + yield* ageLockFile(fs, prepared.lockPath); + yield* triggerSweep(root); + expect(yield* fs.exists(prepared.path)).toBe(true); + + yield* Scope.close(processScope, Exit.void); + yield* ageLockFile(fs, prepared.lockPath); + yield* triggerSweep(root); + expect(yield* fs.exists(prepared.path)).toBe(false); + }), + ), + 20_000, + ); + + it.live( + "keeps the generation pinned until every process-group member is gone, not only the direct child", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-launcher-group-", + }); + const key = "database/postgres-launcher-group"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const prepared = yield* store.prepare({ ...request, key }); + + const environment = yield* Environment.confine(fs, path, `${root}/home`); + const pidPath = path.join(root, "descendant.pid"); + const readyPath = path.join(root, "ready"); + const goPath = path.join(root, "go"); + const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); + // The descendant ignores SIGTERM and records its own pid, then the leader signals + // readiness and waits for the test's own release file before exiting: only the + // group-wide SIGKILL in the launcher's exit handler can end the descendant. + yield* spawnNativeProcess( + { + executable: "/bin/sh", + args: [ + "-c", + // `$!` (captured by the leader right after backgrounding) is the descendant's + // real pid; `$$` read from inside the backgrounded subshell is unreliable here, + // since this shell reuses the leader's own pid for it. + '(trap "" TERM; while true; do sleep 1; done) & echo $! > "$1/descendant.pid"; ' + + 'touch "$1/ready"; while [ ! -f "$1/go" ]; do sleep 0.05; done', + "sh", + root, + ], + environment, + artifactLockPath: prepared.lockPath, + }, + defaultNativeProcessLauncher(), + ).pipe(Scope.provide(processScope)); + + const pollUntil = ( + check: Effect.Effect, + satisfied: (value: A) => boolean, + guard: Duration.Input, + ) => + check.pipe( + Effect.filterOrFail(satisfied, () => "not yet" as const), + Effect.retry({ + schedule: Schedule.spaced("20 millis").pipe(Schedule.upTo({ duration: guard })), + while: () => true, + }), + ); + const isPinBusy = Effect.scoped( + acquireLock(prepared.lockPath, "existing").pipe( + Effect.flatMap(takeExclusiveLock), + Effect.as(false), + Effect.catch((error) => Effect.succeed(isBusy(error))), + ), + ); + const isESRCH = (cause: unknown): boolean => + typeof cause === "object" && + cause !== null && + "code" in cause && + cause.code === "ESRCH"; + class ProcessProbeError extends Data.TaggedError("ProcessProbeError")<{ + readonly cause: unknown; + }> {} + // `true` means still alive; ESRCH resolves to `false` (terminated); any other error + // (an unexpected probe failure) propagates instead of being read either way. + const probeAlive = (pid: number) => + Effect.try({ + try: (): boolean => { + process.kill(pid, 0); + return true; + }, + catch: (cause) => new ProcessProbeError({ cause }), + }).pipe( + Effect.catch((error) => + isESRCH(error.cause) ? Effect.succeed(false) : Effect.fail(error), + ), + ); + /** + * Totalizes two separately-timed probes against the race where `kill(pid, 0)` finds the + * descendant alive but it vanishes before `ps` runs moments later: `ps` finding nothing + * (or failing outright) re-checks with `kill` rather than reading empty output as "still + * running". An unexpected (non-ESRCH) error from either probe fails the check. + */ + const isDescendantTerminated = (pid: number) => + Effect.gen(function* () { + if (!(yield* probeAlive(pid))) return true; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const { output, exitCode } = yield* Effect.scoped( + Effect.gen(function* () { + const inspection = yield* ChildProcess.make( + "ps", + ["-o", "stat=", "-p", String(pid)], + { + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }, + ); + const [stdout, exitCode] = yield* Effect.all( + [ + inspection.stdout.pipe(Stream.decodeText, Stream.mkString), + inspection.exitCode, + ], + { concurrency: 2 }, + ); + return { output: stdout.trim(), exitCode: Number(exitCode) }; + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)), + ); + if (exitCode === 0 && output.length > 0) return output.startsWith("Z"); + return !(yield* probeAlive(pid)); + }); + + // Readiness (the descendant's pid recorded, the leader waiting on `go`) is an + // observable file, not a fixed delay: polls its existence, bounded by a guard timeout. + yield* pollUntil(fs.exists(readyPath), (ready) => ready, "5 seconds"); + const descendantPid = Number((yield* fs.readFileString(pidPath)).trim()); + + expect(yield* isPinBusy).toBe(true); + + yield* fs.writeFileString(goPath, ""); + + // Polls the lock's own availability, not a fixed delay, bounded by a guard timeout. + yield* pollUntil(isPinBusy, (busy) => !busy, "10 seconds"); + + // Correlates the lock's release with the descendant's own termination: a launcher that + // released the pin before actually killing the group would still show it running. + expect(yield* isDescendantTerminated(descendantPid)).toBe(true); + + yield* Scope.close(processScope, Exit.void); + }), + ), + 20_000, + ); +}); + +describe("retirement across release keys", () => { + it.live( + "retires an older release key's stale, unlocked generation when a newer release key is prepared", + () => + withPlatform( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-stack-artifact-cross-key-", + }); + // Keys embed a release version (`Artifacts.ts`'s `artifactKey`), so a CLI upgrade + // resolves a different key entirely rather than a new digest under the same one. + const olderKey = "database/postgres/17.0.0-r0/darwin-arm64"; + const newerKey = "database/postgres/17.0.1-r0/darwin-arm64"; + const store = yield* makeArtifactStore({ cacheRoot: root, source: sourceWriting() }); + const older = yield* store.prepare({ ...request, key: olderKey }); + expect(older.outcome).toBe("downloaded"); + yield* ageLockFile(fs, older.lockPath); + + yield* store.prepare({ ...request, key: newerKey }); + + expect(yield* fs.exists(older.path)).toBe(false); + expect(yield* fs.exists(older.lockPath)).toBe(true); + }), + ), ); }); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index c321aae3c1..5d18634562 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -668,7 +668,10 @@ describe("slim-services artifact source", () => { const fetcher = serving({ "example.test": { archive, manifest: demoManifest } }); const failed = yield* withFetch(fetcher, store.prepare(request).pipe(Effect.exit)); expect(Exit.isFailure(failed)).toBe(true); - expect(yield* fs.exists(`${root}/demo/v1`)).toBe(false); + const entries = yield* fs + .readDirectory(`${root}/demo/v1`) + .pipe(Effect.orElseSucceed(() => [])); + expect(entries.filter((name) => /^[0-9a-f]{64}$/u.test(name))).toEqual([]); current = demo; const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); @@ -823,43 +826,40 @@ describe("native artifact catalog", () => { ), ); - it.live("keys the native cache by release version and ignores a legacy upstream entry", () => - Effect.scoped( - Effect.gen(function* () { - const resolved = yield* resolveArtifact({ service: "rest" }); - const fs = yield* FileSystem.FileSystem; - const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-cache-" }); - const requested: string[] = []; - const seed = (version: string, content: string) => - makeArtifactStore({ cacheRoot, source: fixtureSource(content) }).pipe( - Effect.flatMap((store) => - withFetch( - serving({}, requested), + it.live( + "keys the native cache by content: a fixture seeded under the real pinned digest is used, and an unpinned digest is never requested", + () => + Effect.scoped( + Effect.gen(function* () { + const resolved = yield* resolveArtifact({ service: "rest" }); + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-cache-" }); + const requested: string[] = []; + const digest = resolved.natives["linux-amd64"].archive; + const seeded = yield* withFetch( + serving({}, requested), + makeArtifactStore({ + cacheRoot, + source: { ...fixtureSource("pinned"), checksum: () => Effect.succeed(digest) }, + }).pipe( + Effect.flatMap((store) => store.prepare({ - key: `slim-services/postgrest/${version}/linux-amd64`, + key: `slim-services/postgrest/${resolved.releaseVersion}/linux-amd64`, requiredRuntimePaths: resolved.requiredRuntimePaths, executablePath: resolved.executablePath, }), ), ), ); - const prepare = withFetch( - serving({}, requested), - prepareNativeArtifact({ service: "rest" }, cacheRoot, linux), - ); - - yield* seed(resolved.version, "legacy"); - expect(Exit.isFailure(yield* prepare.pipe(Effect.exit))).toBe(true); - expect(requested).not.toEqual([]); - - const seeded = yield* seed(resolved.releaseVersion, "pinned"); - requested.length = 0; - const prepared = yield* prepare; - expect(prepared.root).toBe(seeded.path); - expect(yield* fs.readFileString(prepared.executable)).toBe("pinned"); - expect(requested).toEqual([]); - }).pipe(Effect.provide(NodeServices.layer)), - ), + const prepared = yield* withFetch( + serving({}, requested), + prepareNativeArtifact({ service: "rest" }, cacheRoot, linux), + ); + expect(prepared.root).toBe(seeded.path); + expect(yield* fs.readFileString(prepared.executable)).toBe("pinned"); + expect(requested).toEqual([]); + }).pipe(Effect.provide(NodeServices.layer)), + ), ); it("catalog has no placeholder pins", () => { diff --git a/packages/stack/src/public.e2e.test.ts b/packages/stack/src/public.e2e.test.ts index f8f33a415c..c27754776b 100644 --- a/packages/stack/src/public.e2e.test.ts +++ b/packages/stack/src/public.e2e.test.ts @@ -3,12 +3,12 @@ import { expect, expectTypeOf, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Path, Redacted, Schema, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; -import { tmpdir } from "node:os"; import { discover, open } from "./effect.ts"; import { postgres } from "./Commands.ts"; import * as PromiseStack from "./index.ts"; import { assertOwnerExited, captureOwnerPid } from "../tests/owner.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; +import { testArtifactCacheRoot } from "../tests/artifact-cache.ts"; for (const runtime of ["node", "bun"] as const) { it.live( @@ -22,7 +22,7 @@ for (const runtime of ["node", "bun"] as const) { [ new URL("../tests/owner-exit-client.ts", import.meta.url).pathname, root, - `${tmpdir()}/supabase-stack-artifacts`, + testArtifactCacheRoot, ], { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, ); @@ -51,7 +51,7 @@ it.live( const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-public-e2e-" }); - const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + const cacheRoot = testArtifactCacheRoot; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( @@ -370,7 +370,7 @@ it.live( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-public-observation-" }); - const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + const cacheRoot = testArtifactCacheRoot; const locations = { stateRoot: `${root}/state`, cacheRoot }; const stack = yield* Effect.tryPromise(() => PromiseStack.create({ ...locations, projectRoot: root, runtime: "native" }), diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts index 9028e812db..da4cb5f163 100644 --- a/packages/stack/src/runtime/Container.integration.test.ts +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, ConfigProvider, + Context, Crypto, Data, Deferred, @@ -13,8 +14,10 @@ import { Layer, Option, Path, + Redacted, Ref, Schema, + Scope, Sink, Stream, } from "effect"; @@ -27,17 +30,32 @@ import { DOCKER_HOST_ALIAS, makeContainerRuntime, makeHostGateway, + resolveEngineTarget, type ContainerProcess, + type EngineTarget, } from "./Container.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import * as Owner from "../Owner.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; +import { reconcileContainerPassword } from "../services/Database.ts"; const image = await Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const file = yield* path.fromFileUrl(new URL("../../../../.bun-version", import.meta.url)); const version = yield* fs.readFileString(file); - return `oven/bun:${version.trim()}-slim`; + return `docker.io/oven/bun:${version.trim()}-slim`; }).pipe(Effect.provide(NodeServices.layer), Effect.runPromise); +// An unpinned target: these tests exercise the container runtime, not endpoint pinning, so an +// empty argv prefix leaves every command exactly as it was before pinning existed. +const containerTarget: EngineTarget = { ...engineTarget, argv: [] }; const ipv4 = /^(?:\d{1,3}\.){3}\d{1,3}$/u; +// One probe for every test that does not assert probe behaviour, so engine load stays bounded. +const sharedHostGateway = await makeHostGateway.pipe( + Effect.provideService(Scope.Scope, await Effect.runPromise(Scope.make())), + Effect.runPromise, +); const stoppableIdleScript = "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)"; @@ -46,11 +64,14 @@ class ContainerTestError extends Data.TaggedError("ContainerTestError")<{ readonly cause?: unknown; }> {} -describe("container process adapter", () => { +describe("container process adapter", { timeout: 120_000 }, () => { it.live("recognizes a cached pinned image without pulling", () => Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + }); yield* runtime.prepare(image); const repositoryDigest = yield* repoDigest(delegate, image); const digest = repositoryDigest.slice(repositoryDigest.indexOf("@") + 1); @@ -58,7 +79,10 @@ describe("container process adapter", () => { const pinnedImage = `${image}@${digest}`; const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); - yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + }).pipe( Effect.flatMap((runtime) => runtime.prepare(pinnedImage)), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), ); @@ -73,7 +97,10 @@ describe("container process adapter", () => { const token = yield* crypto.randomUUIDv4; const pullAttempted = yield* Ref.make(false); const spawner = makePullFailureSpawner(delegate, pullAttempted); - const result = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + const result = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + }).pipe( Effect.flatMap((runtime) => runtime.prepare(`supabase-prepare-regression:${token}`).pipe(Effect.exit), ), @@ -89,7 +116,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const id = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -123,7 +154,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const id = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launchCommand({ image, @@ -160,7 +195,7 @@ describe("container process adapter", () => { Effect.scoped( Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }); @@ -197,7 +232,11 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const hostGateway = yield* makeHostGateway; - const makeRuntime = makeContainerRuntime({ engine: "docker", root: ".", hostGateway }).pipe( + const makeRuntime = makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeHostGatewayProbeSpawner(delegate, probes, () => undefined), @@ -239,13 +278,13 @@ describe("container process adapter", () => { const hostGateway = yield* makeHostGateway; const spawner = makeHostGatewayProbeSpawner(delegate, probes, () => undefined, release); const database = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway, awaitHostGateway: false, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); const service = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway, }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); @@ -280,7 +319,7 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -316,7 +355,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -354,7 +393,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -393,14 +432,14 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeHostGatewayProbeSpawner(delegate, probes, (attempt) => - attempt === 1 ? hostGatewayRejectionScript : podmanHostsScript, + attempt === 1 ? hostGatewayRejectionScript : engineHostsScript, ), ), ); @@ -425,7 +464,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -435,7 +474,7 @@ describe("container process adapter", () => { attempt === 2 ? `console.error("daemon busy"); process.exit(1)` : attempt === 4 - ? podmanHostsScript + ? engineHostsScript : hostGatewayRejectionScript, ), ), @@ -460,7 +499,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, }).pipe( @@ -485,7 +524,9 @@ describe("container process adapter", () => { }), ).pipe(Effect.flip); expect(failure._tag).toBe("ContainerError"); - expect(failure.message).toContain("upgrade Podman or use --runtime podman"); + expect(failure.message).toContain( + "rejects host-gateway in --add-host and maps no IPv4 address", + ); } expect(yield* Ref.get(probes)).toBe(2); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -497,7 +538,7 @@ describe("container process adapter", () => { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const probes = yield* Ref.make(0); const database = yield* makeContainerRuntime({ - engine: "docker", + target: containerTarget, root: ".", hostGateway: yield* makeHostGateway, awaitHostGateway: false, @@ -507,7 +548,7 @@ describe("container process adapter", () => { makeHostGatewayProbeSpawner( makeHostGatewayRejectingCreateSpawner(delegate), probes, - (attempt) => (attempt === 1 ? hostGatewayRejectionScript : podmanHostsScript), + (attempt) => (attempt === 1 ? hostGatewayRejectionScript : engineHostsScript), ), ), ); @@ -529,7 +570,11 @@ describe("container process adapter", () => { it.live("names and labels a service container for compose-style grouping", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -553,7 +598,11 @@ describe("container process adapter", () => { it.live("marks a one-shot container's name and compose labels as a task", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launchCommand({ image, @@ -576,7 +625,11 @@ describe("container process adapter", () => { it.live("labels a created container with the configured test run", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const crypto = yield* Crypto.Crypto; const testRunId = `container-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; @@ -606,7 +659,11 @@ describe("container process adapter", () => { () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const launch = (instanceId: string, marker: string) => runtime.launch({ @@ -642,7 +699,11 @@ describe("container process adapter", () => { it.live("stops a container with its configured stop signal", () => Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -668,7 +729,11 @@ describe("container process adapter", () => { it.live("waits until a discarded container stops before returning", () => Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -695,7 +760,11 @@ describe("container process adapter", () => { it.live("returns cleanup authority when container start fails", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const result = yield* runtime .launch({ @@ -723,7 +792,11 @@ describe("container process adapter", () => { it.live("keeps exit observation shared after a caller cancels its wait", () => Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -751,7 +824,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const delegate = yield* ChildProcessSpawner.ChildProcessSpawner; const released = yield* Deferred.make(); - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeLogFollowerSpawner( @@ -789,7 +866,11 @@ describe("container process adapter", () => { const released = yield* Deferred.make(); yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }).pipe( Effect.provideService( ChildProcessSpawner.ChildProcessSpawner, makeLogFollowerSpawner(delegate, released, "setInterval(() => {}, 1000)"), @@ -819,7 +900,11 @@ describe("container process adapter", () => { yield* Effect.ensuring( Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -851,7 +936,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -888,7 +977,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -930,7 +1023,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -974,7 +1071,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1013,7 +1114,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1060,7 +1165,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1104,7 +1213,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const process = yield* runtime.launch({ image, @@ -1157,7 +1270,11 @@ describe("container process adapter", () => { yield* Effect.ensuring( Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const result = yield* runtime .launch({ @@ -1198,7 +1315,11 @@ describe("container process adapter", () => { Effect.gen(function* () { const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); return yield* runtime .launch({ @@ -1238,7 +1359,11 @@ describe("container process adapter", () => { const spawner = makePendingCreateSpawner(delegate, started, release, present); const result = yield* Effect.scoped( Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }); yield* runtime.prepare(image); const launch = yield* runtime .launch({ @@ -1267,9 +1392,11 @@ describe("container process adapter", () => { it.effect("bounds a hung docker create and removes the container it may still create", () => Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( - Effect.provide(engine.layer), - ); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( runtime.launch({ image, stackId: "i".repeat(64), instanceId: "hung-create", env: {} }), ).pipe(Effect.provide(engine.layer), Effect.exit, Effect.forkChild); @@ -1291,9 +1418,11 @@ describe("container process adapter", () => { it.effect("removes the container a hung docker create may still create when interrupted", () => Effect.gen(function* () { const engine = yield* makeHangingCreateSpawner(); - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }).pipe( - Effect.provide(engine.layer), - ); + const runtime = yield* makeContainerRuntime({ + target: containerTarget, + root: ".", + hostGateway: sharedHostGateway, + }).pipe(Effect.provide(engine.layer)); const launch = yield* Effect.scoped( runtime.launch({ image, stackId: "i".repeat(64), instanceId: "interrupted", env: {} }), ).pipe(Effect.provide(engine.layer), Effect.forkChild); @@ -1304,13 +1433,206 @@ describe("container process adapter", () => { expect(engine.commands).toContainEqual(["rm", "--force", engine.createdName()]); }).pipe(Effect.provide(NodeServices.layer)), ); + + it.live( + "resolveEngineTarget pins the context resolved at startup by name, unaffected by a later context switch", + () => + Effect.gen(function* () { + const contextCalls: Array> = []; + let activeContext = "alpha"; + const handle = (exitCode: number, stdout = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected command"); + if (command.args[0] === "context" && command.args[1] === "show") { + contextCalls.push(command.args); + return Effect.succeed(handle(0, `${activeContext}\n`)); + } + // Identity is resolved through the same pinned context, so "info" arrives prefixed with + // the `--context ` pin this very call resolved, by name, not by endpoint: pinning + // by name (rather than an inspected endpoint) is what keeps a TLS context's own + // certificate material intact for this and every later invocation. + if (command.args[0] === "--context" && command.args[2] === "info") + return Effect.succeed(handle(0, `daemon-at-${command.args[1]}`)); + return Effect.die(`Unexpected docker command: ${command.args.join(" ")}`); + }); + const target = yield* resolveEngineTarget(spawner, "docker"); + // Simulates `docker context use beta` run elsewhere, after this target was resolved. + activeContext = "beta"; + expect(contextCalls).toHaveLength(1); + expect(target.argv).toEqual(["--context", "alpha"]); + expect(target.daemonId).toBe("daemon-at-alpha"); + }).pipe( + Effect.provide( + Layer.merge(NodeServices.layer, ConfigProvider.layer(ConfigProvider.fromEnvRecord({}))), + ), + ), + ); + + it.live( + "keeps helper creation, runtime launches, password reconcile and reconcile pinned to the context an owner resolved at startup, despite a later context switch", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "engine-target-pin-" }); + const commands: Array> = []; + let activeContext = "alpha"; + // An orphaned container found by label alone: listed once (for removal), then gone. + const orphanId = "orphan00000000000000000000000000001"; + let psCalls = 0; + const handle = (exitCode: number, stdout = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: stdout === "" ? Stream.empty : Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + // A permissive fake: every engine call this test's consumers make succeeds, so the only + // thing under test is the argv each one is actually invoked with. + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + return Effect.die("Unexpected command"); + commands.push(command.args); + if (command.args[0] === "context" && command.args[1] === "show") + return Effect.succeed(handle(0, `${activeContext}\n`)); + // Every other call carries the `--context ` pin this test's resolution produced. + const unpinned = command.args.slice(2); + if (unpinned[0] === "info") return Effect.succeed(handle(0, "fake-daemon-id")); + if (unpinned[0] === "version") return Effect.succeed(handle(0, "27.0.0|27.0.0")); + if (unpinned[0] === "inspect") + return Effect.succeed(handle(0, '{"Ports":{},"Status":"running","ExitCode":0}')); + if (unpinned[0] === "run") + return Effect.succeed( + handle(0, unpinned.includes("-i") ? "supabase-helper-ready\n" : "abcdef012345"), + ); + // Two call sites share "ps": the launched container's own `--rm` removal poll (filtered + // by name, always confirmed gone) and the label-based orphan discovery below. + if (unpinned[0] === "ps" && unpinned.some((arg) => arg.includes("com.supabase.stack="))) { + psCalls += 1; + return Effect.succeed(handle(0, psCalls === 1 ? `${orphanId}\n` : "")); + } + if (unpinned[0] === "ps") return Effect.succeed(handle(0, "")); + return Effect.succeed(handle(0)); + }); + + // Resolved once, as an owner's own startup would. + const target = yield* resolveEngineTarget(spawner, "docker"); + expect(target.argv).toEqual(["--context", "alpha"]); + // Simulates `docker context use beta` run elsewhere, after this target was resolved. + activeContext = "beta"; + // Only the resolution above may run unpinned; every consumer exercised below must carry + // the context it pinned, regardless of the later switch. + commands.length = 0; + + const container = yield* makeContainerRuntime({ + target, + root, + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + + // Runtime launches. + yield* container + .prepare(image) + .pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + yield* Effect.scoped( + container.launchCommand({ + image, + stackId: "f".repeat(64), + instanceId: "engine-target-pin-launch", + env: {}, + args: ["-e", "process.exit(0)"], + }), + ).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); + + // Helper creation, sharing the same container runtime a real stack would. + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target, + stackId: "engine-target-pin", + instanceId: "database", + instanceRoot: root, + root, + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + + // Password reconcile: the consumer F2 found spawning the engine unpinned. + yield* reconcileContainerPassword( + target, + "password-reconcile-container", + Redacted.make("engine-target-pin-password"), + spawner, + ); + + // Reconcile: an orphaned container, found and removed by label alone, through the same + // pinned target. + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: path.join(root, "state") })), + StackNamespace.Service, + ); + const saved = { + id: "engine-target-pin", + runtime: "docker" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "engine-target-pin" }, + instances: [], + lifetime: "detached" as const, + composition: { members: [], dependencies: [] }, + }; + yield* state.save(saved); + yield* Owner.sweepContainers(saved, root, target).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + + // Every call these consumers made, after the target was resolved, carries the context + // pinned at startup: asserted of the whole list, not merely of a filtered subset, so a + // consumer that forgets the pin (as the password reconcile once did) fails this test + // instead of being silently excluded from it. + expect(commands.length).toBeGreaterThan(0); + for (const args of commands) expect(args.slice(0, 2)).toEqual(["--context", "alpha"]); + // Each consumer is actually exercised, not vacuously skipped. + expect(commands.some((args) => args.includes("pull"))).toBe(true); + expect(commands.some((args) => args.includes("version"))).toBe(true); + expect(commands.some((args) => args.includes("password-reconcile-container"))).toBe(true); + expect(commands.some((args) => args.includes("rm") && args.includes(orphanId))).toBe(true); + }).pipe( + Effect.provide( + Layer.merge(NodeServices.layer, ConfigProvider.layer(ConfigProvider.fromEnvRecord({}))), + ), + ), + ); }); const repoDigest = (spawner: ChildProcessSpawnerService["Service"], image: string) => Effect.gen(function* () { const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, ["image", "inspect", "--format", "{{range .RepoDigests}}{{println .}}{{end}}", image], { stdin: "ignore" }, ), @@ -1343,7 +1665,7 @@ const makePullFailureSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "pull" ) return Effect.gen(function* () { @@ -1363,13 +1685,13 @@ const hostGatewayRejectionScript = `console.error(${JSON.stringify( 'Error response from daemon: invalid IP address in add-host: "host-gateway"', )}); process.exit(125)`; -const podmanHostsScript = `console.log("10.88.0.1\\thost.containers.internal host.docker.internal")`; +const engineHostsScript = `console.log("10.88.0.1\\thost.docker.internal")`; -/** Rejects a `docker create` that maps the host alias to `host-gateway`, as older Podman does. */ +/** Rejects a `docker create` that maps the host alias to `host-gateway`. */ const makeHostGatewayRejectingCreateSpawner = (delegate: ChildProcessSpawnerService["Service"]) => ChildProcessSpawner.make((command) => ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "create" && command.args.includes(`${DOCKER_HOST_ALIAS}:host-gateway`) ? delegate.spawn( @@ -1393,7 +1715,7 @@ const makeHostGatewayProbeSpawner = ( ChildProcessSpawner.make((command) => { if ( !ChildProcess.isStandardCommand(command) || - command.command !== "docker" || + command.command !== testEngine || command.args[0] !== "run" || !command.args.includes("/etc/hosts") ) @@ -1416,7 +1738,7 @@ const makeStopFailureSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "stop" ) { return Effect.gen(function* () { @@ -1441,7 +1763,7 @@ const makeLogFollowerSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "logs" ) return Effect.gen(function* () { @@ -1460,7 +1782,7 @@ const makeLostRemoveResultSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1494,9 +1816,11 @@ const makeRemoveFailureSpawner = ( if ( failProbe !== undefined && ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "ps" && - command.args.some((arg) => arg.startsWith("name=^/?")) + // Excludes the host-gateway probe's own throwaway "-task-" container, created and awaited + // absent during `launch`, before this test's own instance container ever reaches `remove`. + command.args.some((arg) => arg.startsWith("name=^/?") && !arg.includes("-task-")) ) { return Effect.gen(function* () { if (!(yield* Ref.get(failProbe))) return yield* delegate.spawn(command); @@ -1512,7 +1836,7 @@ const makeRemoveFailureSpawner = ( } if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1538,9 +1862,11 @@ const makePendingRemoveSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "ps" && - command.args.some((arg) => arg.startsWith("name=^/?")) + // Excludes the host-gateway probe's own throwaway "-task-" container, created and awaited + // absent during `launch`, before this test's own instance container ever reaches `remove`. + command.args.some((arg) => arg.startsWith("name=^/?") && !arg.includes("-task-")) ) { return Effect.gen(function* () { const remaining = yield* Ref.get(pendingProbes); @@ -1555,7 +1881,7 @@ const makePendingRemoveSpawner = ( } if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1588,7 +1914,7 @@ const makeInterruptedRemoveSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "rm" ) { return Effect.gen(function* () { @@ -1624,7 +1950,7 @@ const makeCreateFailureSpawner = ( ChildProcessSpawner.make((command) => { if ( ChildProcess.isStandardCommand(command) && - command.command === "docker" && + command.command === testEngine && command.args[0] === "create" ) { return Effect.gen(function* () { @@ -1659,7 +1985,7 @@ const makePendingCreateSpawner = ( present: Ref.Ref, ) => ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command) || command.command !== "docker") + if (!ChildProcess.isStandardCommand(command) || command.command !== testEngine) return delegate.spawn(command); if (command.args[0] === "stop" || command.args[0] === "rm") { return Effect.gen(function* () { @@ -1687,14 +2013,14 @@ const makePendingCreateSpawner = ( }); }); -const successfulHandle = () => +const successfulHandle = (stdout = "") => ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(0), exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), isRunning: Effect.succeed(false), kill: () => Effect.void, stdin: Sink.drain, - stdout: Stream.empty, + stdout: stdout === "" ? Stream.empty : Stream.succeed(new TextEncoder().encode(stdout)), stderr: Stream.empty, all: Stream.empty, getInputFd: () => Sink.drain, @@ -1708,6 +2034,7 @@ const makeHangingCreateSpawner = Effect.fn("ContainerTest.hangingCreateSpawner") const spawner = ChildProcessSpawner.make((command) => { if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); commands.push([...command.args]); + if (command.args[0] === "info") return Effect.succeed(successfulHandle("fake-daemon-id")); if (command.args[0] !== "create") return Effect.succeed(successfulHandle()); // The engine never answers create; the real daemon may or may not have committed it. return Deferred.succeed(createStarted, undefined).pipe( @@ -1759,7 +2086,7 @@ const exists = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", id], { + ChildProcess.make(testEngine, ["inspect", id], { stdin: "ignore", stdout: "ignore", stderr: "ignore", @@ -1772,7 +2099,7 @@ const inspectLabels = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", "--format={{json .Config.Labels}}", id], { + ChildProcess.make(testEngine, ["inspect", "--format={{json .Config.Labels}}", id], { stdin: "ignore", }), ); @@ -1786,7 +2113,7 @@ const inspectExtraHosts = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", "--format={{json .HostConfig.ExtraHosts}}", id], { + ChildProcess.make(testEngine, ["inspect", "--format={{json .HostConfig.ExtraHosts}}", id], { stdin: "ignore", }), ); @@ -1800,7 +2127,7 @@ const removeExternally = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["rm", "--force", id], { + ChildProcess.make(testEngine, ["rm", "--force", id], { stdin: "ignore", stdout: "ignore", stderr: "ignore", @@ -1809,7 +2136,7 @@ const removeExternally = (id: string) => const code = yield* child.exitCode; if (Number(code) !== 0) return yield* new ContainerTestError({ - message: `docker rm --force exited with ${String(code)}`, + message: `${testEngine} rm --force exited with ${String(code)}`, }); }); @@ -1818,7 +2145,7 @@ const idsByInstance = (instanceId: string) => const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, [ "ps", "--all", @@ -1843,7 +2170,7 @@ const removeByInstance = (instanceId: string) => if (ids.length === 0) return; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["rm", "--force", ...ids], { stdin: "ignore" }), + ChildProcess.make(testEngine, ["rm", "--force", ...ids], { stdin: "ignore" }), ); const [stderr, code] = yield* Effect.all([ child.stderr.pipe(Stream.decodeText, Stream.mkString), @@ -1860,7 +2187,7 @@ const running = (id: string) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["inspect", "--format={{.State.Running}}", id], { + ChildProcess.make(testEngine, ["inspect", "--format={{.State.Running}}", id], { stdin: "ignore", }), ); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 092ffafd7c..d64168089d 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -1,6 +1,7 @@ import { withAttemptCount } from "../internal/attempts.ts"; import { Cause, + Config, Crypto, Data, Deferred, @@ -21,13 +22,13 @@ import { } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; +import { CONTAINER_ENV_DIRNAME } from "../namespace/Paths.ts"; import { identifyContainer } from "./ContainerName.ts"; export class ContainerError extends Data.TaggedError("ContainerError")<{ readonly operation: string; readonly message: string; readonly cause?: unknown; - readonly reason?: "engine-unavailable"; }> {} interface ContainerSpec { @@ -54,6 +55,8 @@ interface ContainerSpec { readonly stopGraceSeconds?: number; /** Signal `docker stop` sends first; omitted uses the image's stop signal. */ readonly stopSignal?: "SIGINT" | "SIGTERM"; + /** `uid:gid` the container runs as, overriding the image's own user. */ + readonly user?: string; } export interface ContainerProcess { @@ -88,7 +91,7 @@ export interface ContainerRuntime { ) => Effect.Effect; } -const errorFor = (operation: string, cause: unknown) => +const errorFor = (operation: string, cause: unknown): ContainerError => new ContainerError({ operation, message: cause instanceof Error ? cause.message : String(cause), @@ -100,6 +103,25 @@ const testRunLabelArgs = readTestRunLabelArgs.pipe( Effect.mapError((cause) => errorFor("config", cause)), ); +/** + * Matches an engine CLI that is missing or reports a daemon that is not listening, never one + * that rejects the caller (for example on permissions), so callers can leave work for a retry + * instead of surfacing a hard failure. + */ +export const engineUnreachable = (cause: unknown): boolean => { + const message = cause instanceof Error ? cause.message : String(cause); + return ( + (cause instanceof Object && + "cause" in cause && + cause.cause instanceof PlatformError.PlatformError && + cause.cause.reason._tag === "NotFound" && + cause.cause.reason.method === "spawn") || + /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)|unable to connect to podman socket:[^\n]*the system cannot find the file specified/iu.test( + message, + ) + ); +}; + const rateLimited = (error: ContainerError) => /toomanyrequests|too many requests|rate limit|rate exceeded/iu.test(error.message); @@ -113,42 +135,179 @@ const transientPullFailure = (error: ContainerError) => error.message, ); +/** Runs one engine CLI invocation outside any pinned target, for resolving that target itself. */ +const runRaw = ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + engine: ContainerEngine, + args: ReadonlyArray, +): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make(engine, args, { stdin: "ignore", stdout: "pipe", stderr: "pipe" }), + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor(args[0] ?? "command", stderr.trim() || `Engine exited with ${code}`); + return stdout; + }), + ).pipe( + Effect.timeout("10 seconds"), + Effect.mapError((cause) => + cause instanceof ContainerError ? cause : errorFor(args[0] ?? "command", cause), + ), + ); + +/** The container engine CLI a stack's containers run through. */ +export type ContainerEngine = "docker" | "podman"; + /** - * Matches an engine CLI that is missing or reports a daemon that is not listening, not one that - * rejects the caller. Podman's connection wrappers and Windows' `error during connect` also wrap - * authentication and TLS failures, so only their refused or missing-endpoint causes match. + * The engine endpoint and identity an owner resolves once, at startup, and pins for its entire + * lifetime: the container runtime, the storage helpers, the host-gateway probes and the + * namespace's reconcile loop all share this one target instead of each resolving (and so + * potentially disagreeing on) their own. `argv` is the explicit prefix every invocation carries + * (`--host ` for Docker, `--url`/`--connection` for Podman), since the CLI gives it + * priority over its environment and saved defaults, and since an argument, unlike an environment + * variable, never leaks to subprocesses a launched workload spawns. `rootless` is whether the + * Podman service runs without root, which decides how host uids map into containers. */ -const engineUnreachable = (error: ContainerError) => - (error.cause instanceof PlatformError.PlatformError && - error.cause.reason._tag === "NotFound" && - error.cause.reason.method === "spawn") || - /cannot connect to the docker daemon|connection refused|connect: no such file or directory|error during connect:[^\n]*(?:docker daemon is not running|the system cannot find the file specified)/iu.test( - error.message, +export type EngineTarget = + | { + readonly engine: "docker"; + readonly argv: ReadonlyArray; + readonly daemonId: string; + } + | { + readonly engine: "podman"; + readonly argv: ReadonlyArray; + readonly daemonId: string; + readonly rootless: boolean; + }; + +/** + * The active context's own name, pinned by name rather than by its endpoint alone: `context show` + * honours `DOCKER_CONTEXT` and the config file, and pinning by name keeps that context's own TLS + * material (CA, client certificate and key, skip-verify) intact for every later invocation, which + * extracting just its endpoint would otherwise drop. + */ +const resolveContextName = (spawner: ChildProcessSpawner.ChildProcessSpawner["Service"]) => + Effect.gen(function* () { + const name = (yield* runRaw(spawner, "docker", ["context", "show"])).trim(); + if (name.length === 0) return yield* errorFor("context", "Engine returned no active context"); + return name; + }); + +const nonEmptyEnv = (name: string) => + Config.option(Config.string(name)).pipe( + Effect.map(Option.filter((value) => value.length > 0)), + Effect.orElseSucceed(() => Option.none()), ); -const markUnavailable = (error: ContainerError) => - engineUnreachable(error) - ? new ContainerError({ - operation: error.operation, - message: error.message, - cause: error.cause, - reason: "engine-unavailable", - }) - : error; +const PODMAN_INFO_FORMAT = + "{{.Host.ServiceIsRemote}}|{{.Host.Security.Rootless}}|{{.Host.Hostname}}|{{.Store.GraphRoot}}"; -/** A pull worth retrying: rate-limited or a dropped connection, never an unreachable engine. */ -const retryablePull = (error: ContainerError) => - (rateLimited(error) || transientPullFailure(error)) && !engineUnreachable(error); +/** Podman has no daemon `.ID`; its host name and graph root together identify the service. */ +const podmanInfo = ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + argv: ReadonlyArray, +) => + Effect.gen(function* () { + const output = yield* runRaw(spawner, "podman", [ + ...argv, + "info", + "--format", + PODMAN_INFO_FORMAT, + ]); + const [remote, rootless, hostname, ...graphRoot] = output.trim().split("|"); + const root = graphRoot.join("|"); + if (hostname === undefined || hostname.length === 0 || root.length === 0) + return yield* errorFor("identity", "Engine returned an empty id"); + return { + remote: remote === "true", + rootless: rootless === "true", + daemonId: `${hostname}|${root}`, + }; + }); -const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; +/** + * Pins the connection the Podman CLI would use: `CONTAINER_HOST` or `CONTAINER_CONNECTION` when + * set, otherwise the default saved connection when the CLI talks to a remote service (a + * `podman machine`), and nothing when it talks to a local one, which saved connections never + * redirect. + */ +const resolvePodmanTarget = Effect.fn("Container.resolvePodmanTarget")(function* ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], +): Effect.fn.Return { + const url = yield* nonEmptyEnv("CONTAINER_HOST"); + const connection = yield* nonEmptyEnv("CONTAINER_CONNECTION"); + const explicit: ReadonlyArray | undefined = Option.isSome(url) + ? ["--url", url.value] + : Option.isSome(connection) + ? ["--connection", connection.value] + : undefined; + const bare = explicit === undefined ? yield* podmanInfo(spawner, []) : undefined; + let argv: ReadonlyArray = explicit ?? []; + if (bare?.remote === true) { + const listing = yield* runRaw(spawner, "podman", [ + "system", + "connection", + "list", + "--format", + "{{.Name}}|{{.Default}}", + ]); + const name = listing + .split("\n") + .map((line) => line.trim().split("|")) + .find(([, isDefault]) => isDefault === "true")?.[0]; + if (name === undefined || name.length === 0) + return yield* errorFor("connection", "Podman has no default connection to pin"); + argv = ["--connection", name]; + } + const info = bare !== undefined && argv.length === 0 ? bare : yield* podmanInfo(spawner, argv); + return { engine: "podman", argv, daemonId: info.daemonId, rootless: info.rootless }; +}); + +/** + * Resolves and pins, once for an owner's whole lifetime, the single engine endpoint its commands + * target and that endpoint's own identity. + */ +export const resolveEngineTarget = Effect.fn("Container.resolveEngineTarget")(function* ( + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + engine: ContainerEngine, +): Effect.fn.Return { + yield* Effect.annotateCurrentSpan("container.engine", engine); + if (engine === "podman") return yield* resolvePodmanTarget(spawner); + const host = yield* nonEmptyEnv("DOCKER_HOST"); + const argv: ReadonlyArray = Option.isSome(host) + ? ["--host", host.value] + : ["--context", yield* resolveContextName(spawner)]; + const daemonId = (yield* runRaw(spawner, "docker", [ + ...argv, + "info", + "--format", + "{{.ID}}", + ])).trim(); + if (daemonId.length === 0) return yield* errorFor("identity", "Engine returned an empty id"); + return { engine: "docker", argv, daemonId }; +}); + +/** A pull worth retrying: rate-limited or a dropped connection. */ +const retryablePull = (error: ContainerError) => rateLimited(error) || transientPullFailure(error); const PULL_MAX_RETRIES = 4; /** * Host alias mapped in Docker containers' `/etc/hosts` to the engine's IPv4 host gateway, or to * `host-gateway` for runtimes that do not await the probe; Docker Desktop's `host.docker.internal` - * and `host-gateway` also resolve to an IPv6 address. Engines that reject `host-gateway` get the - * IPv4 address they map to `host.docker.internal` or `host.containers.internal` instead. + * and `host-gateway` also resolve to an IPv6 address. An engine that rejects `host-gateway` gets + * the IPv4 address it maps to `host.docker.internal` instead. */ export const DOCKER_HOST_ALIAS = "host.supabase.internal"; @@ -157,7 +316,7 @@ const HOST_GATEWAY_PROBE_TIMEOUT: Duration.Input = "15 seconds"; const IPV4_ADDRESS = /^(?:\d{1,3}\.){3}\d{1,3}$/u; -/** Names an engine may write into `/etc/hosts` for its host, such as Podman's compat socket. */ +/** The name the engine writes into `/etc/hosts` for its own host. */ const ENGINE_HOST_NAMES = ["host.docker.internal", "host.containers.internal"]; const firstIpv4For = (hosts: string, names: ReadonlyArray) => @@ -169,7 +328,7 @@ const firstIpv4For = (hosts: string, names: ReadonlyArray) => IPV4_ADDRESS.test(address ?? "") && mapped.some((name) => names.includes(name)), )?.[0]; -/** Matches an engine that rejects the `host-gateway` keyword in `--add-host`, such as older Podman. */ +/** Matches an engine that rejects the `host-gateway` keyword in `--add-host`. */ const rejectsHostGateway = (error: ContainerError) => /(?:invalid|unknown|unsupported|bad)[^\n]*add-host[^\n]*host-gateway/iu.test(error.message); @@ -243,6 +402,9 @@ const pullBackoff = Schedule.exponential("2 seconds").pipe(Schedule.jittered); /** `docker create` only writes metadata; a healthy daemon answers well within this bound. */ const CREATE_TIMEOUT: Duration.Input = "2 minutes"; +/** A started container's loopback publish can lag under concurrent engine load before landing. */ +const PORT_PUBLISH_TIMEOUT: Duration.Input = "2 minutes"; + const PublishedPorts = Schema.Record( Schema.String, Schema.NullOr( @@ -255,6 +417,13 @@ const PublishedPorts = Schema.Record( ), ); +/** The published ports together with the run state they were observed in. */ +const PublicationState = Schema.Struct({ + Ports: Schema.NullOr(PublishedPorts), + Status: Schema.String, + ExitCode: Schema.Finite, +}); + const mountField = (key: string, value: string) => { const field = `${key}=${value}`; return /[,"\n\r]/u.test(field) ? `"${field.replaceAll('"', '""')}"` : field; @@ -268,7 +437,7 @@ const mountField = (key: string, value: string) => { * with backoff. */ export const makeContainerRuntime = (options: { - readonly engine: "docker" | "podman"; + readonly target: EngineTarget; readonly root: string; readonly imageMirrors?: (image: string) => ReadonlyArray; /** Omitted gives this runtime its own host-gateway probe. */ @@ -293,6 +462,9 @@ export const makeContainerRuntime = (options: { const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const stackRoot = path.resolve(options.root); + // An owned directory for per-launch environment files, never the real system temp directory. + const containerEnvRoot = path.join(stackRoot, CONTAINER_ENV_DIRNAME); + const engine = options.target.engine; const command = ( args: ReadonlyArray, @@ -300,14 +472,18 @@ export const makeContainerRuntime = (options: { readonly stdin?: "ignore" | "pipe"; readonly forceKillAfter?: Duration.Input; } = {}, - ) => ChildProcess.make(options.engine, args, { stdin: "ignore", ...commandOptions }); + ) => + ChildProcess.make(engine, [...options.target.argv, ...args], { + stdin: "ignore", + ...commandOptions, + }); const run = Effect.fn("Container.command")(function* ( args: ReadonlyArray, commandOptions: { readonly timeout?: Duration.Input } = { timeout: "30 seconds" }, ) { yield* Effect.annotateCurrentSpan({ - "process.executable.name": options.engine, + "process.executable.name": engine, "process.arg_count": args.length, "container.command": args[0] ?? "", }); @@ -410,34 +586,61 @@ export const makeContainerRuntime = (options: { ); const hostGateway = options.hostGateway ?? (yield* makeHostGateway); - /** Reads `/etc/hosts` from a throwaway container of an already present image. */ - const readProbeHosts = (image: string, spec: ContainerSpec, addHost: ReadonlyArray) => - testRunLabelArgs.pipe( - Effect.flatMap((testRunLabel) => - run( - [ - "run", - "--rm", - "--pull", - "never", - ...addHost, - // No instance label: `--rm` removal is asynchronous and must not count as an - // instance container; the stack labels keep it sweepable. - "--label", - `com.supabase.stack=${spec.stackId}`, - "--label", - `com.supabase.stack-root=${stackRoot}`, - ...testRunLabel, - "--entrypoint", - "cat", - image, - "/etc/hosts", - ], - { timeout: undefined }, - ), - ), - Effect.timeout(HOST_GATEWAY_PROBE_TIMEOUT), + /** Polls until the engine confirms a `--rm` container's own asynchronous removal finished. */ + const awaitRemoved = (name: string) => + run( + [ + "ps", + "--all", + "--no-trunc", + "--filter", + // Docker matches this as a regex; `.` is the only metacharacter a name can hold. + `name=^/?${name.replaceAll(".", "\\.")}$`, + "--format", + "{{.State}}", + ], + { timeout: "5 seconds" }, + ).pipe( + Effect.repeat({ + schedule: Schedule.spaced("250 millis"), + while: (output) => output !== "", + }), + Effect.timeout("10 seconds"), + Effect.asVoid, ); + /** Reads `/etc/hosts` from a throwaway, claimed container of an already present image. */ + const readProbeHosts = (image: string, spec: ContainerSpec, addHost: ReadonlyArray) => + Effect.gen(function* () { + const testRunLabel = yield* testRunLabelArgs; + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("identity", cause)), + ); + // Not an instance container: no instance label, so it is never mistaken for one. + const { name } = identifyContainer(spec, token, true); + const hosts = yield* run( + [ + "run", + "--rm", + "--pull", + "never", + "--name", + name, + ...addHost, + "--label", + `com.supabase.stack=${spec.stackId}`, + "--label", + `com.supabase.stack-root=${stackRoot}`, + ...testRunLabel, + "--entrypoint", + "cat", + image, + "/etc/hosts", + ], + { timeout: undefined }, + ); + yield* awaitRemoved(name); + return hosts; + }).pipe(Effect.timeout(HOST_GATEWAY_PROBE_TIMEOUT)); /** Resolves the IPv4 host address the engine writes itself, since it rejects `host-gateway`. */ const engineHostProbe = (image: string, spec: ContainerSpec, rejection: ContainerError) => readProbeHosts(image, spec, []).pipe( @@ -453,7 +656,7 @@ export const makeContainerRuntime = (options: { ? Effect.fail( new ContainerError({ operation: "host-gateway", - message: `The container engine rejects host-gateway in --add-host and maps no IPv4 address to ${ENGINE_HOST_NAMES.join(" or ")}; upgrade Podman or use --runtime podman`, + message: `The container engine rejects host-gateway in --add-host and maps no IPv4 address to ${ENGINE_HOST_NAMES.join(" or ")}`, cause: rejection, }), ) @@ -504,9 +707,21 @@ export const makeContainerRuntime = (options: { if (!Number.isInteger(port) || port < 1 || port > 65535) return yield* errorFor("ports", "Invalid container port"); } - const directory = yield* fs - .makeTempDirectoryScoped({ prefix: "supabase-container-" }) + yield* fs + .makeDirectory(containerEnvRoot, { recursive: true, mode: 0o700 }) .pipe(Effect.mapError((cause) => errorFor("environment", cause))); + // Not `makeTempDirectoryScoped`: its scoped cleanup removes without `force`, then dies on + // any failure, including a confirmed ENOENT when this stack's whole root (this directory's + // owned ancestor) is already gone — turning an ordinary stop into an unrecoverable defect. + // `acquireRelease` keeps the plain `makeTempDirectory` and installing its tolerant, + // `force: true` finalizer uninterruptible together, so an interruption between the two + // can never leave the directory created but unregistered for cleanup. + const directory = yield* Effect.acquireRelease( + fs + .makeTempDirectory({ directory: containerEnvRoot, prefix: "container-" }) + .pipe(Effect.mapError((cause) => errorFor("environment", cause))), + (value) => fs.remove(value, { recursive: true, force: true }).pipe(Effect.ignore), + ); const envPath = path.join(directory, "environment"); yield* fs .writeFileString( @@ -521,8 +736,7 @@ export const makeContainerRuntime = (options: { Effect.mapError((cause) => errorFor("identity", cause)), ); const { name, composeProject, composeService } = identifyContainer(spec, token, oneOff); - const hostAlias = - options.engine === "docker" ? yield* hostAliasTarget(image, spec) : undefined; + const hostAlias = yield* hostAliasTarget(image, spec); const testRunLabel = yield* testRunLabelArgs; const createArgs = (target: string | undefined) => [ "create", @@ -553,6 +767,12 @@ export const makeContainerRuntime = (options: { "--label", `com.docker.compose.service=${composeService}`, ...(oneOff ? ["--label", "com.docker.compose.oneoff=True"] : []), + ...(spec.user === undefined ? [] : ["--user", spec.user]), + // A rootless Podman maps the host uid elsewhere inside the container; keep-id maps it to + // itself so files written through a borrowed bind mount stay owned by the caller. + ...(spec.user !== undefined && options.target.engine === "podman" && options.target.rootless + ? ["--userns=keep-id"] + : []), "--env-file", envPath, ...(spec.mounts ?? []).flatMap((mount) => [ @@ -769,40 +989,77 @@ export const makeContainerRuntime = (options: { getWaiter, ).pipe(Effect.flatMap((fiber) => Fiber.join(fiber))); owned = { ...partial, exitCode }; - const text = yield* run([ - "inspect", - "--format", - "{{json .NetworkSettings.Ports}}", - name, - ]); - const bindings = yield* Schema.decodeEffect(Schema.fromJsonString(PublishedPorts))( - text, - ).pipe(Effect.mapError((cause) => errorFor("inspect", cause))); - const ports: Record = {}; - for (const port of spec.ports ?? []) { - const value = bindings[`${port}/tcp`]?.find( - (binding) => binding.HostIp === "127.0.0.1", - )?.HostPort; - const actual = Number(value); - if (!Number.isInteger(actual) || actual < 1 || actual > 65535) - return yield* errorFor("inspect", `No loopback publication for ${port}`); - ports[port] = actual; - } + const requestedPorts = spec.ports ?? []; + const inspectPublished = Effect.gen(function* () { + const text = yield* run([ + "inspect", + "--format", + '{"Ports":{{json .NetworkSettings.Ports}},"Status":{{json .State.Status}},"ExitCode":{{.State.ExitCode}}}', + name, + ]); + const state = yield* Schema.decodeEffect(Schema.fromJsonString(PublicationState))( + text, + ).pipe(Effect.mapError((cause) => errorFor("inspect", cause))); + const bindings = state.Ports ?? {}; + const published: Record = {}; + const pending: Array = []; + for (const port of requestedPorts) { + const value = bindings[`${port}/tcp`]?.find( + (binding) => binding.HostIp === "127.0.0.1", + )?.HostPort; + const actual = Number(value); + if (Number.isInteger(actual) && actual >= 1 && actual <= 65535) + published[port] = actual; + else pending.push(port); + } + if (pending.length > 0 && state.Status !== "running") + return yield* errorFor( + "inspect", + `Container ${state.Status} (exit code ${state.ExitCode}) before publishing ${pending.join(", ")}`, + ); + return { published, pending }; + }); + // The engine can report a container started before its loopback publish lands, + // so poll for it instead of trusting a single inspect under load. + const awaitPublishedPorts = Effect.fn("Container.awaitPublishedPorts")(function* () { + const observed = yield* Ref.make<{ + readonly published: Record; + readonly pending: ReadonlyArray; + }>({ published: {}, pending: requestedPorts }); + yield* withAttemptCount( + inspectPublished.pipe(Effect.tap((result) => Ref.set(observed, result))), + (counted) => + counted.pipe( + Effect.repeat({ + schedule: Schedule.spaced("250 millis"), + while: (result) => result.pending.length > 0, + }), + ), + ).pipe( + Effect.timeout(PORT_PUBLISH_TIMEOUT), + Effect.catchTag("TimeoutError", () => + Ref.get(observed).pipe( + Effect.flatMap(({ pending }) => + errorFor( + "inspect", + `No loopback publication for ${pending.join(", ")} after ${PORT_PUBLISH_TIMEOUT}`, + ), + ), + ), + ), + ); + return (yield* Ref.get(observed)).published; + }); + const ports = yield* awaitPublishedPorts(); const logProcess = attached ?? (yield* Effect.gen(function* () { // Released on exit, since a release left for service stop can hit a reused pid. const followerScope = yield* Scope.fork(owner); - const follower = yield* spawner - .spawn( - ChildProcess.make(options.engine, ["logs", "--follow", name], { - stdin: "ignore", - }), - ) - .pipe( - Scope.provide(followerScope), - Effect.mapError((cause) => errorFor("logs", cause)), - ); + const follower = yield* spawner.spawn(command(["logs", "--follow", name])).pipe( + Scope.provide(followerScope), + Effect.mapError((cause) => errorFor("logs", cause)), + ); yield* Effect.forkIn( Effect.exit(follower.exitCode).pipe( Effect.andThen(Scope.close(followerScope, Exit.void)), @@ -832,70 +1089,115 @@ export const makeContainerRuntime = (options: { }; }); -const runCleanupCommand = Effect.fn("Container.runCleanupCommand")(function* ( - engine: "docker" | "podman", - args: ReadonlyArray, -) { - yield* Effect.annotateCurrentSpan({ - "process.executable.name": engine, - "process.arg_count": args.length, - "container.command": args[0] ?? "", - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - return yield* Effect.scoped( +/** + * Removes a container by its exact id, succeeding when it is already absent, through the same + * pinned {@link EngineTarget} it was discovered through, so a context switch in between can never + * split the two. + */ +const removeContainerById = Effect.fn("Container.removeContainerById")( + (options: { readonly target: EngineTarget; readonly id: string }) => Effect.gen(function* () { - const child = yield* spawner.spawn( - ChildProcess.make(engine, args, { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make( + options.target.engine, + [...options.target.argv, "rm", "--force", options.id], + { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, + ), + ); + const [stderr, code] = yield* Effect.all( + [child.stderr.pipe(Stream.decodeText, Stream.mkString), child.exitCode], + { concurrency: "unbounded" }, + ); + // Docker reports this exact phrasing for an id that no longer exists. + if (Number(code) !== 0 && !/no such container/iu.test(stderr)) + return yield* errorFor("cleanup", stderr.trim() || `Engine exited with ${code}`); }), - ); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe(Stream.decodeText, Stream.mkString), - child.stderr.pipe(Stream.decodeText, Stream.mkString), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); - if (Number(code) !== 0) - return yield* errorFor(args[0] ?? "cleanup", stderr.trim() || `Engine exited with ${code}`); - return stdout.trim(); - }), - ).pipe( - Effect.timeoutOrElse({ - duration: "30 seconds", - orElse: () => - Effect.fail( - errorFor( - args[0] ?? "cleanup", - `${engine} ${args[0] ?? "command"} did not respond within 30 seconds`, - ), + ).pipe( + Effect.timeout("30 seconds"), + Effect.mapError((cause) => + cause instanceof ContainerError ? cause : errorFor("cleanup", cause), ), + ); }), - Effect.mapError((cause) => - cause instanceof ContainerError ? cause : errorFor(args[0] ?? "cleanup", cause), - ), - ); +); + +/** + * Lists every container carrying this stack's identity and data-root labels, through the pinned + * engine: service containers and every storage helper alike (per-instance and shared), + * independent of any in-memory registry's own bookkeeping. The data-root label is required + * alongside the stack id label because a stack id alone does not identify a stack: two owners can + * share one id while rooted at different data directories. + */ +const listStackContainers = Effect.fn("Container.listStackContainers")(function* (options: { + readonly target: EngineTarget; + readonly stackId: string; + readonly stackRoot: string; +}) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* runRaw(spawner, options.target.engine, [ + ...options.target.argv, + "ps", + "--all", + "--quiet", + "--no-trunc", + "--filter", + `label=com.supabase.stack=${options.stackId}`, + "--filter", + `label=com.supabase.stack-root=${options.stackRoot}`, + ]); + return output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); }); +/** + * Removes every container carrying this stack's identity and data-root labels and reports how + * many remain (normally 0): the registration-independent confirming sweep destroy and stop use + * instead of depending on any helper registry's own bookkeeping. A container that disappears + * between listing and removal is not an error, matching {@link removeContainerById}. + */ +export const removeStackContainers = Effect.fn("Container.removeStackContainers")( + function* (options: { + readonly target: EngineTarget; + readonly stackId: string; + readonly stackRoot: string; + }) { + const ids = yield* listStackContainers(options); + yield* Effect.forEach(ids, (id) => removeContainerById({ target: options.target, id }), { + concurrency: "unbounded", + discard: true, + }); + const remaining = yield* listStackContainers(options); + return remaining; + }, +); + const decodeStackLabels = Schema.decodeEffect( Schema.fromJsonString(Schema.Tuple([Schema.String, Schema.String])), ); -export const listStackContainers = Effect.fn("Container.listStackContainers")(function* ( - engine: "docker" | "podman", +/** + * Lists the stack id and data-root labels of every stack-labelled container on the pinned + * engine, for finding stacks that no registration accounts for. A line that is not a label pair + * is skipped. + */ +export const listStackLabels = Effect.fn("Container.listStackLabels")(function* ( + target: EngineTarget, ) { - const output = yield* runCleanupCommand(engine, [ + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const output = yield* runRaw(spawner, target.engine, [ + ...target.argv, "ps", "--all", "--filter", "label=com.supabase.stack", "--format", '[{{json (.Label "com.supabase.stack")}},{{json (.Label "com.supabase.stack-root")}}]', - ]).pipe(Effect.mapError(markUnavailable)); + ]); const labels = yield* Effect.forEach( output.split("\n").filter((line) => line.length > 0), (line) => decodeStackLabels(line).pipe(Effect.option), @@ -904,60 +1206,3 @@ export const listStackContainers = Effect.fn("Container.listStackContainers")(fu yield* Effect.annotateCurrentSpan("container.count", containers.length); return containers; }); - -export const removeStackContainers = Effect.fn("Container.removeStackContainers")( - (options: { - readonly engine: "docker" | "podman"; - readonly stackId: string; - readonly root: string; - }) => - Effect.gen(function* () { - const stackRoot = options.root; - const run = (args: ReadonlyArray) => runCleanupCommand(options.engine, args); - const filters = [ - "--filter", - `label=com.supabase.stack=${options.stackId}`, - "--filter", - `label=com.supabase.stack-root=${stackRoot}`, - ]; - const list = () => run(["ps", "--all", "--quiet", "--no-trunc", ...filters]); - // Only the initial listing can show the engine itself is unreachable; a later `rm` or - // leftover check failing is a per-container cleanup problem instead. - const ids = (yield* list().pipe(Effect.mapError(markUnavailable))) - .split("\n") - .filter((id) => id.length > 0); - yield* Effect.forEach( - ids, - (id) => - run(["rm", "--force", id]).pipe( - Effect.catchTag("ContainerError", (failure) => - run(["ps", "--all", "--quiet", "--no-trunc", "--filter", `id=${id}`]).pipe( - Effect.flatMap((present) => - present.length === 0 ? Effect.void : Effect.fail(failure), - ), - ), - ), - ), - { concurrency: 1, discard: true }, - ); - const remaining = yield* list(); - if (remaining.length > 0) - return yield* errorFor("cleanup", `Stack containers remain: ${remaining}`); - }), -); - -/** Shell command that removes the same containers as `removeStackContainers`, succeeding when none remain. */ -export const removeStackContainersCommand = (options: { - readonly engine: "docker" | "podman"; - readonly stackId: string; - readonly root: string; -}): string => { - const filters = [ - `label=com.supabase.stack=${options.stackId}`, - `label=com.supabase.stack-root=${options.root}`, - ] - .map((filter) => `--filter ${shellQuote(filter)}`) - .join(" "); - // `sh -c` keeps POSIX word splitting of `$ids` when pasted into shells like zsh that skip it. - return `sh -c ${shellQuote(`ids=$(${options.engine} ps --all --quiet --no-trunc ${filters}) && { [ -z "$ids" ] || ${options.engine} rm --force $ids; }`)}`; -}; diff --git a/packages/stack/src/runtime/Container.unit.test.ts b/packages/stack/src/runtime/Container.unit.test.ts index fbd132bb2e..f0e6a5596f 100644 --- a/packages/stack/src/runtime/Container.unit.test.ts +++ b/packages/stack/src/runtime/Container.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; import { Deferred, Effect, Exit, Fiber, Ref, Scope } from "effect"; -import { ContainerError, makeHostGateway } from "./Container.ts"; +import { makeHostGateway } from "./Container.ts"; const gatewayAddress = "192.168.65.254"; @@ -15,56 +15,6 @@ const gatedProbe = Effect.fnUntraced(function* (result: string | undefined) { }); describe("host gateway", () => { - it.effect("prefetches without waiting and lets a later resolve await the same probe", () => - Effect.gen(function* () { - const gateway = yield* makeHostGateway; - const { probe, runs, release } = yield* gatedProbe(gatewayAddress); - - yield* gateway.prefetch(probe); - const waiting = yield* gateway.resolve(probe).pipe(Effect.forkChild); - yield* release; - - expect(yield* Fiber.join(waiting)).toBe(gatewayAddress); - expect(yield* gateway.resolve(probe)).toBe(gatewayAddress); - expect(yield* Ref.get(runs)).toBe(1); - }), - ); - - it.effect("retries once for a waiter whose background probe finds no target", () => - Effect.gen(function* () { - const gateway = yield* makeHostGateway; - const background = yield* gatedProbe(undefined); - const later = yield* gatedProbe(gatewayAddress); - - yield* gateway.prefetch(background.probe); - const waiting = yield* gateway - .resolve(later.probe) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* background.release; - yield* later.release; - - expect(yield* Fiber.join(waiting)).toBe(gatewayAddress); - expect(yield* Ref.get(later.runs)).toBe(1); - expect(yield* gateway.resolve(later.probe)).toBe(gatewayAddress); - expect(yield* Ref.get(later.runs)).toBe(1); - }), - ); - - it.effect("caches a probe failure for later resolves without probing again", () => - Effect.gen(function* () { - const gateway = yield* makeHostGateway; - const runs = yield* Ref.make(0); - const unsupported = new ContainerError({ operation: "host-gateway", message: "unsupported" }); - const probe = Ref.update(runs, (count) => count + 1).pipe( - Effect.andThen(Effect.fail(unsupported)), - ); - - expect(yield* Effect.flip(gateway.resolve(probe))).toBe(unsupported); - expect(yield* Effect.flip(gateway.resolve(probe))).toBe(unsupported); - expect(yield* Ref.get(runs)).toBe(1); - }), - ); - it.effect("keeps the probe running when a waiting resolve is interrupted", () => Effect.gen(function* () { const gateway = yield* makeHostGateway; diff --git a/packages/stack/src/runtime/ContainerName.unit.test.ts b/packages/stack/src/runtime/ContainerName.unit.test.ts index a663dc7a6b..a9f147dc58 100644 --- a/packages/stack/src/runtime/ContainerName.unit.test.ts +++ b/packages/stack/src/runtime/ContainerName.unit.test.ts @@ -5,26 +5,6 @@ const stackId = "0123456789abcdef0123"; const token = "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d"; describe("identifyContainer", () => { - it("names a service container after its project and service", () => { - expect( - identifyContainer({ stackId, project: "my-app", service: "studio" }, token, false), - ).toEqual({ - name: "supabase-my-app-studio-a1b2c3d4e5f6", - composeProject: "supabase-my-app-0123456789ab", - composeService: "studio", - }); - }); - - it("marks one-shot containers with a task segment but keeps the service for grouping", () => { - const identity = identifyContainer( - { stackId, project: "my-app", service: "auth" }, - token, - true, - ); - expect(identity.name).toBe("supabase-my-app-auth-task-a1b2c3d4e5f6"); - expect(identity.composeService).toBe("auth"); - }); - it("keeps dotted folder names in the container name but not in the compose project", () => { const identity = identifyContainer( { stackId, project: "My.App", service: "rest" }, diff --git a/packages/stack/src/runtime/NativeProcess.ts b/packages/stack/src/runtime/NativeProcess.ts index 1826d5ca67..e4f6650a5b 100644 --- a/packages/stack/src/runtime/NativeProcess.ts +++ b/packages/stack/src/runtime/NativeProcess.ts @@ -3,6 +3,7 @@ import { fileURLToPath } from "node:url"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { PlatformError } from "effect/PlatformError"; import { isBunVirtualPath } from "../internal/dispatch-markers.ts"; +import * as Environment from "../namespace/Environment.ts"; import type { ChildProcessHandle, ExitCode, @@ -12,8 +13,17 @@ import type { export interface NativeProcessSpec { readonly executable: string; readonly args?: ReadonlyArray; + /** Merged over {@link environment}'s confined values; it is rejected if it sets any of them. */ readonly env?: Readonly>; + /** The owned HOME/TMPDIR/XDG/Deno directories this workload is confined to. */ + readonly environment: Environment.NativeEnvironment; readonly cwd?: string; + /** + * The native artifact generation's digest lock file. The launcher takes its own SHARED pin on + * this before spawning the workload, and holds it until the workload's process group has + * exited, including the forced-kill path. + */ + readonly artifactLockPath?: string; readonly stdin?: "ignore" | "pipe"; /** Numeric identity the workload runs as; omitted keeps the launcher's identity. */ readonly uid?: number; @@ -71,7 +81,7 @@ export const defaultNativeProcessLauncher = (): NativeProcessLauncher => ({ args: [nativeLauncherEntrypointFor(import.meta.url)], }); -const encodeSpec = (spec: NativeProcessSpec): Uint8Array => { +const encodeSpec = (spec: NativeProcessSpec, env: Readonly>): Uint8Array => { const timeout = spec.gracefulStopSignal === undefined ? Option.none() @@ -84,10 +94,11 @@ const encodeSpec = (spec: NativeProcessSpec): Uint8Array => { JSON.stringify({ executable: spec.executable, args: spec.args ?? [], - env: spec.env, + env, cwd: spec.cwd, uid: spec.uid, gid: spec.gid, + ...(spec.artifactLockPath === undefined ? {} : { artifactLockPath: spec.artifactLockPath }), ...(Option.isSome(timeout) ? { gracefulStopSignal: spec.gracefulStopSignal, @@ -123,6 +134,7 @@ export const spawnNativeProcess = Effect.fn("NativeProcess.spawn")(function* ( "process.executable.name": spec.executable.split(/[\\/]/u).pop() ?? spec.executable, "process.arg_count": spec.args?.length ?? 0, }); + const env = yield* Environment.apply(spec.environment, spec.env); return yield* Effect.gen(function* () { // Shutdown must precede the spawner's finalizer even when the caller closes in parallel. const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); @@ -344,7 +356,7 @@ export const spawnNativeProcess = Effect.fn("NativeProcess.spawn")(function* ( Effect.orDie, ), ); - yield* Stream.run(Stream.succeed(encodeSpec(spec)), handle.getInputFd(4)); + yield* Stream.run(Stream.succeed(encodeSpec(spec, env)), handle.getInputFd(4)); const groupIdLine = yield* handle .getOutputFd(5) .pipe(Stream.decodeText, Stream.splitLines, Stream.runHead); diff --git a/packages/stack/src/runtime/Session.unit.test.ts b/packages/stack/src/runtime/Session.unit.test.ts index 6abf8f2192..ba725da4cf 100644 --- a/packages/stack/src/runtime/Session.unit.test.ts +++ b/packages/stack/src/runtime/Session.unit.test.ts @@ -1,6 +1,14 @@ import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, PubSub } from "effect"; -import { launchOutputPublisher, mapToServiceError, type LaunchOutput } from "./Session.ts"; +import { Cause, Effect, Exit, Fiber, Option, PubSub, Ref } from "effect"; +import * as TestClock from "effect/testing/TestClock"; +import { + launchOutputPublisher, + mapToServiceError, + processExit, + type LaunchOutput, +} from "./Session.ts"; + +const describeExit = (code: number) => `runtime exited with code ${code}`; describe("mapToServiceError", () => { it("names an error whose cause carries no message", () => { @@ -17,6 +25,24 @@ describe("mapToServiceError", () => { }); }); +it.effect("waits for the stderr tail after a long-running runtime exits", () => + Effect.gen(function* () { + const tail = yield* Ref.make(""); + const drained = yield* Effect.forkChild( + Effect.sleep("2500 millis").pipe(Effect.andThen(Ref.set(tail, "FATAL: data directory"))), + ); + const settled = yield* Effect.forkChild( + processExit(Effect.sleep("2 seconds").pipe(Effect.as(1)), describeExit, { tail, drained }), + ); + yield* TestClock.adjust("3 seconds"); + const exit = yield* Fiber.join(settled); + const error = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) + : undefined; + expect(error?.message).toBe("runtime exited with code 1: FATAL: data directory"); + }), +); + describe("launchOutputPublisher", () => { it.effect( "tags chunks with the launch, a part per process and one sequence across streams and parts", diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts index e528d9d33e..760145b947 100644 --- a/packages/stack/src/runtime/container-image-mirror.integration.test.ts +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -3,8 +3,10 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, Fiber, Layer, Option, Sink, Stream } from "effect"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { makeContainerRuntime } from "./Container.ts"; +import { makeContainerRuntime, type EngineTarget } from "./Container.ts"; +// An unpinned target: these tests exercise image mirroring, not endpoint pinning. +const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; const primary = "ghcr.io/supabase/cli/postgrest:v16.2"; const mirror = "public.ecr.aws/supabase/cli/postgrest:v16.2"; const secondMirror = "registry.test/supabase/cli/postgrest:v16.2"; @@ -45,6 +47,9 @@ const fakeEngine = (options: { const args = [...command.args]; commands.push(args); const ref = args.at(-1) ?? ""; + // Resolved once before any container is created; this stand-in always reports one so launch + // never fails at that earlier step for reasons unrelated to what each test exercises. + if (args[0] === "info") return Effect.succeed(handle(0, "fake-daemon-id")); if (args[0] === "image") return Effect.succeed(handle(0, local.has(ref) ? "sha256:1" : "")); if (args[0] === "pull") { const queued = failures.get(ref); @@ -79,7 +84,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -97,7 +102,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [secondMirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []), }); @@ -116,7 +121,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -133,7 +138,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -150,7 +155,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -168,7 +173,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [primary, mirror] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); @@ -181,7 +186,10 @@ describe("container image mirror", () => { it.effect("retries a rate-limited pull with backoff until the registry accepts it", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 2 } }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: dockerTarget, + root: ".", + }); const prepared = yield* runtime.prepare(primary).pipe(Effect.forkChild); yield* TestClock.adjust("1 minute"); yield* Fiber.join(prepared); @@ -198,7 +206,10 @@ describe("container image mirror", () => { }, }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: dockerTarget, + root: ".", + }); const prepared = yield* runtime.prepare(primary).pipe(Effect.forkChild); yield* TestClock.adjust("1 minute"); yield* Fiber.join(prepared); @@ -220,7 +231,10 @@ describe("container image mirror", () => { failures: { [primary]: [message] }, }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: dockerTarget, + root: ".", + }); const failed = yield* runtime.prepare(primary).pipe(Effect.exit); const error = Exit.isFailure(failed) ? Option.getOrUndefined(Cause.findErrorOption(failed.cause)) @@ -234,7 +248,10 @@ describe("container image mirror", () => { it.effect("reports the rate limit after five throttled attempts", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 10 } }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: dockerTarget, + root: ".", + }); const prepared = yield* runtime.prepare(primary).pipe(Effect.exit, Effect.forkChild); yield* TestClock.adjust("5 minutes"); const failed = yield* Fiber.join(prepared); @@ -249,7 +266,10 @@ describe("container image mirror", () => { it.effect("stops pulling once a concurrent prepare lands the image during backoff", () => { const engine = fakeEngine({ pullable: [primary], throttled: { [primary]: 1 } }); return Effect.gen(function* () { - const runtime = yield* makeContainerRuntime({ engine: "docker", root: "." }); + const runtime = yield* makeContainerRuntime({ + target: dockerTarget, + root: ".", + }); const throttled = yield* runtime.prepare(primary).pipe(Effect.forkChild); yield* TestClock.adjust("1 millis"); yield* runtime.prepare(primary); @@ -263,7 +283,7 @@ describe("container image mirror", () => { const engine = fakeEngine({ pullable: [mirror], throttled: { [primary]: 10 } }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ - engine: "docker", + target: dockerTarget, root: ".", imageMirrors: (image) => (image === primary ? [mirror] : []), }); diff --git a/packages/stack/src/runtime/native-launcher.ts b/packages/stack/src/runtime/native-launcher.ts index 37081c5d34..a55d497378 100644 --- a/packages/stack/src/runtime/native-launcher.ts +++ b/packages/stack/src/runtime/native-launcher.ts @@ -3,9 +3,19 @@ import { Config, ConfigProvider, Duration, Effect, Option, Schema } from "effect"; import { Socket } from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem cannot adopt inherited fd3/fd4; Bun also requires synchronous fd4 reads at this process boundary. -import { createReadStream, readFileSync, writeSync } from "node:fs"; +import { createReadStream, readFileSync, utimesSync, writeSync } from "node:fs"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- workloads join the launcher's process group and need child-only signal forwarding; Effect spawners signal child process groups. import { spawn } from "node:child_process"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- this standalone process has no Effect FileSystem/SQLite service; it opens its own pin connection directly. +import { DatabaseSync } from "node:sqlite"; +import { takeSharedLockSync } from "../namespace/drivers/sqlite-pin.ts"; + +/** Best-effort diagnostic: a write to a gone owner's stderr pipe must never end the launcher and release its pin. */ +const report = (message: string): void => { + try { + writeSync(2, message); + } catch {} +}; interface LaunchSpec { readonly executable: string; @@ -14,6 +24,7 @@ interface LaunchSpec { readonly cwd?: string; readonly uid?: number; readonly gid?: number; + readonly artifactLockPath?: string; readonly gracefulStopSignal?: "SIGTERM" | "SIGINT"; readonly gracefulStopTimeoutMs?: number; } @@ -25,6 +36,7 @@ const LaunchSpecSchema = Schema.Struct({ cwd: Schema.optionalKey(Schema.String), uid: Schema.optionalKey(Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))), gid: Schema.optionalKey(Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0)))), + artifactLockPath: Schema.optionalKey(Schema.String), gracefulStopSignal: Schema.optionalKey(Schema.Literals(["SIGTERM", "SIGINT"])), gracefulStopTimeoutMs: Schema.optionalKey( Schema.Finite.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0))), @@ -48,6 +60,7 @@ const decodeSpec = (bytes: Buffer): LaunchSpec | undefined => { env: value.env, uid: value.uid, gid: value.gid, + artifactLockPath: value.artifactLockPath, ...(value.gracefulStopSignal === undefined ? {} : { gracefulStopSignal: value.gracefulStopSignal }), @@ -58,6 +71,32 @@ const decodeSpec = (bytes: Buffer): LaunchSpec | undefined => { } }; +/** + * Takes this launcher's own SHARED pin on the workload's artifact generation before it spawns, + * independent of whatever pin its owner process holds: the launcher outlives a dead owner, so its + * own kernel lock is what keeps a concurrent retirement sweep from touching a generation whose + * workload is still running. There is no heartbeat; the lock's mtime is touched once, here. + */ +const pinArtifactGeneration = (lockPath: string): (() => void) | undefined => { + try { + const connection = new DatabaseSync(lockPath); + takeSharedLockSync(connection); + // oxlint-disable-next-line effecttsgo/global-date -- this standalone process has no Effect Clock service to source "now" from. + const now = new Date(); + utimesSync(lockPath, now, now); + return () => { + try { + connection.close(); + } catch { + // The workload has already exited; a failure to close is not actionable here. + } + }; + } catch (error) { + report(`Unable to pin the native artifact generation: ${String(error)}\n`); + return undefined; + } +}; + const inheritedEnvironmentNames = [ "PATH", "HOME", @@ -115,7 +154,9 @@ export const runNativeLauncher = (): void => { // before this launcher exits while preserving the workload's exit code. process.kill(-(child?.pid ?? process.pid), signal); } catch { - child?.kill(signal); + try { + child?.kill(signal); + } catch {} } }; @@ -204,6 +245,66 @@ export const runNativeLauncher = (): void => { } else { specGracefulStopSignal = spec.gracefulStopSignal; specGracefulStopTimeoutMs = spec.gracefulStopTimeoutMs; + // Taken before spawning and released only after the workload's group has been sent SIGKILL, + // on every exit path below, so a concurrent retirement sweep never races a running workload. + const releasePin = + spec.artifactLockPath === undefined + ? undefined + : pinArtifactGeneration(spec.artifactLockPath); + if (spec.artifactLockPath !== undefined && releasePin === undefined) { + // The pin could not be taken: never spawn a workload whose generation isn't protected. + // Reporting nothing on fd5 is the same failure the owner already handles when the + // launcher never writes a valid process group. + process.exit(127); + return; + } + /** + * ESRCH (no such process group) counts as success: the group is already gone. A real failure + * (for example EPERM) must never be treated as confirmation that it is. + */ + const killGroupForExit = (pgid: number): boolean => { + if (process.platform === "win32") return true; + try { + process.kill(-pgid, "SIGKILL"); + return true; + } catch (error) { + return ( + typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH" + ); + } + }; + // Once SIGKILL has reached every group member, none can run user code again, and unlinking + // this generation's files afterward cannot affect an fd or mapping a member already holds + // open: releasing the pin right after the kill, without waiting to confirm each member has + // actually been reaped, is safe. A real kill failure must never reach `afterRelease`, since + // exiting closes this process's own fd and reclaims the advisory lock regardless of whether + // `releasePin` was ever called: staying alive and retrying is the only way to keep it held. + const releaseAfterGroupKill = (afterRelease: () => void): void => { + const pgid = child?.pid; + if (releasePin === undefined) { + afterRelease(); + return; + } + if (pgid === undefined) { + releasePin(); + afterRelease(); + return; + } + const attempt = (reported: boolean): void => { + if (killGroupForExit(pgid)) { + releasePin(); + afterRelease(); + return; + } + if (!reported) + report( + "Unable to confirm the native workload's process group is gone; keeping its artifact generation pinned and retrying\n", + ); + // oxlint-disable-next-line effecttsgo/global-timers -- this standalone process has no Effect runtime to schedule through. + setTimeout(() => attempt(true), 1_000); + }; + attempt(false); + }; child = spawn(spec.executable, [...spec.args], { cwd: spec.cwd, env: { ...Effect.runSync(inheritedEnvironment), ...spec.env }, @@ -216,23 +317,26 @@ export const runNativeLauncher = (): void => { writeSync(5, `${child.pid ?? 0}\n`); } catch { terminateWorkloadGroup("SIGKILL"); - process.exit(127); + releaseAfterGroupKill(() => process.exit(127)); + return; } child.on("error", (error) => { - writeSync(2, `Native workload failed to start: ${error.message}\n`); - process.exit(127); + report(`Native workload failed to start: ${error.message}\n`); + releaseAfterGroupKill(() => process.exit(127)); }); child.on("exit", (code, signal) => { childExited = true; terminateWorkloadGroup("SIGKILL"); - if (ownerLossGraceful) { + releaseAfterGroupKill(() => { + if (ownerLossGraceful) { + process.exit(code ?? 1); + return; + } + ownerPipe.destroy(); + if (!gracefulForwarded && signal !== null) + report(`Native workload exited due to signal ${signal}\n`); process.exit(code ?? 1); - return; - } - ownerPipe.destroy(); - if (!gracefulForwarded && signal !== null) - writeSync(2, `Native workload exited due to signal ${signal}\n`); - process.exit(code ?? 1); + }); }); } }; diff --git a/packages/stack/src/runtime/native-process.integration.test.ts b/packages/stack/src/runtime/native-process.integration.test.ts index 0fef4834c0..5d35bc46c1 100644 --- a/packages/stack/src/runtime/native-process.integration.test.ts +++ b/packages/stack/src/runtime/native-process.integration.test.ts @@ -10,6 +10,7 @@ import type { ExitCode } from "effect/unstable/process/ChildProcessSpawner"; import { execFileSync, spawn as spawnProcess } from "node:child_process"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- The test waits for actual inherited-fd and process events. import { once } from "node:events"; +import type { NativeEnvironment } from "../namespace/Environment.ts"; import { defaultNativeProcessLauncher, spawnNativeProcess, @@ -19,6 +20,21 @@ import { const targetPid = 87_035; +/** These tests exercise process lifecycle, not confinement; the paths need not exist. */ +const testEnvironment: NativeEnvironment = { + values: { + HOME: "/tmp/native-process-test-home", + TMPDIR: "/tmp/native-process-test-home/tmp", + TMP: "/tmp/native-process-test-home/tmp", + TEMP: "/tmp/native-process-test-home/tmp", + XDG_CACHE_HOME: "/tmp/native-process-test-home/.cache", + XDG_CONFIG_HOME: "/tmp/native-process-test-home/.config", + XDG_DATA_HOME: "/tmp/native-process-test-home/.local/share", + XDG_STATE_HOME: "/tmp/native-process-test-home/.local/state", + DENO_DIR: "/tmp/native-process-test-home/.cache/deno", + }, +}; + interface FakeProcessOptions { readonly groupOutput?: string; readonly groupExitCode?: number; @@ -133,7 +149,7 @@ const makeSpawner = (options: FakeProcessOptions) => { return spawner; }; -const spec: NativeProcessSpec = { executable: "test-native-process" }; +const spec: NativeProcessSpec = { executable: "test-native-process", environment: testEnvironment }; // Intercepts every signal sent to the positive launcher pid and its negative // group so tests using a placeholder pid never reach a real OS process, for @@ -246,6 +262,7 @@ const descendantSpec = (): NativeProcessSpec => { return { executable: process.execPath, args: ["--input-type=module", "-e", workloadCode], + environment: testEnvironment, }; }; @@ -260,7 +277,9 @@ describe("native process group cleanup", () => { try: async () => { const launcher = defaultNativeProcessLauncher(); const workloadMarker = `fd5-workload-${process.pid}`; - const child = spawnProcess(launcher.command, launcher.args, { + // Carried as a trailing argv entry so the launcher's own command line is checkable + // the same way as the workload's, without the launcher reading or acting on it. + const child = spawnProcess(launcher.command, [...launcher.args, workloadMarker], { detached: true, stdio: ["ignore", "pipe", "pipe", "pipe", "pipe", "pipe"], }); @@ -305,18 +324,40 @@ describe("native process group cleanup", () => { } return rows; }; - const killGroup = (pid: number | undefined) => { - if (pid === undefined) return; + // A recycled PID or PGID can belong to an unrelated process group (another user's, + // in the EPERM case); only a marker match proves it is still this test's own. + const groupCarriesMarker = (groupId: number): boolean => { + try { + return execFileSync("ps", ["-axo", "pgid=,command="], { encoding: "utf8" }) + .split(/\r?\n/) + .some((line) => { + const match = /^\s*(\d+)\s+(.*)$/.exec(line); + return ( + match !== null && + match[1] !== undefined && + match[2] !== undefined && + Number(match[1]) === groupId && + match[2].includes(workloadMarker) + ); + }); + } catch { + return false; + } + }; + const errorCode = (cause: unknown): string | undefined => + typeof cause === "object" && cause !== null && "code" in cause + ? String(cause.code) + : undefined; + const killGroup = (groupId: number | undefined) => { + if (groupId === undefined || !groupCarriesMarker(groupId)) return; try { - process.kill(-pid, "SIGKILL"); + process.kill(-groupId, "SIGKILL"); } catch (cause) { - if ( - typeof cause !== "object" || - cause === null || - !("code" in cause) || - cause.code !== "ESRCH" - ) - throw cause; + const code = errorCode(cause); + // A group that lost its marker between the check and the signal already exited; + // ESRCH or EPERM against a group that still carries it is a real failure. + if ((code === "ESRCH" || code === "EPERM") && !groupCarriesMarker(groupId)) return; + throw cause; } }; const killOwnedGroups = () => { @@ -468,6 +509,7 @@ describe("native process group cleanup", () => { Effect.gen(function* () { const native = yield* spawnNativeProcess({ executable: process.execPath, + environment: testEnvironment, stdin: "pipe", args: [ "--input-type=module", @@ -498,17 +540,22 @@ describe("native process group cleanup", () => { () => Effect.scoped( Effect.gen(function* () { + // Identifies the descendant's own argv so a PID the kernel recycled for an unrelated + // process right after the descendant exited is never mistaken for a still-live member. + const descendantMarker = `native-process-descendant-${process.pid}`; const native = yield* spawnNativeProcess({ executable: process.execPath, + environment: testEnvironment, args: [ "--input-type=module", "-e", [ "import { spawn } from 'node:child_process';", - "const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: ['ignore', 'inherit', 'inherit'] });", + "const descendant = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)', process.argv[1]], { stdio: ['ignore', 'inherit', 'inherit'] });", "process.stdout.write(`READY ${process.pid} ${descendant.pid}\\n`);", "setInterval(() => {}, 1000);", ].join("\n"), + descendantMarker, ], }); const output = yield* Effect.acquireRelease( @@ -538,11 +585,11 @@ describe("native process group cleanup", () => { const match = /READY \d+ (\d+)/.exec(yield* Ref.get(output.text)); expect(match).not.toBeNull(); if (match === null) return; - const [state, status] = yield* Effect.scoped( + const [psOutput, status] = yield* Effect.scoped( Effect.gen(function* () { const descendant = yield* ChildProcess.make( "/bin/ps", - ["-p", match[1] ?? "", "-o", "stat="], + ["-p", match[1] ?? "", "-o", "stat=,command="], { stdin: "ignore", stdout: "pipe", stderr: "ignore" }, ); return yield* Effect.all( @@ -551,9 +598,13 @@ describe("native process group cleanup", () => { ); }), ); + const line = psOutput.trim(); + const state = line.split(/\s+/u)[0] ?? ""; + // A recycled PID belongs to an unrelated process, not the descendant; its argv won't + // carry the marker, and that counts the same as the descendant already being gone. + const stillTracksDescendant = line.includes(descendantMarker); expect( - (Number(status) === 1 && state.trim() === "") || - (Number(status) === 0 && state.trim().startsWith("Z")), + Number(status) === 1 || line === "" || !stillTracksDescendant || state.startsWith("Z"), ).toBe(true); }), ).pipe(Effect.provide(NodeServices.layer)), @@ -717,6 +768,7 @@ describe("native process group cleanup", () => { const realSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; const native = yield* spawnNativeProcess({ executable: process.execPath, + environment: testEnvironment, args: [ "--input-type=module", "-e", @@ -793,7 +845,10 @@ describe("native process group cleanup", () => { it.live("reports a native launcher startup failure when workload spawn fails", () => Effect.scoped( - spawnNativeProcess({ executable: "/definitely/missing/native-workload" }).pipe(Effect.exit), + spawnNativeProcess({ + executable: "/definitely/missing/native-workload", + environment: testEnvironment, + }).pipe(Effect.exit), ).pipe( Effect.provide(NodeServices.layer), Effect.tap((result) => @@ -936,6 +991,7 @@ describe("native process group cleanup", () => { spawnNativeProcess( { executable: process.execPath, + environment: testEnvironment, args: [ "-e", "process.on('SIGTERM', () => {}); process.stdout.write('native-workload-ready\\n'); setInterval(() => {}, 1000)", diff --git a/packages/stack/src/runtime/postgres-user.integration.test.ts b/packages/stack/src/runtime/postgres-user.integration.test.ts new file mode 100644 index 0000000000..bc6d75689c --- /dev/null +++ b/packages/stack/src/runtime/postgres-user.integration.test.ts @@ -0,0 +1,143 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Path } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- the current process's own uid/gid make the chown calls below harmless no-ops. +import { userInfo } from "node:os"; +import { + acquireNativeRuntimeRoot, + handOverNativePostgresFiles, + NativeRuntimeRootBase, + nativeRuntimeRootPath, + type PasswdEntry, +} from "./postgres-user.ts"; + +/** + * Exercises only the symlink-rejection step of the step-down chown, which needs no privilege: the + * handed-off targets are chowned to the current process's own uid/gid, a harmless no-op, so the + * test runs identically whether or not it has root. The cross-uid chown itself (the part that + * actually needs root) is verified by trace, not by this test: `handOverNativePostgresFiles` + * chowns the confined HOME once, with `-R -P` and no nested paths as separate arguments (see + * `chownRecursive` and its single call per target in `runtime/postgres-user.ts`). + */ +describe("handOverNativePostgresFiles", () => { + it.live("refuses a confined HOME replaced by a symlink, without chowning through it", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "postgres-user-symlink-" }); + const self = userInfo(); + const user: PasswdEntry = { + name: self.username, + uid: self.uid, + gid: self.gid, + home: self.homedir, + }; + const dataPath = path.join(root, "data"); + const rootKeyPath = path.join(root, "pgsodium_root.key"); + const socketPath = path.join(root, "socket"); + const hbaPath = path.join(socketPath, "pg_hba.conf"); + const bundleRoot = path.join(root, "bundle"); + const executable = path.join(bundleRoot, "bin", "postgres"); + yield* fs.makeDirectory(dataPath, { recursive: true }); + yield* fs.makeDirectory(socketPath, { recursive: true }); + yield* fs.makeDirectory(path.dirname(executable), { recursive: true }); + yield* fs.writeFileString(rootKeyPath, "root-key"); + yield* fs.writeFileString(hbaPath, "local all all trust\n"); + yield* fs.writeFileString(executable, "#!/bin/sh\n"); + + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "postgres-user-symlink-outside-", + }); + const environmentHome = path.join(root, "home"); + yield* fs.symlink(outside, environmentHome); + + const failure = yield* handOverNativePostgresFiles(user, { + dataPath, + rootKeyPath, + socketPath, + hbaPath, + runtimeRoot: root, + bundleRoot, + executable, + environmentHome, + }).pipe(Effect.flip); + expect(failure.message).toContain("is a symlink"); + expect(yield* fs.readLink(environmentHome).pipe(Effect.isSuccess)).toBe(true); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("acquireNativeRuntimeRoot", () => { + it.live("refuses a pre-existing runtime root that is a symlink, leaving it untouched", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-symlink-", + }); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-outside-", + }); + const root = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + yield* fs.symlink(outside, root); + + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("is a symlink"); + expect(yield* fs.readLink(root)).toBe(outside); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "refuses a pre-existing runtime root that is group- or world-writable, leaving it untouched", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-writable-", + }); + const root = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + yield* fs.makeDirectory(root); + // chmod, unlike the mkdir mode above, is never narrowed by the process umask. + yield* fs.chmod(root, 0o707); + + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("writable by group or others"); + expect((yield* fs.stat(root)).mode & 0o777).toBe(0o707); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("refuses a base that is group- or world-writable without the sticky bit", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* fs.makeTempDirectoryScoped({ + prefix: "native-runtime-root-shared-", + }); + yield* fs.chmod(base, 0o777); + + const failure = yield* acquireNativeRuntimeRoot().pipe( + Effect.flip, + Effect.provide(Layer.succeed(NativeRuntimeRootBase, base)), + ); + expect(failure.message).toContain("writable by group or others without the sticky bit"); + expect(yield* fs.exists(nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0))).toBe( + false, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/stack/src/runtime/postgres-user.ts b/packages/stack/src/runtime/postgres-user.ts index b53302bee1..1c82465fe1 100644 --- a/packages/stack/src/runtime/postgres-user.ts +++ b/packages/stack/src/runtime/postgres-user.ts @@ -1,10 +1,26 @@ -import { Config, Effect, FileSystem, Option, Path } from "effect"; +import { Config, Context, Crypto, Effect, FileSystem, Option, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { PlatformError } from "effect/PlatformError"; +import { contentDigestHex } from "../internal/content-digest.ts"; +import { lstatPath } from "../namespace/drivers/FileSystem.ts"; import { ServiceError } from "../Service.ts"; /** Names the non-root system user that runs native PostgreSQL when the stack itself runs as root. */ export const NATIVE_POSTGRES_USER_ENV = "SUPABASE_NATIVE_POSTGRES_USER"; +/** + * Base directory of the per-uid native runtime root. Always the system `/tmp` outside tests: its + * protection is the administrator's, so no user-chosen base widens what recovery may delete. + */ +export const NativeRuntimeRootBase = Context.Reference( + "@supabase/stack/NativeRuntimeRootBase", + { defaultValue: () => "/tmp" }, +); + +/** The per-uid private root every native socket directory nests under. */ +export const nativeRuntimeRootPath = (path: Path.Path, base: string, uid: number): string => + path.join(base, `supabase-${uid}`); + export interface PasswdEntry { readonly name: string; readonly uid: number; @@ -122,6 +138,143 @@ const ancestorsOf = (path: Path.Path, start: string): ReadonlyArray => { return parent === start ? [start] : [start, ...ancestorsOf(path, parent)]; }; +const isAlreadyExists = (error: PlatformError): boolean => error.reason._tag === "AlreadyExists"; + +/** + * Confirms `target`'s unresolved state (one `lstat`, so a symlink is never followed) is a + * directory worth trusting: owned by `uid`, or by root when `allowRootOwner`, and never writable by + * group or others unless `allowStickyWritable` and the sticky bit guard it, as `/tmp` itself is. + */ +const confirmPrivateDirectory = Effect.fn("NativePostgresUser.confirmPrivateDirectory")(function* ( + target: string, + uid: number, + policy: { readonly allowRootOwner: boolean; readonly allowStickyWritable: boolean }, +) { + const info = yield* lstatPath(target).pipe( + Effect.mapError( + (cause) => + new ServiceError({ operation: "launch", message: `Unable to inspect ${target}`, cause }), + ), + ); + if (info?.type === "SymbolicLink") + return yield* new ServiceError({ + operation: "launch", + message: `${target} is a symlink; refusing to use it for the native runtime root`, + }); + if (info === undefined || info.type !== "Directory") + return yield* new ServiceError({ + operation: "launch", + message: `${target} is not a directory; refusing to use it for the native runtime root`, + }); + if (info.uid !== uid && !(policy.allowRootOwner && info.uid === 0)) + return yield* new ServiceError({ + operation: "launch", + message: `${target} belongs to uid ${String(info.uid)}, not the current user; refusing to use it for the native runtime root`, + }); + const writable = (info.mode & 0o022) !== 0; + const sticky = (info.mode & 0o1000) !== 0; + if (writable && !(policy.allowStickyWritable && sticky)) + return yield* new ServiceError({ + operation: "launch", + message: `${target} is writable by group or others${policy.allowStickyWritable ? " without the sticky bit" : ""}; refusing to use it for the native runtime root`, + }); + return info; +}); + +/** + * Resolves and confirms the per-uid root native socket directories nest under: the configured base, + * canonicalized with `realPath` so a symlink or replacement planted after this check can't retarget + * it, then its `supabase-` leaf, confirmed owned by this uid alone with no write bit at all. + * `create` additionally makes the leaf (mode 0700) first, tolerating a race with another of our own + * processes. + */ +const resolveNativeRuntimeRoot = (create: boolean) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* NativeRuntimeRootBase; + const uid = process.getuid?.() ?? 0; + const canonicalBase = yield* fs + .realPath(base) + .pipe( + Effect.mapError( + (cause) => + new ServiceError({ operation: "launch", message: `Unable to resolve ${base}`, cause }), + ), + ); + yield* confirmPrivateDirectory(canonicalBase, uid, { + allowRootOwner: true, + allowStickyWritable: true, + }); + const root = nativeRuntimeRootPath(path, canonicalBase, uid); + if (create) + yield* fs.makeDirectory(root, { mode: 0o700 }).pipe( + Effect.catchIf(isAlreadyExists, () => Effect.void), + Effect.mapError( + (cause) => + new ServiceError({ operation: "launch", message: `Unable to create ${root}`, cause }), + ), + ); + yield* confirmPrivateDirectory(root, uid, { + allowRootOwner: false, + allowStickyWritable: false, + }); + return root; + }); + +/** Acquires this process's native runtime root for a launch, creating the leaf if missing. */ +export const acquireNativeRuntimeRoot = Effect.fn("NativePostgresUser.acquireRuntimeRoot")(() => + resolveNativeRuntimeRoot(true), +); + +/** Resolves the native runtime root for recovery's containment check, without creating it. */ +const resolveNativeRuntimeRootForRecovery = Effect.fn( + "NativePostgresUser.resolveRuntimeRootForRecovery", +)(() => resolveNativeRuntimeRoot(false)); + +/** + * The socket directory of one native database instance: a function of its data root and id alone, + * so launch and recovery agree on it with nothing persisted. PostgreSQL limits a Unix socket path + * (this directory plus `/.s.PGSQL.5432`) to 103 bytes, which bounds the runtime root base. + */ +export const nativeSocketDirectoryPath = ( + crypto: Crypto.Crypto, + path: Path.Path, + runtimeRoot: string, + dataRoot: string, + instanceId: string, +) => + contentDigestHex(crypto, `${dataRoot}\0${instanceId}`).pipe( + Effect.map((digest) => path.join(runtimeRoot, `pg-${digest}`)), + ); + +/** + * Removes the socket directory of one native database instance. A runtime root that is missing is + * nothing to remove; one that is not safe to trust (a symlink, foreign owner or writable by others) + * is warned about and left untouched. + */ +export const removeNativeSocketDirectory = Effect.fn("NativePostgresUser.removeSocketDirectory")( + function* (dataRoot: string, instanceId: string) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const runtimeRoot = yield* resolveNativeRuntimeRootForRecovery().pipe(Effect.option); + if (Option.isNone(runtimeRoot)) { + const base = yield* NativeRuntimeRootBase; + const leaf = nativeRuntimeRootPath(path, base, process.getuid?.() ?? 0); + if ((yield* lstatPath(leaf)) !== undefined) + yield* Effect.logWarning( + `${leaf} is not a trusted native runtime root; leaving its socket directories`, + ); + return; + } + yield* fs.remove( + yield* nativeSocketDirectoryPath(crypto, path, runtimeRoot.value, dataRoot, instanceId), + { recursive: true, force: true }, + ); + }, +); + /** * Restricts a directory to its owner but keeps an existing traverse-only grant, because a * stepped-down PostgreSQL resolves bundle and data paths through the cache and state roots while it runs. @@ -144,6 +297,25 @@ const allowTraverse = Effect.fn("NativePostgresUser.allowTraverse")(function* ( } }); +/** + * A directory inside a published artifact generation keeps the archive's own mode for life, so + * the step-down handover checks for the traverse bit (o+x) there instead of granting it: granting + * it would mutate an already-published, content-addressed generation. + */ +const requireTraverse = Effect.fn("NativePostgresUser.requireTraverse")(function* ( + directories: ReadonlyArray, +) { + const fs = yield* FileSystem.FileSystem; + for (const directory of directories) { + const { mode } = yield* fs.stat(directory); + if ((mode & 0o001) === 0) + return yield* new ServiceError({ + operation: "launch", + message: `${directory} is missing its traverse bit (o+x); re-prepare the artifact`, + }); + } +}); + const requireOwnedByRootOr = Effect.fn("NativePostgresUser.requireOwnedByRootOr")(function* ( user: PasswdEntry, target: string, @@ -191,7 +363,38 @@ export const openNativePostgresInstance = Effect.fn("NativePostgresUser.openInst /** The bundle's first-boot init runs `chmod +x` on this script, which only its owner may do. */ const GETKEY_SCRIPT = "share/supabase-cli/config/pgsodium_getkey.sh"; -/** Hands the instance data, key, socket, HBA file, and the bundle's getkey script to the PostgreSQL user. */ +/** `chown -R -P uid:gid target`, failing with a message naming `target` and the recipient user. */ +const chownRecursive = Effect.fn("NativePostgresUser.chownRecursive")(function* ( + user: PasswdEntry, + target: string, +) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // -P never follows symlinks inside the tree, and directories change owner after their contents. + const chownArgs = ["-R", "-P", `${user.uid}:${user.gid}`, target]; + yield* Effect.annotateCurrentSpan({ + "process.executable.name": "chown", + "process.arg_count": chownArgs.length, + }); + const status = yield* spawner.exitCode( + ChildProcess.make("chown", chownArgs, { stdin: "ignore", stdout: "ignore", stderr: "ignore" }), + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(status)); + if (Number(status) !== 0) + return yield* new ServiceError({ + operation: "launch", + message: `chown exited with ${String(status)} while handing ${target} to '${user.name}'`, + }); +}); + +/** + * Hands the instance data, key, socket, HBA file, the bundle's getkey script, and a confined + * native `environmentHome` (if any) to the PostgreSQL user, each with its own `chown -R -P` call + * so a nested directory is never also listed as a separate, redundant target. `runtimeRoot` and its + * ancestors only gain traverse, so the step-down user reaches its socket directory under a + * restrictive configured base without write access to any of them. An ancestor inside the + * published generation (the bundle root and anything between it and the executable or getkey) + * is only required to already have traverse, never granted it: that generation is immutable. + */ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOverFiles")(function* ( user: PasswdEntry, paths: { @@ -199,13 +402,15 @@ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOve readonly rootKeyPath: string; readonly socketPath: string; readonly hbaPath: string; + readonly runtimeRoot: string; readonly bundleRoot: string; readonly executable: string; + /** A confined native environment root (HOME); chowned on its own, never as a nested target. */ + readonly environmentHome?: string; }, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const getkey = path.join(paths.bundleRoot, GETKEY_SCRIPT); const hasGetkey = yield* fs.exists(getkey); const targets = [paths.dataPath, paths.rootKeyPath, paths.socketPath, paths.hbaPath]; @@ -213,27 +418,28 @@ export const handOverNativePostgresFiles = Effect.fn("NativePostgresUser.handOve yield* requireOwnedByRootOr(user, getkey); targets.push(getkey); } - // -P never follows symlinks inside the tree, and directories change owner after their contents. - const chownArgs = ["-R", "-P", `${user.uid}:${user.gid}`, ...targets]; - yield* Effect.annotateCurrentSpan({ - "process.executable.name": "chown", - "process.arg_count": chownArgs.length, - }); - const status = yield* spawner.exitCode( - ChildProcess.make("chown", chownArgs, { - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - }), - ); - yield* Effect.annotateCurrentSpan("process.exit_code", Number(status)); - if (Number(status) !== 0) - return yield* new ServiceError({ - operation: "launch", - message: `chown exited with ${String(status)} while handing files to '${user.name}'`, - }); + for (const target of targets) yield* chownRecursive(user, target); + if (paths.environmentHome !== undefined) { + // A prior launch's confined HOME could have been replaced by a symlink or junction between + // runs; refuse to chown through it rather than trust reuse. + if ((yield* lstatPath(paths.environmentHome))?.type === "SymbolicLink") + return yield* new ServiceError({ + operation: "launch", + message: `${paths.environmentHome} is a symlink; refusing to hand it to '${user.name}'`, + }); + yield* chownRecursive(user, paths.environmentHome); + } + const insideGeneration = (candidate: string) => + candidate === paths.bundleRoot || candidate.startsWith(`${paths.bundleRoot}${path.sep}`); + const executableAncestors = ancestorsOf(path, path.dirname(paths.executable)); + const getkeyAncestors = hasGetkey ? ancestorsOf(path, path.dirname(getkey)) : []; + yield* requireTraverse([ + ...executableAncestors.filter(insideGeneration), + ...getkeyAncestors.filter(insideGeneration), + ]); yield* allowTraverse(user, [ - ...ancestorsOf(path, path.dirname(paths.executable)), - ...(hasGetkey ? ancestorsOf(path, path.dirname(getkey)) : []), + ...ancestorsOf(path, paths.runtimeRoot), + ...executableAncestors.filter((candidate) => !insideGeneration(candidate)), + ...getkeyAncestors.filter((candidate) => !insideGeneration(candidate)), ]); }); diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index ccb807e3ec..6a82f853c0 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -73,6 +73,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ const backend = yield* backendConnection(databaseUrl); return { DATABASE_URL: databaseUrl, + PHX_HTTP_IP: container ? "0.0.0.0" : "127.0.0.1", ...(http === undefined ? {} : { PORT: String(http.port), PHX_HTTP_PORT: String(http.port) }), @@ -84,12 +85,15 @@ export const makeSpec = (): ProcessRecipeSpec => ({ DB_PASSWORD: backend.password, LOGFLARE_SUPABASE_MODE: "true", LOGFLARE_SINGLE_TENANT: "true", - ...(container ? {} : { LOGFLARE_GRPC_PORT: "0" }), + ...(container ? {} : { LOGFLARE_GRPC_PORT: "0", LOGFLARE_GRPC_IP: "127.0.0.1" }), ...(creation.config.apiKey === undefined ? {} : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), POSTGRES_BACKEND_URL: backend.url, POSTGRES_BACKEND_SCHEMA: schema, + // Logflare waits this period after SIGTERM before stopping (a load-balancer drain delay), + // longer than the stack's stop grace. + ERL_AFLAGS: "-eval application:set_env(logflare,sigterm_shutdown_grace_period_ms,0).", }; }), args: () => Effect.succeed(["start"]), diff --git a/packages/stack/src/services/AnalyticsImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsImgproxy.integration.test.ts index 9ad83b0bc4..ed9085b1c4 100644 --- a/packages/stack/src/services/AnalyticsImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsImgproxy.integration.test.ts @@ -2,10 +2,12 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-analytics", @@ -17,7 +19,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -29,6 +32,11 @@ describe("service catalog", () => { const fs = yield* FileSystem.FileSystem; const client = yield* HttpClient.HttpClient; const root = yield* makeDockerDatabaseRoot("catalog-optional-data-"); + // Ownership is by location: Imgproxy's served directory is a caller path and must live + // outside the stack's data root, not merely outside the service's own instance root. + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-optional-data-caller-", + }); const secret = "catalog-optional-data-secret-with-at-least-32-chars"; const databaseRecipe = yield* makeServiceRecipe( { @@ -41,9 +49,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -58,9 +65,8 @@ describe("service catalog", () => { config: { databaseUrl, backend: "postgres", apiKey: "catalog-analytics" }, }, dockerOptions(root), - Effect.succeed([]), ); - const analytics = yield* makeService(analyticsRecipe.definition, { + const analytics = yield* makeStandaloneService(analyticsRecipe.definition, { id: "analytics", config: analyticsRecipe.creation, }); @@ -69,19 +75,18 @@ describe("service catalog", () => { const analyticsEndpoint = yield* analyticsRecipe.endpoint("http"); const analyticsResponse = yield* client.execute( HttpClientRequest.get( - `http://${analyticsEndpoint.host}:${analyticsEndpoint.port}/health`, + `http://${httpHost(analyticsEndpoint)}:${analyticsEndpoint.port}/health`, ), ); expect(analyticsResponse.status).toBe(200); - const imageRoot = `${root}/images`; + const imageRoot = `${callerRoot}/images`; yield* fs.makeDirectory(imageRoot, { recursive: true }); const imgproxyRecipe = yield* makeServiceRecipe( { service: "imgproxy", config: { filePath: imageRoot } }, dockerOptions(root), - Effect.succeed([]), ); - const imgproxy = yield* makeService(imgproxyRecipe.definition, { + const imgproxy = yield* makeStandaloneService(imgproxyRecipe.definition, { id: "imgproxy", config: imgproxyRecipe.creation, }); @@ -90,7 +95,7 @@ describe("service catalog", () => { const imgproxyEndpoint = yield* imgproxyRecipe.endpoint("http"); const imgproxyResponse = yield* client.execute( HttpClientRequest.get( - `http://${imgproxyEndpoint.host}:${imgproxyEndpoint.port}/health`, + `http://${httpHost(imgproxyEndpoint)}:${imgproxyEndpoint.port}/health`, ), ); expect(imgproxyResponse.status).toBe(200); diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts index a28f9e6571..d271baabd3 100644 --- a/packages/stack/src/services/AuthStorage.integration.test.ts +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -3,10 +3,12 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Layer, Redacted, Schema } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; const options = (root: string) => ({ stackId: "catalog-auth-storage", @@ -18,7 +20,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -30,6 +33,11 @@ describe("service catalog", () => { const fs = yield* FileSystem.FileSystem; const client = yield* HttpClient.HttpClient; const root = yield* makeDockerDatabaseRoot("catalog-auth-storage-"); + // Ownership is by location: Storage's filePath is a caller path and must live outside + // the stack's data root, not merely outside the service's own instance root. + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-auth-storage-caller-", + }); const secret = "catalog-auth-storage-secret-with-at-least-32-chars"; const databaseRecipe = yield* makeServiceRecipe( { @@ -42,9 +50,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -58,9 +65,8 @@ describe("service catalog", () => { config: { databaseUrl, jwtSecret: secret, jwtExpiry: 3600 }, }, dockerOptions(root), - Effect.succeed([]), ); - const auth = yield* makeService(authRecipe.definition, { + const auth = yield* makeStandaloneService(authRecipe.definition, { id: "auth", config: authRecipe.creation, }); @@ -70,7 +76,7 @@ describe("service catalog", () => { const email = "catalog@example.test"; const password = "catalog-password-123"; const signupRequest = yield* HttpClientRequest.bodyJson({ email, password })( - HttpClientRequest.post(`http://${authEndpoint.host}:${authEndpoint.port}/signup`), + HttpClientRequest.post(`http://${httpHost(authEndpoint)}:${authEndpoint.port}/signup`), ); const signup = yield* client.execute(signupRequest); expect(signup.status).toBe(200); @@ -84,7 +90,7 @@ describe("service catalog", () => { password, })( HttpClientRequest.post( - `http://${authEndpoint.host}:${authEndpoint.port}/token?grant_type=password`, + `http://${httpHost(authEndpoint)}:${authEndpoint.port}/token?grant_type=password`, ), ); const login = yield* client.execute(loginRequest); @@ -95,14 +101,13 @@ describe("service catalog", () => { )(loginBody); expect(loginToken.access_token.length).toBeGreaterThan(0); - const storageRoot = `${root}/storage`; + const storageRoot = `${callerRoot}/storage`; yield* fs.makeDirectory(storageRoot, { recursive: true }); const imgproxyRecipe = yield* makeServiceRecipe( { service: "imgproxy", config: { filePath: storageRoot } }, dockerOptions(root), - Effect.succeed([]), ); - const imgproxy = yield* makeService(imgproxyRecipe.definition, { + const imgproxy = yield* makeStandaloneService(imgproxyRecipe.definition, { id: "imgproxy", config: imgproxyRecipe.creation, }); @@ -120,9 +125,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const storage = yield* makeService(storageRecipe.definition, { + const storage = yield* makeStandaloneService(storageRecipe.definition, { id: "storage", config: storageRecipe.creation, }); @@ -139,7 +143,7 @@ describe("service catalog", () => { ); const bucketRequest = yield* HttpClientRequest.bodyJson({ name: "catalog" })( HttpClientRequest.post( - `http://${storageEndpoint.host}:${storageEndpoint.port}/bucket`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/bucket`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), @@ -149,7 +153,7 @@ describe("service catalog", () => { expect(bucket.status).toBe(200); const uploadRequest = HttpClientRequest.bodyText( HttpClientRequest.post( - `http://${storageEndpoint.host}:${storageEndpoint.port}/object/catalog/hello.txt`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/object/catalog/hello.txt`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), @@ -167,7 +171,7 @@ describe("service catalog", () => { ); const imageRequest = HttpClientRequest.bodyUint8Array( HttpClientRequest.post( - `http://${storageEndpoint.host}:${storageEndpoint.port}/object/catalog/source.png`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/object/catalog/source.png`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), @@ -179,7 +183,7 @@ describe("service catalog", () => { expect(imageUpload.status).toBe(200); const transformedImage = yield* client.execute( HttpClientRequest.get( - `http://${storageEndpoint.host}:${storageEndpoint.port}/render/image/authenticated/catalog/source.png?width=1&height=1`, + `http://${httpHost(storageEndpoint)}:${storageEndpoint.port}/render/image/authenticated/catalog/source.png?width=1&height=1`, ).pipe( HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), HttpClientRequest.setHeader("apikey", serviceToken), diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts index 5cad63c214..be06f948a4 100644 --- a/packages/stack/src/services/Catalog.integration.test.ts +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -24,7 +24,6 @@ describe("service catalog", () => { endpoints: { smtp: { port: "auto" } }, }, options(root), - Effect.succeed([]), ).pipe(Effect.exit); expect(Exit.isFailure(result)).toBe(true); const misplacedDatabaseVersion = yield* makeServiceRecipe( @@ -39,7 +38,6 @@ describe("service catalog", () => { }, }, options(root), - Effect.succeed([]), ).pipe(Effect.exit); expect(Exit.isFailure(misplacedDatabaseVersion)).toBe(true); }), @@ -58,7 +56,6 @@ describe("service catalog", () => { config: { databaseUrl: "postgres://db" }, }, options(root), - Effect.succeed([]), ), ); expect(error.operation).toBe("config"); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index c7d9e4375c..ab2f540c92 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -2,9 +2,7 @@ import { Crypto, Effect, FileSystem, Path, Ref, Schema } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; import { makeContainerRuntime } from "../runtime/Container.ts"; -import { randomPortSpanStart, reserveNativePort } from "../Ports.ts"; import { ServiceError, type ServiceDefinition } from "../Service.ts"; -import type * as State from "../State.ts"; import { makeDatabase, DatabaseConfig, @@ -29,8 +27,10 @@ import { type CatalogOptions, type CatalogRecipe as RecipeCatalogRecipe, type ProcessRecipeResult, + type ServiceEndpoint, } from "./Recipe.ts"; import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; +import { borrow } from "../namespace/Paths.ts"; import { missingInput } from "./ServiceConfig.ts"; import type { SnapshotScope } from "./DatabaseSnapshot.ts"; import { slimImageMirrors, type ServiceKind } from "../Artifacts.ts"; @@ -268,6 +268,13 @@ const databaseRecipe = ( endpoint: (name) => name === "sql" ? component.endpoint.pipe( + // node-postgres appends the `.s.PGSQL.` suffix itself, so the database keeps the + // directory and port private; the catalog's endpoint carries the full socket filename. + Effect.map((value): ServiceEndpoint => + value.kind === "unix" + ? { ...value, path: `${value.path}/.s.PGSQL.${value.port}` } + : value, + ), Effect.mapError( (cause) => new CatalogError({ @@ -289,11 +296,7 @@ const databaseRecipe = ( }); export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( - ( - input: unknown, - options: CatalogOptions, - readPortClaims: Effect.Effect, State.StateError>, - ) => + (input: unknown, options: CatalogOptions) => Effect.gen(function* () { const endpointError = validateEndpointNames(input); if (endpointError !== undefined) return yield* endpointError; @@ -324,6 +327,7 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( runtime: options.runtime, ...(options.helpers === undefined ? {} : { helpers: options.helpers }), ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), + ...(options.engineTarget === undefined ? {} : { engineTarget: options.engineTarget }), }).pipe( Effect.mapError( (cause) => @@ -338,14 +342,19 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( return databaseRecipe(creation, component); } const container = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeContainerRuntime({ - engine: options.runtime, + target: options.engineTarget, root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), }); + const borrowCallerPath = (candidate: string) => + borrow(fs, path, candidate, options.root, (operation, cause) => { + const message = cause instanceof Error ? cause.message : String(cause); + return new ServiceError({ operation, message, cause }); + }); const deps: ProcessDependencies = { fs, path, @@ -353,9 +362,7 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( client, spawner, container, - readPortClaims, - reserveNativePort: (key, claims, excluded) => - reserveNativePort(claims, key, randomPortSpanStart(crypto), excluded), + borrowCallerPath, }; switch (creation.service) { case "rest": diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 553f105ed0..4c23f04320 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -17,11 +17,14 @@ import { Schema, Stream, } from "effect"; -import { tmpdir } from "node:os"; +import { createHash } from "node:crypto"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; -import { makeDockerDatabaseRoot, runDocker } from "../../tests/docker-fixture.ts"; +import { makeDockerDatabaseRoot, runEngine } from "../../tests/docker-fixture.ts"; +import { observeContainerStop, cleanStopEvents } from "../../tests/engine-events.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const config: DatabaseConfig = { version: "17", @@ -31,7 +34,7 @@ const config: DatabaseConfig = { rootKey: Redacted.make("a".repeat(64)), }; -const artifactCacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const artifactCacheRoot = testArtifactCacheRoot; const query = ( endpoint: BackendEndpoint, @@ -59,8 +62,8 @@ const query = ( describe("database component", { timeout: 180_000 }, () => { for (const target of [ { runtime: "native", version: "17" }, - { runtime: "docker", version: "15" }, - { runtime: "docker", version: "17" }, + { runtime: testEngine, version: "15" }, + { runtime: testEngine, version: "17" }, ] as const) it.live( `preserves default encryption keys and Vault secrets after ${target.runtime} PostgreSQL ${target.version} recreation`, @@ -70,7 +73,7 @@ describe("database component", { timeout: 180_000 }, () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = - target.runtime === "docker" + target.runtime !== "native" ? yield* makeDockerDatabaseRoot("stack-default-key-", "default-key-test") : yield* fs.makeTempDirectoryScoped({ prefix: "stack-default-key-" }); const recipe = yield* makeDatabase({ @@ -79,6 +82,7 @@ describe("database component", { timeout: 180_000 }, () => { root, cacheRoot: artifactCacheRoot, runtime: target.runtime, + ...(target.runtime !== "native" ? { engineTarget } : {}), }); const defaults: DatabaseConfig = { healthTimeoutMs: 120_000, @@ -87,7 +91,7 @@ describe("database component", { timeout: 180_000 }, () => { jwtSecret: config.jwtSecret, jwtExpiry: config.jwtExpiry, }; - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: "database", config: defaults, }); @@ -138,7 +142,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: target.runtime, }); - const secondService = yield* makeService(second.definition, { + const secondService = yield* makeStandaloneService(second.definition, { id: "database-second", config: { ...defaults, rootKey: Redacted.make("b".repeat(64)) }, }); @@ -172,7 +176,10 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: "native", }); - const service = yield* makeService(database.definition, { id: "database:hba", config }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:hba", + config, + }); yield* service.start; yield* service.ready; const endpoint = yield* database.endpoint; @@ -251,7 +258,10 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: "native", }); - const service = yield* makeService(database.definition, { id: "database:tls", config }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:tls", + config, + }); yield* service.start.pipe( Effect.provide( ConfigProvider.layerAdd(ConfigProvider.fromEnvRecord({ SSL_CERT_FILE: bundle }), { @@ -317,7 +327,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: "native", }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:trust-env", config, }); @@ -366,7 +376,7 @@ describe("database component", { timeout: 180_000 }, () => { runtime: "native", }); const databaseConfig = { ...config, version }; - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:cron", config: databaseConfig, }); @@ -457,7 +467,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot: artifactCacheRoot, runtime: "native", }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:probe", config: { ...config, healthTimeoutMs: 0 }, }); @@ -468,6 +478,50 @@ describe("database component", { timeout: 180_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + it.live( + "replaces a socket directory a killed owner left behind and removes its own on stop", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-leftover-" }); + // The derived name is computed here independently of production code. + const socketDirectory = `${yield* fs.realPath("/tmp")}/supabase-${process.getuid?.() ?? 0}/pg-${createHash( + "sha256", + ) + .update(`${root}\0leftover`) + .digest("hex") + .slice(0, 16)}`; + yield* fs.makeDirectory(socketDirectory, { recursive: true, mode: 0o700 }); + yield* fs.writeFileString(`${socketDirectory}/stale.marker`, "left by a killed owner\n"); + const database = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "leftover", + root, + cacheRoot: artifactCacheRoot, + runtime: "native", + }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:leftover", + config, + }); + + yield* service.start; + yield* service.ready; + + const endpoint = yield* database.endpoint; + expect(endpoint).toMatchObject({ kind: "unix", path: socketDirectory }); + expect(yield* fs.exists(`${socketDirectory}/stale.marker`)).toBe(false); + expect(yield* query(endpoint, config.databasePassword, "SELECT 1 AS ok")).toEqual([ + { ok: 1 }, + ]); + yield* service.stop; + expect(yield* fs.exists(socketDirectory)).toBe(false); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + it.live( "persists SQL data across exact-session stop and reopen, isolates instances, and validates restart before stopping", () => @@ -485,7 +539,7 @@ describe("database component", { timeout: 180_000 }, () => { runtime: "native", }); const loggedConfig: DatabaseConfig = { ...config, settings: { log_statement: "all" } }; - const service = yield* makeService(first.definition, { + const service = yield* makeStandaloneService(first.definition, { id: "database:first", config: loggedConfig, }); @@ -560,7 +614,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot, runtime: "native", }); - const reopenedService = yield* makeService(reopened.definition, { + const reopenedService = yield* makeStandaloneService(reopened.definition, { id: "database:first-reopened", config, }); @@ -586,7 +640,7 @@ describe("database component", { timeout: 180_000 }, () => { cacheRoot, runtime: "native", }); - const secondService = yield* makeService(second.definition, { + const secondService = yield* makeStandaloneService(second.definition, { id: "database:second", config, }); @@ -638,9 +692,10 @@ describe("database component", { timeout: 180_000 }, () => { instanceId: "database", root, cacheRoot, - runtime: "docker", + runtime: testEngine, + engineTarget, }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:container", config: databaseConfig, }); @@ -693,10 +748,11 @@ describe("database component", { timeout: 180_000 }, () => { instanceId: "database", root, cacheRoot, - runtime: "docker", + runtime: testEngine, + engineTarget, }); const replacementPassword = Redacted.make("reopened-target-password"); - const reopenedService = yield* makeService(reopened.definition, { + const reopenedService = yield* makeStandaloneService(reopened.definition, { id: "database:reopened", config: { ...databaseConfig, databasePassword: replacementPassword }, }); @@ -717,48 +773,53 @@ describe("database component", { timeout: 180_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); - it.live("groups the database and its storage helper under one compose project", () => - Effect.scoped( - Effect.gen(function* () { - const path = yield* Path.Path; - const stackId = "stack-compose-group"; - const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); - const database = yield* makeDatabase({ - stackId, - instanceId: "database", - project: "my.app", - root, - cacheRoot: artifactCacheRoot, - runtime: "docker", - }); - const service = yield* makeService(database.definition, { - id: "database:group", - config, - }); - yield* service.start; - yield* service.ready; - const listed = yield* runDocker([ - "ps", - "--filter", - `label=com.supabase.stack-root=${path.resolve(root)}`, - "--format", - '{{.Names}}|{{.Label "com.docker.compose.project"}}|{{.Label "com.docker.compose.service"}}', - ]); - const rows = listed.output - .split("\n") - .filter((line) => line.trim().length > 0) - .map((line) => line.trim().split("|")); - expect(rows.some(([name]) => name?.startsWith("supabase-db-helper-"))).toBe(true); - expect(new Set(rows.map(([, project]) => project))).toEqual( - new Set(["supabase-my-app-stack-compos"]), - ); - expect(new Set(rows.map(([, , group]) => group))).toEqual( - new Set(["database", "database-helper"]), - ); - yield* service.destroy; - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - ); + // The storage helper joins the compose project only with the Docker volume backend. + describe.runIf(testEngine === "docker")("Docker volume storage", () => { + it.live("groups the database and its storage helper under one compose project", () => + Effect.scoped( + Effect.gen(function* () { + const path = yield* Path.Path; + const stackId = "stack-compose-group"; + const root = yield* makeDockerDatabaseRoot("stack-database-group-", stackId); + const database = yield* makeDatabase({ + stackId, + instanceId: "database", + project: "my.app", + root, + cacheRoot: artifactCacheRoot, + runtime: testEngine, + engineTarget, + }); + const service = yield* makeStandaloneService(database.definition, { + id: "database:group", + config, + }); + yield* service.start; + yield* service.ready; + const listed = yield* runEngine([ + ...engineTarget.argv, + "ps", + "--filter", + `label=com.supabase.stack-root=${path.resolve(root)}`, + "--format", + '{{.Names}}|{{.Label "com.docker.compose.project"}}|{{.Label "com.docker.compose.service"}}', + ]); + const rows = listed.output + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => line.trim().split("|")); + expect(rows.some(([name]) => name?.startsWith("supabase-db-helper-"))).toBe(true); + expect(new Set(rows.map(([, project]) => project))).toEqual( + new Set(["supabase-my-app-stack-compos"]), + ); + expect(new Set(rows.map(([, , group]) => group))).toEqual( + new Set(["database", "database-helper"]), + ); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + }); it.live("shuts PostgreSQL down fast while a client stays connected across stop", () => Effect.scoped( @@ -771,9 +832,10 @@ describe("database component", { timeout: 180_000 }, () => { instanceId: "database", root, cacheRoot: artifactCacheRoot, - runtime: "docker", + runtime: testEngine, + engineTarget, }); - const service = yield* makeService(database.definition, { + const service = yield* makeStandaloneService(database.definition, { id: "database:shutdown", config: { ...config, stopGraceSeconds: 30 }, }); @@ -781,7 +843,8 @@ describe("database component", { timeout: 180_000 }, () => { yield* service.ready; const endpoint = yield* database.endpoint; if (endpoint.kind !== "tcp") return yield* Effect.die("Expected a TCP endpoint"); - const listed = yield* runDocker([ + const listed = yield* runEngine([ + ...engineTarget.argv, "ps", "--filter", `label=com.supabase.stack-root=${path.resolve(root)}`, @@ -797,13 +860,19 @@ describe("database component", { timeout: 180_000 }, () => { .filter((name) => name.length > 0 && !name.startsWith("supabase-db-helper-")); expect(containers).toHaveLength(1); const container = containers.join(""); - const startedAt = yield* runDocker([ + const startedAt = yield* runEngine([ + ...engineTarget.argv, "inspect", "--format", - "{{.State.StartedAt}}", + "{{json .State.StartedAt}}", container, ]); + const observed = yield* observeContainerStop( + container, + startedAt.output.trim().replaceAll('"', ""), + ); + yield* Effect.scoped( Effect.gen(function* () { const services = yield* Layer.build( @@ -823,26 +892,10 @@ describe("database component", { timeout: 180_000 }, () => { }), ); - const stoppedAt = yield* runDocker(["info", "--format", "{{.SystemTime}}"]); - const events = yield* runDocker([ - "events", - "--since", - startedAt.output.trim(), - "--until", - stoppedAt.output.trim(), - "--filter", - `container=${container}`, - "--filter", - "event=kill", - "--filter", - "event=die", - "--format", - '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}', - ]); expect( - events.output.trim().split("\n"), + yield* Fiber.join(observed), "stop sends SIGINT and PostgreSQL exits cleanly", - ).toEqual(["kill 2", "die 0"]); + ).toEqual(cleanStopEvents); yield* service.destroy; }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index db7417d5a8..f718783deb 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -6,8 +6,6 @@ import { Crypto, Data, Effect, - Exit, - Fiber, FileSystem, Layer, Option, @@ -28,12 +26,14 @@ import { prepareNativeArtifact, postgresVersion, resolveArtifact, + useNativeArtifact, type PreparedNativeArtifact, } from "../Artifacts.ts"; import { makeContainerRuntime, type ContainerProcess, type ContainerRuntime, + type EngineTarget, type HostGateway, } from "../runtime/Container.ts"; import { DatabaseBootstrapError, runDatabaseBootstrap } from "../runtime/DatabaseBootstrap.ts"; @@ -43,7 +43,7 @@ import { } from "../runtime/PostgresDatabaseSession.ts"; import { mapToServiceError, - processExit as sharedProcessExit, + processExit, publishProcessLogs, runtimeSessionFromContainer, launchOutputPublisher, @@ -66,17 +66,17 @@ import { } from "../runtime/NativeProcess.ts"; import type { StackId } from "../identity/StackId.ts"; import { + acquireNativeRuntimeRoot, + nativeSocketDirectoryPath, + removeNativeSocketDirectory, handOverNativePostgresFiles, openNativePostgresInstance, resolveNativePostgresUser, type PasswdEntry, } from "../runtime/postgres-user.ts"; import { EndpointIntent, serviceCreation, type CatalogLogs } from "./Recipe.ts"; -import { - containerInstancePath, - ensureOwnedInstanceRoot, - removeOwnedInstanceRoot, -} from "./InstanceRoot.ts"; +import * as Environment from "../namespace/Environment.ts"; +import { containerInstancePath, destroyOwnedRoot } from "../namespace/Paths.ts"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { instanceSnapshotsDirectory, @@ -144,6 +144,8 @@ export interface DatabaseOptions { readonly helpers?: DockerHelperRegistry; /** Shares one host-gateway probe across this host's container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; } export interface DatabaseComponent { @@ -189,71 +191,66 @@ const databaseError = (operation: string, cause: unknown): DatabaseError => const describePostgresExit = (code: number) => `PostgreSQL exited with code ${code}`; -/** Settles a native PostgreSQL exit, waiting briefly after it for the stderr tail to drain. */ -export const processExit = ( - exitCode: Effect.Effect, - stderr?: { - readonly tail: Ref.Ref; - readonly drained: Fiber.Fiber; - }, -): Effect.Effect> => - sharedProcessExit(exitCode, describePostgresExit, stderr); - -const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPassword")(function* ( - engine: "docker" | "podman", - id: string, - password: Redacted.Redacted, - spawner: ChildProcessSpawnerService["Service"], -) { - const args = [ - "exec", - "-i", - id, - "/opt/postgres/bin/psql", - "-h", - "/tmp", - "-p", - "5432", - "-U", - "supabase_admin", - "-d", - "postgres", - "-X", - "-v", - "ON_ERROR_STOP=1", - ]; - yield* Effect.annotateCurrentSpan({ - "process.executable.name": engine, - "process.arg_count": args.length, - }); - // Each retry overwrites `process.exit_code`, so the span reports the last attempt's status. - return yield* withAttemptCount( - Effect.scoped( - Effect.gen(function* () { - const child = yield* spawner.spawn(ChildProcess.make(engine, args, { stdin: "pipe" })); - const statement = `BEGIN; SET LOCAL log_statement = 'none'; SET LOCAL log_min_error_statement = 'panic'; SET LOCAL log_min_duration_statement = -1; SET LOCAL log_min_duration_sample = -1; SET LOCAL standard_conforming_strings = on; ALTER ROLE supabase_admin PASSWORD '${Redacted.value(password).replaceAll("'", "''")}'; COMMIT;`; - const [, , , code] = yield* Effect.all( - [ - Stream.make(new TextEncoder().encode(statement)).pipe(Stream.run(child.stdin)), - child.stdout.pipe(Stream.runDrain), - child.stderr.pipe(Stream.runDrain), - child.exitCode, - ], - { concurrency: "unbounded" }, - ); - yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); - if (Number(code) !== 0) - return yield* errorFor("health", "Local database credential setup has not succeeded"); - }), - ), - (counted) => - counted.pipe( - Effect.mapError(() => errorFor("health", "Local database credential setup failed")), - Effect.retry(Schedule.spaced("250 millis")), +/** Exported for the cross-cutting pin test in {@link "../runtime/Container.integration.test.ts"}. */ +export const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPassword")( + function* ( + target: EngineTarget, + id: string, + password: Redacted.Redacted, + spawner: ChildProcessSpawnerService["Service"], + ) { + const args = [ + "exec", + "-i", + id, + "/opt/postgres/bin/psql", + "-h", + "/tmp", + "-p", + "5432", + "-U", + "supabase_admin", + "-d", + "postgres", + "-X", + "-v", + "ON_ERROR_STOP=1", + ]; + yield* Effect.annotateCurrentSpan({ + "process.executable.name": target.engine, + "process.arg_count": args.length, + }); + // Each retry overwrites `process.exit_code`, so the span reports the last attempt's status. + return yield* withAttemptCount( + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make(target.engine, [...target.argv, ...args], { stdin: "pipe" }), + ); + const statement = `BEGIN; SET LOCAL log_statement = 'none'; SET LOCAL log_min_error_statement = 'panic'; SET LOCAL log_min_duration_statement = -1; SET LOCAL log_min_duration_sample = -1; SET LOCAL standard_conforming_strings = on; ALTER ROLE supabase_admin PASSWORD '${Redacted.value(password).replaceAll("'", "''")}'; COMMIT;`; + const [, , , code] = yield* Effect.all( + [ + Stream.make(new TextEncoder().encode(statement)).pipe(Stream.run(child.stdin)), + child.stdout.pipe(Stream.runDrain), + child.stderr.pipe(Stream.runDrain), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); + if (Number(code) !== 0) + return yield* errorFor("health", "Local database credential setup has not succeeded"); + }), ), - { traced: true }, - ); -}); + (counted) => + counted.pipe( + Effect.mapError(() => errorFor("health", "Local database credential setup failed")), + Effect.retry(Schedule.spaced("250 millis")), + ), + { traced: true }, + ); + }, +); /** Idempotent readiness reconciliation, so a session also re-runs it as its probe. */ const health = Effect.fn("Database.health")(function* ( @@ -367,53 +364,6 @@ const publishLogs = publishProcessLogs; const runtimeFromContainer = (process: ContainerProcess, discard: boolean): RuntimeSession => runtimeSessionFromContainer(process, describePostgresExit, { discard }); -const databaseOwnerFileName = ".supabase-database-owner.json"; - -const ensureOwnedRoot = ( - fs: FileSystem.FileSystem, - path: Path.Path, - parentRoot: string, - stackId: string, - instanceId: string, -): Effect.Effect => - ensureOwnedInstanceRoot( - { - fs, - path, - parentRoot, - stackId, - instanceId, - ownerFileName: databaseOwnerFileName, - label: "Database root", - }, - databaseError, - ); - -const removeOwnedRoot = ( - fs: FileSystem.FileSystem, - path: Path.Path, - parentRoot: string, - stackId: string, - instanceId: string, - removeData: Effect.Effect, - /** Root entries kept with the owner marker; when empty the root itself is removed. */ - keep: ReadonlyArray = [], -): Effect.Effect => - removeOwnedInstanceRoot( - { - fs, - path, - parentRoot, - stackId, - instanceId, - ownerFileName: databaseOwnerFileName, - label: "Database root", - }, - removeData, - errorFor, - keep, - ); - const hostCaBundles = [ "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", @@ -516,17 +466,19 @@ const nativeProcess = ( readonly configPath: string | undefined; }, settings: ReadonlyArray, - context: ServiceInstanceContext, + scope: Scope.Closeable, stackId: string, instanceId: string, spawner: ChildProcessSpawnerService["Service"], user: PasswdEntry | undefined, + environment: Environment.NativeEnvironment, trustStore: Readonly>, fs: FileSystem.FileSystem, ): Effect.Effect => { const identity = { stackId, workloadId: instanceId }; const spec = { executable: artifact.executable, + artifactLockPath: artifact.lockPath, ...(user === undefined ? {} : { uid: user.uid, gid: user.gid, cwd: "/" }), args: [ "-D", @@ -546,13 +498,13 @@ const nativeProcess = ( ], env: { ...trustStore, - ...(user === undefined ? {} : { HOME: user.home }), PGDATA: paths.dataPath, PGSODIUM_KEY_FILE: paths.rootKeyPath, POSTGRES_USER: "supabase_admin", POSTGRES_DB: "postgres", POSTGRES_PASSWORD: Redacted.value(config.databasePassword), }, + environment, gracefulStopSignal: "SIGINT", gracefulStopTimeout: "15 seconds", } satisfies NativeProcessSpec; @@ -573,7 +525,7 @@ const nativeProcess = ( identity, ); }).pipe( - Scope.provide(context.scope), + Scope.provide(scope), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(FileSystem.FileSystem, fs), Effect.mapError((cause) => errorFor("launch", cause)), @@ -601,16 +553,20 @@ export const makeDatabase = ( const client = yield* HttpClient.HttpClient; const logs = yield* PubSub.sliding(256); const endpoint = yield* Ref.make(undefined); - const prepared = yield* Ref.make>(new Map()); if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(String(options.stackId))) return yield* databaseError("identity", "Invalid stack id"); - yield* ensureOwnedRoot(fs, path, options.root, String(options.stackId), options.instanceId); + // Ownership is by location: instanceRoot is owned simply by being under options.root, with no + // separate marker to establish or verify. Still created eagerly here, since the storage + // marker write below assumes the directory exists rather than creating it itself. const instanceRoot = path.join(options.root, options.instanceId); + yield* fs + .makeDirectory(instanceRoot, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => databaseError("root", cause))); const container: ContainerRuntime | undefined = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeContainerRuntime({ - engine: options.runtime, + target: options.engineTarget, root: options.root, imageMirrors: slimImageMirrors, ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), @@ -618,10 +574,11 @@ export const makeDatabase = ( awaitHostGateway: false, }); const storage: DockerDatabaseStorage | undefined = - options.runtime === "native" + options.engineTarget === undefined ? undefined : yield* makeDockerDatabaseStorage({ runtime: options.runtime, + target: options.engineTarget, stackId: String(options.stackId), instanceId: options.instanceId, ...(options.project === undefined ? {} : { project: options.project }), @@ -745,15 +702,14 @@ export const makeDatabase = ( }), ); if (options.runtime === "native") + // Ahead-of-time warm-up only: downloads and publishes the generation but pins nothing. + // `launch` resolves (or prepares) and pins its own copy right before it spawns. return prepareNativeArtifact({ service: "database", version }, options.cacheRoot).pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), Effect.provideService(Crypto.Crypto, crypto), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), Effect.provideService(HttpClient.HttpClient, client), - Effect.tap((artifact) => - Ref.update(prepared, (map) => new Map(map).set(version, artifact)), - ), Effect.asVoid, Effect.mapError((cause) => errorFor("prepare", cause)), ); @@ -928,6 +884,13 @@ export const makeDatabase = ( stepDownUser === undefined ? instanceRoot : yield* asRoot(openNativePostgresInstance(stepDownUser, instanceRoot)); + // A dedicated subdirectory, never the owned root itself, so a step-down chown of it + // never touches `nativeRoot`'s own root-only ownership that the next launch verifies. + const environment = yield* Environment.confine( + fs, + path, + path.join(nativeRoot, "home"), + ).pipe(Effect.mapError((cause) => errorFor("launch", cause))); const dataPath = path.join(nativeRoot, "data"); yield* fs .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) @@ -936,12 +899,26 @@ export const makeDatabase = ( yield* fs .writeFileString(rootKeyPath, Redacted.value(config.rootKey), { mode: 0o600 }) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); - // PostgreSQL limits Unix socket paths to 103 bytes, independently of the user's state root. - const socketPath = yield* Effect.acquireRelease( - fs.makeTempDirectory({ directory: "/tmp", prefix: "supabase-pg-" }), - (directory) => + // PostgreSQL limits Unix socket paths to 103 bytes, so this directory nests under the + // short per-uid native runtime root (acquired and validated below) rather than under + // instanceRoot. Ownership by location there keeps a foreign uid from ever creating or + // replacing it, and its derived name lets recovery find a leftover without a journal. + const runtimeRoot = yield* asRoot(acquireNativeRuntimeRoot()); + const socketPath = yield* nativeSocketDirectoryPath( + crypto, + path, + runtimeRoot, + options.root, + options.instanceId, + ).pipe(Effect.mapError((cause) => errorFor("launch", cause))); + // A directory already at the derived name is a previous run's leftover. + yield* Effect.acquireRelease( + fs + .remove(socketPath, { recursive: true, force: true }) + .pipe(Effect.andThen(fs.makeDirectory(socketPath, { mode: 0o700 }))), + () => fs - .remove(directory, { recursive: true, force: true }) + .remove(socketPath, { recursive: true, force: true }) .pipe(Effect.catch((cause) => Effect.logError(cause))), ).pipe( Scope.provide(context.scope), @@ -968,9 +945,25 @@ export const makeDatabase = ( ) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); } - const artifact = (yield* Ref.get(prepared)).get(config.version); - if (artifact === undefined) - return yield* errorFor("launch", `Artifact ${config.version} was not prepared`); + // `context.scope` finalizes its direct children in parallel (it is forked "parallel" + // in Service.ts), so registering the pin there directly would race its release + // against the native process's own cleanup. A "sequential" child scope finalizes + // LIFO instead: the pin is registered on it first, and the native process below is + // also scoped to it (not to `context.scope`), so closing it always runs the + // process's cleanup before releasing the pin. + const launchScope = yield* Scope.fork(context.scope, "sequential"); + const artifact = yield* useNativeArtifact( + { service: "database", version: config.version }, + options.cacheRoot, + ).pipe( + Scope.provide(launchScope), + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HttpClient.HttpClient, client), + Effect.mapError((cause) => errorFor("launch", cause)), + ); if (stepDownUser !== undefined) yield* asRoot( handOverNativePostgresFiles(stepDownUser, { @@ -978,8 +971,10 @@ export const makeDatabase = ( rootKeyPath, socketPath, hbaPath, + runtimeRoot, bundleRoot: artifact.root, executable: artifact.executable, + environmentHome: environment.values.HOME, }), ); const process = yield* nativeProcess( @@ -987,11 +982,12 @@ export const makeDatabase = ( config, { dataPath, socketPath, hbaPath, rootKeyPath, configPath }, settings, - context, + launchScope, String(options.stackId), options.instanceId, spawner, stepDownUser, + environment, yield* nativeTrustStore.pipe( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), @@ -1021,7 +1017,10 @@ export const makeDatabase = ( return { health: setup, probe: setup, - exit: processExit(process.exitCode, { tail: stderrTail, drained: stderrDrained }), + exit: processExit(process.exitCode, describePostgresExit, { + tail: stderrTail, + drained: stderrDrained, + }), stop: process.kill.pipe(Effect.mapError((cause) => errorFor("stop", cause))), remove: fs.remove(socketPath, { recursive: true, force: true }).pipe( Effect.mapError((cause) => errorFor("remove", cause)), @@ -1037,15 +1036,13 @@ export const makeDatabase = ( yield* Ref.set(endpoint, selectedEndpoint); yield* publishLogs(launched, publish, context.scope); const session = runtimeFromContainer(launched, config.stopGraceSeconds === 0); + const engineTarget = options.engineTarget; + if (engineTarget === undefined) + return yield* errorFor("launch", "Container runtime is unavailable"); const setup = health( selectedEndpoint, config, - reconcileContainerPassword( - options.runtime, - launched.id, - config.databasePassword, - spawner, - ), + reconcileContainerPassword(engineTarget, launched.id, config.databasePassword, spawner), { fs, instanceRoot, @@ -1068,24 +1065,37 @@ export const makeDatabase = ( }), ); + // Removed before the registration that lets recovery recompute its name, so a failure here + // leaves the instance registered to retry. + const removeSocketDirectory = + options.runtime === "native" + ? removeNativeSocketDirectory(options.root, options.instanceId).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.mapError((cause) => errorFor("destroy", cause)), + ) + : Effect.void; + const definition: ServiceDefinition = { prepare, launch, removeData: (context) => - removeOwnedRoot( + destroyOwnedRoot( fs, path, + instanceRoot, options.root, - String(options.stackId), - options.instanceId, - Effect.gen(function* () { - if (storage !== undefined) { - yield* storage.destroyData(postgresVersion(context.config.version)); - return; - } - if (container === undefined || !(yield* fs.exists(path.join(instanceRoot, "data")))) - return; - }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + removeSocketDirectory.pipe( + Effect.andThen( + storage === undefined + ? Effect.void + : storage + .destroyData(postgresVersion(context.config.version)) + .pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + ), + ), + errorFor, ), }; const resetData = Effect.fn("Database.resetData")(( @@ -1093,22 +1103,11 @@ export const makeDatabase = ( ) => { const clear: Effect.Effect = storage === undefined - ? removeOwnedRoot( - fs, - path, - options.root, - String(options.stackId), - options.instanceId, - Effect.void, - [instanceSnapshotsDirectory], - ) + ? destroyOwnedRoot(fs, path, instanceRoot, options.root, Effect.void, errorFor, [ + instanceSnapshotsDirectory, + ]) : storage.removeData(postgresVersion(context.config.version)); - return clear.pipe( - Effect.andThen( - ensureOwnedRoot(fs, path, options.root, String(options.stackId), options.instanceId), - ), - Effect.mapError((cause) => errorFor("reset", cause)), - ); + return clear.pipe(Effect.mapError((cause) => errorFor("reset", cause))); }); return { definition, diff --git a/packages/stack/src/services/Database.unit.test.ts b/packages/stack/src/services/Database.unit.test.ts deleted file mode 100644 index 0b0cee5ce8..0000000000 --- a/packages/stack/src/services/Database.unit.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Fiber, Option, Ref } from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import { processExit } from "./Database.ts"; - -it.effect("waits for the stderr tail after a long-running native PostgreSQL exits", () => - Effect.gen(function* () { - const tail = yield* Ref.make(""); - const drained = yield* Effect.forkChild( - Effect.sleep("2500 millis").pipe(Effect.andThen(Ref.set(tail, "FATAL: data directory"))), - ); - const settled = yield* Effect.forkChild( - processExit(Effect.sleep("2 seconds").pipe(Effect.as(1)), { tail, drained }), - ); - yield* TestClock.adjust("3 seconds"); - const exit = yield* Fiber.join(settled); - const error = Exit.isFailure(exit) - ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) - : undefined; - expect(error?.message).toBe("PostgreSQL exited with code 1: FATAL: data directory"); - }), -); diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts index a89e190335..add24aa49f 100644 --- a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -3,11 +3,12 @@ import { describe, expect, it } from "@effect/vitest"; import { Crypto, Data, Effect, Exit, FileSystem, Path, Ref, Schema, Scope, Stream } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; -import { makeContainerRuntime } from "../runtime/Container.ts"; +import { makeContainerRuntime, resolveEngineTarget } from "../runtime/Container.ts"; import { makeDockerDatabaseStorage } from "../storage/DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; import { shellQuote } from "../storage/DockerSnapshotBackend.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { testEngine } from "../../tests/test-engine.ts"; import { makeDatabaseSnapshots, type SnapshotScope } from "./DatabaseSnapshot.ts"; // Derived from the real catalog (not hardcoded), so a Postgres pin bump never makes this go stale. @@ -183,7 +184,11 @@ const docker = Effect.fnUntraced(function* () { const root = yield* makeDockerDatabaseRoot("database-snapshot-contract-", stackId); const cacheRoot = path.resolve(root, "../../../cache"); const image = (yield* resolveArtifact({ service: "database", version }).pipe(Effect.orDie)).image; - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const target = yield* resolveEngineTarget(spawner, "docker"); + const container = yield* makeContainerRuntime({ + target, + root, + }); const helpers = yield* makeDockerHelperRegistry( `snapshot-contract-${yield* crypto.randomUUIDv4}`, ); @@ -221,8 +226,11 @@ const docker = Effect.fnUntraced(function* () { ), ); const partialCopy = rewriting(spawner, (command, args) => { - const script = args.at(-1) ?? ""; - if (command !== "docker" || args[0] !== "exec" || !script.includes("/usr/local/bin/cp")) + // Every engine invocation now carries the pinned `target.argv` prefix (e.g. `--host `) + // ahead of the real subcommand; skip past it before matching on "exec". + const unpinned = args.slice(target.argv.length); + const script = unpinned.at(-1) ?? ""; + if (command !== "docker" || unpinned[0] !== "exec" || !script.includes("/usr/local/bin/cp")) return undefined; return [ command, @@ -240,6 +248,7 @@ const docker = Effect.fnUntraced(function* () { yield* fs.makeDirectory(instanceRoot, { recursive: true }); const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", + target, stackId, instanceId: name, instanceRoot, @@ -291,9 +300,10 @@ const docker = Effect.fnUntraced(function* () { return engine; }); +// The Docker case exercises the Docker volume backend; Podman keeps data in host directories. const engines = [ { name: "native", make: native }, - { name: "docker", make: docker }, + ...(testEngine === "docker" ? [{ name: "docker", make: docker }] : []), ]; const Descriptor = Schema.fromJsonString(Schema.Struct({ logicalKey: Schema.String })); diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index eb335fcaa1..ed6ba06108 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -17,6 +17,7 @@ import type { PlatformError } from "effect/PlatformError"; import { ChildProcessSpawner } from "effect/unstable/process"; import { postgresVersion } from "../Artifacts.ts"; import { failureMessage } from "../internal/failure-message.ts"; +import { lstatPath } from "../namespace/drivers/FileSystem.ts"; import { copyDirectory, type DirectoryCopyError } from "../storage/DirectoryCopy.ts"; import type { DatabaseRuntime } from "./Database.ts"; @@ -411,7 +412,7 @@ const makeNativeSnapshotBackend = Effect.fnUntraced(function* (options: { fs.makeDirectory(directory, { recursive: true, mode: 0o700 }).pipe(Effect.as(proceed)), Clear: ({ directory }) => Effect.gen(function* () { - if (yield* fs.readLink(directory).pipe(Effect.isSuccess)) + if ((yield* lstatPath(directory))?.type === "SymbolicLink") return yield* errorFor("clear", `${directory} is a symbolic link`); yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); for (const name of yield* fs.readDirectory(directory)) diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 0b667ca653..f85a21d5be 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -1,4 +1,4 @@ -import { NodeCrypto, NodeHttpClient, NodePath, NodeServices } from "@effect/platform-node"; +import { NodeHttpClient, NodePath, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Cause, @@ -16,23 +16,15 @@ import { } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { FunctionsBootstrapOwner } from "../functions/FunctionsBootstrap.ts"; import { ContainerError, type ContainerRuntime } from "../runtime/Container.ts"; -import { randomPortSpanStart, reserveNativePort } from "../Ports.ts"; -import { makeService } from "../Service.ts"; -import type * as State from "../State.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import * as Functions from "./Functions.ts"; - -// No saved stacks to consult; this test never launches the native backend it configures. -const testReadPortClaims = Effect.succeed([]); -const testReserveNativePort = ( - key: string, - claims: ReadonlyArray, - excluded: ReadonlySet, -) => - Effect.flatMap(Crypto.Crypto, (crypto) => - reserveNativePort(claims, key, randomPortSpanStart(crypto), excluded), - ).pipe(Effect.provide(NodeCrypto.layer)); +import { makeProcessRecipe } from "./ProcessRecipe.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const options = (root: string) => ({ stackId: "catalog-functions", @@ -44,18 +36,19 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); -const dockerInfo = Effect.scoped( +const engineInfo = Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", ["info"], { stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, ["info"], { stdout: "pipe", stderr: "pipe" }), ); return yield* Stream.mkString(Stream.decodeText(child.all)); }), -).pipe(Effect.orElseSucceed(() => "docker info unavailable")); +).pipe(Effect.orElseSucceed(() => `${testEngine} info unavailable`)); describe("service catalog", () => { it.live( @@ -68,7 +61,12 @@ describe("service catalog", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-functions-" }); const stackId = "b".repeat(64); const instanceId = "functions-instance"; - const functionsRoot = root + "/user-functions"; + // Ownership is by location: a borrowed caller path must live outside the whole stack + // data root, so the user's functions project lives in its own, separate tree. + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "catalog-functions-caller-", + }); + const functionsRoot = callerRoot + "/user-functions"; yield* fs.makeDirectory(functionsRoot + "/hello", { recursive: true }); yield* fs.writeFileString( functionsRoot + "/hello/index.ts", @@ -104,10 +102,13 @@ describe("service catalog", () => { inspector: true, }, }, - { ...dockerOptions(root), stackId, instanceId }, - Effect.succeed([]), + { + ...dockerOptions(root), + stackId, + instanceId, + }, ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: instanceId, config: recipe.creation, }); @@ -115,7 +116,7 @@ describe("service catalog", () => { yield* instance.ready; const endpoint = yield* recipe.endpoint("http"); const response = yield* client.execute( - HttpClientRequest.get("http://" + endpoint.host + ":" + endpoint.port + "/hello"), + HttpClientRequest.get("http://" + httpHost(endpoint) + ":" + endpoint.port + "/hello"), ); expect(response.status).toBe(200); expect(yield* response.json).toEqual( @@ -134,7 +135,7 @@ describe("service catalog", () => { const inspector = yield* recipe.endpoint("inspector"); const inspectorResponse = yield* client.execute( HttpClientRequest.get( - "http://" + inspector.host + ":" + inspector.port + "/json/version", + "http://" + httpHost(inspector) + ":" + inspector.port + "/json/version", ), ); expect(inspectorResponse.status).toBe(200); @@ -144,6 +145,56 @@ describe("service catalog", () => { { timeout: 120_000 }, ); + it.live("destroy survives a symlink planted where its runtime directory used to be", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "functions-symlink-" }); + const stackId = "c".repeat(64); + const instanceId = "functions-symlink-instance"; + const callerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-symlink-caller-", + }); + const functionsRoot = `${callerRoot}/user-functions`; + yield* fs.makeDirectory(functionsRoot, { recursive: true }); + yield* fs.writeFileString(`${functionsRoot}/index.ts`, "export default {};"); + const recipe = yield* makeServiceRecipe( + { + service: "functions", + config: { functionsRoot, databaseUrl: "postgres://functions-db" }, + }, + { + ...options(root), + stackId, + instanceId, + }, + ); + const config = recipe.creation; + yield* recipe.definition.prepare?.(config) ?? Effect.void; + const instanceRoot = path.join(root, instanceId); + const runtimeDir = path.join(instanceRoot, "runtime"); + expect(yield* fs.exists(runtimeDir)).toBe(true); + const outside = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-symlink-outside-", + }); + // removeData reaches for `/functions` specifically, not the whole runtime + // directory; the sentinel sits there so this test proves that previously endangered + // data survives, not just the symlink target's top level. + const sentinel = path.join(outside, "functions", "sentinel"); + yield* fs.makeDirectory(path.join(outside, "functions"), { recursive: true }); + yield* fs.writeFileString(sentinel, "do not remove me"); + // The planted attack: the recipe's own runtime directory replaced by a symlink. + yield* fs.remove(runtimeDir, { recursive: true, force: true }); + yield* fs.symlink(outside, runtimeDir); + const scope = yield* Scope.make(); + yield* recipe.definition.removeData({ id: instanceId, config, scope }); + yield* Scope.close(scope, Exit.void); + expect(yield* fs.exists(sentinel)).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + const ancestors = [ { name: "a package.json with an unreadable sibling", @@ -178,6 +229,11 @@ describe("service catalog", () => { prefix: "functions-ancestor-", }); yield* ancestor.setup(fs, root); + // Separate from the stack's own data root: this test's ancestor walk needs + // functionsRoot nested under `root`, which a borrowed caller path may not be. + const stackRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-ancestor-stack-", + }); const functionsRoot = `${root}/project/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.makeDirectory(`${functionsRoot}/_shared`, { recursive: true }); @@ -202,12 +258,11 @@ describe("service catalog", () => { }, }, { - ...options(root), + ...options(stackRoot), stackId: "d".repeat(64), instanceId: "ancestor", - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, - Effect.succeed([]), ); const logs = yield* Ref.make(""); yield* Stream.fromSubscription(yield* recipe.logs).pipe( @@ -216,7 +271,7 @@ describe("service catalog", () => { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "ancestor", config: recipe.creation, }); @@ -225,7 +280,9 @@ describe("service catalog", () => { Effect.tapError(() => Ref.get(logs).pipe(Effect.flatMap(Effect.logError))), ); const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.get(`http://${endpoint.host}:${endpoint.port}/hello`); + const response = yield* client.get( + `http://${httpHost(endpoint)}:${endpoint.port}/hello`, + ); expect(response.status, yield* Ref.get(logs)).toBe(200); expect(yield* response.text).toBe("shared"); yield* instance.stop; @@ -247,6 +304,9 @@ describe("service catalog", () => { directory: temporaryRoot, prefix: "functions-deno-config-", }); + const stackRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-deno-config-stack-", + }); const functionsRoot = `${root}/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.writeFileString( @@ -271,12 +331,11 @@ describe("service catalog", () => { }, }, { - ...options(root), + ...options(stackRoot), stackId: "e".repeat(64), instanceId: "deno-config", - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, - Effect.succeed([]), ); const logs = yield* Ref.make(""); yield* Stream.fromSubscription(yield* recipe.logs).pipe( @@ -285,7 +344,7 @@ describe("service catalog", () => { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "deno-config", config: recipe.creation, }); @@ -295,7 +354,7 @@ describe("service catalog", () => { ); const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.get(`http://${endpoint.host}:${endpoint.port}/hello`); + const response = yield* client.get(`http://${httpHost(endpoint)}:${endpoint.port}/hello`); expect(response.status, yield* Ref.get(logs)).toBe(200); expect(yield* response.text).toBe("config"); @@ -318,6 +377,9 @@ describe("service catalog", () => { directory: temporaryRoot, prefix: "functions-plain-deno-config-", }); + const stackRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "functions-plain-deno-config-stack-", + }); const functionsRoot = `${root}/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.makeDirectory(`${functionsRoot}/_shared`, { recursive: true }); @@ -343,12 +405,11 @@ describe("service catalog", () => { }, }, { - ...options(root), + ...options(stackRoot), stackId: "f".repeat(64), instanceId: "plain-deno-config", - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, }, - Effect.succeed([]), ); const logs = yield* Ref.make(""); const warned = yield* Deferred.make(); @@ -364,7 +425,7 @@ describe("service catalog", () => { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "plain-deno-config", config: recipe.creation, }); @@ -374,7 +435,7 @@ describe("service catalog", () => { ); const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.get(`http://${endpoint.host}:${endpoint.port}/hello`); + const response = yield* client.get(`http://${httpHost(endpoint)}:${endpoint.port}/hello`); yield* Deferred.await(warned).pipe( Effect.timeout("30 seconds"), Effect.tapError(() => Ref.get(logs).pipe(Effect.flatMap(Effect.logError))), @@ -391,7 +452,7 @@ describe("service catalog", () => { ); }); -for (const runtime of ["native", "docker"] as const) { +for (const runtime of ["native", testEngine] as const) { it.live( `serves configured function files, secrets and JWT policies in ${runtime}`, () => @@ -406,6 +467,10 @@ for (const runtime of ["native", "docker"] as const) { directory: temporaryRoot, prefix: "functions-configured-", }); + const stackRoot = yield* fs.makeTempDirectoryScoped({ + directory: temporaryRoot, + prefix: "functions-configured-stack-", + }); const filesRoot = `${root}/project`; const functionsRoot = `${filesRoot}/supabase/functions`; yield* fs.makeDirectory(`${functionsRoot}/locked`, { recursive: true }); @@ -453,13 +518,13 @@ for (const runtime of ["native", "docker"] as const) { }, }, { - ...options(root), + ...options(stackRoot), stackId: "c".repeat(64), instanceId: "configured", runtime, - cacheRoot: "/tmp/supabase-stack-artifacts", + ...(runtime !== "native" ? { engineTarget } : {}), + cacheRoot: testArtifactCacheRoot, }, - Effect.succeed([]), ); const logs = yield* Ref.make(""); yield* Stream.fromSubscription(yield* recipe.logs).pipe( @@ -468,7 +533,7 @@ for (const runtime of ["native", "docker"] as const) { ), Effect.forkScoped({ startImmediately: true }), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "configured", config: recipe.creation, }); @@ -484,7 +549,7 @@ for (const runtime of ["native", "docker"] as const) { ), ); const endpoint = yield* recipe.endpoint("http"); - const base = `http://${endpoint.host}:${endpoint.port}`; + const base = `http://${httpHost(endpoint)}:${endpoint.port}`; const response = yield* client.execute(HttpClientRequest.get(`${base}/hello`)); const responseText = yield* response.text; expect(response.status, responseText).toBe(200); @@ -511,8 +576,8 @@ for (const runtime of ["native", "docker"] as const) { Effect.tapCause(() => Effect.gen(function* () { yield* Effect.logError(`Functions ${runtime} logs:\n${yield* Ref.get(logs)}`); - if (runtime === "docker") - yield* Effect.logError(`docker info:\n${yield* dockerInfo}`); + if (runtime !== "native") + yield* Effect.logError(`${testEngine} info:\n${yield* engineInfo}`); }), ), ); @@ -554,7 +619,23 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win }, }, }; - const recipe = yield* Functions.makeRecipe( + // A real bootstrap owner would join the Win32 stack root below with real fs calls, writing a + // garbled path on this POSIX test host; a fake owner keeps the test to what it exercises + // here, the Windows caller-path translation in `env`/`args`. + const fakeBootstrap: FunctionsBootstrapOwner = { + root: "/fake-bootstrap-root", + write: () => Effect.succeed("/fake-bootstrap-root/generation-fake/index.ts"), + }; + const deps = { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), + fs: yield* FileSystem.FileSystem, + path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), + crypto: yield* Crypto.Crypto, + client: yield* HttpClient.HttpClient, + spawner: yield* ChildProcessSpawner.ChildProcessSpawner, + container, + }; + const recipe = yield* makeProcessRecipe( { stackId: "e".repeat(64), instanceId: "windows", @@ -562,16 +643,8 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win cacheRoot: "C:\\Users\\dev\\cache", runtime: "docker", }, - { - fs: yield* FileSystem.FileSystem, - path: yield* Path.Path.pipe(Effect.provide(NodePath.layerWin32)), - crypto: yield* Crypto.Crypto, - client: yield* HttpClient.HttpClient, - spawner: yield* ChildProcessSpawner.ChildProcessSpawner, - container, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, - }, + deps, + Functions.makeSpec(fakeBootstrap, deps.path, deps.fs), ); const scope = yield* Scope.make(); yield* recipe.definition @@ -583,7 +656,7 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win "/__supabase_project/supabase/functions", ); expect(spec?.env.SUPABASE_INTERNAL_FUNCTIONS_FILES_ROOT).toBe("/__supabase_project"); - expect(spec?.args).toContain("--main-service=/__supabase_functions"); + expect(spec?.args).toContain("--main-service=/__supabase_bootstrap/generation-fake"); const config = yield* Schema.decodeUnknownEffect( Schema.fromJsonString( Schema.Record( diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts index cbc0df1386..91f0b26f7a 100644 --- a/packages/stack/src/services/Functions.ts +++ b/packages/stack/src/services/Functions.ts @@ -1,4 +1,4 @@ -import { Crypto, Effect, FileSystem, Path, Ref, Schema } from "effect"; +import { Crypto, Effect, FileSystem, Path, Schema } from "effect"; import { makeFunctionsBootstrapOwner, type FunctionsBootstrapOwner, @@ -63,8 +63,10 @@ const FunctionsRuntimeConfigJson = Schema.fromJsonString( Schema.Record(Schema.String, FunctionSettings), ); -const makeSpec = ( - functionsRoot: Ref.Ref, +/** Fixed container mount point for `bootstrap.root`; `args` picks the generation underneath it. */ +const containerBootstrapRoot = "/__supabase_bootstrap"; + +export const makeSpec = ( bootstrap: FunctionsBootstrapOwner, path: Path.Path, fs: FileSystem.FileSystem, @@ -131,6 +133,9 @@ const makeSpec = ( creation.config.serviceRoleKey ?? (jwt === undefined ? undefined : yield* serviceJwt("service_role", jwt)); return { + // The image has no passwd entry: Docker then defaults HOME to /, Podman leaves it unset, + // and the runtime needs it to place its Deno cache. + ...(container ? { HOME: "/" } : {}), ...creation.config.env, ...(http === undefined ? {} : { EDGE_RUNTIME_PORT: String(http.port) }), SUPABASE_INTERNAL_FUNCTIONS_ROOT: root, @@ -174,18 +179,29 @@ const makeSpec = ( }), args: (creation, endpoints, context) => Effect.gen(function* () { - const override = yield* Ref.get(functionsRoot); + const published = yield* bootstrap + .write({ content: creation.config.bootstrap ?? defaultFunctionsBootstrap }) + .pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "launch", + message: "Unable to publish Functions bootstrap", + cause, + }), + ), + ); const root = context.container - ? override === undefined - ? "/__supabase_functions" - : "/__supabase_bootstrap" - : (override ?? creation.config.functionsRoot); + ? `${containerBootstrapRoot}/${path.basename(path.dirname(published))}` + : path.dirname(published); const http = endpoints.get("http"); const inspector = endpoints.get("inspector"); return [ "start", `--main-service=${root}`, - ...(http === undefined ? [] : [`--port=${http.port}`]), + ...(http === undefined + ? [] + : [`--port=${http.port}`, `--ip=${context.container ? "0.0.0.0" : "127.0.0.1"}`]), ...(creation.config.policy === undefined ? [] : [`--policy=${creation.config.policy}`]), ...(creation.config.inspector === true && inspector !== undefined ? [`--inspect=${context.container ? "0.0.0.0" : "127.0.0.1"}:${inspector.port}`] @@ -193,22 +209,20 @@ const makeSpec = ( ]; }), mounts: (creation) => - Effect.gen(function* () { - const override = yield* Ref.get(functionsRoot); - const files = creation.config.filesRoot; - const source = files ?? creation.config.functionsRoot; - const target = files === undefined ? "/__supabase_functions" : "/__supabase_project"; - return [ - { source, target, readOnly: true }, - ...(override === undefined - ? [] - : [{ source: override, target: "/__supabase_bootstrap", readOnly: true }]), - ]; - }), + Effect.succeed([ + { + source: creation.config.filesRoot ?? creation.config.functionsRoot, + target: + creation.config.filesRoot === undefined ? "/__supabase_functions" : "/__supabase_project", + readOnly: true, + }, + { source: bootstrap.root, target: containerBootstrapRoot, readOnly: true }, + ]), startupCommands: [], + // Prefetches the bootstrap so a later start need not pay for it; `args` publishes idempotently. prepare: (creation) => bootstrap.write({ content: creation.config.bootstrap ?? defaultFunctionsBootstrap }).pipe( - Effect.flatMap((target) => Ref.set(functionsRoot, path.dirname(target))), + Effect.asVoid, Effect.mapError( (cause) => new ServiceError({ @@ -218,17 +232,14 @@ const makeSpec = ( }), ), ), - removeData: () => - bootstrap.cleanupAll.pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "destroy", - message: "Unable to remove Functions bootstrap", - cause, - }), - ), - ), + // No removeData: the bootstrap directory lives under instanceRoot, which ProcessRecipe's + // destroyOwnedRoot already removes as a single recursive delete starting from a root it has + // just confirmed is real; a selective removeData reaching into a nested path on its own could + // instead follow a symlink an attacker (or a race) planted there, deleting outside it. + callerPaths: (creation) => + creation.config.filesRoot === undefined + ? [creation.config.functionsRoot] + : [creation.config.functionsRoot, creation.config.filesRoot], }); export const makeRecipe = Effect.fn("Functions.makeRecipe")( @@ -237,7 +248,6 @@ export const makeRecipe = Effect.fn("Functions.makeRecipe")( deps: ProcessDependencies, ): Effect.Effect, CatalogError> => Effect.gen(function* () { - const functionsRoot = yield* Ref.make(undefined); const stackId = yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( Effect.mapError( (cause) => @@ -267,10 +277,6 @@ export const makeRecipe = Effect.fn("Functions.makeRecipe")( }), ), ); - return yield* makeProcessRecipe( - options, - deps, - makeSpec(functionsRoot, bootstrap, deps.path, deps.fs), - ); + return yield* makeProcessRecipe(options, deps, makeSpec(bootstrap, deps.path, deps.fs)); }), ); diff --git a/packages/stack/src/services/Imgproxy.ts b/packages/stack/src/services/Imgproxy.ts index 7c497b7760..cc5dbbca72 100644 --- a/packages/stack/src/services/Imgproxy.ts +++ b/packages/stack/src/services/Imgproxy.ts @@ -35,4 +35,6 @@ export const makeSpec = (): ProcessRecipeSpec => ({ : [{ source: creation.config.filePath, target: "/mnt", readOnly: true }], ), startupCommands: [], + callerPaths: (creation) => + creation.config.filePath === undefined ? [] : [creation.config.filePath], }); diff --git a/packages/stack/src/services/Initialization.ts b/packages/stack/src/services/Initialization.ts index 7e67c439f5..591a1c7f05 100644 --- a/packages/stack/src/services/Initialization.ts +++ b/packages/stack/src/services/Initialization.ts @@ -2,7 +2,7 @@ import { Effect } from "effect"; import { resolveArtifact } from "../Artifacts.ts"; import type { InitializationCommand, ResolvedCommand } from "../Commands.ts"; import { ServiceError } from "../Service.ts"; -import type { StackCredentials } from "../State.ts"; +import type { StackCredentials } from "../StackNamespace.ts"; import * as Auth from "./Auth.ts"; import * as Realtime from "./Realtime.ts"; import * as Storage from "./Storage.ts"; diff --git a/packages/stack/src/services/InstanceRoot.integration.test.ts b/packages/stack/src/services/InstanceRoot.integration.test.ts deleted file mode 100644 index 4d8f003f3e..0000000000 --- a/packages/stack/src/services/InstanceRoot.integration.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Data, Effect, Exit, FileSystem, Option, Path, PlatformError } from "effect"; -import { ensureOwnedInstanceRoot, type OwnedInstanceRootParams } from "./InstanceRoot.ts"; - -class TestInstanceRootError extends Data.TaggedError("TestInstanceRootError")<{ - readonly operation: string; - readonly cause: unknown; -}> {} - -const onError = (operation: string, cause: unknown) => - new TestInstanceRootError({ operation, cause }); - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -describe("InstanceRoot", () => { - it.live("lets concurrent first claims of a fresh root all succeed", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "instance-root-race-" }); - const params: OwnedInstanceRootParams = { - fs, - path, - parentRoot, - stackId: "stack", - instanceId: "instance", - ownerFileName: ".supabase-instance-owner.json", - label: "Instance root", - }; - const results = yield* Effect.all( - Array.from({ length: 10 }, () => Effect.exit(ensureOwnedInstanceRoot(params, onError))), - { concurrency: "unbounded" }, - ); - expect(results.every(Exit.isSuccess)).toBe(true); - const marker = yield* fs.readFileString( - path.join(parentRoot, "instance", ".supabase-instance-owner.json"), - ); - expect(marker).toBe('{"stackId":"stack","instanceId":"instance"}'); - }), - ), - ); - - it.live("lets exactly one of concurrent conflicting claims of a fresh root win", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "instance-root-conflict-" }); - const ownerFileName = ".supabase-instance-owner.json"; - const paramsFor = (stackId: string): OwnedInstanceRootParams => ({ - fs, - path, - parentRoot, - stackId, - instanceId: "instance", - ownerFileName, - label: "Instance root", - }); - const stackIds = Array.from({ length: 10 }, (_, i) => `stack-${i}`); - const results = yield* Effect.all( - stackIds.map((stackId) => - Effect.exit(ensureOwnedInstanceRoot(paramsFor(stackId), onError)), - ), - { concurrency: "unbounded" }, - ); - const failures = results.filter(Exit.isFailure); - expect(results.filter(Exit.isSuccess)).toHaveLength(1); - expect(failures).toHaveLength(9); - for (const failure of failures) - expect(Option.getOrUndefined(Cause.findErrorOption(failure.cause))?.cause).toBe( - "Instance root belongs to another instance", - ); - const marker = yield* fs.readFileString(path.join(parentRoot, "instance", ownerFileName)); - expect( - stackIds.map((stackId) => JSON.stringify({ stackId, instanceId: "instance" })), - ).toContain(marker); - }), - ), - ); - - it.live("ignores an orphaned staging file when claiming a fresh root", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* fs.makeTempDirectoryScoped({ prefix: "instance-root-orphan-" }); - const ownerFileName = ".supabase-instance-owner.json"; - const root = path.join(parentRoot, "instance"); - yield* fs.makeDirectory(root, { recursive: true }); - yield* fs.writeFileString(path.join(root, `.${ownerFileName}.orphan.tmp`), "leftover"); - const params: OwnedInstanceRootParams = { - fs, - path, - parentRoot, - stackId: "stack", - instanceId: "instance", - ownerFileName, - label: "Instance root", - }; - yield* ensureOwnedInstanceRoot(params, onError); - const marker = yield* fs.readFileString(path.join(root, ownerFileName)); - expect(marker).toBe('{"stackId":"stack","instanceId":"instance"}'); - }), - ), - ); - - it.live("falls back to an exclusive create when hard links are unsupported", () => - run( - Effect.gen(function* () { - const realFs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const parentRoot = yield* realFs.makeTempDirectoryScoped({ - prefix: "instance-root-fallback-", - }); - const ownerFileName = ".supabase-instance-owner.json"; - const fs: FileSystem.FileSystem = { - ...realFs, - link: () => - Effect.fail( - PlatformError.systemError({ - _tag: "Unknown", - module: "FileSystem", - method: "link", - description: "injected unsupported-link failure", - cause: Object.assign(new Error("Operation not supported"), { code: "EPERM" }), - }), - ), - }; - const params: OwnedInstanceRootParams = { - fs, - path, - parentRoot, - stackId: "stack", - instanceId: "instance", - ownerFileName, - label: "Instance root", - }; - yield* ensureOwnedInstanceRoot(params, onError); - const root = path.join(parentRoot, "instance"); - const marker = yield* realFs.readFileString(path.join(root, ownerFileName)); - expect(marker).toBe('{"stackId":"stack","instanceId":"instance"}'); - expect(yield* realFs.readDirectory(root)).toEqual([ownerFileName]); - }), - ), - ); -}); diff --git a/packages/stack/src/services/InstanceRoot.ts b/packages/stack/src/services/InstanceRoot.ts deleted file mode 100644 index 4f547c6753..0000000000 --- a/packages/stack/src/services/InstanceRoot.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { Effect, Predicate, type FileSystem, type Path } from "effect"; -import type { PlatformError } from "effect/PlatformError"; -import { randomUUID } from "node:crypto"; - -/** The container mount target for an instance-scoped directory owned on the host. */ -export const containerInstancePath = "/instance"; - -const safeInstanceIdPattern = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u; - -/** True when `instanceId` is safe to join to a root as one path segment, without traversal. */ -const isSafeInstanceId = (instanceId: string): boolean => safeInstanceIdPattern.test(instanceId); - -/** Name a staging file for `ownerFileName`, recognizable so directory scans can ignore it. */ -const stagingFileName = (ownerFileName: string) => `.${ownerFileName}.${randomUUID()}.tmp`; - -/** True when `entry` looks like one of our own staging files for `ownerFileName`. */ -const isStagingEntry = (ownerFileName: string, entry: string): boolean => - entry.startsWith(`.${ownerFileName}.`) && entry.endsWith(".tmp"); - -/** True when `error` signals the filesystem does not support hard links. */ -const isUnsupportedLink = (error: unknown): boolean => { - if (!Predicate.hasProperty(error, "cause")) return false; - const code = - Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" - ? error.cause.code - : undefined; - return code === "EPERM" || code === "ENOTSUP" || code === "EOPNOTSUPP"; -}; - -export interface OwnedInstanceRootParams { - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - /** Parent directory the instance-scoped root is joined under, as `/`. */ - readonly parentRoot: string; - readonly stackId: string; - readonly instanceId: string; - readonly ownerFileName: string; - /** Names the owned instance root in error messages, e.g. "Database root". */ - readonly label: string; -} - -/** Claims `/` for one stack+instance pair, failing if another instance already owns it. */ -export const ensureOwnedInstanceRoot = Effect.fn("InstanceRoot.ensureOwnedInstanceRoot")(( - params: OwnedInstanceRootParams, - onError: (operation: string, cause: unknown) => E, -): Effect.Effect => { - const { fs, path, parentRoot, stackId, instanceId, ownerFileName, label } = params; - const root = path.join(parentRoot, instanceId); - const ownerFile = path.join(root, ownerFileName); - const marker = JSON.stringify({ stackId, instanceId }); - const claimExistingMarker = Effect.gen(function* () { - const existing = yield* fs - .readFileString(ownerFile) - .pipe(Effect.mapError((cause) => onError("data", cause))); - if (existing !== marker) - return yield* Effect.fail(onError("data", `${label} belongs to another instance`)); - }); - const claimIfAlreadyExists = (effect: Effect.Effect) => - effect.pipe( - Effect.catchIf( - (error) => error.reason._tag === "AlreadyExists", - () => Effect.void, - ), - ); - // Staging the full marker next to `ownerFile` and hard-linking it into place keeps `wx`'s - // exclusivity without ever exposing the partial content a direct write there could risk. - const writeMarkerAtomically = Effect.suspend(() => { - const tempFile = path.join(root, stagingFileName(ownerFileName)); - return fs.writeFileString(tempFile, marker, { mode: 0o600, flag: "wx" }).pipe( - Effect.andThen(() => - fs.link(tempFile, ownerFile).pipe( - claimIfAlreadyExists, - // Filesystems without hard links (e.g. exFAT, some network shares) fall back to the - // direct exclusive create this replaced; it stays correct, just racy on those alone. - Effect.catchIf(isUnsupportedLink, () => - fs - .writeFileString(ownerFile, marker, { mode: 0o600, flag: "wx" }) - .pipe(claimIfAlreadyExists), - ), - ), - ), - Effect.ensuring(fs.remove(tempFile, { force: true }).pipe(Effect.ignore)), - Effect.mapError((cause) => onError("data", cause)), - ); - }); - return Effect.gen(function* () { - if (!isSafeInstanceId(instanceId)) - return yield* Effect.fail(onError("data", `${label} instance id is not a safe path segment`)); - yield* fs - .makeDirectory(root, { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => onError("data", cause))); - const present = yield* fs - .exists(ownerFile) - .pipe(Effect.mapError((cause) => onError("data", cause))); - if (present) return yield* claimExistingMarker; - const entries = yield* fs.readDirectory(root).pipe( - Effect.map((entries) => entries.filter((entry) => !isStagingEntry(ownerFileName, entry))), - Effect.mapError((cause) => onError("data", cause)), - ); - if (entries.length > 0) { - // A concurrent first claim may have written the marker between the `exists` and - // `readDirectory` calls above; an otherwise-empty root is still safe to compare. - if (entries.length === 1 && entries[0] === ownerFileName) return yield* claimExistingMarker; - return yield* Effect.fail(onError("data", `${label} is non-empty and unmarked`)); - } - yield* writeMarkerAtomically; - return yield* claimExistingMarker; - }); -}); - -/** Removes `/` for one stack+instance pair after running `removeData`, keeping any `keep` entries. */ -export const removeOwnedInstanceRoot = Effect.fn("InstanceRoot.removeOwnedInstanceRoot")(( - params: OwnedInstanceRootParams, - removeData: Effect.Effect, - onError: (operation: string, cause: unknown) => E, - /** Root entries kept with the owner marker; when empty the root itself is removed. */ - keep: ReadonlyArray = [], -): Effect.Effect => { - const { fs, path, parentRoot, stackId, instanceId, ownerFileName, label } = params; - const root = path.join(parentRoot, instanceId); - const ownerFile = path.join(root, ownerFileName); - const marker = JSON.stringify({ stackId, instanceId }); - return Effect.gen(function* () { - if (!isSafeInstanceId(instanceId)) - return yield* Effect.fail( - onError("destroy", `${label} instance id is not a safe path segment`), - ); - const present = yield* fs - .exists(ownerFile) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - if (!present) { - if (yield* fs.exists(root).pipe(Effect.mapError((cause) => onError("destroy", cause)))) - return yield* Effect.fail(onError("destroy", `${label} is unmarked`)); - return; - } - const existing = yield* fs - .readFileString(ownerFile) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - if (existing !== marker) - return yield* Effect.fail(onError("destroy", `${label} belongs to another instance`)); - yield* removeData; - if (keep.length === 0) - return yield* fs - .remove(root, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - const names = yield* fs - .readDirectory(root) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - for (const name of names) - if (name !== path.basename(ownerFile) && !keep.includes(name)) - yield* fs - .remove(path.join(root, name), { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => onError("destroy", cause))); - }); -}); diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts index 1b6b262f88..4965058a7a 100644 --- a/packages/stack/src/services/Mail.integration.test.ts +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -2,8 +2,10 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Layer, Path } from "effect"; import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ stackId: "catalog-mail", @@ -15,7 +17,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); interface MailpitMessages { @@ -64,9 +67,8 @@ describe("service catalog", () => { const recipe = yield* makeServiceRecipe( { service: "mail", config: {} }, dockerOptions(root), - Effect.succeed([]), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -74,7 +76,7 @@ describe("service catalog", () => { yield* instance.ready; const endpoint = yield* recipe.endpoint("http"); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}/readyz`), + HttpClientRequest.get(`http://${httpHost(endpoint)}:${endpoint.port}/readyz`), ); expect(response.status).toBe(200); yield* instance.stop; @@ -91,18 +93,14 @@ describe("service catalog", () => { const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-mail-isolation-" }); const recipeFor = (instanceId: string) => - makeServiceRecipe( - { service: "mail", config: {} }, - { ...options(root), instanceId }, - Effect.succeed([]), - ); + makeServiceRecipe({ service: "mail", config: {} }, { ...options(root), instanceId }); const first = yield* recipeFor("mail-a"); const second = yield* recipeFor("mail-b"); - const firstInstance = yield* makeService(first.definition, { + const firstInstance = yield* makeStandaloneService(first.definition, { id: "mail-a", config: first.creation, }); - const secondInstance = yield* makeService(second.definition, { + const secondInstance = yield* makeStandaloneService(second.definition, { id: "mail-b", config: second.creation, }); @@ -126,12 +124,8 @@ describe("service catalog", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-mail-persistence-" }); - const recipe = yield* makeServiceRecipe( - { service: "mail", config: {} }, - options(root), - Effect.succeed([]), - ); - const instance = yield* makeService(recipe.definition, { + const recipe = yield* makeServiceRecipe({ service: "mail", config: {} }, options(root)); + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -164,9 +158,8 @@ describe("service catalog", () => { const recipe = yield* makeServiceRecipe( { service: "mail", config: {} }, dockerOptions(root), - Effect.succeed([]), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -195,12 +188,8 @@ describe("service catalog", () => { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-mail-destroy-" }); - const recipe = yield* makeServiceRecipe( - { service: "mail", config: {} }, - options(root), - Effect.succeed([]), - ); - const instance = yield* makeService(recipe.definition, { + const recipe = yield* makeServiceRecipe({ service: "mail", config: {} }, options(root)); + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); @@ -223,9 +212,8 @@ describe("service catalog", () => { const recipe = yield* makeServiceRecipe( { service: "mail", config: {} }, { ...options(root), instanceId: "../escaped" }, - Effect.succeed([]), ); - const instance = yield* makeService(recipe.definition, { + const instance = yield* makeStandaloneService(recipe.definition, { id: "mail", config: recipe.creation, }); diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts index e853e7f01f..9cb5ebbc88 100644 --- a/packages/stack/src/services/Pooler.integration.test.ts +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -2,24 +2,27 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Context, Effect, Layer, Redacted } from "effect"; import { PgClient } from "@effect/sql-pg"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const options = (root: string) => ({ stackId: "catalog-pooler", instanceId: "instance", root, - cacheRoot: "/tmp/supabase-stack-artifacts", + cacheRoot: testArtifactCacheRoot, runtime: "native" as const, }); const dockerOptions = (root: string) => ({ ...options(root), cacheRoot: `${root}/cache`, - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -41,9 +44,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -55,7 +57,7 @@ describe("service catalog", () => { const databaseRelay = yield* makeDockerTcpRelay(databaseRecipe.endpoint("sql")); const dockerDatabaseUrl = `postgresql://supabase_admin:postgres@${databaseRelay.host}:${databaseRelay.port}/_supabase`; const nativeDatabaseUrl = `postgresql://supabase_admin:postgres@${databaseEndpoint.host}:${databaseEndpoint.port}/_supabase`; - for (const runtime of ["native", "docker"] as const) { + for (const runtime of ["native", testEngine] as const) { for (const poolMode of ["transaction", "session"] as const) { const tenant = `catalog-${runtime}-${poolMode}`; const poolerRecipe = yield* makeServiceRecipe( @@ -71,9 +73,8 @@ describe("service catalog", () => { }, }, runtime === "native" ? options(root) : dockerOptions(root), - Effect.succeed([]), ); - const pooler = yield* makeService(poolerRecipe.definition, { + const pooler = yield* makeStandaloneService(poolerRecipe.definition, { id: `pooler-${runtime}-${poolMode}`, config: poolerRecipe.creation, }); diff --git a/packages/stack/src/services/Pooler.ts b/packages/stack/src/services/Pooler.ts index 7e58fb7b11..b49f41ff34 100644 --- a/packages/stack/src/services/Pooler.ts +++ b/packages/stack/src/services/Pooler.ts @@ -55,6 +55,7 @@ const environment: ProcessRecipeSpec["env"] = (creation, endpoints, co PROXY_PORT: "0", SESSION_PROXY_PORTS: "0", TRANSACTION_PROXY_PORTS: "0", + SUPAVISOR_BIND_IP: "127.0.0.1", }), ...(sql === undefined ? {} diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index d91b480694..a5989f7efb 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -1,4 +1,4 @@ -import { NodeCrypto, NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { PlatformError, @@ -25,8 +25,7 @@ import * as Net from "node:net"; // oxlint-disable-next-line effecttsgo/node-builtin-import -- the collision fixture owns a local HTTP listener. import * as NodeHttp from "node:http"; import { catalogPins, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; -import { randomPortSpanStart, reserveNativePort } from "../Ports.ts"; -import type * as State from "../State.ts"; +import { nativePortBase, nativePortSpan } from "../Ports.ts"; import { makeArtifactStore, type ArtifactRequest, @@ -39,7 +38,9 @@ import { type ContainerProcess, type ContainerRuntime, } from "../runtime/Container.ts"; -import { makeService, ServiceError } from "../Service.ts"; +import { ServiceError } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; import { makeProcessRecipe, type ProcessDependencies, @@ -100,17 +101,6 @@ const isPortOccupied = (port: number): Effect.Effect => }); }); -// No saved stacks to consult. -const testReadPortClaims = Effect.succeed([]); -const testReserveNativePort = ( - key: string, - claims: ReadonlyArray, - excluded: ReadonlySet, -) => - Effect.flatMap(Crypto.Crypto, (crypto) => - reserveNativePort(claims, key, randomPortSpanStart(crypto), excluded), - ).pipe(Effect.provide(NodeCrypto.layer)); - describe("ProcessRecipe launch cleanup", () => { for (const scenario of [ "partial launch", @@ -201,18 +191,19 @@ describe("ProcessRecipe launch cleanup", () => { client: yield* HttpClient.HttpClient, spawner: yield* ChildProcessSpawner.ChildProcessSpawner, container, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(options, dependencies, spec); - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: `rest-process-recipe-${scenario}`, config: creation, }); expect(Exit.isFailure(yield* service.start.pipe(Effect.exit))).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopping"); - expect((yield* service.get).cleanupError?.operation).toBe("stop"); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopping", + error: { operation: "launch" }, + }); expect(yield* Ref.get(startupLaunches)).toBe(1); expect(yield* Ref.get(serviceLaunches)).toBe(0); expect(yield* Ref.get(active)).toBe(1); @@ -290,25 +281,25 @@ describe("ProcessRecipe launch cleanup", () => { client, spawner, container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(nativeOptions, dependencies, nativeSpec); - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: "rest-process-recipe-native", config: creation, }); const failure = yield* service.start.pipe(Effect.exit); expect(Exit.isFailure(failure)).toBe(true); - expect((yield* service.get).lifecycle).toBe("stopping"); - expect((yield* service.get).cleanupError?.operation).toBe("stop"); + expect(yield* service.get).toMatchObject({ + lifecycle: "stopping", + error: { operation: "launch" }, + }); expect(startupWrapped).toBe(true); yield* Ref.set(cleanupFailure, false); yield* service.stop; expect((yield* service.get).lifecycle).toBe("stopped"); - expect((yield* service.get).cleanupError).toBeUndefined(); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ), ); @@ -402,11 +393,10 @@ describe("ProcessRecipe launch cleanup", () => { client, spawner, container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), } satisfies ProcessDependencies; const recipe = yield* makeProcessRecipe(nativeOptions, dependencies, nativeSpec); - const service = yield* makeService(recipe.definition, { + const service = yield* makeStandaloneService(recipe.definition, { id: "rest-process-recipe-port-order", config: creation, }); @@ -419,6 +409,10 @@ describe("ProcessRecipe launch cleanup", () => { expect(blocker?.listening).toBe(true); expect(ports).toHaveLength(3); expect(ports.every((port) => port > 0)).toBe(true); + for (const port of ports) { + expect(port).toBeGreaterThanOrEqual(nativePortBase); + expect(port).toBeLessThan(nativePortBase + nativePortSpan); + } yield* service.stop; }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ), @@ -470,18 +464,17 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { runtime: "docker", }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs: yield* FileSystem.FileSystem, path: yield* Path.Path, crypto: yield* Crypto.Crypto, client: yield* HttpClient.HttpClient, spawner: yield* ChildProcessSpawner.ChildProcessSpawner, container, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, Realtime.makeSpec(), ); - return yield* makeService(recipe.definition, { id: "realtime", config: creation }); + return yield* makeStandaloneService(recipe.definition, { id: "realtime", config: creation }); }); const platform = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -506,15 +499,19 @@ const nativeFixtureArtifact = Effect.fn(function* ( : undefined; if (target === undefined) return yield* Effect.fail(`Unsupported test platform: ${platformName}`); - const { releaseVersion } = yield* resolveArtifact({ service: artifact.service }); + const { releaseVersion, natives } = yield* resolveArtifact({ service: artifact.service }); const request: ArtifactRequest = { key: `slim-services/${artifact.name}/${releaseVersion}/${target}`, requiredRuntimePaths: Object.keys(artifact.files), executablePath: artifact.executablePath, }; const fs = yield* FileSystem.FileSystem; + // The generation directory is content-addressed by this digest: it must match what the real + // artifact source resolves for this platform, or this fixture is invisible to consumers that + // resolve the real catalog source for the same key. + const digest = natives[target].archive; const source: ArtifactSource = { - checksum: () => Effect.succeed("0".repeat(64)), + checksum: () => Effect.succeed(digest), materialize: (_entry, destination) => Effect.gen(function* () { yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); @@ -675,14 +672,13 @@ const nativeRestRecipe = Effect.fn(function* ( platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, client, spawner, container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, { ...spec, @@ -756,7 +752,10 @@ describe("process recipe startup", () => { `server.listen(Number(process.env.PORT), "127.0.0.1");\n`, collidingSpawner, ); - const service = yield* makeService(recipe.definition, { id: "rest", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "rest", + config: creation, + }); yield* service.start; yield* service.ready; @@ -766,7 +765,7 @@ describe("process recipe startup", () => { expect(yield* Ref.get(taken)).toHaveLength(2); expect(yield* Ref.get(taken)).not.toContain(endpoint?.port); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint?.host}:${endpoint?.port}/`), + HttpClientRequest.get(`http://${endpoint && httpHost(endpoint)}:${endpoint?.port}/`), ); expect(yield* response.text).toBe("owned-fixture"); yield* service.stop; @@ -810,7 +809,10 @@ describe("process recipe startup", () => { {}, (line) => line.includes("listener bound"), ); - const service = yield* makeService(recipe.definition, { id: "rest", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "rest", + config: creation, + }); yield* service.start; const failure = yield* Effect.flip(service.ready); @@ -855,7 +857,10 @@ describe("process recipe startup", () => { spawner, { PID_FILE: pidFile }, ); - const service = yield* makeService(recipe.definition, { id: "rest", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "rest", + config: creation, + }); yield* service.start; const failure = yield* Effect.flip(service.ready); @@ -952,14 +957,13 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, client, spawner: interceptingSpawner, container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, Pooler.makeSpec(), ); @@ -985,7 +989,9 @@ describe("process recipe startup", () => { ); expect(yield* competingResponse.text).toBe("competing-listener"); const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint?.host}:${endpoint?.port}/api/health`), + HttpClientRequest.get( + `http://${endpoint && httpHost(endpoint)}:${endpoint?.port}/api/health`, + ), ); expect(yield* response.text).toBe(`owned-fixture:${endpoint?.port}`); expect(yield* Ref.get(collisionInstalled)).toBe(true); @@ -994,79 +1000,6 @@ describe("process recipe startup", () => { ).pipe(Effect.provide(platform)), ); - it.live("reserves a native backend port below the default ephemeral range", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const client = yield* HttpClient.HttpClient; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "process-recipe-port-range-" }); - const cacheRoot = path.join(root, "cache"); - yield* nativePoolerArtifact(cacheRoot); - const creation: Pooler.Creation = { - service: "pooler", - config: { - databaseUrl: "postgresql://postgres:postgres@127.0.0.1:5432/postgres", - jwtSecret: "pooler-port-range-test-secret-with-more-than-32-characters", - tenant: "port-range-test", - poolMode: "transaction", - }, - }; - // Pooler reserves "http" and "sql" concurrently; sharing one scan start forces both - // to contend for the same first candidate so the reservation has to skip one of them. - const sharedStart = Effect.succeed(yield* randomPortSpanStart(crypto)); - const contendingReserveNativePort = ( - key: string, - claims: ReadonlyArray, - excluded: ReadonlySet, - ) => reserveNativePort(claims, key, sharedStart, excluded); - const recipe = yield* makeProcessRecipe( - { - stackId: "process-recipe-port-range", - instanceId: "instance", - root, - cacheRoot, - runtime: "native", - platform: { os: process.platform, arch: process.arch }, - }, - { - fs, - path, - crypto, - client, - spawner, - container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: contendingReserveNativePort, - }, - Pooler.makeSpec(), - ); - if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); - const scope = yield* Scope.fork(yield* Effect.scope, "sequential"); - const runtime = yield* recipe.definition.launch({ - id: "pooler", - launchId: 1, - config: creation, - scope, - }); - yield* runtime.health; - const endpoints = yield* Ref.get(recipe.endpoints); - const endpoint = endpoints.get("http"); - // Below Linux's default ephemeral range, so the released probe port isn't handed to an - // outgoing connection there; a custom host dynamic range can still overlap (ADR 0017). - expect(endpoint?.port).toBeGreaterThanOrEqual(20000); - expect(endpoint?.port).toBeLessThan(32768); - // Pooler reserves "http" and "sql" concurrently; they must never settle on the same port. - const sql = endpoints.get("sql"); - expect(sql?.port).toBeDefined(); - expect(sql?.port).not.toBe(endpoint?.port); - yield* runtime.stop; - }), - ).pipe(Effect.provide(platform)), - ); - it.live("stops after three consecutive native Pooler port collisions", () => Effect.scoped( Effect.gen(function* () { @@ -1101,18 +1034,20 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, client, spawner: countingSpawner(spawner, mainLaunches, startupLaunches, true), container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, Pooler.makeSpec(), ); - const service = yield* makeService(recipe.definition, { id: "pooler", config: creation }); + const service = yield* makeStandaloneService(recipe.definition, { + id: "pooler", + config: creation, + }); const subscribed = yield* Deferred.make(); const exitObserved = yield* Deferred.make(); yield* service.observation.pipe( @@ -1236,14 +1171,13 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, client, spawner: deadlineSpawner, container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, Pooler.makeSpec(), ); @@ -1304,14 +1238,13 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, client, spawner: countingSpawner(spawner, mainLaunches, startupLaunches), container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, Pooler.makeSpec(), ); @@ -1414,14 +1347,13 @@ describe("process recipe startup", () => { platform: { os: process.platform, arch: process.arch }, }, { + borrowCallerPath: () => Effect.die("borrowCallerPath not exercised in this test"), fs, path, crypto, client, spawner: failingSpawner, container: undefined, - readPortClaims: testReadPortClaims, - reserveNativePort: testReserveNativePort, }, Pooler.makeSpec(), ); diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 1845cef70e..af855f6e30 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -20,9 +20,15 @@ import { import { ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { prepareNativeArtifact, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; -import { accepts, type NativePortReservation, type PortError } from "../Ports.ts"; -import type * as State from "../State.ts"; +import { + prepareNativeArtifact, + resolveArtifact, + useNativeArtifact, + type ServiceKind, +} from "../Artifacts.ts"; +import * as Environment from "../namespace/Environment.ts"; +import { containerInstancePath, destroyOwnedRoot, type BorrowedPath } from "../namespace/Paths.ts"; +import { accepts, reserveNativePort } from "../Ports.ts"; import { type ContainerError, type ContainerProcess, @@ -50,11 +56,6 @@ import { type RuntimeSession, type ServiceLaunchContext, } from "../Service.ts"; -import { - containerInstancePath, - ensureOwnedInstanceRoot, - removeOwnedInstanceRoot, -} from "./InstanceRoot.ts"; import { type CatalogOptions, type ProcessRecipeResult, @@ -68,6 +69,22 @@ interface RecipeMount { readonly readOnly: boolean; } +/** `uid:gid` a container runs as when it writes into a borrowed host path, so the caller keeps ownership. */ +const hostUser: string | undefined = + process.getuid === undefined || process.getgid === undefined + ? undefined + : `${process.getuid()}:${process.getgid()}`; + +/** The host-user override for a launch whose mounts write into one of its recipe's caller paths. */ +const userForMounts = ( + mounts: ReadonlyArray, + callerPaths: ReadonlyArray, +): string | undefined => + hostUser !== undefined && + mounts.some((mount) => !mount.readOnly && callerPaths.includes(mount.source)) + ? hostUser + : undefined; + export interface StartupCommand { readonly args: ReadonlyArray; readonly nativeExecutable?: string; @@ -112,6 +129,11 @@ export interface ProcessRecipeSpec Effect.Effect; /** Claims an owned instance directory, mounted at `/instance` in containers. */ readonly instanceDirectory?: boolean; + /** + * Every caller-supplied path this recipe reads or mounts, validated against the stack's data + * root before `prepare` and launch run (see `namespace/Paths.borrow`). + */ + readonly callerPaths?: (creation: C) => ReadonlyArray; } export interface ResolvedStartupCommand { @@ -168,12 +190,8 @@ export interface ProcessDependencies { readonly client: HttpClient.HttpClient; readonly spawner: ChildProcessSpawnerService["Service"]; readonly container: ContainerRuntime | undefined; - readonly readPortClaims: Effect.Effect, State.StateError>; - readonly reserveNativePort: ( - key: string, - claims: ReadonlyArray, - excluded: ReadonlySet, - ) => Effect.Effect; + /** Validates a caller-supplied path against the stack's data root; see `namespace/Paths.borrow`. */ + readonly borrowCallerPath: (candidate: string) => Effect.Effect; } const serviceError = mapToServiceError; @@ -194,9 +212,16 @@ const runtimeFromNative = (process: NativeProcess): RuntimeSession => ({ remove: Effect.void, }); +const emptyNativePortSet: ReadonlySet = new Set(); + const startupTimeoutSeconds = 60; const nativeLaunchAttempts = 3; const probeTimeout = Duration.seconds(10); +/** + * Bounds one readiness HTTP call: a request that connects but never answers is retried instead + * of spending the whole check on that one attempt. + */ +const readinessAttemptTimeout = Duration.seconds(3); const outputDrainGrace = Duration.seconds(2); const startupOutputTailLines = 20; const startupOutputLineChars = 1_000; @@ -353,7 +378,7 @@ const readiness = Effect.fn("ProcessRecipe.readiness")(function* ( timeout: Duration.Input = "60 seconds", ) { const attempt = client - .execute(HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}${path}`)) + .execute(HttpClientRequest.get(`http://127.0.0.1:${endpoint.port}${path}`)) .pipe( Effect.flatMap((response) => (response.status >= 200 && response.status < 300) || response.status === 401 @@ -362,6 +387,7 @@ const readiness = Effect.fn("ProcessRecipe.readiness")(function* ( new ServiceError({ operation: "health", message: `HTTP ${response.status}` }), ), ), + Effect.timeout(readinessAttemptTimeout), ); return yield* withAttemptCount(attempt, (counted) => counted.pipe( @@ -379,34 +405,37 @@ export const makeProcessRecipe = spec: ProcessRecipeSpec, ): Effect.Effect> => Effect.gen(function* () { - const prepared = yield* Ref.make(undefined); - const preparedRoot = yield* Ref.make(undefined); const endpoints = yield* Ref.make>(new Map()); const logs = yield* PubSub.sliding(256); + // Ownership is by location: instanceRoot is owned simply by being under options.root, created + // on demand by whatever first writes under it (no separate marker to establish or verify). const instanceRoot = deps.path.join(options.root, options.instanceId); - const ownedInstanceRoot = { - fs: deps.fs, - path: deps.path, - parentRoot: options.root, - stackId: options.stackId, - instanceId: options.instanceId, - ownerFileName: ".supabase-instance-owner.json", - label: "Instance root", - }; const nativeInstanceDir = spec.instanceDirectory === true ? instanceRoot : undefined; const containerInstanceDir = spec.instanceDirectory === true ? containerInstancePath : undefined; + // A dedicated, fully namespace-owned subdirectory for native confinement, so it never mixes + // with the recipe's own owned files. + const environmentRoot = deps.path.join(instanceRoot, ".supabase-environment"); const prepare = Effect.fn("ProcessRecipe.prepare")(function* (candidate: C) { + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(options.instanceId)) + return yield* serviceError("identity", "Instance id is not a safe path segment"); if (spec.instanceDirectory === true) - yield* ensureOwnedInstanceRoot(ownedInstanceRoot, serviceError); + yield* deps.fs + .makeDirectory(instanceRoot, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => serviceError("prepare", cause))); + if (spec.callerPaths !== undefined) + for (const candidatePath of spec.callerPaths(candidate)) + yield* deps.borrowCallerPath(candidatePath); if (spec.prepare !== undefined) yield* spec.prepare(candidate); const resolved = yield* resolveArtifact({ service: candidate.service, version: candidate.version, }).pipe(Effect.mapError((cause) => serviceError("prepare", cause))); if (options.runtime === "native") { - const artifact = yield* prepareNativeArtifact( + // Ahead-of-time warm-up only: downloads and publishes the generation but pins nothing. + // `launch` resolves (or prepares) and pins its own copy right before it spawns. + yield* prepareNativeArtifact( { service: candidate.service, version: candidate.version }, options.cacheRoot, options.platform, @@ -418,8 +447,6 @@ export const makeProcessRecipe = Effect.provideService(HttpClient.HttpClient, deps.client), Effect.mapError((cause) => serviceError("prepare", cause)), ); - yield* Ref.set(prepared, artifact.executable); - yield* Ref.set(preparedRoot, artifact.root); } else { if (deps.container === undefined) return yield* serviceError("prepare", "Container runtime unavailable"); @@ -435,12 +462,30 @@ export const makeProcessRecipe = ([name]) => spec.enabledPort === undefined || spec.enabledPort(context.config, name), ); if (options.runtime === "native") { - const executable = yield* Ref.get(prepared); - const artifactRoot = yield* Ref.get(preparedRoot); - if (executable === undefined) - return yield* serviceError("launch", "Artifact was not prepared"); - if (artifactRoot === undefined) - return yield* serviceError("launch", "Artifact root was not prepared"); + // `context.scope` finalizes its direct children in parallel (it is forked "parallel" in + // Service.ts), so registering the pin there directly would race its release against the + // spawned processes' own cleanup. A "sequential" child scope finalizes LIFO instead: the + // pin is registered on it first, every native process scope below forks from it (not from + // `context.scope`), so closing it always runs their cleanup before releasing the pin. + const launchScope = yield* Scope.fork(context.scope, "sequential"); + const artifact = yield* useNativeArtifact( + { service: context.config.service, version: context.config.version }, + options.cacheRoot, + options.platform, + ).pipe( + Scope.provide(launchScope), + Effect.provideService(FileSystem.FileSystem, deps.fs), + Effect.provideService(Path.Path, deps.path), + Effect.provideService(Crypto.Crypto, deps.crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), + Effect.provideService(HttpClient.HttpClient, deps.client), + Effect.mapError((cause) => serviceError("launch", cause)), + ); + const executable = artifact.executable; + const artifactRoot = artifact.root; + const environment = yield* Environment.confine(deps.fs, deps.path, environmentRoot).pipe( + Effect.mapError((cause) => serviceError("launch", cause)), + ); const keyFor = (name: string) => `${options.instanceId}:${context.id}:${name}`; // A port a prior attempt lost stays excluded so a retry advances instead of repeating it. const excludedByKey = yield* Ref.make>>(new Map()); @@ -449,17 +494,11 @@ export const makeProcessRecipe = ) { const portScope = yield* Scope.fork(parent, "sequential"); const excluded = yield* Ref.get(excludedByKey); - // Read once and share across this batch; a later retry attempt re-reads it. - const claims = yield* deps.readPortClaims.pipe( - Effect.mapError((cause) => serviceError("launch", cause)), - ); const reservations = yield* Effect.forEach( portNames, ([name]) => - deps.reserveNativePort(keyFor(name), claims, excluded.get(keyFor(name)) ?? new Set()), - // Each reservation holds its bound probe listener until the batch releases, so two - // endpoints never settle on the same port even when reserved concurrently. - { concurrency: "unbounded" }, + reserveNativePort(keyFor(name), excluded.get(keyFor(name)) ?? emptyNativePortSet), + { concurrency: 1 }, ).pipe( Scope.provide(portScope), Effect.mapError((cause) => serviceError("launch", cause)), @@ -485,7 +524,7 @@ export const makeProcessRecipe = } | undefined; if (spec.startupCommands.length > 0) { - const startupScope = yield* Scope.fork(context.scope, "sequential"); + const startupScope = yield* Scope.fork(launchScope, "sequential"); const reservation = spec.nativeStartupEnv === undefined ? undefined @@ -506,7 +545,9 @@ export const makeProcessRecipe = executable: `${artifactRoot}/bin/${command.executable}`, args: command.args, env: command.env, + environment, cwd: artifactRoot, + artifactLockPath: artifact.lockPath, }, defaultNativeProcessLauncher(), { @@ -543,7 +584,7 @@ export const makeProcessRecipe = readonly portScope: Scope.Closeable; readonly endpoints: ReadonlyMap; }) { - const scope = yield* Scope.fork(context.scope, "sequential"); + const scope = yield* Scope.fork(launchScope, "sequential"); const reservation = held ?? (yield* reserveEndpoints(scope)); const selected = reservation.endpoints; const args = yield* spec.args(context.config, selected, { @@ -557,6 +598,8 @@ export const makeProcessRecipe = executable, args, env, + environment, + artifactLockPath: artifact.lockPath, gracefulStopSignal: "SIGTERM", gracefulStopTimeout: "5 seconds", }, @@ -750,6 +793,7 @@ export const makeProcessRecipe = entrypoint: command.entrypoint, args: command.args, mounts: command.mounts, + user: userForMounts(command.mounts, spec.callerPaths?.(context.config) ?? []), }) .pipe( Effect.catchTag("ContainerLaunchError", ({ failure, process }) => @@ -793,6 +837,12 @@ export const makeProcessRecipe = runtime: runtimeFromContainer(startupProcess), }); } + const launchMounts = [ + ...(yield* spec.mounts(context.config, { container: true })), + ...(spec.instanceDirectory === true + ? [{ source: instanceRoot, target: containerInstancePath, readOnly: false }] + : []), + ]; const launched = yield* deps.container .launch({ image: resolved.image, @@ -803,12 +853,8 @@ export const makeProcessRecipe = env: yield* spec.env(context.config, containerDesired, true, containerInstanceDir), entrypoint: spec.containerEntrypoint?.(context.config), args: yield* spec.args(context.config, containerDesired, { container: true }), - mounts: [ - ...(yield* spec.mounts(context.config, { container: true })), - ...(spec.instanceDirectory === true - ? [{ source: instanceRoot, target: containerInstancePath, readOnly: false }] - : []), - ], + mounts: launchMounts, + user: userForMounts(launchMounts, spec.callerPaths?.(context.config) ?? []), ports: [...containerDesired.values()].map((endpoint) => endpoint.port), }) .pipe( @@ -860,14 +906,16 @@ export const makeProcessRecipe = definition: { prepare, launch, + // instanceRoot (environmentRoot's parent, and the recipe's own owned files) is removed as + // one recursive delete; a recipe with nothing on the host filesystem simply finds it absent. removeData: (context) => - (spec.instanceDirectory === true - ? removeOwnedInstanceRoot( - ownedInstanceRoot, - spec.removeData?.(context.config) ?? Effect.void, - serviceError, - ) - : (spec.removeData?.(context.config) ?? Effect.void) + destroyOwnedRoot( + deps.fs, + deps.path, + instanceRoot, + options.root, + spec.removeData?.(context.config) ?? Effect.void, + serviceError, ).pipe(Effect.andThen(Ref.set(endpoints, new Map()))), }, endpoints, diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts index 06922425c0..1daa650085 100644 --- a/packages/stack/src/services/Realtime.integration.test.ts +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -2,10 +2,12 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; const options = (root: string) => ({ stackId: "catalog-realtime", @@ -17,7 +19,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -40,9 +43,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -54,9 +56,8 @@ describe("service catalog", () => { const realtimeRecipe = yield* makeServiceRecipe( { service: "realtime", config: { databaseUrl, jwtSecret: secret } }, dockerOptions(root), - Effect.succeed([]), ); - const realtime = yield* makeService(realtimeRecipe.definition, { + const realtime = yield* makeStandaloneService(realtimeRecipe.definition, { id: "realtime", config: realtimeRecipe.creation, }); @@ -65,7 +66,7 @@ describe("service catalog", () => { const realtimeEndpoint = yield* realtimeRecipe.endpoint("http"); const realtimeResponse = yield* client.execute( HttpClientRequest.get( - `http://${realtimeEndpoint.host}:${realtimeEndpoint.port}/healthcheck`, + `http://${httpHost(realtimeEndpoint)}:${realtimeEndpoint.port}/healthcheck`, ), ); expect(realtimeResponse.status).toBe(200); @@ -73,9 +74,8 @@ describe("service catalog", () => { const pgmetaRecipe = yield* makeServiceRecipe( { service: "pgmeta", config: { databaseUrl } }, dockerOptions(root), - Effect.succeed([]), ); - const pgmeta = yield* makeService(pgmetaRecipe.definition, { + const pgmeta = yield* makeStandaloneService(pgmetaRecipe.definition, { id: "pgmeta", config: pgmetaRecipe.creation, }); @@ -84,7 +84,9 @@ describe("service catalog", () => { const pgmetaEndpoint = yield* pgmetaRecipe.endpoint("http"); const pgmetaRelay = yield* makeDockerHttpRelay(pgmetaRecipe.endpoint("http")); const schemasResponse = yield* client.execute( - HttpClientRequest.get(`http://${pgmetaEndpoint.host}:${pgmetaEndpoint.port}/schemas`), + HttpClientRequest.get( + `http://${httpHost(pgmetaEndpoint)}:${pgmetaEndpoint.port}/schemas`, + ), ); expect(schemasResponse.status).toBe(200); expect(yield* schemasResponse.text).toContain("public"); @@ -102,9 +104,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const studio = yield* makeService(studioRecipe.definition, { + const studio = yield* makeStandaloneService(studioRecipe.definition, { id: "studio", config: studioRecipe.creation, }); @@ -113,13 +114,13 @@ describe("service catalog", () => { const studioEndpoint = yield* studioRecipe.endpoint("http"); const profileResponse = yield* client.execute( HttpClientRequest.get( - `http://${studioEndpoint.host}:${studioEndpoint.port}/api/platform/profile`, + `http://${httpHost(studioEndpoint)}:${studioEndpoint.port}/api/platform/profile`, ), ); expect(profileResponse.status).toBe(200); const queryResponse = yield* client.execute( HttpClientRequest.post( - `http://${studioEndpoint.host}:${studioEndpoint.port}/api/platform/pg-meta/default/query`, + `http://${httpHost(studioEndpoint)}:${studioEndpoint.port}/api/platform/pg-meta/default/query`, ).pipe(HttpClientRequest.bodyJsonUnsafe({ query: "select current_user" })), ); expect(queryResponse.status).toBe(200); diff --git a/packages/stack/src/services/Realtime.ts b/packages/stack/src/services/Realtime.ts index 87a7a46f35..231b8e3d13 100644 --- a/packages/stack/src/services/Realtime.ts +++ b/packages/stack/src/services/Realtime.ts @@ -52,6 +52,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ return { DATABASE_URL: databaseUrl, ...(http === undefined ? {} : { PORT: String(http.port) }), + ...(container ? {} : { PHX_HTTP_IP: "127.0.0.1" }), DB_URL: databaseUrl, DB_HOST: db.host, DB_PORT: db.port, diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index e6e9aa3a00..2de538fa53 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -3,7 +3,7 @@ import type { Effect, PubSub, Ref, Scope } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; import type { LaunchOutput } from "../runtime/Session.ts"; import type { ServiceDefinition } from "../Service.ts"; -import type { HostGateway } from "../runtime/Container.ts"; +import type { EngineTarget, HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; type CatalogRuntime = "native" | "docker" | "podman"; @@ -15,12 +15,10 @@ export class CatalogError extends Data.TaggedError("CatalogError")<{ readonly cause?: unknown; }> {} -export interface ServiceEndpoint { - readonly kind: "tcp" | "unix"; - readonly host?: "127.0.0.1"; - readonly path?: string; - readonly port: number; -} +export type ServiceEndpoint = + | { readonly kind: "tcp"; readonly host?: "127.0.0.1"; readonly port: number } + /** `path` is the full socket filename, for example `/.s.PGSQL.`. */ + | { readonly kind: "unix"; readonly path: string; readonly port: number }; export const EndpointIntent = Schema.Struct({ port: Schema.Union([Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), Schema.Literal("auto")]), @@ -56,6 +54,8 @@ export interface CatalogOptions { readonly helpers?: DockerHelperRegistry; /** Shares one host-gateway probe across this host's container runtimes. */ readonly hostGateway?: HostGateway; + /** The engine endpoint and identity the owner resolved once at startup; absent when native. */ + readonly engineTarget?: EngineTarget; } /** Subscribes to a recipe's launch output; chunks published before the subscription are missed. */ diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts index 0f902e8bbe..1dbcdb52b8 100644 --- a/packages/stack/src/services/Rest.integration.test.ts +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -4,27 +4,29 @@ import { Context, Effect, FileSystem, Layer, Path, Redacted, Ref } from "effect" import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; -import { tmpdir } from "node:os"; import * as Network from "../Network.ts"; -import * as State from "../State.ts"; -import { makeService } from "../Service.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; import { ProxyError } from "../Proxy.ts"; import { makeServiceRecipe } from "./Catalog.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { httpHost } from "../../tests/helpers/endpoint.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; const makeTestState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const makeTestNetwork = (options: { readonly stackId: string; readonly runtime: Network.NetworkRuntime; - readonly state: State.Interface; + readonly state: StackNamespace.Interface; }) => Layer.build( Network.layer({ stackId: options.stackId, runtime: options.runtime }).pipe( - Layer.provide(Layer.succeed(State.Service, options.state)), + Layer.provide(Layer.succeed(StackNamespace.Service, options.state)), ), ).pipe(Effect.map((context) => Context.get(context, Network.Service))); @@ -38,7 +40,8 @@ const options = (root: string) => ({ const dockerOptions = (root: string) => ({ ...options(root), - runtime: "docker" as const, + runtime: testEngine, + engineTarget, }); describe("service catalog", () => { @@ -55,13 +58,12 @@ describe("service catalog", () => { yield* state.save({ id: stackId, identity: { projectRoot: root, branchContext: "test", stackName: "catalog" }, - runtime: "docker", + runtime: testEngine, instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); - const network = yield* makeTestNetwork({ stackId, runtime: "docker", state }); + const network = yield* makeTestNetwork({ stackId, runtime: testEngine, state }); const secret = "catalog-rest-secret-with-at-least-32-chars"; const databaseRecipe = yield* makeServiceRecipe( { @@ -74,9 +76,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); @@ -140,9 +141,8 @@ describe("service catalog", () => { }, }, dockerOptions(root), - Effect.succeed([]), ); - const rest = yield* makeService(restRecipe.definition, { + const rest = yield* makeStandaloneService(restRecipe.definition, { id: "rest", config: restRecipe.creation, }); @@ -156,7 +156,7 @@ describe("service catalog", () => { enabled: Ref.get(restActive), backend: restRecipe.endpoint("http").pipe( Effect.flatMap((endpoint) => - endpoint.host === undefined + endpoint.kind !== "tcp" || endpoint.host === undefined ? Effect.fail(new ProxyError({ message: "REST endpoint has no host" })) : Effect.succeed({ host: endpoint.host, port: endpoint.port }), ), @@ -193,7 +193,7 @@ describe("service catalog", () => { yield* Ref.set(databaseActive, false); yield* databaseNamespace.close; yield* databaseNamespace.release; - yield* network.release; + yield* network.releaseStack; }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 120_000 }, @@ -203,6 +203,7 @@ describe("service catalog", () => { Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const client = yield* HttpClient.HttpClient; const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-rest-native-" }); const stackId = "catalog-native"; @@ -217,21 +218,23 @@ describe("service catalog", () => { jwtExpiry: 3600, }, }, - { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, - Effect.succeed([]), + { ...options(root), stackId, cacheRoot: testArtifactCacheRoot }, ); - const database = yield* makeService(databaseRecipe.definition, { + const database = yield* makeStandaloneService(databaseRecipe.definition, { id: "database", config: databaseRecipe.creation, }); yield* database.start; yield* database.ready; const databaseEndpoint = yield* databaseRecipe.endpoint("sql"); - if (databaseEndpoint.kind !== "unix" || databaseEndpoint.path === undefined) + if (databaseEndpoint.kind !== "unix") return yield* new ProxyError({ message: "Native database did not expose a Unix socket" }); + // node-postgres appends the `.s.PGSQL.` suffix itself, so it must receive the + // socket directory rather than the endpoint's full socket filename. + const databaseSocketDir = path.dirname(databaseEndpoint.path); const databaseLayer = yield* Layer.build( PgClient.layer({ - host: databaseEndpoint.path, + host: databaseSocketDir, port: databaseEndpoint.port, database: "postgres", username: "supabase_admin", @@ -253,15 +256,14 @@ describe("service catalog", () => { config: { databaseUrl: "postgresql://supabase_admin:postgres@127.0.0.1/postgres?host=" + - encodeURIComponent(databaseEndpoint.path), + encodeURIComponent(databaseSocketDir), jwtSecret: secret, anonRole: "anon", }, }, - { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, - Effect.succeed([]), + { ...options(root), stackId, cacheRoot: testArtifactCacheRoot }, ); - const rest = yield* makeService(restRecipe.definition, { + const rest = yield* makeStandaloneService(restRecipe.definition, { id: "rest", config: restRecipe.creation, }); @@ -277,7 +279,7 @@ describe("service catalog", () => { ); const response = yield* client.execute( HttpClientRequest.get( - "http://" + endpoint.host + ":" + endpoint.port + "/catalog_native_probe", + "http://" + httpHost(endpoint) + ":" + endpoint.port + "/catalog_native_probe", ).pipe(HttpClientRequest.setHeader("Authorization", "Bearer " + token)), ); const responseBody = yield* response.text; diff --git a/packages/stack/src/services/Rest.ts b/packages/stack/src/services/Rest.ts index 0d913c6158..3eac29824b 100644 --- a/packages/stack/src/services/Rest.ts +++ b/packages/stack/src/services/Rest.ts @@ -27,7 +27,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ executable: "bin/postgrest", ports: { http: 3000 }, healthPath: "/", - env: (creation, endpoints) => + env: (creation, endpoints, container) => Effect.gen(function* () { const databaseUrl = yield* requiredInput("rest", "databaseUrl", creation.config.databaseUrl); const http = endpoints.get("http"); @@ -35,6 +35,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ return { DATABASE_URL: databaseUrl, PGRST_DB_URI: databaseUrl, + PGRST_SERVER_HOST: container ? "0.0.0.0" : "127.0.0.1", ...(http === undefined ? {} : { PGRST_SERVER_PORT: String(http.port) }), PGRST_DB_SCHEMAS: creation.config.schemas ?? "public,graphql_public", ...(creation.config.extraSearchPath === undefined diff --git a/packages/stack/src/services/ServiceConfig.ts b/packages/stack/src/services/ServiceConfig.ts index 72156d14a0..58ee7f9f66 100644 --- a/packages/stack/src/services/ServiceConfig.ts +++ b/packages/stack/src/services/ServiceConfig.ts @@ -7,7 +7,7 @@ import { DEFAULT_LOCAL_SECRET_KEY, DEFAULT_SIGNING_KEY, } from "../Defaults.ts"; -import type { StackCredentials, StackKeysInput } from "../State.ts"; +import type { StackCredentials, StackKeysInput } from "../StackNamespace.ts"; const serviceError = (operation: string, cause: unknown): ServiceError => cause instanceof ServiceError diff --git a/packages/stack/src/services/Storage.ts b/packages/stack/src/services/Storage.ts index 2018dfad64..7e97c0d68d 100644 --- a/packages/stack/src/services/Storage.ts +++ b/packages/stack/src/services/Storage.ts @@ -63,6 +63,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ const filePath = container ? "/mnt" : creation.config.filePath; return { DATABASE_URL: databaseUrl, + SERVER_HOST: container ? "0.0.0.0" : "127.0.0.1", ...(http === undefined ? {} : { STORAGE_PORT: String(http.port), PORT: String(http.port) }), ANON_KEY: creation.config.anonKey ?? anon, SERVICE_KEY: creation.config.serviceRoleKey ?? service, @@ -124,4 +125,5 @@ export const makeSpec = (): ProcessRecipeSpec => ({ mounts: (creation, _context) => Effect.succeed([{ source: creation.config.filePath, target: "/mnt", readOnly: false }]), startupCommands: [initializationCommand], + callerPaths: (creation) => [creation.config.filePath], }); diff --git a/packages/stack/src/services/StorageGateway.integration.test.ts b/packages/stack/src/services/StorageGateway.integration.test.ts index 857e6b9460..e5fef13ec2 100644 --- a/packages/stack/src/services/StorageGateway.integration.test.ts +++ b/packages/stack/src/services/StorageGateway.integration.test.ts @@ -1,19 +1,20 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Context, Crypto, Effect, FileSystem, Layer, Path, Redacted } from "effect"; +import { Context, Crypto, Effect, FileSystem, Layer, Option, Path, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { tmpdir } from "node:os"; import { DEFAULT_LOCAL_S3_ACCESS_KEY_ID, DEFAULT_LOCAL_S3_REGION, DEFAULT_LOCAL_S3_SECRET_ACCESS_KEY, } from "../Defaults.ts"; -import * as State from "../State.ts"; -import type { SavedStack } from "../State.ts"; +import * as StackNamespace from "../StackNamespace.ts"; +import type { SavedStack } from "../StackNamespace.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; import { ownerFor } from "../../tests/owner-rpc.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; +import { testArtifactCacheRoot } from "../../tests/artifact-cache.ts"; -const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const cacheRoot = testArtifactCacheRoot; const jwtSecret = "storage-gateway-secret-with-at-least-32-chars"; const s3Credentials = { accessKeyId: DEFAULT_LOCAL_S3_ACCESS_KEY_ID, @@ -31,10 +32,15 @@ const layout = Effect.fnUntraced(function* (runtime: SavedStack["runtime"], stac const dataRoot = yield* makeDockerDatabaseRoot("storage-gateway-docker-", stackId).pipe( Effect.flatMap(fs.realPath), ); + // Ownership is by location: Storage's filePath is a caller path and must live outside the + // stack's data root, not merely outside the service's own instance root. + const storageRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "storage-gateway-docker-caller-", + }); return { stateRoot: path.dirname(path.dirname(dataRoot)), dataRoot, - storageRoot: `${dataRoot}/storage`, + storageRoot, }; }); @@ -52,11 +58,19 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }; - const state = Context.get(yield* Layer.build(State.layer({ root: stateRoot })), State.Service); + const state = Context.get( + yield* Layer.build(StackNamespace.layer({ root: stateRoot })), + StackNamespace.Service, + ); yield* state.save(saved); - const owner = yield* ownerFor({ saved, state, root: dataRoot, cacheRoot }); + const owner = yield* ownerFor({ + saved, + state, + root: dataRoot, + cacheRoot, + ...(runtime !== "native" ? { engineTarget } : {}), + }); yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); const created = yield* owner.rpc.supabaseComposition({ services: [ @@ -83,17 +97,17 @@ const serveStorage = Effect.fnUntraced(function* (runtime: SavedStack["runtime"] const { url } = yield* owner.rpc.credentials({ id: storage.id, from: "host" }); if (url === undefined) return yield* Effect.die("Storage gateway URL missing"); const { serviceRoleKey } = yield* owner.getStackCredentials; - return { url, serviceRoleKey }; + return { url, serviceRoleKey, storageRoot }; }); -for (const runtime of ["native", "docker"] as const) +for (const runtime of ["native", testEngine] as const) it.live( `accepts S3 requests signed over the gateway path and resumes TUS uploads there (${runtime})`, () => Effect.scoped( Effect.gen(function* () { const client = yield* HttpClient.HttpClient; - const { url, serviceRoleKey } = yield* serveStorage(runtime); + const { url, serviceRoleKey, storageRoot } = yield* serveStorage(runtime); const bucket = "gateway"; const createBucket = yield* client.execute( @@ -147,6 +161,22 @@ for (const runtime of ["native", "docker"] as const) ); expect(patch.status, yield* patch.text).toBe(204); expect(patch.headers["upload-offset"]).toBe("5"); + + if (runtime !== "native") { + // The container writes into storageRoot, a borrowed host directory; the host user + // must still own what it created there, not the engine's own root (see Container's + // `user` field). + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const entries = yield* fs.readDirectory(storageRoot, { recursive: true }); + const stats = yield* Effect.forEach(entries, (entry) => + fs.stat(path.join(storageRoot, entry)), + ); + const files = stats.filter((info) => info.type === "File"); + expect(files.length).toBeGreaterThan(0); + for (const info of files) + expect(Option.getOrUndefined(info.uid)).toBe(process.getuid?.()); + } }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, diff --git a/packages/stack/src/services/Studio.integration.test.ts b/packages/stack/src/services/Studio.integration.test.ts new file mode 100644 index 0000000000..29d665162b --- /dev/null +++ b/packages/stack/src/services/Studio.integration.test.ts @@ -0,0 +1,40 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer } from "effect"; +import { makeStandaloneService } from "../../tests/standalone-service.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; + +const options = (root: string) => ({ + stackId: "catalog-studio", + instanceId: "studio", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +describe("studio recipe", () => { + it.live( + "rejects a functionsRoot that resolves inside a sibling instance under the same data root", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-studio-reject-" }); + // Ownership is by location: Studio's functionsRoot is a caller path and must live + // outside the stack's data root, not merely outside this instance's own instance root. + const siblingInstance = `${root}/functions-instance`; + yield* fs.makeDirectory(siblingInstance, { recursive: true }); + const recipe = yield* makeServiceRecipe( + { service: "studio", config: { functionsRoot: siblingInstance } }, + options(root), + ); + const studio = yield* makeStandaloneService(recipe.definition, { + id: "studio", + config: recipe.creation, + }); + const failure = yield* studio.start.pipe(Effect.flip); + expect(failure.message).toContain("resolves inside the owned data root"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); diff --git a/packages/stack/src/services/Studio.ts b/packages/stack/src/services/Studio.ts index 133e1f24ea..30a57e44e9 100644 --- a/packages/stack/src/services/Studio.ts +++ b/packages/stack/src/services/Studio.ts @@ -129,4 +129,8 @@ export const makeSpec = (): ProcessRecipeSpec => ({ ]), ]), startupCommands: [], + callerPaths: (creation) => [ + ...(creation.config.functionsRoot === undefined ? [] : [creation.config.functionsRoot]), + ...(creation.config.snippetsRoot === undefined ? [] : [creation.config.snippetsRoot]), + ], }); diff --git a/packages/stack/src/shutdown.integration.test.ts b/packages/stack/src/shutdown.integration.test.ts index 905d95750c..f9f3d7a8a6 100644 --- a/packages/stack/src/shutdown.integration.test.ts +++ b/packages/stack/src/shutdown.integration.test.ts @@ -6,7 +6,7 @@ import { fileURLToPath } from "node:url"; import * as PromiseStack from "./index.ts"; import { HostEndpoint } from "./HostProcess.ts"; import { StackError } from "./Rpc.ts"; -import * as State from "./State.ts"; +import * as StackNamespace from "./StackNamespace.ts"; import { open as openEffect } from "./effect.ts"; const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -51,8 +51,8 @@ const withHeldOwner = ( const stateRoot = `${root}/state`; const cacheRoot = `${root}/cache`; const id = "shutdown-held"; - const context = yield* Layer.build(State.layer({ root: stateRoot })); - const state = Context.get(context, State.Service); + const context = yield* Layer.build(StackNamespace.layer({ root: stateRoot })); + const state = Context.get(context, StackNamespace.Service); yield* state.save({ id, runtime: "native", @@ -60,7 +60,6 @@ const withHeldOwner = ( instances: [], lifetime: "detached", composition: { members: [], dependencies: [] }, - ports: [], }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; return yield* Effect.acquireUseRelease( diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 6f2ff7fce0..a7cc8ce481 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -23,13 +23,17 @@ import { import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import * as TestClock from "effect/testing/TestClock"; import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; -import { makeContainerRuntime } from "../runtime/Container.ts"; +import { makeContainerRuntime, type EngineTarget } from "../runtime/Container.ts"; import { makeDatabaseSnapshots } from "../services/DatabaseSnapshot.ts"; import { makeDockerDatabaseStorage } from "./DockerDatabaseStorage.ts"; import { makeDockerHelperRegistry } from "./DockerHelperRegistry.ts"; +import { engineTarget, testEngine } from "../../tests/engine-target.ts"; import { removeTestRunVolumes } from "../../tests/docker-volume-run.ts"; import type { DockerHelperRegistry } from "./DockerHelperRegistry.ts"; +// An unpinned target: these tests exercise storage marker/volume logic, not endpoint pinning. +const dockerTarget: EngineTarget = { engine: "docker", argv: [], daemonId: "test-daemon-id" }; +const containerTarget: EngineTarget = { ...engineTarget, argv: [] }; const postgresImage = (version: string) => resolveArtifact({ service: "database", version: postgresVersion(version) }).pipe( Effect.map(({ image }) => image), @@ -46,9 +50,19 @@ const Marker = Schema.Struct({ class DockerTestError extends Data.TaggedError("DockerTestError")<{ readonly message: string; }> {} +class PublishBarrierError extends Data.TaggedError("PublishBarrierError")<{ + readonly message: string; +}> {} + +const dockerStoragePublishLoopFixture = fileURLToPath( + new URL("../../tests/docker-storage-publish-loop-fixture.ts", import.meta.url), +); const helperMirror = "registry.test/supabase/postgres:17"; -const fakeHelperEngine = () => { +const fakeHelperEngine = ( + onRun: () => Effect.Effect = () => Effect.void, + { volumePresent = false }: { readonly volumePresent?: boolean } = {}, +) => { const commands: string[][] = []; const local = new Set(); const handle = (exitCode: number, stdout = "", stderr = "") => @@ -65,39 +79,46 @@ const fakeHelperEngine = () => { getOutputFd: () => Stream.empty, unref: Effect.succeed(Effect.void), }); - const spawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); - const args = [...command.args]; - commands.push(args); - if (args[0] === "image") - return Effect.succeed(handle(0, local.has(args.at(-1) ?? "") ? "sha256:1" : "")); - if (args[0] === "pull") { - if (args.at(-1) !== helperMirror) - return Effect.succeed(handle(1, "", `denied: ${args.at(-1)}`)); - local.add(helperMirror); - return Effect.succeed(handle(0)); - } - if (args[0] === "inspect") return Effect.succeed(handle(1, "", "no such container")); - if (args[0] === "run") { - const shellIndex = args.indexOf("/bin/sh"); - const image = args[shellIndex - 1]; - return image !== undefined && local.has(image) - ? Effect.succeed( - handle(0, args.includes("-i") ? "supabase-helper-ready\n" : "abcdef0123456789"), - ) - : Effect.succeed(handle(1, "", `Unable to find image '${image ?? ""}' locally`)); - } - if (args[0] === "exec" || args[0] === "rm") return Effect.succeed(handle(0, "done")); - return Effect.succeed(handle(1, "", `unexpected engine command: ${args[0] ?? ""}`)); - }); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("unexpected piped command"); + const args = [...command.args]; + commands.push(args); + if (args[0] === "info") return handle(0, "fake-daemon-id"); + if (args[0] === "version") return handle(0, "27.0.0|27.0.0"); + if (args[0] === "volume" && args[1] === "inspect") + return volumePresent ? handle(0) : handle(1, "", "no such volume"); + if (args[0] === "volume" && args[1] === "create") return handle(0); + if (args[0] === "image") return handle(0, local.has(args.at(-1) ?? "") ? "sha256:1" : ""); + if (args[0] === "pull") { + if (args.at(-1) !== helperMirror) return handle(1, "", `denied: ${args.at(-1)}`); + local.add(helperMirror); + return handle(0); + } + if (args[0] === "inspect") return handle(1, "", "no such container"); + if (args[0] === "run") { + const shellIndex = args.indexOf("/bin/sh"); + const image = args[shellIndex - 1]; + if (image === undefined || !local.has(image)) + return handle(1, "", `Unable to find image '${image ?? ""}' locally`); + // Fires once the create command the claim must precede is observed, so the claim's + // presence at this exact point is a deterministic fact, not a timing guess. + yield* onRun(); + return handle(0, args.includes("-i") ? "supabase-helper-ready\n" : "abcdef0123456789"); + } + if (args[0] === "exec" || args[0] === "rm") return handle(0, "done"); + return handle(1, "", `unexpected engine command: ${args[0] ?? ""}`); + }), + ); return { commands, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; }; -const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray) => +const engine = Effect.fn("StorageTest.engine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn(ChildProcess.make("docker", args, { stdin: "ignore" })); + const child = yield* spawner.spawn(ChildProcess.make(testEngine, args, { stdin: "ignore" })); const [stdout, stderr, code] = yield* Effect.all( [ child.stdout.pipe( @@ -120,7 +141,7 @@ const docker = Effect.fn("DockerStorageTest.docker")((args: ReadonlyArray `'${value.replaceAll("'", "'\\''")}'`; -/** Includes events emitted before the subscription starts. */ const awaitDestroyed = Effect.fn("DockerStorageTest.awaitDestroyed")((id: string, since: number) => Effect.scoped( Effect.gen(function* () { @@ -173,7 +193,7 @@ const ownerDeathScenario = (shared: boolean) => const stackId = `storage-owner-death-${yield* crypto.randomUUIDv4}`; const filter = `label=com.supabase.stack=${stackId}`; // Pull before spawning so a cold image cache does not consume the owner's readiness timeout. - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ target: dockerTarget, root }); yield* container.prepareImage(yield* postgresImage("17")); const child = yield* spawner.spawn( ChildProcess.make( @@ -189,8 +209,8 @@ const ownerDeathScenario = (shared: boolean) => ); yield* Effect.addFinalizer(() => Effect.gen(function* () { - const ids = yield* docker(["ps", "-aq", "--filter", filter]); - if (ids.length > 0) yield* docker(["rm", "-f", ...ids.split("\n")]); + const ids = yield* engine(["ps", "-aq", "--filter", filter]); + if (ids.length > 0) yield* engine(["rm", "-f", ...ids.split("\n")]); if (!shared) return; const markerText = yield* fs .readFileString(`${root}/state/stack/data/database/.supabase-database-storage.json`) @@ -200,7 +220,7 @@ const ownerDeathScenario = (shared: boolean) => markerText.value, ).pipe(Effect.option); if (Option.isSome(marker) && marker.value.volume !== undefined) - yield* docker(["volume", "rm", marker.value.volume]); + yield* engine(["volume", "rm", marker.value.volume]); }).pipe(Effect.ignore), ); const stderrOutput = yield* child.stderr.pipe( @@ -229,7 +249,7 @@ const ownerDeathScenario = (shared: boolean) => message: `Owner exited before helper was ready (stdout: ${Exit.isSuccess(ready) && Option.isSome(ready.value) ? ready.value.value : ""}, stderr: ${stderr.trim() || ""}, exit: ${Exit.isSuccess(exited) ? exited.value : "killed by signal"}${Exit.isFailure(ready) ? `, cause: ${ready.cause}` : ""})`, }); } - const id = yield* docker([ + const id = yield* engine([ "ps", "--filter", filter, @@ -245,7 +265,7 @@ const ownerDeathScenario = (shared: boolean) => ); if (shared) expect( - yield* docker([ + yield* engine([ "inspect", "--format", '{{index .Config.Labels "com.supabase.stack-helper"}}', @@ -264,297 +284,551 @@ const ownerDeathScenario = (shared: boolean) => orElse: () => new DockerTestError({ message: `Orphaned database helper: ${id}` }), }), ); - expect(yield* docker(["ps", "-aq", "--filter", filter])).toBe(""); + expect(yield* engine(["ps", "-aq", "--filter", filter])).toBe(""); }), ).pipe(Effect.provide(NodeServices.layer)); -describe("Docker database storage", { timeout: 120_000 }, () => { - it.live.skipIf(process.platform === "win32")( - "removes a database helper when its owner is killed", - () => ownerDeathScenario(false), - ); +// The volume backend, helper registry and volume-labelling contracts exist only on Docker; Podman +// always stores data in host directories, covered by the host storage suite below. +describe.runIf(testEngine === "docker")( + "Docker volume database storage", + { timeout: 120_000 }, + () => { + it.live.skipIf(process.platform === "win32")( + "removes a database helper when its owner is killed", + () => ownerDeathScenario(false), + ); - it.live.skipIf(process.platform === "win32")( - "removes a shared volume helper when its owner is killed", - () => ownerDeathScenario(true), - ); + it.live.skipIf(process.platform === "win32")( + "removes a shared volume helper when its owner is killed", + () => ownerDeathScenario(true), + ); - it.live("waits for helper creation to settle before cleaning up an interrupted operation", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-helper-create-cancel-" }); - const started = yield* Deferred.make(); - const release = yield* Deferred.make(); - const present = yield* Ref.make(false); - const removals = yield* Ref.make(0); - const spawner = ChildProcessSpawner.make((command) => - Effect.gen(function* () { - if (!ChildProcess.isStandardCommand(command)) - return yield* Effect.die("Unexpected command"); - const creating = command.args[0] === "run"; - if (creating) yield* Deferred.succeed(started, undefined); - if (command.args[0] === "rm") { - yield* Ref.update(removals, (count) => count + 1); - yield* Ref.set(present, false); - } - return ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: creating - ? Stream.fromEffect( - Deferred.await(release).pipe( - Effect.andThen(Ref.set(present, true)), - Effect.as(new TextEncoder().encode("supabase-helper-ready\n")), - ), - ) - : Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), + it.live("waits for helper creation to settle before cleaning up an interrupted operation", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "docker-helper-create-cancel-", + }); + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + const present = yield* Ref.make(false); + const removals = yield* Ref.make(0); + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("Unexpected command"); + const creating = command.args[0] === "run"; + if (creating) yield* Deferred.succeed(started, undefined); + if (command.args[0] === "rm") { + yield* Ref.update(removals, (count) => count + 1); + yield* Ref.set(present, false); + } + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: creating + ? Stream.fromEffect( + Deferred.await(release).pipe( + Effect.andThen(Ref.set(present, true)), + Effect.as(new TextEncoder().encode("supabase-helper-ready\n")), + ), + ) + : Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "helper-create-cancel", + instanceId: "database", + instanceRoot: root, + root, + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + spawner, + container: { + prepare: () => Effect.void, + prepareImage: (image) => Effect.succeed(image), + launch: () => Effect.die("unused"), + launchCommand: () => Effect.die("unused"), + }, + }); + yield* storage.prepare("17"); + const operation = yield* storage.removeData("17").pipe(Effect.forkScoped); + yield* Deferred.await(started); + yield* Effect.sync(() => operation.interruptUnsafe()); + yield* Effect.yieldNow; + expect(yield* Ref.get(removals)).toBe(0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.await(operation); + expect(yield* Ref.get(removals)).toBe(1); + expect(yield* Ref.get(present)).toBe(false); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("reports a helper readiness timeout alongside Docker warnings", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-helper-timeout-" }); + const waiting = yield* Deferred.make(); + const warning = + "WARNING: The requested image's platform does not match the host platform"; + const spawner = ChildProcessSpawner.make((command) => + Effect.gen(function* () { + if (!ChildProcess.isStandardCommand(command)) + return yield* Effect.die("Unexpected command"); + const creating = command.args[0] === "run"; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(0), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: creating + ? Stream.fromEffect(Deferred.succeed(waiting, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.never), + ) + : Stream.empty, + stderr: creating + ? Stream.succeed(new TextEncoder().encode(`${warning}\n`)) + : Stream.empty, + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + }), + ); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "helper-timeout", + instanceId: "database", + instanceRoot: root, + root, + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + spawner, + container: { + prepare: () => Effect.void, + prepareImage: (image) => Effect.succeed(image), + launch: () => Effect.die("unused"), + launchCommand: () => Effect.die("unused"), + }, + }); + yield* storage.prepare("17"); + const operation = yield* storage.removeData("17").pipe(Effect.flip, Effect.forkScoped); + yield* Deferred.await(waiting); + yield* TestClock.adjust("30 seconds"); + const failure = yield* Fiber.join(operation); + expect(failure.message).toBe( + `Database helper did not become ready within 30 seconds: ${warning}`, + ); + }), + ).pipe(Effect.provide([NodeServices.layer, TestClock.layer()])), + ); + + it.live("starts a storage helper with the mirror image selected during preparation", () => { + const engine = fakeHelperEngine(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-mirror-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + imageMirrors: () => [helperMirror], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-helper-mirror", + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + + yield* storage.prepare("17"); + yield* storage.removeData("17"); + + const run = engine.commands.find((args) => args[0] === "run"); + const shellIndex = run?.indexOf("/bin/sh") ?? -1; + expect(run?.[shellIndex - 1]).toBe(helperMirror); + expect( + engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), + ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("round-trips PostgreSQL 15 data with its catalog image", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("15"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-pg15-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; + const makeStorage = (instanceId: string, instanceRoot: string) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, }); - }), - ); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", - stackId: "helper-create-cancel", - instanceId: "database", - instanceRoot: root, - root, - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - spawner, - container: { - prepare: () => Effect.void, - prepareImage: (image) => Effect.succeed(image), - launch: () => Effect.die("unused"), - launchCommand: () => Effect.die("unused"), - }, - }); - yield* storage.prepare("17"); - const operation = yield* storage.removeData("17").pipe(Effect.forkScoped); - yield* Deferred.await(started); - yield* Effect.sync(() => operation.interruptUnsafe()); - yield* Effect.yieldNow; - expect(yield* Ref.get(removals)).toBe(0); - yield* Deferred.succeed(release, undefined); - yield* Fiber.await(operation); - expect(yield* Ref.get(removals)).toBe(1); - expect(yield* Ref.get(present)).toBe(false); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("15").pipe(Effect.ignore); + yield* target.destroyData("15").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* engine(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); - it.effect("reports a helper readiness timeout alongside Docker warnings", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-helper-timeout-" }); - const waiting = yield* Deferred.make(); - const warning = "WARNING: The requested image's platform does not match the host platform"; - const spawner = ChildProcessSpawner.make((command) => - Effect.gen(function* () { - if (!ChildProcess.isStandardCommand(command)) - return yield* Effect.die("Unexpected command"); - const creating = command.args[0] === "run"; - return ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(0), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: creating - ? Stream.fromEffect(Deferred.succeed(waiting, undefined)).pipe( - Stream.drain, - Stream.concat(Stream.never), - ) - : Stream.empty, - stderr: creating - ? Stream.succeed(new TextEncoder().encode(`${warning}\n`)) - : Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), + yield* source.prepare("15"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; printf 15 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf pg15-source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + ]); + yield* source.markInitialized("15"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"15","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("15", "restore-me"); + + yield* target.prepare("15"); + expect(yield* target.restoreSnapshot("15", "restore-me")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /store/PG_VERSION)" = 15 && test "$(cat /store/fixture)" = pg15-source', + ]); + yield* target.destroyData("15"); + yield* source.destroyData("15"); + yield* engine(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("round-trips stopped data and protects managed namespaces", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const makeStorageAt = ( + instanceId: string, + instanceRoot: string, + storageRootValue: string, + cacheRootValue: string, + ) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-test", + instanceId, + instanceRoot, + root: storageRootValue, + cacheRoot: cacheRootValue, + fs, + path, + crypto, + container, + spawner, }); - }), - ); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", - stackId: "helper-timeout", - instanceId: "database", - instanceRoot: root, - root, - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - spawner, - container: { - prepare: () => Effect.void, - prepareImage: (image) => Effect.succeed(image), - launch: () => Effect.die("unused"), - launchCommand: () => Effect.die("unused"), - }, - }); - yield* storage.prepare("17"); - const operation = yield* storage.removeData("17").pipe(Effect.flip, Effect.forkScoped); - yield* Deferred.await(waiting); - yield* TestClock.adjust("30 seconds"); - const failure = yield* Fiber.join(operation); - expect(failure.message).toBe( - `Database helper did not become ready within 30 seconds: ${warning}`, - ); - }), - ).pipe(Effect.provide([NodeServices.layer, TestClock.layer()])), - ); + const makeStorage = (instanceId: string, instanceRoot: string) => + makeStorageAt(instanceId, instanceRoot, storageRoot, cacheRoot); + const source = yield* makeStorage("source", sourceRoot); + const target = yield* makeStorage("target", targetRoot); + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + yield* source.destroyData("17").pipe(Effect.ignore); + yield* target.destroyData("17").pipe(Effect.ignore); + const marker = yield* fs + .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isSome(marker)) { + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + marker.value, + ).pipe(Effect.option); + if (Option.isSome(parsed) && parsed.value.volume !== undefined) + yield* engine(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); + } + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); + yield* source.prepare("17"); + const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}; /usr/bin/busybox setfattr -n user.storage-smoke -v preserved ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, + ]); + yield* source.markInitialized("17"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("17", "roundtrip"); - it.live("starts a storage helper with the mirror image selected during preparation", () => { - const engine = fakeHelperEngine(); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-helper-mirror-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ - engine: "podman", - root, - imageMirrors: () => [helperMirror], - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", - stackId: "storage-helper-mirror", - instanceId: "database", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); + const copiedRoot = path.join(root, "copied-state", "stack", "data", "source"); + const copiedStorageRoot = path.join(root, "copied-state", "stack", "data"); + const copiedCacheRoot = path.join(root, "copied-cache"); + yield* fs.makeDirectory(copiedRoot, { recursive: true }); + yield* fs.makeDirectory(copiedCacheRoot, { recursive: true }); + yield* fs.writeFileString( + path.join(copiedRoot, ".supabase-database-storage.json"), + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + const copied = yield* makeStorageAt( + "source", + copiedRoot, + copiedStorageRoot, + copiedCacheRoot, + ); + const mountError = yield* copied.mount("17").pipe(Effect.flip); + expect(mountError.message).toContain("another state directory"); + const removeError = yield* copied.removeData("17").pipe(Effect.flip); + expect(removeError.message).toContain("another state directory"); + const destroyError = yield* copied.destroyData("17").pipe(Effect.flip); + expect(destroyError.message).toContain("another state directory"); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(cat /store/${sourceMarker.namespace}/data/fixture)" = source`, + ]); - yield* storage.prepare("17"); - yield* storage.removeData("17"); + yield* target.prepare("17"); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); + const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(targetMarker.volume).toBe(sourceMarker.volume); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, + helperImage, + "/bin/sh", + "-c", + `set -eu; test "$(cat /store/fixture)" = source; /usr/bin/busybox setfattr -x user.storage-smoke /store/fixture`, + ]); - const run = engine.commands.find((args) => args[0] === "run"); - const shellIndex = run?.indexOf("/bin/sh") ?? -1; - expect(run?.[shellIndex - 1]).toBe(helperMirror); - expect( - engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), - ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); - }); + yield* target.removeData("unsupported"); + const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), + ); + expect(resetMarker.volume).toBe(sourceMarker.volume); + expect(resetMarker.initialized).toBe(false); - it.live("starts a shared volume helper with the mirror image selected during preparation", () => { - const engine = fakeHelperEngine(); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "storage-shared-helper-mirror-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const stackId = "shared-helper-mirror"; - const instanceId = "database"; - const cachePath = yield* fs.realPath(cacheRoot); - const cacheHash = yield* crypto.digest("SHA-256", new TextEncoder().encode(cachePath)); - const cacheNamespace = `cache-${Array.from(cacheHash, (byte) => - byte.toString(16).padStart(2, "0"), - ) - .join("") - .slice(0, 32)}`; - const marker = yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ - backend: "docker", - volume: "database-volume", - namespace: `instance-${stackId}-${instanceId}`, - cacheNamespace, - initialized: false, - }); - yield* fs.writeFileString( - path.join(instanceRoot, ".supabase-database-storage.json"), - marker, - ); - const container = yield* makeContainerRuntime({ - engine: "podman", - root, - imageMirrors: () => [helperMirror], - }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const helpers = yield* makeDockerHelperRegistry("mirror-test"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", - stackId, - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - helpers, - }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + 'descriptor=$(/usr/bin/busybox find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', + ]); + const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( + Effect.matchEffect({ + onFailure: () => Effect.succeed(true), + onSuccess: () => Effect.succeed(false), + }), + ); + expect(corrupt).toBe(true); + yield* source.saveSnapshot("17", "roundtrip"); - yield* storage.prepare("17"); + yield* target.restoreSnapshot("17", "roundtrip"); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `rm -rf ${quote(`/store/${targetMarker.namespace}/data`)}`, + ]); + const missing = yield* target.prepare("17").pipe(Effect.exit); + expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); - const run = engine.commands.find((args) => args[0] === "run"); - const shellIndex = run?.indexOf("/bin/sh") ?? -1; - expect(run?.[shellIndex - 1]).toBe(helperMirror); - expect( - engine.commands.filter((args) => args[0] === "pull").map((args) => args.at(-1)), - ).toEqual([expect.stringContaining("ghcr.io/supabase/cli/postgres:"), helperMirror]); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); - }); + yield* target.removeData("17"); + yield* source.saveSnapshot("17", "retention-first"); + yield* source.saveSnapshot("17", "retention-second"); + yield* source.saveSnapshot("17", "retention-third"); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceMarker.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `test "$(/usr/bin/busybox find ${quote(`/store/${sourceMarker.cacheNamespace}/entries`)} -mindepth 1 -maxdepth 1 -type d | /usr/bin/busybox wc -l)" -eq 3`, + ]); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(false); + yield* source.saveSnapshot("17", "roundtrip"); + yield* source.destroyData("17"); + expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); + yield* target.destroyData("17"); + yield* engine(["volume", "rm", sourceMarker.volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - it.live("round-trips PostgreSQL 15 data with its catalog image", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("15"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-pg15-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const sourceRoot = path.join(storageRoot, "source"); - const targetRoot = path.join(storageRoot, "target"); - yield* fs.makeDirectory(sourceRoot, { recursive: true }); - yield* fs.makeDirectory(targetRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-pg15-${yield* crypto.randomUUIDv4}`; - const makeStorage = (instanceId: string, instanceRoot: string) => - makeDockerDatabaseStorage({ + it.live("labels its volume with the configured test run", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-test-run-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-test-run-${yield* crypto.randomUUIDv4}`; + const testRunId = `storage-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; + // This run id overrides the ambient one, so the shared run teardown never sees the volume. + yield* Effect.addFinalizer(() => removeTestRunVolumes(testRunId).pipe(Effect.orDie)); + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", + target: dockerTarget, stackId, - instanceId, + instanceId: "database", instanceRoot, root: storageRoot, cacheRoot, @@ -564,563 +838,254 @@ describe("Docker database storage", { timeout: 120_000 }, () => { container, spawner, }); - const source = yield* makeStorage("source", sourceRoot); - const target = yield* makeStorage("target", targetRoot); - const ownerScope = yield* Scope.Scope; - yield* Scope.addFinalizer( - ownerScope, - Effect.gen(function* () { - yield* source.destroyData("15").pipe(Effect.ignore); - yield* target.destroyData("15").pipe(Effect.ignore); - const marker = yield* fs - .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) - .pipe(Effect.option); - if (Option.isSome(marker)) { - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - marker.value, - ).pipe(Effect.option); - if (Option.isSome(parsed) && parsed.value.volume !== undefined) - yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); - } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.ignore, - ), - ); - - yield* source.prepare("15"); - const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `set -eu; printf 15 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf pg15-source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, - ]); - yield* source.markInitialized("15"); - yield* fs.writeFileString( - path.join(sourceRoot, ".supabase-database-ready.json"), - '{"version":"15","runtime":"docker","profile":"supabase"}', - ); - yield* source.saveSnapshot("15", "restore-me"); + yield* storage + .prepare("17") + .pipe( + Effect.provide( + ConfigProvider.layer( + ConfigProvider.fromEnvRecord({ SUPABASE_STACK_TEST_RUN: testRunId }), + ), + ), + ); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const labels = yield* engine([ + "volume", + "inspect", + "--format", + '{{ index .Labels "com.supabase.stack-test-run" }}', + marker.volume, + ]); + expect(labels).toBe(testRunId); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - yield* target.prepare("15"); - expect(yield* target.restoreSnapshot("15", "restore-me")).toBe(true); - const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), - ); - expect(targetMarker.volume).toBe(sourceMarker.volume); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, - helperImage, - "/bin/sh", - "-c", - 'test "$(cat /store/PG_VERSION)" = 15 && test "$(cat /store/fixture)" = pg15-source', - ]); - yield* target.destroyData("15"); - yield* source.destroyData("15"); - yield* docker(["volume", "rm", sourceMarker.volume]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("round-trips stopped data and protects managed namespaces", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-" }); - const stateRoot = path.join(root, "state"); - const storageRoot = path.join(stateRoot, "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const sourceRoot = path.join(storageRoot, "source"); - const targetRoot = path.join(storageRoot, "target"); - yield* fs.makeDirectory(sourceRoot, { recursive: true }); - yield* fs.makeDirectory(targetRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const makeStorageAt = ( - instanceId: string, - instanceRoot: string, - storageRootValue: string, - cacheRootValue: string, - ) => - makeDockerDatabaseStorage({ + it.live("removes an unprepared storage without requiring Docker", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); + const instanceRoot = path.join(root, "state", "stack", "data", "empty"); + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", - stackId: "storage-test", - instanceId, + target: dockerTarget, + stackId: "storage-empty-test", + instanceId: "empty", instanceRoot, - root: storageRootValue, - cacheRoot: cacheRootValue, + root: path.join(root, "state", "stack", "data"), + cacheRoot: path.join(root, "cache"), fs, path, crypto, - container, + container: undefined, spawner, }); - const makeStorage = (instanceId: string, instanceRoot: string) => - makeStorageAt(instanceId, instanceRoot, storageRoot, cacheRoot); - const source = yield* makeStorage("source", sourceRoot); - const target = yield* makeStorage("target", targetRoot); - const ownerScope = yield* Scope.Scope; - yield* Scope.addFinalizer( - ownerScope, - Effect.gen(function* () { - yield* source.destroyData("17").pipe(Effect.ignore); - yield* target.destroyData("17").pipe(Effect.ignore); - const marker = yield* fs - .readFileString(path.join(sourceRoot, ".supabase-database-storage.json")) - .pipe(Effect.option); - if (Option.isSome(marker)) { - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - marker.value, - ).pipe(Effect.option); - if (Option.isSome(parsed) && parsed.value.volume !== undefined) - yield* docker(["volume", "rm", parsed.value.volume]).pipe(Effect.ignore); - } - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.ignore, - ), - ); - yield* source.prepare("17"); - const sourceMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - if (sourceMarker.backend !== "docker" || sourceMarker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `set -eu; printf 17 > ${quote(`/store/${sourceMarker.namespace}/data/PG_VERSION`)}; printf source > ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}; /usr/bin/busybox setfattr -n user.storage-smoke -v preserved ${quote(`/store/${sourceMarker.namespace}/data/fixture`)}`, - ]); - yield* source.markInitialized("17"); - yield* fs.writeFileString( - path.join(sourceRoot, ".supabase-database-ready.json"), - '{"version":"17","runtime":"docker","profile":"supabase"}', - ); - yield* source.saveSnapshot("17", "roundtrip"); + yield* storage.removeData("17"); + yield* storage.destroyData("17"); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + false, + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); - const copiedRoot = path.join(root, "copied-state", "stack", "data", "source"); - const copiedStorageRoot = path.join(root, "copied-state", "stack", "data"); - const copiedCacheRoot = path.join(root, "copied-cache"); - yield* fs.makeDirectory(copiedRoot, { recursive: true }); - yield* fs.makeDirectory(copiedCacheRoot, { recursive: true }); - yield* fs.writeFileString( - path.join(copiedRoot, ".supabase-database-storage.json"), - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - const copied = yield* makeStorageAt( - "source", - copiedRoot, - copiedStorageRoot, - copiedCacheRoot, - ); - const mountError = yield* copied.mount("17").pipe(Effect.flip); - expect(mountError.message).toContain("another state directory"); - const removeError = yield* copied.removeData("17").pipe(Effect.flip); - expect(removeError.message).toContain("another state directory"); - const destroyError = yield* copied.destroyData("17").pipe(Effect.flip); - expect(destroyError.message).toContain("another state directory"); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `test "$(cat /store/${sourceMarker.namespace}/data/fixture)" = source`, - ]); + it.live( + "keeps the prior marker decodable when a real kill lands between staging and publishing an update", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-crash-" }); + const instanceRoot = path.join(root, "state", "stack", "data", "crash"); + const dataRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const helperImage = yield* postgresImage("17"); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-crash-test", + instanceId: "crash", + instanceRoot, + root: dataRoot, + cacheRoot, + fs, + path, + crypto, + container: yield* makeContainerRuntime({ + target: dockerTarget, + root: instanceRoot, + }), + spawner, + }); + // Removes the real volume the seeding step below creates, rather than leaving it for + // the global test-run teardown, since a container that mounted it may not yet be + // reaped by the time that teardown runs. + yield* Effect.addFinalizer(() => storage.destroyData("17").pipe(Effect.ignore)); + yield* storage.prepare("17"); + const before = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(before.initialized).toBe(false); + if (before.backend !== "docker" || before.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = before.volume; + // The writer below is killed, not interrupted, so its own one-off helper container (the + // PG_VERSION check's, and any the killed markInitialized retry below starts) never runs + // its own cleanup; removed here instead of leaving it attached when `destroyData` above + // (registered first, so it runs after this) tries to remove the volume. + yield* Effect.addFinalizer(() => + engine(["ps", "--all", "--quiet", "--filter", `volume=${volume}`]).pipe( + Effect.flatMap((listed) => { + const ids = listed + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0); + return ids.length === 0 + ? Effect.void + : engine(["rm", "--force", ...ids]).pipe(Effect.asVoid); + }), + Effect.ignore, + ), + ); + // `markInitialized` (below, and inside the killed writer) checks this file through a + // helper container; this test is about the marker's own crash-safety, not PostgreSQL's, + // so it is seeded directly rather than through a real database launch. + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${before.volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `set -eu; mkdir -p ${quote(`/store/${before.namespace}/data`)}; printf 17 > ${quote(`/store/${before.namespace}/data/PG_VERSION`)}`, + ]); - yield* target.prepare("17"); - expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); - const targetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), - ); - expect(targetMarker.volume).toBe(sourceMarker.volume); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store,volume-subpath=${targetMarker.namespace}/data`, - helperImage, - "/bin/sh", - "-c", - `set -eu; test "$(cat /store/fixture)" = source; /usr/bin/busybox setfattr -x user.storage-smoke /store/fixture`, - ]); + const writer = yield* spawner.spawn( + ChildProcess.make( + process.execPath, + [dockerStoragePublishLoopFixture, instanceRoot, dataRoot, cacheRoot], + { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, + ), + ); + const ready = yield* Deferred.make(); + const stderr = yield* Ref.make(""); + const diagnostics = yield* writer.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + const output = yield* writer.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.tap((line) => + line === "about-to-publish" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + Stream.runDrain, + Effect.forkScoped, + ); + const writerFailure = (reason: string) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail(new PublishBarrierError({ message: `${reason}: ${text}` })), + ), + ); + // The fixture announces this boundary itself, right before the real rename call; + // killing here lands in the staging-to-publish gap on every run, not "most of the time". + yield* Deferred.await(ready).pipe( + Effect.raceFirst( + writer.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + writerFailure(`Writer exited before readiness: ${String(cause)}`), + onSuccess: (code) => writerFailure(`Writer exited before readiness (${code})`), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => writerFailure("Writer did not reach the publish boundary"), + }), + ); + yield* Effect.sync(() => process.kill(Number(writer.pid), "SIGKILL")); + yield* writer.exitCode.pipe(Effect.ignore); + yield* Fiber.join(output); + yield* Fiber.join(diagnostics); - yield* target.removeData("unsupported"); - const resetMarker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(targetRoot, ".supabase-database-storage.json")), - ); - expect(resetMarker.volume).toBe(sourceMarker.volume); - expect(resetMarker.initialized).toBe(false); + // The killed writer never reached the rename: the prior marker is untouched, and only + // its staging file, never a partial target, is left behind. + const afterKill = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(afterKill).toEqual(before); + expect( + (yield* fs.readDirectory(instanceRoot)).filter((entry) => entry.endsWith(".tmp")), + ).toHaveLength(1); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - 'descriptor=$(/usr/bin/busybox find /store -name descriptor.json -print -quit); printf corrupt > "$descriptor"', - ]); - const corrupt = yield* target.restoreSnapshot("17", "roundtrip").pipe( - Effect.matchEffect({ - onFailure: () => Effect.succeed(true), - onSuccess: () => Effect.succeed(false), + yield* storage.markInitialized("17"); + const afterRetry = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), + ); + expect(afterRetry.initialized).toBe(true); }), - ); - expect(corrupt).toBe(true); - yield* source.saveSnapshot("17", "roundtrip"); - - yield* target.restoreSnapshot("17", "roundtrip"); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `rm -rf ${quote(`/store/${targetMarker.namespace}/data`)}`, - ]); - const missing = yield* target.prepare("17").pipe(Effect.exit); - expect(missing).toSatisfy((exit) => Exit.isFailure(exit)); - - yield* target.removeData("17"); - yield* source.saveSnapshot("17", "retention-first"); - yield* source.saveSnapshot("17", "retention-second"); - yield* source.saveSnapshot("17", "retention-third"); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceMarker.volume},dst=/store`, - helperImage, - "/bin/sh", - "-c", - `test "$(/usr/bin/busybox find ${quote(`/store/${sourceMarker.cacheNamespace}/entries`)} -mindepth 1 -maxdepth 1 -type d | /usr/bin/busybox wc -l)" -eq 3`, - ]); - expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(false); - yield* source.saveSnapshot("17", "roundtrip"); - yield* source.destroyData("17"); - expect(yield* target.restoreSnapshot("17", "roundtrip")).toBe(true); - yield* target.destroyData("17"); - yield* docker(["volume", "rm", sourceMarker.volume]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + ).pipe(Effect.provide(NodeServices.layer)), + 20_000, + ); - it.live("labels its volume with the configured test run", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-test-run-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-test-run-${yield* crypto.randomUUIDv4}`; - const testRunId = `storage-test-run-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; - // This run id overrides the ambient one, so the shared run teardown never sees the volume. - yield* Effect.addFinalizer(() => removeTestRunVolumes(testRunId).pipe(Effect.orDie)); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - stackId, - instanceId: "database", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); - yield* storage - .prepare("17") - .pipe( - Effect.provide( - ConfigProvider.layer( - ConfigProvider.fromEnvRecord({ SUPABASE_STACK_TEST_RUN: testRunId }), - ), - ), + it.live("refuses to start unmarked data and removes it through the helper on destroy", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "unmarked"); + const dataRoot = path.join(instanceRoot, "data"); + const checkpointsRoot = path.join(instanceRoot, ".supabase-snapshots"); + yield* fs.makeDirectory(path.join(dataRoot, "base"), { recursive: true }); + yield* fs.writeFileString(path.join(dataRoot, "base", "fixture"), "unmarked"); + yield* fs.makeDirectory(path.join(checkpointsRoot, "entries", "checkpoint", "data"), { + recursive: true, + }); + yield* fs.writeFileString( + path.join(checkpointsRoot, "entries", "checkpoint", "data", "PG_VERSION"), + "17", ); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), - ); - if (marker.backend !== "docker" || marker.volume === undefined) - return yield* new DockerTestError({ message: "Docker test selected host fallback" }); - const labels = yield* docker([ - "volume", - "inspect", - "--format", - '{{ index .Labels "com.supabase.stack-test-run" }}', - marker.volume, - ]); - expect(labels).toBe(testRunId); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("removes an unprepared storage without requiring Docker", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-empty-" }); - const instanceRoot = path.join(root, "state", "stack", "data", "empty"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - stackId: "storage-empty-test", - instanceId: "empty", - instanceRoot, - root: path.join(root, "state", "stack", "data"), - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - container: undefined, - spawner, - }); - yield* storage.removeData("17"); - yield* storage.destroyData("17"); - expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( - false, - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("refuses to start unmarked data and removes it through the helper on destroy", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "unmarked"); - const dataRoot = path.join(instanceRoot, "data"); - const checkpointsRoot = path.join(instanceRoot, ".supabase-snapshots"); - yield* fs.makeDirectory(path.join(dataRoot, "base"), { recursive: true }); - yield* fs.writeFileString(path.join(dataRoot, "base", "fixture"), "unmarked"); - yield* fs.makeDirectory(path.join(checkpointsRoot, "entries", "checkpoint", "data"), { - recursive: true, - }); - yield* fs.writeFileString( - path.join(checkpointsRoot, "entries", "checkpoint", "data", "PG_VERSION"), - "17", - ); - yield* docker([ - "run", - "--rm", - "--mount", - `type=bind,src=${instanceRoot},dst=/instance`, - helperImage, - "/bin/sh", - "-c", - "chown -R 100:101 /instance/data /instance/.supabase-snapshots; chmod -R 700 /instance/data /instance/.supabase-snapshots", - ]); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - stackId: `storage-unmarked-${yield* crypto.randomUUIDv4}`, - instanceId: "unmarked", - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - }); - const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); - - const failure = yield* storage.prepare("17").pipe(Effect.flip); - expect(failure.message).toContain("without a storage marker"); - expect(yield* fs.exists(markerPath)).toBe(false); - - yield* storage.destroyData("17"); - expect(yield* fs.exists(dataRoot)).toBe(false); - expect(yield* fs.exists(checkpointsRoot)).toBe(false); - expect(yield* fs.exists(markerPath)).toBe(false); - yield* fs.remove(cacheRoot, { recursive: true, force: true }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("refuses to adopt unmarked Podman data at startup", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "podman-storage-unmarked-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const instanceRoot = path.join(storageRoot, "unmarked"); - const dataRoot = path.join(instanceRoot, "data"); - yield* fs.makeDirectory(dataRoot, { recursive: true }); - yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "podman", - stackId: "storage-podman-unmarked", - instanceId: "unmarked", - instanceRoot, - root: storageRoot, - cacheRoot: path.join(root, "cache"), - fs, - path, - crypto, - container: yield* makeContainerRuntime({ engine: "podman", root }), - spawner: yield* ChildProcessSpawner.ChildProcessSpawner, - }); - - const failure = yield* storage.prepare("17").pipe(Effect.flip); - expect(failure.message).toContain("without a storage marker"); - expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( - false, - ); - expect(yield* fs.readFileString(path.join(dataRoot, "PG_VERSION"))).toBe("17\n"); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("recovers after an owned helper is removed externally", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "recovery"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; - const instanceId = "recovery"; - const helperScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(helperScope, Exit.void)); - const helperRegistry = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4).pipe( - Effect.provideService(Scope.Scope, helperScope), - ); - const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", - stackId, - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot, - fs, - path, - crypto, - container, - spawner, - helpers: helperRegistry, - }); - yield* storage.prepare("17"); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), - ); - const volume = marker.volume; - if (volume === undefined) - return yield* new DockerTestError({ message: "Docker storage volume missing" }); - const helpers = yield* docker([ - "ps", - "--filter", - "label=com.supabase.stack=" + stackId, - "--format", - "{{.Names}}", - ]); - const helper = helpers.split("\n").find((name) => name.length > 0); - if (helper === undefined) - return yield* new DockerTestError({ message: "Owned helper was not discoverable" }); - yield* docker(["rm", "-f", helper]); - yield* storage.prepare("17"); - yield* storage.destroyData("17"); - yield* Scope.close(helperScope, Exit.void); - yield* docker(["volume", "rm", volume]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("keeps another owner's volume helper running when one owner closes", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-owner-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "database"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-helper-owner-${yield* crypto.randomUUIDv4}`; - const firstScope = yield* Scope.make(); - const secondScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => - Effect.gen(function* () { - yield* Scope.close(secondScope, Exit.void); - yield* Scope.close(firstScope, Exit.void); - const markerText = yield* fs - .readFileString(path.join(instanceRoot, ".supabase-database-storage.json")) - .pipe(Effect.option); - if (Option.isNone(markerText)) return; - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - markerText.value, - ).pipe(Effect.option); - if (Option.isSome(marker) && marker.value.volume !== undefined) - yield* docker(["volume", "rm", marker.value.volume]).pipe(Effect.ignore); - }).pipe(Effect.ignore), - ); - const firstHelpers = yield* makeDockerHelperRegistry("first-owner").pipe( - Effect.provideService(Scope.Scope, firstScope), - ); - const secondHelpers = yield* makeDockerHelperRegistry("second-owner").pipe( - Effect.provideService(Scope.Scope, secondScope), - ); - const makeStorage = (helpers: DockerHelperRegistry) => - makeDockerDatabaseStorage({ + yield* engine([ + "run", + "--rm", + "--mount", + `type=bind,src=${instanceRoot},dst=/instance`, + helperImage, + "/bin/sh", + "-c", + "chown -R 100:101 /instance/data /instance/.supabase-snapshots; chmod -R 700 /instance/data /instance/.supabase-snapshots", + ]); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", - stackId, - instanceId: "database", + target: dockerTarget, + stackId: `storage-unmarked-${yield* crypto.randomUUIDv4}`, + instanceId: "unmarked", instanceRoot, root: storageRoot, cacheRoot, @@ -1129,67 +1094,91 @@ describe("Docker database storage", { timeout: 120_000 }, () => { crypto, container, spawner, - helpers, }); - const firstStorage = yield* makeStorage(firstHelpers).pipe( - Effect.provideService(Scope.Scope, firstScope), - ); - const secondStorage = yield* makeStorage(secondHelpers).pipe( - Effect.provideService(Scope.Scope, secondScope), - ); - yield* firstStorage.prepare("17"); - yield* secondStorage.prepare("17"); - const runningHelpers = (yield* docker([ - "ps", - "--filter", - `label=com.supabase.stack=${stackId}`, - "--format", - "{{.Names}}", - ])) - .split("\n") - .filter((name) => name.length > 0); - expect(runningHelpers).toHaveLength(2); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); - yield* Scope.close(firstScope, Exit.void); - const remainingHelpers = (yield* docker([ - "ps", - "--filter", - `label=com.supabase.stack=${stackId}`, - "--format", - "{{.Names}}", - ])) - .split("\n") - .filter((name) => name.length > 0); - expect(remainingHelpers).toHaveLength(1); - expect(runningHelpers).toContain(remainingHelpers[0]); - yield* secondStorage.prepare("17"); - yield* secondStorage.destroyData("17"); - yield* Scope.close(secondScope, Exit.void); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + const failure = yield* storage.prepare("17").pipe(Effect.flip); + expect(failure.message).toContain("without a storage marker"); + expect(yield* fs.exists(markerPath)).toBe(false); - it.live("destroys an initialized storage after its volume is removed", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); - const storageRoot = path.join(root, "state", "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const instanceRoot = path.join(storageRoot, "missing"); - yield* fs.makeDirectory(instanceRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; - const makeStorage = () => - makeDockerDatabaseStorage({ + yield* storage.destroyData("17"); + expect(yield* fs.exists(dataRoot)).toBe(false); + expect(yield* fs.exists(checkpointsRoot)).toBe(false); + expect(yield* fs.exists(markerPath)).toBe(false); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("refuses to adopt unmarked Docker data at startup", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-unmarked-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const instanceRoot = path.join(storageRoot, "unmarked"); + const dataRoot = path.join(instanceRoot, "data"); + yield* fs.makeDirectory(dataRoot, { recursive: true }); + yield* fs.writeFileString(path.join(dataRoot, "PG_VERSION"), "17\n"); + const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", + target: dockerTarget, + stackId: "storage-docker-unmarked", + instanceId: "unmarked", + instanceRoot, + root: storageRoot, + cacheRoot: path.join(root, "cache"), + fs, + path, + crypto, + container: yield* makeContainerRuntime({ + target: dockerTarget, + root, + }), + spawner: yield* ChildProcessSpawner.ChildProcessSpawner, + }); + + const failure = yield* storage.prepare("17").pipe(Effect.flip); + expect(failure.message).toContain("without a storage marker"); + expect(yield* fs.exists(path.join(instanceRoot, ".supabase-database-storage.json"))).toBe( + false, + ); + expect(yield* fs.readFileString(path.join(dataRoot, "PG_VERSION"))).toBe("17\n"); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("recovers after an owned helper is removed externally", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-helper-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "recovery"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-helper-recovery-${yield* crypto.randomUUIDv4}`; + const instanceId = "recovery"; + const helperScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(helperScope, Exit.void)); + const helperRegistry = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4).pipe( + Effect.provideService(Scope.Scope, helperScope), + ); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, stackId, - instanceId: "missing", + instanceId, instanceRoot, root: storageRoot, cacheRoot, @@ -1198,22 +1187,266 @@ describe("Docker database storage", { timeout: 120_000 }, () => { crypto, container, spawner, + helpers: helperRegistry, + }); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + const volume = marker.volume; + if (volume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + const helpers = yield* engine([ + "ps", + "--filter", + "label=com.supabase.stack=" + stackId, + "--format", + "{{.Names}}", + ]); + const helper = helpers.split("\n").find((name) => name.length > 0); + if (helper === undefined) + return yield* new DockerTestError({ message: "Owned helper was not discoverable" }); + yield* engine(["rm", "-f", helper]); + yield* storage.prepare("17"); + yield* storage.destroyData("17"); + yield* Scope.close(helperScope, Exit.void); + yield* engine(["volume", "rm", volume]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("keeps another owner's volume helper running when one owner closes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "docker-storage-helper-owner-", }); - let volume: string | undefined; - yield* Effect.scoped( + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "database"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-helper-owner-${yield* crypto.randomUUIDv4}`; + const firstScope = yield* Scope.make(); + const secondScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* Scope.close(secondScope, Exit.void); + yield* Scope.close(firstScope, Exit.void); + const markerText = yield* fs + .readFileString(path.join(instanceRoot, ".supabase-database-storage.json")) + .pipe(Effect.option); + if (Option.isNone(markerText)) return; + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + markerText.value, + ).pipe(Effect.option); + if (Option.isSome(marker) && marker.value.volume !== undefined) + yield* engine(["volume", "rm", marker.value.volume]).pipe(Effect.ignore); + }).pipe(Effect.ignore), + ); + const firstHelpers = yield* makeDockerHelperRegistry("first-owner").pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + const secondHelpers = yield* makeDockerHelperRegistry("second-owner").pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + const makeStorage = (helpers: DockerHelperRegistry) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "database", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + helpers, + }); + const firstStorage = yield* makeStorage(firstHelpers).pipe( + Effect.provideService(Scope.Scope, firstScope), + ); + const secondStorage = yield* makeStorage(secondHelpers).pipe( + Effect.provideService(Scope.Scope, secondScope), + ); + yield* firstStorage.prepare("17"); + yield* secondStorage.prepare("17"); + const runningHelpers = (yield* engine([ + "ps", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--format", + "{{.Names}}", + ])) + .split("\n") + .filter((name) => name.length > 0); + expect(runningHelpers).toHaveLength(2); + + yield* Scope.close(firstScope, Exit.void); + const remainingHelpers = (yield* engine([ + "ps", + "--filter", + `label=com.supabase.stack=${stackId}`, + "--format", + "{{.Names}}", + ])) + .split("\n") + .filter((name) => name.length > 0); + expect(remainingHelpers).toHaveLength(1); + expect(runningHelpers).toContain(remainingHelpers[0]); + yield* secondStorage.prepare("17"); + yield* secondStorage.destroyData("17"); + yield* Scope.close(secondScope, Exit.void); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("destroys an initialized storage after its volume is removed", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-missing-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "missing"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-missing-${yield* crypto.randomUUIDv4}`; + const makeStorage = () => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "missing", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + let volume: string | undefined; + yield* Effect.scoped( + Effect.gen(function* () { + const storage = yield* makeStorage(); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + ), + ); + volume = marker.volume; + if (volume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + }), + ); + if (volume === undefined) + return yield* new DockerTestError({ message: "Missing volume" }); + const removedVolume = volume; + yield* engine(["volume", "rm", removedVolume]); + yield* Effect.scoped( + Effect.gen(function* () { + const storage = yield* makeStorage(); + const prepare = yield* storage.prepare("17").pipe(Effect.exit); + expect(prepare).toSatisfy((exit) => Exit.isFailure(exit)); + const removeError = yield* storage.removeData("17").pipe(Effect.flip); + expect(removeError.message).toMatch(/no such volume/iu); + expect( + yield* engine(["volume", "inspect", removedVolume]).pipe(Effect.exit), + ).toSatisfy((exit) => Exit.isFailure(exit)); + expect(yield* storage.destroyData("17").pipe(Effect.exit)).toSatisfy((exit) => + Exit.isSuccess(exit), + ); + }), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "destroys the owned namespace in the shared volume after its on-disk marker is deleted, using the identity resolved in memory", + () => + Effect.scoped( Effect.gen(function* () { - const storage = yield* makeStorage(); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-no-marker-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "sleeping"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-no-marker-${yield* crypto.randomUUIDv4}`; + // One storage object spans the whole case, the way an Owner keeps one per instance for + // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "sleeping", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); yield* storage.prepare("17"); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + yield* fs.readFileString(markerPath), ); - volume = marker.volume; - if (volume === undefined) - return yield* new DockerTestError({ message: "Docker storage volume missing" }); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = marker.volume; const mount = yield* storage.mount("17"); if (mount.type !== "volume" || mount.volumeSubpath === undefined) return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); - yield* docker([ + yield* engine([ "run", "--rm", "--mount", @@ -1224,30 +1457,245 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "printf 17 > /data/PG_VERSION", ]); yield* storage.markInitialized("17"); + + // The database sleeps (the registered instance stays stopped); its storage object keeps + // running, so this is the only change: the registration's on-disk marker is now gone. + yield* fs.remove(markerPath); + + yield* storage.destroyData("17"); + + const remaining = yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, + ]); + expect(remaining.trim()).toBe("absent"); + yield* engine(["volume", "rm", volume]); }), - ); - if (volume === undefined) return yield* new DockerTestError({ message: "Missing volume" }); - const removedVolume = volume; - yield* docker(["volume", "rm", removedVolume]); - yield* Effect.scoped( + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "destroys the owned namespace after its whole state root is deleted, with no filesystem dependency for an identity already resolved in memory", + () => + Effect.scoped( Effect.gen(function* () { - const storage = yield* makeStorage(); - const prepare = yield* storage.prepare("17").pipe(Effect.exit); - expect(prepare).toSatisfy((exit) => Exit.isFailure(exit)); - const removeError = yield* storage.removeData("17").pipe(Effect.flip); - expect(removeError.message).toMatch(/no such volume/iu); - expect(yield* docker(["volume", "inspect", removedVolume]).pipe(Effect.exit)).toSatisfy( - (exit) => Exit.isFailure(exit), - ); - expect(yield* storage.destroyData("17").pipe(Effect.exit)).toSatisfy((exit) => - Exit.isSuccess(exit), + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + // `root` is deleted entirely below; nest it under the scope's own temp dir so the test + // framework's own cleanup of that outer, still-present directory does not fail. + const tempDir = yield* fs.makeTempDirectoryScoped({ + prefix: "docker-storage-no-root-", + }); + const root = path.join(tempDir, "state-root"); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "sleeping"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const stackId = `storage-no-root-${yield* crypto.randomUUIDv4}`; + // One storage object spans the whole case, the way an Owner keeps one per instance for + // the stack's lifetime: its resolved identity stays in memory across the sleep below. + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId, + instanceId: "sleeping", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.prepare("17"); + const markerPath = path.join(instanceRoot, ".supabase-database-storage.json"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(markerPath), ); + if (marker.backend !== "docker" || marker.volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + const volume = marker.volume; + const mount = yield* storage.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION", + ]); + yield* storage.markInitialized("17"); + + // The database sleeps; its storage object keeps running (the resolved identity is + // already in memory), then the whole state root disappears from under the live owner. + yield* fs.remove(root, { recursive: true, force: true }); + + yield* storage.destroyData("17"); + + const remaining = yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `[ -e ${quote(`/store/${marker.namespace}`)} ] && echo present || echo absent`, + ]); + expect(remaining.trim()).toBe("absent"); + yield* engine(["volume", "rm", volume]); }), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("reopens managed storage and preserves snapshots after source destruction", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); + const stateRoot = path.join(root, "state"); + const storageRoot = path.join(stateRoot, "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const sourceRoot = path.join(storageRoot, "source"); + const targetRoot = path.join(storageRoot, "target"); + yield* fs.makeDirectory(sourceRoot, { recursive: true }); + yield* fs.makeDirectory(targetRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const container = yield* makeContainerRuntime({ + target: dockerTarget, + root, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const makeStorageWithCache = ( + instanceId: string, + instanceRoot: string, + cacheRootValue: string, + ) => + makeDockerDatabaseStorage({ + runtime: "docker", + target: dockerTarget, + stackId: "storage-reopen-test", + instanceId, + instanceRoot, + root: storageRoot, + cacheRoot: cacheRootValue, + fs, + path, + crypto, + container, + spawner, + }); + const makeStorage = (instanceId: string, instanceRoot: string) => + makeStorageWithCache(instanceId, instanceRoot, cacheRoot); + let sourceVolume: string | undefined; + const ownerScope = yield* Scope.Scope; + yield* Scope.addFinalizer( + ownerScope, + Effect.gen(function* () { + if (sourceVolume !== undefined) yield* engine(["volume", "rm", sourceVolume]); + yield* fs.remove(cacheRoot, { recursive: true, force: true }); + }).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.catchCause((cause) => Effect.die(cause)), + ), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const source = yield* makeStorage("source", sourceRoot); + yield* source.prepare("17"); + const mount = yield* source.mount("17"); + if (mount.type !== "volume" || mount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), + ); + sourceVolume = marker.volume; + if (sourceVolume === undefined) + return yield* new DockerTestError({ message: "Docker storage volume missing" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceVolume},dst=/data,volume-subpath=${mount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + "printf 17 > /data/PG_VERSION; printf reopened > /data/fixture", + ]); + yield* source.markInitialized("17"); + yield* fs.writeFileString( + path.join(sourceRoot, ".supabase-database-ready.json"), + '{"version":"17","runtime":"docker","profile":"supabase"}', + ); + yield* source.saveSnapshot("17", "reopened"); + }), + ); + yield* Effect.scoped( + Effect.gen(function* () { + const changedCacheRoot = path.join(root, "changed-cache"); + yield* fs.makeDirectory(changedCacheRoot, { recursive: true }); + const changedCache = yield* makeStorageWithCache( + "source", + sourceRoot, + changedCacheRoot, + ); + yield* changedCache.prepare("17"); + yield* changedCache.removeData("17"); + expect(yield* changedCache.restoreSnapshot("17", "reopened")).toBe(false); + const source = yield* makeStorage("source", sourceRoot); + yield* source.prepare("17"); + yield* source.removeData("17"); + expect(yield* source.restoreSnapshot("17", "reopened")).toBe(true); + yield* source.saveSnapshot("17", "reopened-again"); + yield* source.destroyData("17"); + const target = yield* makeStorage("target", targetRoot); + yield* target.prepare("17"); + expect(yield* target.restoreSnapshot("17", "reopened-again")).toBe(true); + const targetMount = yield* target.mount("17"); + if (targetMount.type !== "volume" || targetMount.volumeSubpath === undefined) + return yield* new DockerTestError({ message: "Docker target volume unavailable" }); + yield* engine([ + "run", + "--rm", + "--mount", + `type=volume,src=${sourceVolume ?? ""},dst=/data,volume-subpath=${targetMount.volumeSubpath}`, + helperImage, + "/bin/sh", + "-c", + 'test "$(cat /data/fixture)" = reopened', + ]); + yield* target.destroyData("17"); + }), + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + }, +); + +describe("Host database storage", { timeout: 120_000 }, () => { it.live("handles root-owned host data through helper operations", () => Effect.scoped( Effect.gen(function* () { @@ -1280,11 +1728,14 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* fs.writeFileString(path.join(dataRoot, "fixture"), "adopted"); yield* fs.writeFileString( path.join(instanceRoot, ".supabase-database-ready.json"), - '{"version":"17","runtime":"docker","profile":"supabase"}', + `{"version":"17","runtime":"${testEngine}","profile":"supabase"}`, ); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const container = yield* makeContainerRuntime({ + target: containerTarget, + root, + }); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - yield* docker([ + yield* engine([ "run", "--rm", "--mount", @@ -1295,7 +1746,8 @@ describe("Docker database storage", { timeout: 120_000 }, () => { "chown -R 100:101 /instance/data; chmod 700 /instance/data", ]); const storage = yield* makeDockerDatabaseStorage({ - runtime: "docker", + runtime: testEngine, + target: containerTarget, stackId: "storage-host-test", instanceId: "adopted", instanceRoot, @@ -1321,7 +1773,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ); expect(marker.backend).toBe("host"); expect(marker.initialized).toBe(true); - const expectedOwnership = yield* docker([ + const expectedOwnership = yield* engine([ "run", "--rm", "--mount", @@ -1355,7 +1807,7 @@ describe("Docker database storage", { timeout: 120_000 }, () => { yield* storage.removeData("17"); expect(yield* storage.restoreSnapshot("17", "adopted")).toBe(true); expect( - yield* docker([ + yield* engine([ "run", "--rm", "--mount", @@ -1379,126 +1831,4 @@ describe("Docker database storage", { timeout: 120_000 }, () => { }), ).pipe(Effect.provide(NodeServices.layer)), ); - - it.live("reopens managed storage and preserves snapshots after source destruction", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const helperImage = yield* postgresImage("17"); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-reopen-" }); - const stateRoot = path.join(root, "state"); - const storageRoot = path.join(stateRoot, "stack", "data"); - const cacheRoot = path.join(root, "cache"); - const sourceRoot = path.join(storageRoot, "source"); - const targetRoot = path.join(storageRoot, "target"); - yield* fs.makeDirectory(sourceRoot, { recursive: true }); - yield* fs.makeDirectory(targetRoot, { recursive: true }); - yield* fs.makeDirectory(cacheRoot, { recursive: true }); - const container = yield* makeContainerRuntime({ engine: "docker", root }); - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const makeStorageWithCache = ( - instanceId: string, - instanceRoot: string, - cacheRootValue: string, - ) => - makeDockerDatabaseStorage({ - runtime: "docker", - stackId: "storage-reopen-test", - instanceId, - instanceRoot, - root: storageRoot, - cacheRoot: cacheRootValue, - fs, - path, - crypto, - container, - spawner, - }); - const makeStorage = (instanceId: string, instanceRoot: string) => - makeStorageWithCache(instanceId, instanceRoot, cacheRoot); - let sourceVolume: string | undefined; - const ownerScope = yield* Scope.Scope; - yield* Scope.addFinalizer( - ownerScope, - Effect.gen(function* () { - if (sourceVolume !== undefined) yield* docker(["volume", "rm", sourceVolume]); - yield* fs.remove(cacheRoot, { recursive: true, force: true }); - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.catchCause((cause) => Effect.die(cause)), - ), - ); - yield* Effect.scoped( - Effect.gen(function* () { - const source = yield* makeStorage("source", sourceRoot); - yield* source.prepare("17"); - const mount = yield* source.mount("17"); - if (mount.type !== "volume" || mount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker volume backend unavailable" }); - const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( - yield* fs.readFileString(path.join(sourceRoot, ".supabase-database-storage.json")), - ); - sourceVolume = marker.volume; - if (sourceVolume === undefined) - return yield* new DockerTestError({ message: "Docker storage volume missing" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceVolume},dst=/data,volume-subpath=${mount.volumeSubpath}`, - helperImage, - "/bin/sh", - "-c", - "printf 17 > /data/PG_VERSION; printf reopened > /data/fixture", - ]); - yield* source.markInitialized("17"); - yield* fs.writeFileString( - path.join(sourceRoot, ".supabase-database-ready.json"), - '{"version":"17","runtime":"docker","profile":"supabase"}', - ); - yield* source.saveSnapshot("17", "reopened"); - }), - ); - yield* Effect.scoped( - Effect.gen(function* () { - const changedCacheRoot = path.join(root, "changed-cache"); - yield* fs.makeDirectory(changedCacheRoot, { recursive: true }); - const changedCache = yield* makeStorageWithCache( - "source", - sourceRoot, - changedCacheRoot, - ); - yield* changedCache.prepare("17"); - yield* changedCache.removeData("17"); - expect(yield* changedCache.restoreSnapshot("17", "reopened")).toBe(false); - const source = yield* makeStorage("source", sourceRoot); - yield* source.prepare("17"); - yield* source.removeData("17"); - expect(yield* source.restoreSnapshot("17", "reopened")).toBe(true); - yield* source.saveSnapshot("17", "reopened-again"); - yield* source.destroyData("17"); - const target = yield* makeStorage("target", targetRoot); - yield* target.prepare("17"); - expect(yield* target.restoreSnapshot("17", "reopened-again")).toBe(true); - const targetMount = yield* target.mount("17"); - if (targetMount.type !== "volume" || targetMount.volumeSubpath === undefined) - return yield* new DockerTestError({ message: "Docker target volume unavailable" }); - yield* docker([ - "run", - "--rm", - "--mount", - `type=volume,src=${sourceVolume ?? ""},dst=/data,volume-subpath=${targetMount.volumeSubpath}`, - helperImage, - "/bin/sh", - "-c", - 'test "$(cat /data/fixture)" = reopened', - ]); - yield* target.destroyData("17"); - }), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); }); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index ce1fa866c3..fea5ee6e65 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -18,7 +18,8 @@ import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/u import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; import { failureMessage } from "../internal/failure-message.ts"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; -import type { ContainerRuntime } from "../runtime/Container.ts"; +import * as Publication from "../namespace/Publication.ts"; +import type { ContainerRuntime, EngineTarget } from "../runtime/Container.ts"; import { composeProjectFor } from "../runtime/ContainerName.ts"; import type { DatabaseRuntime } from "../services/Database.ts"; import { @@ -57,7 +58,11 @@ interface ResolvedDaemon extends DaemonIdentity { export class DockerDatabaseStorageError extends Schema.TaggedError()( "DockerDatabaseStorageError", - { operation: Schema.String, message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, + { + operation: Schema.String, + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + }, ) {} export interface DatabaseStorageMount { @@ -99,11 +104,7 @@ const errorFor = (operation: string, cause: unknown) => message: cause.message, cause, }) - : new DockerDatabaseStorageError({ - operation, - message: failureMessage(cause), - cause, - }); + : new DockerDatabaseStorageError({ operation, message: failureMessage(cause), cause }); const parseMajor = (version: string): number | undefined => { const major = Number(version.trim().split(".")[0]); @@ -122,6 +123,8 @@ const testRunLabelArgs = Effect.fn("DockerDatabaseStorage.testRunLabelArgs")(fun export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make")( (options: { readonly runtime: DatabaseRuntime; + /** The engine endpoint and identity the owner resolved once at startup. */ + readonly target: EngineTarget; readonly stackId: string; readonly instanceId: string; readonly project?: string; @@ -137,6 +140,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }): Effect.Effect => Effect.gen(function* () { const markerPath = options.path.join(options.instanceRoot, ".supabase-database-storage.json"); + // Atomic (stage, fsync, rename, fsync the directory), so a kill mid-write leaves the prior + // marker fully decodable rather than a partially written one. + const publishMarker = (content: string) => + Publication.publish(options.fs, options.path, { + target: markerPath, + content, + }).pipe(Effect.mapError((cause) => errorFor("marker", cause))); const composeHelperLabels = [ "--label", `com.docker.compose.project=${composeProjectFor(options.stackId, options.project)}`, @@ -204,7 +214,8 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") cacheNamespace: "native", initialized: false, }; - if (options.runtime !== "docker") { + // Podman has no volume-subpath backend; its data always lives in host directories. + if (options.target.engine === "podman") { const present = yield* options.fs .exists(markerPath) .pipe(Effect.mapError((cause) => errorFor("marker", cause))); @@ -218,17 +229,13 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") if (present) { const marker = yield* options.fs.readFileString(markerPath).pipe( Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker))), - Effect.flatMap(validateMarker), Effect.mapError((cause) => errorFor("marker", cause)), + Effect.flatMap(validateMarker), ); - if (marker.cacheNamespace !== cacheNamespace) { - const updated = { ...marker, cacheNamespace }; - yield* options.fs - .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); - return updated; - } - return marker; + if (marker.cacheNamespace === cacheNamespace) return marker; + const updated = { ...marker, cacheNamespace }; + yield* publishMarker(yield* encodeMarker(updated)); + return updated; } yield* rejectUnmarkedData; const value: Marker = { @@ -237,10 +244,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") cacheNamespace, initialized: false, }; - const encoded = yield* encodeMarker(value); - yield* options.fs - .writeFileString(markerPath, encoded, { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + yield* publishMarker(yield* encodeMarker(value)); return value; } let resolved = yield* resolveDaemon(false); @@ -307,10 +311,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ...validMarker, cacheNamespace: `cache-${resolved.cacheDigest.slice(0, 32)}`, }; - if (updated !== validMarker) - yield* options.fs - .writeFileString(markerPath, yield* encodeMarker(updated), { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + if (updated !== validMarker) yield* publishMarker(yield* encodeMarker(updated)); return updated; } yield* rejectUnmarkedData; @@ -353,9 +354,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.asVoid, ); } - yield* options.fs - .writeFileString(markerPath, yield* encodeMarker(value), { mode: 0o600 }) - .pipe(Effect.mapError((cause) => errorFor("marker", cause))); + yield* publishMarker(yield* encodeMarker(value)); return value; }); yield* Ref.set(selectedCache, value); @@ -368,11 +367,15 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") Effect.scoped( Effect.gen(function* () { yield* Effect.annotateCurrentSpan({ - "process.executable.name": options.runtime, + "process.executable.name": options.target.engine, "process.arg_count": args.length, }); const child = yield* options.spawner - .spawn(ChildProcess.make(options.runtime, args, { stdin: "ignore" })) + .spawn( + ChildProcess.make(options.target.engine, [...options.target.argv, ...args], { + stdin: "ignore", + }), + ) .pipe(Effect.mapError((cause) => errorFor("engine", cause))); const [stdout, stderr, code] = yield* Effect.all( [ @@ -416,9 +419,11 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") : Effect.fail(cause), ), ); - const probeDaemonIdentity: Effect.Effect = + // The daemon id is the owner's own pinned target, resolved once at startup; only the + // version, which that target's identity says nothing about, still needs the daemon itself. + const probeDaemonVersion: Effect.Effect = Effect.gen(function* () { - const daemonId = (yield* engineCommand(["info", "--format", "{{.ID}}"])).trim(); + const daemonId = options.target.daemonId; const version = yield* engineCommand([ "version", "--format", @@ -427,7 +432,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const [clientVersion, serverVersion] = version.split("|"); const clientMajor = parseMajor(clientVersion ?? ""); const serverMajor = parseMajor(serverVersion ?? ""); - if (daemonId.length === 0 || clientMajor === undefined || serverMajor === undefined) + if (clientMajor === undefined || serverMajor === undefined) return yield* errorFor("engine", "Docker returned an invalid version"); const identity = { daemonId, clientMajor, serverMajor }; // The next process, including a shadow diff, reuses this instead of asking the daemon. @@ -488,7 +493,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") } } } - const resolved = yield* describe(yield* probeDaemonIdentity, true, false); + const resolved = yield* describe(yield* probeDaemonVersion, true, false); yield* Ref.set(daemonIdentityRef, resolved); return resolved; }); @@ -548,8 +553,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") const child = yield* options.spawner .spawn( ChildProcess.make( - options.runtime, + options.target.engine, [ + ...options.target.argv, "run", "--rm", "-i", @@ -816,14 +822,33 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") .pipe(Effect.mapError((cause) => errorFor("marker", cause))); return present ? Option.some(yield* getMarker) : Option.none(); }); + // The in-memory resolution from `selected`, kept for the instance's lifetime across sleeps and + // generations once resolved, independent of the on-disk marker: a marker deleted after + // resolution must not hide a namespace this process still knows it owns. + const resolvedFromMemory = Ref.get(selectedCache); const getMarkerForRemoval = Effect.gen(function* () { + const cached = yield* resolvedFromMemory; + if (cached !== undefined) return Option.some(cached); const existing = yield* getMarkerIfPresent; - if (Option.isSome(existing) && options.runtime === "docker") { + if (Option.isSome(existing) && options.runtime !== "native") { yield* selected; return Option.some(yield* getMarker); } return existing; }); + /** + * Destroy tolerates a missing volume by design (`validateDockerMarkerIdentity` plus its own + * volume-existence check below), so this skips `selected`'s stricter resolution and only adds + * the in-memory shortcut on top of the existing on-disk read. `validated` tells the caller + * whether `selected` already confirmed this identity in memory: a live owner's cleanup must + * not then re-derive it from the filesystem (`realPath(stateRoot)` in + * `validateDockerMarkerIdentity`), only a new owner discovering from disk needs that check. + */ + const getMarkerForDestroy = Effect.gen(function* () { + const cached = yield* resolvedFromMemory; + if (cached !== undefined) return { marker: Option.some(cached), validated: true as const }; + return { marker: yield* getMarkerIfPresent, validated: false as const }; + }); const readyMarkerPath = options.path.join( options.instanceRoot, ".supabase-database-ready.json", @@ -854,11 +879,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* removeReadyMarker; }); const writeMarker = (marker: Marker) => - encodeMarker(marker).pipe( - Effect.flatMap((encoded) => - options.fs.writeFileString(markerPath, encoded, { mode: 0o600 }), - ), - ); + encodeMarker(marker).pipe(Effect.flatMap((encoded) => publishMarker(encoded))); const setup = Effect.fn("DockerDatabaseStorage.prepare")((version: string) => Effect.gen(function* () { yield* selected; @@ -976,7 +997,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ); const destroyData = Effect.fn("DockerDatabaseStorage.destroyData")((version: string) => Effect.gen(function* () { - const markerOption = yield* getMarkerIfPresent; + const { marker: markerOption, validated } = yield* getMarkerForDestroy; if (Option.isNone(markerOption)) { yield* removeUnmarkedData(version, { checkpoints: true }); return yield* removeHelper(); @@ -992,7 +1013,9 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") yield* removeReadyMarker; yield* removeHelper(); } else { - yield* validateDockerMarkerIdentity(marker); + // Memory-sourced identities were already validated by `selected` when first resolved; + // only a marker discovered fresh from disk (a new owner recovering) needs this check. + if (!validated) yield* validateDockerMarkerIdentity(marker); if (marker.volume === undefined) return yield* errorFor("destroy", "Recorded Docker storage volume is missing"); const volumeExists = yield* engineCommand(["volume", "inspect", marker.volume]).pipe( @@ -1121,43 +1144,3 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") }; }), ); - -/** Shell commands that delete the Docker-volume data recorded by the database instances under a stack's data root. */ -export const volumeDataCleanupCommands = Effect.fn( - "DockerDatabaseStorage.volumeDataCleanupCommands", -)( - function* (options: { - readonly engine: "docker" | "podman"; - readonly root: string; - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - }) { - if (!(yield* options.fs.exists(options.root))) return []; - const markers: Array = []; - for (const entry of yield* options.fs.readDirectory(options.root)) { - const markerPath = options.path.join(options.root, entry, ".supabase-database-storage.json"); - if (!(yield* options.fs.exists(markerPath))) continue; - const marker = yield* options.fs - .readFileString(markerPath) - .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(Marker)))); - if (marker.backend !== "docker") continue; - if ( - marker.volume === undefined || - !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u.test(marker.volume) || - !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/u.test(marker.namespace) - ) - return yield* errorFor( - "destroy", - `Recorded database storage identity is invalid: ${markerPath}`, - ); - markers.push(marker); - } - if (markers.length === 0) return []; - const { image } = yield* resolveArtifact({ service: "database" }); - return markers.map( - (marker) => - `${options.engine} run --rm --mount ${shellQuote(`type=volume,src=${marker.volume},dst=/store`)} ${shellQuote(image)} /bin/sh -c ${shellQuote(`rm -rf /store/${marker.namespace}`)}`, - ); - }, - Effect.mapError((cause) => errorFor("destroy", cause)), -); diff --git a/packages/stack/src/testing.integration.test.ts b/packages/stack/src/testing.integration.test.ts index 8f92f1b45e..4e139594e1 100644 --- a/packages/stack/src/testing.integration.test.ts +++ b/packages/stack/src/testing.integration.test.ts @@ -102,19 +102,21 @@ it.live( yield* test.checkpoint("0"); const root = path.join(stateRoot, test.stack.id); - const owners = (yield* fs.readDirectory(root, { recursive: true })).filter((file) => - file.endsWith(".supabase-database-owner.json"), + const readyMarkers = (yield* fs.readDirectory(root, { recursive: true })).filter((file) => + file.endsWith(".supabase-database-ready.json"), ); - expect(owners).toHaveLength(1); - const ownerFile = path.join(root, owners[0]!); - const originalMarker = yield* fs.readFileString(ownerFile); - // Released before the stack's teardown, which destroys only data carrying its own marker. + expect(readyMarkers).toHaveLength(1); + const instanceRoot = path.dirname(path.join(root, readyMarkers[0]!)); + const outside = yield* fs.makeTempDirectoryScoped({ prefix: "reset-failure-outside-" }); + // Ownership is by location: replacing the owned root with a symlink makes resetData's + // destroy step refuse, simulating a failure partway through reset before the checkpoint + // restore runs. Released before the stack's teardown, which tolerates a root that is simply + // absent. yield* Effect.acquireRelease( - fs.writeFileString( - ownerFile, - `{"stackId":"not-this-stack","instanceId":"not-this-instance"}`, - ), - () => fs.writeFileString(ownerFile, originalMarker).pipe(Effect.orDie), + fs + .remove(instanceRoot, { recursive: true, force: true }) + .pipe(Effect.andThen(fs.symlink(outside, instanceRoot))), + () => fs.remove(instanceRoot, { force: true }).pipe(Effect.orDie), ); const failure = yield* test.reset("0").pipe(Effect.flip); @@ -167,6 +169,58 @@ it.live( { timeout: 120_000 }, ); +it.live( + "destroys a stack after the snapshot running at the time finishes, removing its data", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateRoot = yield* fs.makeTempDirectoryScoped({ prefix: "stack-testing-snapshot-" }); + const test = yield* makeTestStack({ runtime: "native", stateRoot }); + const database = test.services.database; + const { databaseUrl } = yield* database.credentials(); + if (databaseUrl === undefined) return yield* Effect.die("Database URL missing"); + // Enough data that the snapshot is still copying when the destroy arrives. + yield* sql( + test.stack, + databaseUrl, + "CREATE TABLE filler AS SELECT g, repeat('x', 200) AS value FROM generate_series(1, 200000) g", + ); + yield* database.stop; + const subscribed = yield* Deferred.make(); + const storing = yield* Deferred.make(); + yield* database.followStatus.pipe( + Stream.runForEach((status) => + Deferred.succeed(subscribed, undefined).pipe( + Effect.andThen( + status.currentOperation === "storage" + ? Deferred.succeed(storing, undefined) + : Effect.void, + ), + ), + ), + Effect.forkScoped, + ); + yield* Deferred.await(subscribed); + + const snapshot = yield* database + .saveSnapshot("before-destroy", { scope: "instance" }) + .pipe(Effect.forkScoped); + yield* Deferred.await(storing); + yield* test.stack.destroy; + + expect(Exit.isSuccess(yield* Fiber.await(snapshot))).toBe(true); + expect(yield* discover({ stateRoot })).toEqual([]); + expect(yield* fs.exists(path.join(stateRoot, test.stack.id, "state.json"))).toBe(false); + const data = path.join(stateRoot, test.stack.id, "data"); + expect((yield* fs.exists(data)) ? yield* fs.readDirectory(data) : []).toEqual([]); + }).pipe( + Effect.scoped, + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + { timeout: 120_000 }, +); + it.live( "names the failure, service states and owner log, then removes the stack when test startup fails", () => diff --git a/packages/stack/src/testing.ts b/packages/stack/src/testing.ts index 531b3c4a25..4824e3a7c5 100644 --- a/packages/stack/src/testing.ts +++ b/packages/stack/src/testing.ts @@ -120,6 +120,8 @@ const runtimeOverride = Config.option( Config.literals(["native", "docker", "podman"], "SUPABASE_STACK_TEST_RUNTIME"), ); +const cacheHome = Config.option(Config.nonEmptyString("XDG_CACHE_HOME")); + const testFailure = (operation: string) => (cause: unknown) => stackError(operation, cause); /** Local-development configuration for each kind, with every endpoint on an automatic port. */ @@ -170,7 +172,12 @@ const make = Effect.fn("TestStack.make")( // Shared roots are created owner-only, so each OS user needs its own. const user = process.getuid?.() ?? userInfo().username; const stateRoot = options.stateRoot ?? path.join(tmpdir(), `supabase-stack-tests-${user}`); - const cacheRoot = options.cacheRoot ?? path.join(tmpdir(), `supabase-stack-artifacts-${user}`); + const cacheRoot = + options.cacheRoot ?? + path.join( + Option.getOrElse(yield* cacheHome, () => path.join(homedir(), ".cache")), + "supabase-stack-artifacts", + ); // Native Edge Runtime on Linux cannot read project files below /tmp. const temporaryRoot = fs.makeTempDirectoryScoped({ prefix: "supabase-test-stack-", diff --git a/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts b/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts index a18f68ff23..e3d6e380e3 100644 --- a/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts +++ b/packages/stack/src/whole-stack.docker.idle-parallel.e2e.test.ts @@ -1,9 +1,15 @@ import { it } from "@effect/vitest"; import { parallel } from "../tests/whole-stack/scenarios.ts"; -import { idleWake } from "../tests/whole-stack/idle.ts"; +import { functionsIdleWake, idleWake, restartIdleWake } from "../tests/whole-stack/idle.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live("Docker: idle and wake", () => run(idleWake("docker")), { timeout: 10 * 60_000 }); +it.live("Docker: restart preserves wake and idles again", () => run(restartIdleWake("docker")), { + timeout: 10 * 60_000, +}); +it.live("Docker: functions idles and wakes", () => run(functionsIdleWake("docker")), { + timeout: 10 * 60_000, +}); it.live("Docker: parallel stack isolation", () => run(parallel("docker")), { timeout: 30 * 60_000, }); diff --git a/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts index 53aa5d5dd0..47a825ef47 100644 --- a/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts +++ b/packages/stack/src/whole-stack.docker.lifecycle.e2e.test.ts @@ -1,5 +1,5 @@ import { it } from "@effect/vitest"; -import { allEager, defaultLifecycle } from "../tests/whole-stack/scenarios.ts"; +import { allEager, defaultLifecycle, writeConfinement } from "../tests/whole-stack/scenarios.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live( @@ -10,3 +10,8 @@ it.live( }, ); it.live("Docker: all services eager", () => run(allEager("docker")), { timeout: 15 * 60_000 }); +it.live( + "Docker: writes stay confined to the state root and artifact cache", + () => run(writeConfinement("docker")), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts b/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts index 16fdcce8be..0993cf6e2e 100644 --- a/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts +++ b/packages/stack/src/whole-stack.native.idle-parallel.e2e.test.ts @@ -1,9 +1,15 @@ import { it } from "@effect/vitest"; import { parallel } from "../tests/whole-stack/scenarios.ts"; -import { idleWake } from "../tests/whole-stack/idle.ts"; +import { functionsIdleWake, idleWake, restartIdleWake } from "../tests/whole-stack/idle.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live("native: idle and wake", () => run(idleWake("native")), { timeout: 10 * 60_000 }); +it.live("native: restart preserves wake and idles again", () => run(restartIdleWake("native")), { + timeout: 10 * 60_000, +}); +it.live("native: functions idles and wakes", () => run(functionsIdleWake("native")), { + timeout: 10 * 60_000, +}); it.live("native: parallel stack isolation", () => run(parallel("native")), { timeout: 30 * 60_000, }); diff --git a/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts index d78a02f169..089f96b2f1 100644 --- a/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts +++ b/packages/stack/src/whole-stack.native.lifecycle.e2e.test.ts @@ -1,5 +1,10 @@ import { it } from "@effect/vitest"; -import { allEager, defaultLifecycle } from "../tests/whole-stack/scenarios.ts"; +import { + allEager, + defaultLifecycle, + loopbackOnly, + writeConfinement, +} from "../tests/whole-stack/scenarios.ts"; import { run } from "../tests/whole-stack/fixture.ts"; it.live( @@ -10,3 +15,13 @@ it.live( }, ); it.live("native: all services eager", () => run(allEager("native")), { timeout: 15 * 60_000 }); +it.live( + "native: writes stay confined to the state root and artifact cache", + () => run(writeConfinement("native")), + { timeout: 15 * 60_000 }, +); +it.live.skipIf(process.platform === "win32")( + "native: every workload listener binds loopback only", + () => run(loopbackOnly), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts b/packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts new file mode 100644 index 0000000000..bd6d8a968e --- /dev/null +++ b/packages/stack/src/whole-stack.podman.idle-parallel.e2e.test.ts @@ -0,0 +1,15 @@ +import { it } from "@effect/vitest"; +import { parallel } from "../tests/whole-stack/scenarios.ts"; +import { functionsIdleWake, idleWake, restartIdleWake } from "../tests/whole-stack/idle.ts"; +import { run } from "../tests/whole-stack/fixture.ts"; + +it.live("Podman: idle and wake", () => run(idleWake("podman")), { timeout: 10 * 60_000 }); +it.live("Podman: restart preserves wake and idles again", () => run(restartIdleWake("podman")), { + timeout: 10 * 60_000, +}); +it.live("Podman: functions idles and wakes", () => run(functionsIdleWake("podman")), { + timeout: 10 * 60_000, +}); +it.live("Podman: parallel stack isolation", () => run(parallel("podman")), { + timeout: 30 * 60_000, +}); diff --git a/packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts b/packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts new file mode 100644 index 0000000000..b94bbc7aea --- /dev/null +++ b/packages/stack/src/whole-stack.podman.lifecycle.e2e.test.ts @@ -0,0 +1,17 @@ +import { it } from "@effect/vitest"; +import { allEager, defaultLifecycle, writeConfinement } from "../tests/whole-stack/scenarios.ts"; +import { run } from "../tests/whole-stack/fixture.ts"; + +it.live( + "Podman: default database eager lifecycle and reopen", + () => run(defaultLifecycle("podman")), + { + timeout: 15 * 60_000, + }, +); +it.live("Podman: all services eager", () => run(allEager("podman")), { timeout: 15 * 60_000 }); +it.live( + "Podman: writes stay confined to the state root and artifact cache", + () => run(writeConfinement("podman")), + { timeout: 15 * 60_000 }, +); diff --git a/packages/stack/tests/artifact-cache.ts b/packages/stack/tests/artifact-cache.ts new file mode 100644 index 0000000000..9853e7ae48 --- /dev/null +++ b/packages/stack/tests/artifact-cache.ts @@ -0,0 +1,9 @@ +import { homedir } from "node:os"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. +import { join } from "node:path"; // oxlint-disable-line effecttsgo/node-builtin-import -- plain constant shared by test files. + +/** Shared native artifact cache for tests; OS temp directories age out files inside published generations. */ +export const testArtifactCacheRoot = join( + // oxlint-disable-next-line effecttsgo/process-env -- plain constant shared by test files. + process.env.XDG_CACHE_HOME || join(homedir(), ".cache"), + "supabase-stack-artifacts", +); diff --git a/packages/stack/tests/artifact-store-fixture.ts b/packages/stack/tests/artifact-store-fixture.ts new file mode 100644 index 0000000000..3fe0f3b006 --- /dev/null +++ b/packages/stack/tests/artifact-store-fixture.ts @@ -0,0 +1,93 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { Console, Crypto, Effect, FileSystem, Layer } from "effect"; +import { + makeArtifactStore, + type ArtifactRequest, + type ArtifactSource, +} from "../src/preparation/ArtifactStore.ts"; +import { ArtifactIntegrityError, PreparationError } from "../src/preparation/Errors.ts"; +import { verifySha256 } from "../src/preparation/Integrity.ts"; + +const mapMaterializeError = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + cause instanceof PreparationError || cause instanceof ArtifactIntegrityError + ? cause + : new PreparationError({ + message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + ), + ); + +/** + * Cross-process fixture for `artifacts.integration.test.ts`: real preparer/pin races need real + * separate processes, not fibers in one process. Modes: + * - `prepare`: ahead-of-time `store.prepare`, prints `prepared:::`, exits. + * - `use`: scoped `store.use`, prints `used:::`, then blocks forever so + * the parent controls the pin's release by killing this process. + * - `stall`: `store.prepare` whose materialize prints `started` once extraction has begun, then + * blocks forever, so the parent can synchronize a SIGKILL precisely mid-extraction without + * polling. + */ +const [mode, cacheRoot, key] = process.argv.slice(2); +if (mode === undefined || cacheRoot === undefined || key === undefined) + throw new Error("Usage: artifact-store-fixture "); + +const archive = new TextEncoder().encode("archive"); +const archiveSha256 = "0eb3e36bfb24dcd9bb1d1bece1531216b59539a8fde17ee80224af0653c92aa3"; + +const request: ArtifactRequest = { + key, + requiredRuntimePaths: ["bin/postgres", "etc/postgres.conf"], + executablePath: "bin/postgres", +}; + +const fastSource: ArtifactSource = { + checksum: () => Effect.succeed(archiveSha256), + materialize: (_request, destination, expectedSha256) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* fs.makeDirectory(`${destination}/etc`, { recursive: true }); + yield* fs.writeFileString(`${destination}/bin/postgres`, "native postgres", { mode: 0o755 }); + yield* fs.writeFileString(`${destination}/etc/postgres.conf`, "config", { mode: 0o644 }); + const crypto = yield* Crypto.Crypto; + yield* verifySha256(archive, expectedSha256).pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + }).pipe(mapMaterializeError), +}; + +const stallingSource: ArtifactSource = { + checksum: () => Effect.succeed(archiveSha256), + materialize: (_request, destination) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${destination}/bin`, { recursive: true }); + yield* Console.log("started"); + return yield* Effect.never; + }).pipe(mapMaterializeError), +}; + +const program = Effect.scoped( + Effect.gen(function* () { + const store = yield* makeArtifactStore({ + cacheRoot, + source: mode === "stall" ? stallingSource : fastSource, + }); + if (mode === "prepare" || mode === "stall") { + const prepared = yield* store.prepare(request); + yield* Console.log(`prepared:${prepared.outcome}:${prepared.path}:${prepared.sha256}`); + return; + } + if (mode === "use") { + const prepared = yield* store.use(request); + yield* Console.log(`used:${prepared.outcome}:${prepared.path}:${prepared.lockPath}`); + return yield* Effect.never; + } + throw new Error(`Unknown fixture mode: ${mode}`); + }), +).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))); + +await Effect.runPromise(program); diff --git a/packages/stack/tests/compiled-dispatch-fixture.ts b/packages/stack/tests/compiled-dispatch-fixture.ts index a4ac872efd..40d950ab50 100644 --- a/packages/stack/tests/compiled-dispatch-fixture.ts +++ b/packages/stack/tests/compiled-dispatch-fixture.ts @@ -13,7 +13,7 @@ import { type HostEndpoint, } from "../src/HostProcess.ts"; import { spawnNativeProcess } from "../src/runtime/NativeProcess.ts"; -import * as State from "../src/State.ts"; +import * as StackNamespace from "../src/StackNamespace.ts"; const argv = process.argv.slice(2); class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} @@ -27,7 +27,7 @@ if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispa throw new Error("Owner fixture arguments missing"); const program = Effect.scoped( Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; const access = yield* mode === "owner" ? launchHost(state, { stateRoot, cacheRoot: cacheRoot ?? stateRoot, stackId }) : connectHost(state, stackId); @@ -47,7 +47,7 @@ if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispa Layer.mergeAll( NodeServices.layer, NodeHttpClient.layerNodeHttp, - State.layer({ root: stateRoot }).pipe(Layer.provide(NodeServices.layer)), + StackNamespace.layer({ root: stateRoot }).pipe(Layer.provide(NodeServices.layer)), ), ), ); @@ -65,6 +65,19 @@ if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispa "sh", pidMarker, ], + environment: { + values: { + HOME: "/tmp", + TMPDIR: "/tmp", + TMP: "/tmp", + TEMP: "/tmp", + XDG_CACHE_HOME: "/tmp", + XDG_CONFIG_HOME: "/tmp", + XDG_DATA_HOME: "/tmp", + XDG_STATE_HOME: "/tmp", + DENO_DIR: "/tmp", + }, + }, gracefulStopSignal: "SIGTERM", gracefulStopTimeout: "2 seconds", }); diff --git a/packages/stack/tests/docker-cleanup.ts b/packages/stack/tests/docker-cleanup.ts index cf64d569b6..a779e3b21c 100644 --- a/packages/stack/tests/docker-cleanup.ts +++ b/packages/stack/tests/docker-cleanup.ts @@ -1,5 +1,6 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Effect, Stream } from "effect"; +import { testEngine } from "./test-engine.ts"; export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: string) => Effect.scoped( @@ -7,7 +8,7 @@ export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: stri const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( ChildProcess.make( - "docker", + testEngine, [ "run", "--rm", @@ -15,7 +16,7 @@ export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: stri "0", "--mount", `type=bind,src=${root},dst=/mnt`, - "busybox:1.36", + "docker.io/library/busybox:1.36", "find", "/mnt", "-mindepth", @@ -30,7 +31,7 @@ export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: stri { concurrency: "unbounded" }, ); if (Number(code) !== 0) - return yield* Effect.die(`Docker test cleanup exited with ${code}: ${output}`); + return yield* Effect.die(`${testEngine} test cleanup exited with ${code}: ${output}`); }), ).pipe(Effect.catchCause(Effect.die)), ); diff --git a/packages/stack/tests/docker-fixture.integration.test.ts b/packages/stack/tests/docker-fixture.integration.test.ts index 17303cf6ab..444ecdd2d6 100644 --- a/packages/stack/tests/docker-fixture.integration.test.ts +++ b/packages/stack/tests/docker-fixture.integration.test.ts @@ -3,9 +3,10 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Context, Effect, FileSystem, Layer, Redacted, Schema, Stream } from "effect"; -import { makeService } from "../src/Service.ts"; +import { makeStandaloneService } from "./standalone-service.ts"; import { makeDatabase, type BackendEndpoint } from "../src/services/Database.ts"; import { makeDockerDatabaseRoot } from "./docker-fixture.ts"; +import { engineTarget, testEngine } from "./engine-target.ts"; const marker = Schema.Struct({ backend: Schema.Literal("docker"), volume: Schema.String }); const password = Redacted.make("supabase-test-password"); @@ -53,89 +54,98 @@ const volumeExists = Effect.fn("DockerFixture.volumeExists")((volume: string) => ), ); -describe("Docker database fixture isolation", { timeout: 180_000 }, () => { - it.live("isolates equal identities across separate owned volumes", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const firstRoot = yield* makeDockerDatabaseRoot( - "docker-fixture-first-", - "docker-fixture-isolation", - ); - const secondRoot = yield* makeDockerDatabaseRoot( - "docker-fixture-second-", - "docker-fixture-isolation", - ); - const make = (root: string) => - makeDatabase({ - stackId: "docker-fixture-isolation", - instanceId: "database", - root, - cacheRoot: `${root}/cache`, - runtime: "docker", +describe.runIf(testEngine === "docker")( + "Docker database fixture isolation", + { timeout: 180_000 }, + () => { + it.live("isolates equal identities across separate owned volumes", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const firstRoot = yield* makeDockerDatabaseRoot( + "docker-fixture-first-", + "docker-fixture-isolation", + ); + const secondRoot = yield* makeDockerDatabaseRoot( + "docker-fixture-second-", + "docker-fixture-isolation", + ); + const make = (root: string) => + makeDatabase({ + stackId: "docker-fixture-isolation", + instanceId: "database", + root, + cacheRoot: `${root}/cache`, + runtime: "docker", + engineTarget, + }); + const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); + const [firstService, secondService] = yield* Effect.all([ + makeStandaloneService(first.definition, { + id: "docker-fixture-first", + config: { + version: "17", + databasePassword: password, + jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + }), + makeStandaloneService(second.definition, { + id: "docker-fixture-second", + config: { + version: "17", + databasePassword: password, + jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + }), + ]); + yield* Effect.all([firstService.start, secondService.start], { + concurrency: "unbounded", }); - const [first, second] = yield* Effect.all([make(firstRoot), make(secondRoot)]); - const [firstService, secondService] = yield* Effect.all([ - makeService(first.definition, { - id: "docker-fixture-first", - config: { - version: "17", - databasePassword: password, - jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), - jwtExpiry: 3600, - }, - }), - makeService(second.definition, { - id: "docker-fixture-second", - config: { - version: "17", - databasePassword: password, - jwtSecret: Redacted.make("fixture-secret-with-at-least-32-chars"), - jwtExpiry: 3600, - }, - }), - ]); - yield* Effect.all([firstService.start, secondService.start], { concurrency: "unbounded" }); - yield* Effect.all([firstService.ready, secondService.ready], { concurrency: "unbounded" }); - const [firstEndpoint, secondEndpoint] = yield* Effect.all([ - first.endpoint, - second.endpoint, - ]); - yield* query(firstEndpoint, "CREATE TABLE fixture_isolation (value text)"); - yield* query(firstEndpoint, "INSERT INTO fixture_isolation VALUES ('first')"); - yield* query(secondEndpoint, "CREATE TABLE fixture_isolation (value text)"); - yield* query(secondEndpoint, "INSERT INTO fixture_isolation VALUES ('second')"); - expect(yield* query(firstEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ - { value: "first" }, - ]); - expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ - { value: "second" }, - ]); - yield* firstService.stop; - expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ - { value: "second" }, - ]); - const markers = yield* Effect.all( - [firstRoot, secondRoot].map((root) => - fs - .readFileString(`${root}/database/.supabase-database-storage.json`) - .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(marker)))), + yield* Effect.all([firstService.ready, secondService.ready], { + concurrency: "unbounded", + }); + const [firstEndpoint, secondEndpoint] = yield* Effect.all([ + first.endpoint, + second.endpoint, + ]); + yield* query(firstEndpoint, "CREATE TABLE fixture_isolation (value text)"); + yield* query(firstEndpoint, "INSERT INTO fixture_isolation VALUES ('first')"); + yield* query(secondEndpoint, "CREATE TABLE fixture_isolation (value text)"); + yield* query(secondEndpoint, "INSERT INTO fixture_isolation VALUES ('second')"); + expect(yield* query(firstEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ + { value: "first" }, + ]); + expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ + { value: "second" }, + ]); + yield* firstService.stop; + expect(yield* query(secondEndpoint, "SELECT value FROM fixture_isolation")).toEqual([ + { value: "second" }, + ]); + const markers = yield* Effect.all( + [firstRoot, secondRoot].map((root) => + fs + .readFileString(`${root}/database/.supabase-database-storage.json`) + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(marker)))), + ), + ); + yield* secondService.destroy; + yield* firstService.destroy; + return markers.map((entry) => entry.volume); + }), + ).pipe( + Effect.flatMap((volumes) => + Effect.forEach(volumes, (volume) => + volumeExists(volume).pipe(Effect.map((exists) => ({ volume, exists }))), ), - ); - yield* secondService.destroy; - yield* firstService.destroy; - return markers.map((entry) => entry.volume); - }), - ).pipe( - Effect.flatMap((volumes) => - Effect.forEach(volumes, (volume) => - volumeExists(volume).pipe(Effect.map((exists) => ({ volume, exists }))), ), + Effect.tap((results) => + Effect.sync(() => results.forEach(({ exists }) => expect(exists).toBe(true))), + ), + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), ), - Effect.tap((results) => - Effect.sync(() => results.forEach(({ exists }) => expect(exists).toBe(true))), - ), - Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), - ), - ); -}); + ); + }, +); diff --git a/packages/stack/tests/docker-fixture.ts b/packages/stack/tests/docker-fixture.ts index 490269ef50..8138cfc91f 100644 --- a/packages/stack/tests/docker-fixture.ts +++ b/packages/stack/tests/docker-fixture.ts @@ -1,14 +1,18 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Effect, FileSystem, Layer, Schema, Sink, Stream } from "effect"; import { cleanupDockerRoot } from "./docker-cleanup.ts"; +import { testEngine } from "./test-engine.ts"; -/** Runs a Docker CLI command and returns its combined output and exit code. */ -export const runDocker = Effect.fn("DockerTest.runDocker")((args: ReadonlyArray) => +/** + * Runs a command of the selected test engine's CLI and returns its combined output and exit code. + * It does not pin the engine target, so it works before an engine is known to be reachable. + */ +export const runEngine = Effect.fn("DockerTest.runEngine")((args: ReadonlyArray) => Effect.scoped( Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const child = yield* spawner.spawn( - ChildProcess.make("docker", args, { stdout: "pipe", stderr: "pipe" }), + ChildProcess.make(testEngine, args, { stdout: "pipe", stderr: "pipe" }), ); const [stdout, stderr, code] = yield* Effect.all( [ @@ -55,12 +59,17 @@ const encodeLabels = Schema.encodeEffect( Schema.fromJsonString(Schema.Tuple([Schema.String, Schema.String])), ); +/** The connection options `resolveEngineTarget` pins in front of a subcommand. */ +const TARGET_OPTIONS = ["--context", "--host", "--url", "--connection"]; + /** - * Docker and Podman engines that each start with `containers`. `ps --format` prints their labels - * as JSON lines, then every string entry as is; `ps --quiet` prints the ids of those matching every - * `--filter`; `rm --force ` removes one. A command fails with the given stderr when `failures` - * names its engine (`docker`) or its engine and subcommand (`podman rm`). Every command is recorded - * and passed to `onCommand` before it is answered; `remaining` lists an engine's containers. + * Docker and Podman engines that each start with `containers`. Both answer the commands + * `resolveEngineTarget` runs; a target's connection options are not part of a command's subcommand. + * `ps --format` prints their labels as JSON lines, then every string entry as is; `ps --quiet` + * prints the ids of those matching every `--filter`; `rm --force ` removes one. A command fails + * with the given stderr when `failures` names its engine (`docker`) or its engine and subcommand + * (`podman rm`). Every command is recorded and passed to `onCommand` before it is answered; + * `remaining` lists an engine's containers. */ export const engineStub = ( containers: ReadonlyArray, @@ -77,7 +86,10 @@ export const engineStub = ( return yield* Effect.die("Unexpected child process command"); const argv = [command.command, ...command.args]; commands.push(argv); - const [subcommand] = command.args; + const args = TARGET_OPTIONS.includes(command.args[0] ?? "") + ? command.args.slice(2) + : command.args; + const [subcommand] = args; const failure = failures[`${command.command} ${subcommand}`] ?? failures[command.command]; const inventory = inventories.get(command.command) ?? []; const filters = command.args.filter((_, index) => command.args[index - 1] === "--filter"); @@ -91,17 +103,25 @@ export const engineStub = ( ].includes(filter), ), ); + const listing = Effect.gen(function* () { + if (!command.args.includes("--format")) return listed.map(({ id }) => id).join("\n"); + const labels = yield* Effect.forEach(listed, ({ stackId, root }) => + encodeLabels([stackId, root]).pipe(Effect.orDie), + ); + return [...labels, ...noise].join("\n"); + }); + const identity = + command.command === "podman" ? "false|false|stub-host|/stub/graph" : "stub-id"; const stdout = - failure !== undefined || subcommand !== "ps" + failure !== undefined ? "" - : command.args.includes("--format") - ? [ - ...(yield* Effect.forEach(listed, ({ stackId, root }) => - encodeLabels([stackId, root]).pipe(Effect.orDie), - )), - ...noise, - ].join("\n") - : listed.map(({ id }) => id).join("\n"); + : subcommand === "context" + ? "stub-context" + : subcommand === "info" + ? identity + : subcommand === "ps" + ? yield* listing + : ""; if (failure === undefined && subcommand === "rm") inventories.set( command.command, diff --git a/packages/stack/tests/docker-relay.ts b/packages/stack/tests/docker-relay.ts index 003922d0a5..7490ae3123 100644 --- a/packages/stack/tests/docker-relay.ts +++ b/packages/stack/tests/docker-relay.ts @@ -1,6 +1,7 @@ import { Effect, Predicate } from "effect"; import { bindTcp, serveTcp, ProxyError } from "../src/Proxy.ts"; import { makeHttpProxy } from "../src/HttpProxy.ts"; +import { DOCKER_HOST_ALIAS } from "../src/runtime/Container.ts"; import type { ServiceEndpoint } from "../src/services/Recipe.ts"; const address = (endpoint: Effect.Effect) => @@ -22,7 +23,7 @@ export const makeDockerTcpRelay = Effect.fn("DockerRelay.makeTcp")( if (!Predicate.isTagged(listener.address, "TcpAddress")) return yield* Effect.die("Expected TCP relay listener"); yield* serveTcp(listener, address(endpoint), "docker-relay").pipe(Effect.forkScoped); - return { host: "host.docker.internal", port: listener.address.port }; + return { host: DOCKER_HOST_ALIAS, port: listener.address.port }; }), ); @@ -31,6 +32,6 @@ export const makeDockerHttpRelay = Effect.fn("DockerRelay.makeHttp")( Effect.gen(function* () { const proxy = yield* makeHttpProxy({ host: "0.0.0.0", port: 0 }); yield* proxy.setRoutes([{ id: "service", prefix: "/", target: address(endpoint) }]); - return { host: "host.docker.internal", port: proxy.port }; + return { host: DOCKER_HOST_ALIAS, port: proxy.port }; }), ); diff --git a/packages/stack/tests/docker-storage-publish-loop-fixture.ts b/packages/stack/tests/docker-storage-publish-loop-fixture.ts new file mode 100644 index 0000000000..dea0d8d925 --- /dev/null +++ b/packages/stack/tests/docker-storage-publish-loop-fixture.ts @@ -0,0 +1,61 @@ +import { NodeServices } from "@effect/platform-node"; +import { Console, Crypto, Effect, FileSystem, Layer, Path } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { makeContainerRuntime } from "../src/runtime/Container.ts"; +import { makeDockerDatabaseStorage } from "../src/storage/DockerDatabaseStorage.ts"; + +const [instanceRoot, root, cacheRoot] = process.argv.slice(2); +if (instanceRoot === undefined || root === undefined || cacheRoot === undefined) + throw new Error("instanceRoot, root or cacheRoot missing"); + +/** + * Announces readiness on stdout the instant it is about to replace the storage marker (the real + * staging-to-publish boundary Publication.publish uses), then hangs so a caller can kill this + * process exactly there on every run. + */ +const barrierFs = Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return { + ...fs, + rename: (from: string, to: string) => + to.endsWith(".supabase-database-storage.json") + ? Console.log("about-to-publish").pipe(Effect.andThen(Effect.never)) + : fs.rename(from, to), + }; + }), +); + +const program = Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + // Matches the caller's own target exactly, so this fixture's identity agrees with the + // marker the caller's `prepare` already published through it. + const target = { engine: "docker" as const, argv: [], daemonId: "test-daemon-id" }; + const container = yield* makeContainerRuntime({ + target, + root: instanceRoot, + }); + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + target, + stackId: "storage-crash-test", + instanceId: "crash", + instanceRoot, + root, + cacheRoot, + fs, + path, + crypto, + container, + spawner, + }); + yield* storage.markInitialized("17"); + }), +).pipe(Effect.provide(barrierFs.pipe(Layer.provideMerge(NodeServices.layer)))); + +await Effect.runPromise(program); diff --git a/packages/stack/tests/docker-volume-run.integration.test.ts b/packages/stack/tests/docker-volume-run.integration.test.ts index 6233923a8f..3cb07456d5 100644 --- a/packages/stack/tests/docker-volume-run.integration.test.ts +++ b/packages/stack/tests/docker-volume-run.integration.test.ts @@ -2,7 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Crypto, DateTime, Effect, FileSystem, Path } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { runDocker } from "./docker-fixture.ts"; +import { runEngine } from "./docker-fixture.ts"; import { encodeRunMarker, markerDirectory, @@ -12,12 +12,12 @@ import { } from "./docker-volume-run.ts"; const volumeExists = Effect.fn("DockerVolumeRunTest.volumeExists")((name: string) => - runDocker(["volume", "inspect", name]).pipe(Effect.map((result) => result.code === 0)), + runEngine(["volume", "inspect", name]).pipe(Effect.map((result) => result.code === 0)), ); const createVolume = Effect.fn("DockerVolumeRunTest.createVolume")( (name: string, labels: ReadonlyArray) => - runDocker(["volume", "create", ...labels.flatMap((label) => ["--label", label]), name]).pipe( + runEngine(["volume", "create", ...labels.flatMap((label) => ["--label", label]), name]).pipe( Effect.flatMap((result) => result.code === 0 ? Effect.void @@ -27,7 +27,7 @@ const createVolume = Effect.fn("DockerVolumeRunTest.createVolume")( ); const removeVolume = Effect.fn("DockerVolumeRunTest.removeVolume")((name: string) => - runDocker(["volume", "rm", name]).pipe( + runEngine(["volume", "rm", name]).pipe( Effect.flatMap((result) => result.code === 0 || /no such volume/iu.test(result.output) ? Effect.void @@ -38,17 +38,17 @@ const removeVolume = Effect.fn("DockerVolumeRunTest.removeVolume")((name: string ); const containerExists = Effect.fn("DockerVolumeRunTest.containerExists")((name: string) => - runDocker(["inspect", name]).pipe(Effect.map((result) => result.code === 0)), + runEngine(["inspect", name]).pipe(Effect.map((result) => result.code === 0)), ); const createContainer = Effect.fn("DockerVolumeRunTest.createContainer")( (name: string, labels: ReadonlyArray) => - runDocker([ + runEngine([ "create", "--name", name, ...labels.flatMap((label) => ["--label", label]), - "busybox:1.36", + "docker.io/library/busybox:1.36", "true", ]).pipe( Effect.flatMap((result) => @@ -60,7 +60,7 @@ const createContainer = Effect.fn("DockerVolumeRunTest.createContainer")( ); const removeContainer = Effect.fn("DockerVolumeRunTest.removeContainer")((name: string) => - runDocker(["rm", "--force", "--volumes", name]).pipe( + runEngine(["rm", "--force", "--volumes", name]).pipe( Effect.flatMap((result) => result.code === 0 || /no such container/iu.test(result.output) ? Effect.void diff --git a/packages/stack/tests/docker-volume-run.ts b/packages/stack/tests/docker-volume-run.ts index 14d469ae63..398c555c39 100644 --- a/packages/stack/tests/docker-volume-run.ts +++ b/packages/stack/tests/docker-volume-run.ts @@ -6,7 +6,7 @@ import { testRunEnvVar as stackTestRunEnvVar, testRunLabelKey, } from "../src/internal/test-run-label.ts"; -import { runDocker } from "./docker-fixture.ts"; +import { runEngine } from "./docker-fixture.ts"; /** * Docker test state roots must be private to a run (see `makeDockerDatabaseRoot` and @@ -44,7 +44,7 @@ const isDead = (pid: number): boolean => { } }; -const dockerUnavailable = runDocker(["info", "--format", "{{.ID}}"]).pipe( +const dockerUnavailable = runEngine(["version"]).pipe( Effect.map((result) => result.code !== 0), Effect.catchCause(() => Effect.succeed(true)), ); @@ -62,7 +62,7 @@ export const removeTestRunVolumes = Effect.fn("DockerVolumeRun.removeTestRunVolu ); return; } - const listed = yield* runDocker([ + const listed = yield* runEngine([ "volume", "ls", "-q", @@ -79,7 +79,7 @@ export const removeTestRunVolumes = Effect.fn("DockerVolumeRun.removeTestRunVolu .filter((line) => line.length > 0); if (volumes.length === 0) return; const removals = yield* Effect.forEach(volumes, (volume) => - runDocker(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), + runEngine(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), ); const failed = removals.filter(({ result }) => result.code !== 0); if (failed.length > 0) @@ -97,7 +97,7 @@ const benign = (output: string) => /no such (?:container|volume)/iu.test(output) /** Removes a dead run's labelled containers; `Option.none` on success, a message otherwise. */ const removeDeadRunContainers = Effect.fn("DockerVolumeRun.removeDeadRunContainers")((id: string) => Effect.gen(function* () { - const listed = yield* runDocker([ + const listed = yield* runEngine([ "ps", "--all", "--quiet", @@ -111,7 +111,7 @@ const removeDeadRunContainers = Effect.fn("DockerVolumeRun.removeDeadRunContaine .map((line) => line.trim()) .filter((line) => line.length > 0); if (ids.length === 0) return Option.none(); - const removed = yield* runDocker(["rm", "--force", "--volumes", ...ids]); + const removed = yield* runEngine(["rm", "--force", "--volumes", ...ids]); return removed.code === 0 || benign(removed.output) ? Option.none() : Option.some(`docker rm failed: ${removed.output.trim()}`); @@ -121,7 +121,7 @@ const removeDeadRunContainers = Effect.fn("DockerVolumeRun.removeDeadRunContaine /** Removes a dead run's labelled, stack-managed volumes; `Option.none` on success. */ const removeDeadRunVolumes = Effect.fn("DockerVolumeRun.removeDeadRunVolumes")((id: string) => Effect.gen(function* () { - const listed = yield* runDocker([ + const listed = yield* runEngine([ "volume", "ls", "-q", @@ -137,7 +137,7 @@ const removeDeadRunVolumes = Effect.fn("DockerVolumeRun.removeDeadRunVolumes")(( .filter((line) => line.length > 0); if (volumes.length === 0) return Option.none(); const removals = yield* Effect.forEach(volumes, (volume) => - runDocker(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), + runEngine(["volume", "rm", volume]).pipe(Effect.map((result) => ({ volume, result }))), ); const failed = removals.filter(({ result }) => result.code !== 0 && !benign(result.output)); return failed.length === 0 diff --git a/packages/stack/tests/engine-events.ts b/packages/stack/tests/engine-events.ts new file mode 100644 index 0000000000..b435e92aaa --- /dev/null +++ b/packages/stack/tests/engine-events.ts @@ -0,0 +1,110 @@ +import { Data, Effect, Option, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { engineTarget, testEngine } from "./engine-target.ts"; + +class EngineEventsError extends Data.TaggedError("EngineEventsError")<{ + readonly message: string; +}> {} + +const eventLines = Effect.fn("EngineEvents.eventLines")((args: ReadonlyArray) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const events = yield* spawner.spawn( + ChildProcess.make(testEngine, [...engineTarget.argv, "events", ...args], { + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }), + ); + return events.stdout.pipe(Stream.decodeText, Stream.splitLines); + }), +); + +/** + * Completes when the engine reports the container removed, including a removal that happened + * after `sinceSeconds` and before the stream attached. Docker names that event `destroy`; Podman + * names it `remove`. + */ +export const awaitContainerRemoved = (id: string, sinceSeconds: number) => + Effect.scoped( + Effect.gen(function* () { + const lines = yield* eventLines( + testEngine === "docker" + ? [ + "--since", + String(sinceSeconds), + "--filter", + `container=${id}`, + "--filter", + "event=destroy", + "--format", + "{{.Actor.ID}}", + ] + : [ + "--since", + String(sinceSeconds), + "--filter", + `container=${id}`, + "--filter", + "event=remove", + "--format", + "{{.ID}}", + ], + ); + const removed = yield* lines.pipe( + Stream.filter((line) => line.trim() === id), + Stream.runHead, + ); + if (Option.isNone(removed)) + return yield* new EngineEventsError({ message: `${testEngine} event stream ended` }); + }), + ); + +/** + * What the engine reports for a container stopped with SIGINT that exits cleanly. Docker's stop + * sends the signal through the daemon, so it reports `kill` with the signal before `die`; Podman's + * stop signals the container directly and reports only `died` with the exit code. + */ +export const cleanStopEvents: ReadonlyArray = + testEngine === "docker" ? ["kill 2", "die 0"] : ["died 0"]; + +/** + * Subscribes to the container's stop events from `since` (replaying any logged before the stream + * attaches) and resolves, once its terminal event arrives, to every line observed. Subscribing + * before triggering the stop avoids Docker Desktop's intermittently empty bounded replays. + */ +export const observeContainerStop = Effect.fn("EngineEvents.observeContainerStop")(function* ( + container: string, + since: string, +) { + const lines = yield* eventLines( + testEngine === "docker" + ? [ + "--since", + since, + "--filter", + `container=${container}`, + "--filter", + "event=kill", + "--filter", + "event=die", + "--format", + '{{.Action}} {{index .Actor.Attributes "signal"}}{{index .Actor.Attributes "exitCode"}}', + ] + : [ + "--since", + since, + "--filter", + `container=${container}`, + "--filter", + "event=died", + "--format", + "{{.Status}} {{.ContainerExitCode}}", + ], + ); + return yield* lines.pipe( + Stream.takeUntil((line) => line.startsWith(testEngine === "docker" ? "die " : "died ")), + Stream.runCollect, + Effect.forkScoped, + ); +}); diff --git a/packages/stack/tests/engine-target.ts b/packages/stack/tests/engine-target.ts new file mode 100644 index 0000000000..7bab3eb8e8 --- /dev/null +++ b/packages/stack/tests/engine-target.ts @@ -0,0 +1,16 @@ +import { NodeServices } from "@effect/platform-node"; +import { Effect } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { resolveEngineTarget, type EngineTarget } from "../src/runtime/Container.ts"; +import { testEngine } from "./test-engine.ts"; + +export { testEngine }; + +/** + * Resolves the selected engine target once per test file, for `it.live` tests that exercise a + * service catalog or a database against the real engine rather than a fake one. + */ +export const engineTarget: EngineTarget = await Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* resolveEngineTarget(spawner, testEngine); +}).pipe(Effect.provide(NodeServices.layer), Effect.runPromise); diff --git a/packages/stack/tests/gated-docker-stop-fixture.ts b/packages/stack/tests/gated-docker-stop-fixture.ts new file mode 100644 index 0000000000..b3796d7ea8 --- /dev/null +++ b/packages/stack/tests/gated-docker-stop-fixture.ts @@ -0,0 +1,91 @@ +import { Cause, Effect, Exit, Option, Schema } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. +import { closeSync, writeSync } from "node:fs"; +import { runStackHost, StackHostError } from "../src/StackHost.ts"; + +const writeLine = (value: unknown) => + Effect.gen(function* () { + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))( + value, + ).pipe( + Effect.mapError( + (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + ), + ); + yield* Effect.try({ + try: () => writeSync(3, Buffer.from(`${serialized}\n`, "utf8")), + catch: (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + }); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + try { + closeSync(3); + } catch { + // The parent may close the inherited descriptor during cancellation. + } + }), + ), + ); + +/** + * A real owner entrypoint, identical to `internal/host-process.ts`, except it puts + * `/bin` first on PATH so a test can interpose on the container engine commands the owner runs. + */ +const [stateRoot, cacheRoot, stackId, gateDir, ...rest] = process.argv.slice(2); + +const program = Effect.gen(function* () { + if ( + stateRoot === undefined || + cacheRoot === undefined || + stackId === undefined || + gateDir === undefined || + rest.length > 0 + ) + return yield* new StackHostError({ + operation: "startup", + message: "Expected stateRoot, cacheRoot, stackId and gateDir", + }); + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- the PATH shim must reach the child processes the owner spawns. + process.env.PATH = `${gateDir}/bin:${process.env.PATH ?? ""}`; + let reported = false; + const report = (value: unknown) => + Effect.suspend(() => { + if (reported) return Effect.void; + reported = true; + return writeLine(value); + }); + yield* runStackHost({ + stateRoot, + cacheRoot, + stackId, + onReady: ({ endpoint, secret }) => report({ type: "ready", endpoint, secret }), + }).pipe( + Effect.catchCause((cause) => { + const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); + return report({ + type: "error", + message: failure?.message ?? Cause.pretty(cause), + ...(failure?.reason === undefined ? {} : { reason: failure.reason }), + }).pipe(Effect.exit, Effect.andThen(Effect.failCause(cause))); + }), + ); +}); + +const flushed = (stream: NodeJS.WriteStream) => + Effect.callback((resume) => { + stream.write("", () => resume(Effect.void)); + }).pipe(Effect.timeoutOption("1 second")); + +await Effect.runPromise( + program.pipe( + Effect.exit, + Effect.tap((exit) => + Exit.isFailure(exit) + ? Effect.sync(() => process.stderr.write(`${Cause.pretty(exit.cause)}\n`)) + : Effect.void, + ), + Effect.tap(() => Effect.all([flushed(process.stdout), flushed(process.stderr)])), + Effect.flatMap((exit) => Effect.sync(() => process.exit(Exit.isSuccess(exit) ? 0 : 1))), + ), +); diff --git a/packages/stack/tests/helper-owner-fixture.ts b/packages/stack/tests/helper-owner-fixture.ts index fb8fecabd6..abe91ba5cb 100644 --- a/packages/stack/tests/helper-owner-fixture.ts +++ b/packages/stack/tests/helper-owner-fixture.ts @@ -37,10 +37,12 @@ const run = Effect.gen(function* () { }), { mode: 0o600 }, ); - const container = yield* makeContainerRuntime({ engine: "docker", root }); + const target = { engine: "docker" as const, argv: [], daemonId: "test-daemon-id" }; + const container = yield* makeContainerRuntime({ target, root }); const helpers = shared ? yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4) : undefined; const storage = yield* makeDockerDatabaseStorage({ runtime: "docker", + target, stackId, instanceId: "database", instanceRoot, diff --git a/packages/stack/tests/helpers/endpoint.ts b/packages/stack/tests/helpers/endpoint.ts new file mode 100644 index 0000000000..54fb86feec --- /dev/null +++ b/packages/stack/tests/helpers/endpoint.ts @@ -0,0 +1,5 @@ +import type { ServiceEndpoint } from "../../src/services/Recipe.ts"; + +/** The host to dial for an HTTP-readiness endpoint, which is always TCP in practice. */ +export const httpHost = (endpoint: ServiceEndpoint): string => + endpoint.kind === "unix" ? endpoint.path : (endpoint.host ?? "127.0.0.1"); diff --git a/packages/stack/tests/helpers/git.ts b/packages/stack/tests/helpers/git.ts index 439c946cab..cf4bb07714 100644 --- a/packages/stack/tests/helpers/git.ts +++ b/packages/stack/tests/helpers/git.ts @@ -20,6 +20,11 @@ const decode = (chunks: ReadonlyArray): string => { return new TextDecoder().decode(bytes); }; +/** Disables auto-gc/maintenance so Git never forks a detached writer into `.git/` after this + * command returns; test fixtures remove the repository right after these commands finish, and a + * detached `gc --auto`/`maintenance run --auto` would still be touching objects and packs then. */ +const NO_AUTO_MAINTENANCE = ["-c", "gc.auto=0", "-c", "maintenance.auto=false"]; + /** Runs Git while draining both output streams, preserving diagnostics on failure. */ export const runGit = ( cwd: string, @@ -28,7 +33,7 @@ export const runGit = ( Effect.scoped( Effect.gen(function* () { const command = `git ${args.join(" ")}`; - const child = yield* ChildProcess.make("git", [...args], { + const child = yield* ChildProcess.make("git", [...NO_AUTO_MAINTENANCE, ...args], { cwd, stdout: "pipe", stderr: "pipe", diff --git a/packages/stack/tests/host-process-fixture.ts b/packages/stack/tests/host-process-fixture.ts index f0210334c5..ba571ad63e 100644 --- a/packages/stack/tests/host-process-fixture.ts +++ b/packages/stack/tests/host-process-fixture.ts @@ -16,13 +16,13 @@ import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; import { closeSync, createReadStream, writeSync } from "node:fs"; import { currentRelease, launchHost, authorizes, type HostEndpoint } from "../src/HostProcess.ts"; import { bindControl } from "../src/StackHost.ts"; -import * as State from "../src/State.ts"; +import * as StackNamespace from "../src/StackNamespace.ts"; class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} const makeState = (root: string) => - Layer.build(State.layer({ root })).pipe( - Effect.map((context) => Context.get(context, State.Service)), + Layer.build(StackNamespace.layer({ root })).pipe( + Effect.map((context) => Context.get(context, StackNamespace.Service)), ); const awaitBarrier = NodeStream.fromReadable({ @@ -56,11 +56,15 @@ const owner = Effect.scoped( const path = yield* Path.Path; const stack = yield* state.read(stackId); if (stack === undefined) return yield* new FixtureError({ message: "stack is not registered" }); - if (!(yield* state.lease(stackId))) { - report({ type: "error", message: "lease held", reason: "lease-held" }); - return; - } - yield* state.retractHolder(stackId); + const lease = yield* state.acquireLease(stackId).pipe( + Effect.catchTag("Namespace.LeaseHeldError", () => + Effect.sync(() => { + report({ type: "error", message: "lease held", reason: "lease-held" }); + }).pipe(Effect.as(undefined)), + ), + ); + if (lease === undefined) return; + yield* lease.retractHolder; if (stack.identity.stackName === "slow-handshake") { const marker = path.join(stateRoot, "slow-handshake.pid"); yield* fs.writeFileString(`${marker}.tmp`, String(process.pid)); @@ -97,7 +101,7 @@ const owner = Effect.scoped( }), ) .pipe(Effect.forkScoped); - yield* state.publishHolder(stackId, { + yield* lease.publishHolder({ role: "owner", secret, port: endpoint.port, @@ -109,7 +113,7 @@ const owner = Effect.scoped( report({ type: "ready", endpoint, secret }); if (mode === "held") yield* awaitBarrier; else yield* Deferred.await(shutdown); - yield* state.retractHolder(stackId); + yield* lease.retractHolder; }), ); diff --git a/packages/stack/tests/state-lock-fixture.ts b/packages/stack/tests/namespace-lock-fixture.ts similarity index 69% rename from packages/stack/tests/state-lock-fixture.ts rename to packages/stack/tests/namespace-lock-fixture.ts index d78b5d0e24..322b15e7be 100644 --- a/packages/stack/tests/state-lock-fixture.ts +++ b/packages/stack/tests/namespace-lock-fixture.ts @@ -1,9 +1,8 @@ import { NodeServices, NodeSocketServer } from "@effect/platform-node"; import { Console, Effect } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import * as State from "../src/State.ts"; -import { isBunVirtualPath } from "../src/internal/dispatch-markers.ts"; import { fileURLToPath } from "node:url"; +import * as StackNamespace from "../src/StackNamespace.ts"; const [mode, root, id] = process.argv.slice(2); if (root === undefined) throw new Error("Registry root missing"); @@ -19,7 +18,7 @@ const program = Effect.scoped( return yield* Effect.never; } yield* Effect.gen(function* () { - const state = yield* State.Service; + const state = yield* StackNamespace.Service; yield* state.withLock( Effect.gen(function* () { if (mode === "write") { @@ -34,24 +33,20 @@ const program = Effect.scoped( } const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; yield* spawner.spawn( - ChildProcess.make( - process.execPath, - [...(isBunVirtualPath(fixturePath) ? [] : [fixturePath]), "child", root], - { - // Keep the child independent on Windows so holder termination tests lock recovery. - detached: process.platform === "win32", - stdin: "ignore", - stdout: "inherit", - stderr: "inherit", - forceKillAfter: "1 second", - }, - ), + ChildProcess.make(process.execPath, [fixturePath, "child", root], { + // Keep the child independent on Windows so holder termination tests lock recovery. + detached: process.platform === "win32", + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", + forceKillAfter: "1 second", + }), ); yield* Console.log("locked"); return yield* Effect.never; }), ); - }).pipe(Effect.provide(State.layer({ root }))); + }).pipe(Effect.provide(StackNamespace.layer({ root }))); }), ).pipe(Effect.provide(NodeServices.layer)); diff --git a/packages/stack/tests/owner-rpc.ts b/packages/stack/tests/owner-rpc.ts index fd8d254423..933d4571b3 100644 --- a/packages/stack/tests/owner-rpc.ts +++ b/packages/stack/tests/owner-rpc.ts @@ -2,8 +2,9 @@ import { Context, Effect, Layer } from "effect"; import { RpcTest } from "effect/unstable/rpc"; import * as Owner from "../src/Owner.ts"; import { OwnerRpc } from "../src/Rpc.ts"; -import { Service as StateService } from "../src/State.ts"; -import type { Interface as StateInterface, SavedStack } from "../src/State.ts"; +import type { EngineTarget } from "../src/runtime/Container.ts"; +import { Service as StateService } from "../src/StackNamespace.ts"; +import type { Interface as StateInterface, SavedStack } from "../src/StackNamespace.ts"; /** Builds an in-process owner and an RPC client bound to its handlers. */ export const ownerFor = (options: { @@ -11,6 +12,7 @@ export const ownerFor = (options: { readonly state: StateInterface; readonly root: string; readonly cacheRoot: string; + readonly engineTarget?: EngineTarget; }) => Effect.gen(function* () { const { state, ...layerOptions } = options; diff --git a/packages/stack/tests/owner.ts b/packages/stack/tests/owner.ts index 1678d75ab4..e76297721b 100644 --- a/packages/stack/tests/owner.ts +++ b/packages/stack/tests/owner.ts @@ -1,9 +1,16 @@ import { expect } from "@effect/vitest"; -import { Deferred, Effect, Fiber, Option, Predicate, Stream } from "effect"; +import { Deferred, Effect, Fiber, FileSystem, Option, Stream } from "effect"; import { ChildProcess } from "effect/unstable/process"; import { fileURLToPath } from "node:url"; import { discover, type StackLocations } from "../src/effect.ts"; -import { HostProcessError, shutdownHost, type HostAccess } from "../src/HostProcess.ts"; +import { + hasReason, + HostProcessError, + ownerExitProbe, + shutdownHost, + waitForOwnerExit, + type HostAccess, +} from "../src/HostProcess.ts"; /** Requests shutdown through the owner's stable endpoint, failing with the owner's refusal. */ export const shutdownOwner = Effect.fn("Test.shutdownOwner")(function* ( @@ -72,17 +79,12 @@ export const captureOwnerPid = Effect.fn("Test.captureOwnerPid")(function* ( return owner.pid; }); +/** Waits for the owner to leave the process table: its lease is released before it exits. */ export const assertOwnerExited = Effect.fn("Test.assertOwnerExited")(function* (pid: number) { - const exists = yield* Effect.try({ - try: () => process.kill(pid, 0), - catch: (cause) => - new HostProcessError({ operation: "test-owner", message: String(cause), cause }), - }).pipe( + const fs = yield* FileSystem.FileSystem; + const exited = yield* waitForOwnerExit(pid, ownerExitProbe(fs)).pipe( Effect.as(true), - Effect.catchIf( - ({ cause }) => Predicate.hasProperty(cause, "code") && cause.code === "ESRCH", - () => Effect.succeed(false), - ), + Effect.catchIf(hasReason("owner-exit-pending"), () => Effect.succeed(false)), ); - expect(exists, `Owner process ${pid} survived successful teardown`).toBe(false); + expect(exited, `Owner process ${pid} survived successful teardown`).toBe(true); }); diff --git a/packages/stack/tests/publish-loop-fixture.ts b/packages/stack/tests/publish-loop-fixture.ts new file mode 100644 index 0000000000..a6bb57e3bc --- /dev/null +++ b/packages/stack/tests/publish-loop-fixture.ts @@ -0,0 +1,40 @@ +import { NodeServices } from "@effect/platform-node"; +import { Console, Effect, FileSystem, Layer } from "effect"; +import * as StackNamespace from "../src/StackNamespace.ts"; + +const [root, id] = process.argv.slice(2); +if (root === undefined || id === undefined) throw new Error("Registry root or stack id missing"); + +/** + * Publishes one revision of the saved stack, announcing readiness on stdout the instant it is + * about to replace the target document (the real staging-to-publish boundary Publication.publish + * uses), then hangs so a caller can kill this process exactly there on every run. + */ +const barrierFs = Layer.effect( + FileSystem.FileSystem, + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return { + ...fs, + rename: () => Console.log("about-to-publish").pipe(Effect.andThen(Effect.never)), + }; + }), +); + +const program = Effect.scoped( + Effect.gen(function* () { + const state = yield* StackNamespace.Service; + const saved = yield* state.read(id); + if (saved === undefined) return yield* Effect.die(`Stack ${id} is not registered`); + yield* state.save({ ...saved, identity: { ...saved.identity, stackName: "revision-1" } }); + }), +).pipe( + Effect.provide( + StackNamespace.layer({ root }).pipe( + Layer.provide(barrierFs), + Layer.provide(NodeServices.layer), + ), + ), +); + +await Effect.runPromise(program); diff --git a/packages/stack/tests/release-fifo.ts b/packages/stack/tests/release-fifo.ts new file mode 100644 index 0000000000..346163bb89 --- /dev/null +++ b/packages/stack/tests/release-fifo.ts @@ -0,0 +1,21 @@ +import { Effect } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- the FIFO write end must stay open read-write so releasing never blocks. +import { closeSync, constants, openSync, writeSync } from "node:fs"; + +/** + * Holds an existing FIFO open read-write for the scope, so a release never blocks on a missing + * reader and one written before a reader opens is buffered. `release` is idempotent. + */ +export const holdReleaseFifo = (path: string) => + Effect.gen(function* () { + const holder = openSync(path, constants.O_RDWR); + let released = false; + yield* Effect.addFinalizer(() => Effect.sync(() => closeSync(holder))); + return { + release: Effect.sync(() => { + if (released) return; + released = true; + writeSync(holder, "\n"); + }), + }; + }); diff --git a/packages/stack/tests/short-registration-poll-fixture.ts b/packages/stack/tests/short-registration-poll-fixture.ts new file mode 100644 index 0000000000..32d3226646 --- /dev/null +++ b/packages/stack/tests/short-registration-poll-fixture.ts @@ -0,0 +1,5 @@ +import { runHostProcess } from "../src/internal/host-process.ts"; + +/** An owner whose registration-loss poll runs every 200ms, so ownership-loss tests need not wait the production interval. */ +if (import.meta.main) + await runHostProcess(process.argv.slice(2), { registrationCheckInterval: "200 millis" }); diff --git a/packages/stack/tests/standalone-service.ts b/packages/stack/tests/standalone-service.ts new file mode 100644 index 0000000000..935b26ebfe --- /dev/null +++ b/packages/stack/tests/standalone-service.ts @@ -0,0 +1,63 @@ +import { Effect, Ref } from "effect"; +import * as Orchestrator from "../src/Orchestrator.ts"; +import { makeService, type ServiceDefinition, type ServiceError } from "../src/Service.ts"; + +/** + * Runs one service definition under its own lifecycle authority, as a lazy member with no idle + * timeout: an explicit start launches it once, and a failed launch is not retried on its own. + */ +export const makeStandaloneService = Effect.fn("Test.makeStandaloneService")(function* ( + definition: ServiceDefinition, + options: { readonly id: string; readonly config: Config }, +) { + const { id } = options; + const orchestrator = yield* Orchestrator.make(); + const current = yield* Ref.make(options.config); + const candidate = yield* Ref.make(options.config); + const core = yield* makeService(definition, { + id, + config: options.config, + report: orchestrator.report, + }); + // The orchestrator only passes a restart candidate through; the typed value lives here. + const configFor = (restarted: unknown) => + restarted === undefined ? Ref.get(current) : Ref.get(candidate); + yield* orchestrator.register({ + id, + service: id, + core, + launch: (generation, _inputs, restarted) => + configFor(restarted).pipe( + Effect.tap((config) => Ref.set(current, config)), + Effect.flatMap((config) => core.launch(generation, config)), + ), + prepare: (_inputs, restarted) => configFor(restarted).pipe(Effect.flatMap(core.prepare)), + bind: Effect.void, + close: Effect.void, + confirmRemoved: Effect.void, + release: Effect.void, + releasePorts: Effect.void, + hasEndpoint: true, + inputs: [], + outputs: {}, + }); + yield* orchestrator.configure({ members: [{ id, activation: "lazy" }], dependencies: [] }); + return { + id, + get: Effect.all([orchestrator.status(id), core.get]).pipe( + Effect.map(([status, execution]) => ({ ...status, config: execution.config })), + ), + observation: orchestrator.changes(id), + /** Whether the orchestrator still lists the instance. */ + listed: orchestrator.instances.pipe(Effect.map((entries) => entries.some((e) => e.id === id))), + start: orchestrator.start(id), + ready: orchestrator.ready(id), + stop: orchestrator.stop(id), + restart: (config?: Config) => + config === undefined + ? orchestrator.restart(id) + : Ref.set(candidate, config).pipe(Effect.andThen(orchestrator.restart(id, config))), + destroy: orchestrator.destroy(id), + storage: (operation: Effect.Effect) => orchestrator.storage(id, operation), + }; +}); diff --git a/packages/stack/tests/test-engine.ts b/packages/stack/tests/test-engine.ts new file mode 100644 index 0000000000..fcfac1159b --- /dev/null +++ b/packages/stack/tests/test-engine.ts @@ -0,0 +1,12 @@ +import { Config, Effect } from "effect"; +import type { ContainerEngine } from "../src/runtime/Container.ts"; + +/** + * The container engine real-engine tests run against: `docker` unless `SUPABASE_STACK_TEST_ENGINE` + * selects `podman`. Resolving it never contacts the engine. + */ +export const testEngine: ContainerEngine = Effect.runSync( + Config.literals(["docker", "podman"], "SUPABASE_STACK_TEST_ENGINE").pipe( + Config.withDefault("docker"), + ), +); diff --git a/packages/stack/tests/watch-entry.ts b/packages/stack/tests/watch-entry.ts new file mode 100644 index 0000000000..731a08f122 --- /dev/null +++ b/packages/stack/tests/watch-entry.ts @@ -0,0 +1,38 @@ +import { Effect, Path } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem.watch's async stat before attaching makes attachment unobservable. +import { existsSync, watch } from "node:fs"; + +/** + * Watches `entry` under `directory` and returns, within the caller's scope, an effect that + * completes once the entry's existence equals `present`. Subscribe before triggering the change, + * then await the returned effect after the trigger: attaching the watcher happens synchronously + * here, and existence is only checked afterward, so no event between attachment and the trigger + * can be missed. This does not use `FileSystem.watch`: its `stat` runs asynchronously before the + * underlying `fs.watch` attaches, so attachment itself is unobservable and an event in that gap + * is lost until the caller's timeout fires. + */ +export const watchEntry = Effect.fn("Test.watchEntry")(function* ( + directory: string, + entry: string, + present: boolean, +) { + const path = yield* Path.Path; + const target = path.join(directory, entry); + const listeners = new Set<() => void>(); + yield* Effect.acquireRelease( + Effect.sync(() => watch(directory, () => listeners.forEach((listener) => listener()))), + (watcher) => Effect.sync(() => watcher.close()), + ); + return yield* Effect.succeed( + Effect.callback((resume) => { + const check = () => { + if (existsSync(target) === present) resume(Effect.void); + }; + check(); + listeners.add(check); + return Effect.sync(() => { + listeners.delete(check); + }); + }), + ); +}); diff --git a/packages/stack/tests/whole-stack/confinement.ts b/packages/stack/tests/whole-stack/confinement.ts new file mode 100644 index 0000000000..03a61a0d7b --- /dev/null +++ b/packages/stack/tests/whole-stack/confinement.ts @@ -0,0 +1,187 @@ +import { expect } from "@effect/vitest"; +import { Effect, FileSystem, Option, Path } from "effect"; +import type { PlatformError } from "effect/PlatformError"; + +interface TreeEntry { + readonly isFile: boolean; + readonly size: bigint; + readonly mtimeMs: number; +} + +export type Tree = ReadonlyMap; + +/** + * Snapshots every path under `root`, or an empty tree when `root` does not exist yet. A directory + * listed in `opaque` is recorded but never entered, for data a container engine owns under a + * mapped uid that the host user cannot traverse. + */ +export const snapshotTree = Effect.fn("Confinement.snapshotTree")(function* ( + root: string, + opaque: ReadonlyArray = [], +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + if (!(yield* fs.exists(root))) return new Map(); + const entries = new Map(); + const visit = (directory: string): Effect.Effect => + Effect.gen(function* () { + const names = yield* fs.readDirectory(directory); + yield* Effect.forEach( + names, + (name) => + Effect.gen(function* () { + const full = path.join(directory, name); + const info = yield* fs.stat(full).pipe(Effect.option); + // A path can disappear between listing and stat (e.g. a service's own temp file); treat + // it as absent rather than failing the whole snapshot. + if (Option.isNone(info)) { + entries.set(full, { isFile: false, size: 0n, mtimeMs: 0 }); + return; + } + entries.set(full, { + isFile: info.value.type === "File", + size: info.value.size, + mtimeMs: Option.match(info.value.mtime, { + onNone: () => 0, + onSome: (date) => date.getTime(), + }), + }); + // A symlink is recorded but not followed, matching a recursive directory listing. + if ( + info.value.type === "Directory" && + !opaque.includes(full) && + (yield* fs.readLink(full).pipe(Effect.isFailure)) + ) + yield* visit(full); + }), + { concurrency: 8, discard: true }, + ); + }).pipe( + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.void, + ), + ); + yield* visit(root); + return entries as Tree; +}); + +export interface ConfinementViolation { + readonly path: string; + readonly kind: "created" | "modified"; +} + +/** Flags every path that is new or changed between two snapshots of the same root. */ +export const diffTrees = (before: Tree, after: Tree): ReadonlyArray => { + const violations: Array = []; + for (const [entryPath, entry] of after) { + const previous = before.get(entryPath); + if (previous === undefined) violations.push({ path: entryPath, kind: "created" }); + else if (entry.isFile && (entry.size !== previous.size || entry.mtimeMs !== previous.mtimeMs)) + violations.push({ path: entryPath, kind: "modified" }); + } + return violations; +}; + +/** Fails, listing every offending path, when a violation falls outside the allowed roots. */ +export const assertConfinedTo = ( + violations: ReadonlyArray, + allowedRoots: ReadonlyArray, +): void => { + const isAllowed = (candidate: string) => + allowedRoots.some((root) => candidate === root || candidate.startsWith(`${root}/`)); + const offending = violations.filter((violation) => !isAllowed(violation.path)); + expect( + offending, + `Stack wrote outside its allowed roots (${allowedRoots.join(", ")}):\n${offending + .map((violation) => ` ${violation.kind}: ${violation.path}`) + .join("\n")}`, + ).toEqual([]); +}; + +/** + * Negative control: proves `assertConfinedTo` actually flags an unauthorized write under `root`, + * for both a write that persists and one created and then removed again before the next snapshot + * (the gap a before/after-only check would otherwise miss). Cleans up after itself so it leaves no + * trace for the scenario's own, later checks. + */ +export const proveConfinementDetectsViolations = Effect.fn( + "Confinement.proveConfinementDetectsViolations", +)(function* (root: string, baseline: Tree, allowedRoots: ReadonlyArray) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const persistentMarker = path.join(root, ".confinement-control-persistent"); + const transientMarker = path.join(root, ".confinement-control-transient"); + yield* fs.writeFileString(persistentMarker, "control"); + yield* fs.writeFileString(transientMarker, "control"); + const withBothMarkers = diffTrees(baseline, yield* snapshotTree(root)); + expect(withBothMarkers.map((violation) => violation.path).sort()).toEqual( + [persistentMarker, transientMarker].sort(), + ); + expect(() => assertConfinedTo(withBothMarkers, allowedRoots)).toThrow(); + yield* fs.remove(transientMarker, { force: true }); + const afterTransientRemoved = diffTrees(baseline, yield* snapshotTree(root)); + // The residual gap: a write made and undone strictly between two snapshot points is invisible + // to a diff taken only afterward, unlike the persistent one. + expect(afterTransientRemoved.map((violation) => violation.path)).toEqual([persistentMarker]); + yield* fs.remove(persistentMarker, { force: true }); +}); + +/** + * Environment names the stack, its workloads, or the test harness's own Bun runtime resolve a + * user-level root from. + */ +const sandboxEnvNames = [ + "HOME", + "TMPDIR", + "TMP", + "TEMP", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + // Bun's own JIT transpile cache; a test-harness concern, not a stack confinement one. + "BUN_RUNTIME_TRANSPILER_CACHE_PATH", + // Pre-resolved, in the real environment, to the endpoint the engine's own context store would + // have answered: once set, the owner's pinned target resolution skips that store entirely, so + // it never needs `~/.docker` under the sandboxed HOME below. Exposing that real config directory + // instead would also expose its credential store, which a real pull would then invoke under the + // sandboxed HOME, writing outside what this test allows. + "DOCKER_HOST", +] as const; + +type SandboxEnvName = (typeof sandboxEnvNames)[number]; + +export type SandboxEnvironment = Readonly>; + +/** + * Runs `use` with the given variables set on the process environment, so a detached stack owner + * spawned during `use` inherits them, and restores the previous values once `use` completes. + */ +export const withSandboxEnvironment = ( + values: SandboxEnvironment, + use: Effect.Effect, +): Effect.Effect => + Effect.acquireUseRelease( + Effect.sync(() => { + const previous: Partial> = {}; + for (const name of sandboxEnvNames) { + // oxlint-disable-next-line effecttsgo/process-env-in-effect, effecttsgo/process-env -- the detached stack owner and the harness's own Bun process inherit these at spawn time; this is not application config. + previous[name] = process.env[name]; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- see above. + process.env[name] = values[name]; + } + return previous; + }), + () => use, + (previous) => + Effect.sync(() => { + for (const name of sandboxEnvNames) { + const value = previous[name]; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + if (value === undefined) delete process.env[name]; + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- restores the mutation made above. + else process.env[name] = value; + } + }), + ); diff --git a/packages/stack/tests/whole-stack/fixture.ts b/packages/stack/tests/whole-stack/fixture.ts index c34c578377..cdba91b694 100644 --- a/packages/stack/tests/whole-stack/fixture.ts +++ b/packages/stack/tests/whole-stack/fixture.ts @@ -13,12 +13,15 @@ import { Stream, } from "effect"; import { postgres } from "../../src/Commands.ts"; -import { homedir, tmpdir } from "node:os"; +import { homedir } from "node:os"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { create, type Stack } from "../../src/effect.ts"; +import type { ContainerEngine } from "../../src/runtime/Container.ts"; import type { Observation } from "../../src/Rpc.ts"; import { cleanupDockerRoot } from "../docker-cleanup.ts"; import { destroyTestStack } from "../stack-cleanup.ts"; +import { testEngine } from "../test-engine.ts"; +import { testArtifactCacheRoot } from "../artifact-cache.ts"; type AnyService = Effect.Success[number]; @@ -37,7 +40,7 @@ export const serviceNames = [ "pooler", ] as const; -export type Runtime = "native" | "docker"; +export type Runtime = "native" | ContainerEngine; export type WholeStack = Readonly<{ readonly stack: Stack; readonly services: ReadonlyArray; @@ -110,6 +113,12 @@ const ownerLogTail = Effect.fn("WholeStack.ownerLogTail")( export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => Effect.gen(function* () { + // The test helpers observe and clean up through the selected engine, so a container runtime + // must be that engine. + if (runtime !== "native" && runtime !== testEngine) + return yield* Effect.die( + `Whole-stack ${runtime} scenarios need SUPABASE_STACK_TEST_ENGINE=${runtime}`, + ); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-whole-${runtime}-`, @@ -127,7 +136,7 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => const secret = `whole-stack-${yield* crypto.randomUUIDv4}-secret`; const locations = { stateRoot: `${root}/state`, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + cacheRoot: testArtifactCacheRoot, }; const stack = yield* create({ projectRoot: root, @@ -140,7 +149,7 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => Effect.gen(function* () { const current = yield* Ref.get(owner); const destroy = Option.isSome(current) ? destroyTestStack(current.value) : Effect.void; - yield* runtime === "docker" + yield* runtime !== "native" ? destroy.pipe(Effect.ensuring(cleanupDockerRoot(storageRoot))) : destroy; }), diff --git a/packages/stack/tests/whole-stack/idle.ts b/packages/stack/tests/whole-stack/idle.ts index 67e3f46289..e8bdb13e4c 100644 --- a/packages/stack/tests/whole-stack/idle.ts +++ b/packages/stack/tests/whole-stack/idle.ts @@ -1,5 +1,5 @@ import { expect } from "@effect/vitest"; -import { Context, Effect, Fiber, Layer, Redacted, Schema } from "effect"; +import { Context, Effect, Exit, Fiber, Layer, Option, Redacted, Schema, Stream } from "effect"; import { PgClient } from "@effect/sql-pg"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { SignJWT } from "jose"; @@ -11,8 +11,10 @@ import { waitForLifecycle, wholeStack, type Runtime, + type WholeStack, } from "./fixture.ts"; import { subscribeRealtime } from "./websocket.ts"; +import { killWorkload } from "./workloads.ts"; class IdleProbeError extends Schema.TaggedError()("IdleProbeError", { message: Schema.String, @@ -107,13 +109,13 @@ export const idleWake = (runtime: Runtime) => { id: fixture.stack.id, value: "awake" }, { ...headers, "content-type": "application/json", prefer: "return=minimal" }, ); - expect(insert.status).toBe(201); + expect(insert.status, insert.body).toBe(201); yield* Fiber.join(firstChange); const rest = service(fixture, "rest"); yield* waitForLifecycle(rest, "stopped"); expect((yield* rest.status).wakeEnabled).toBe(true); const authHealth = yield* requestWithHeaders(`${authUrl}/health`, {}); - expect(authHealth.status).toBe(200); + expect(authHealth.status, authHealth.body).toBe(200); expect((yield* rest.status).lifecycle).toBe("stopped"); expect((yield* database.status).lifecycle).toBe("running"); expect((yield* service(fixture, "realtime").status).lifecycle).toBe("running"); @@ -125,7 +127,7 @@ export const idleWake = (runtime: Runtime) => `${restUrl}/idle_items?id=eq.${fixture.stack.id}`, headers, ); - expect(wake.status).toBe(200); + expect(wake.status, wake.body).toBe(200); expect((yield* rest.status).lifecycle).toBe("running"); const marker = `${fixture.stack.id}-change`; const changed = yield* jsonRequest( @@ -134,7 +136,7 @@ export const idleWake = (runtime: Runtime) => { id: marker, value: "wake" }, { ...headers, "content-type": "application/json", prefer: "return=minimal" }, ); - expect(changed.status).toBe(201); + expect(changed.status, changed.body).toBe(201); const change = yield* Fiber.join(nextChange); const changeRow = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -168,7 +170,7 @@ export const idleWake = (runtime: Runtime) => `${restUrl}/idle_items?id=eq.${fixture.stack.id}`, headers, ); - expect(wake.status).toBe(200); + expect(wake.status, wake.body).toBe(200); const marker = `${fixture.stack.id}-reconnect`; const changed = yield* jsonRequest( "POST", @@ -176,7 +178,7 @@ export const idleWake = (runtime: Runtime) => { id: marker, value: "reconnect" }, { ...headers, "content-type": "application/json", prefer: "return=minimal" }, ); - expect(changed.status).toBe(201); + expect(changed.status, changed.body).toBe(201); const change = yield* Fiber.join(nextChange); const changeRow = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -188,3 +190,117 @@ export const idleWake = (runtime: Runtime) => yield* waitForLifecycle(service(fixture, "pooler"), "stopped"); }), ); + +/** Only the given member carries a short idle timeout; the rest of the composition stays lazy but untimed. */ +const configureSoleTimedMember = (fixture: WholeStack, timedId: string) => + Effect.gen(function* () { + const database = service(fixture, "database"); + const configuration = yield* fixture.stack.composition.describe; + yield* fixture.stack.composition.configure({ + ...configuration, + members: configuration.members.map((member) => + member.id === database.id + ? { id: member.id, activation: "eager" as const } + : member.id === timedId + ? { id: member.id, activation: "lazy" as const, idleMillis } + : { id: member.id, activation: "lazy" as const }, + ), + }); + }); + +/** A restart through the public API must preserve activation intent: the member still idle-sleeps + * afterwards and wakes again on the next request (regression for Service.restart disarming wake). */ +export const restartIdleWake = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* wholeStack(runtime); + const rest = service(fixture, "rest"); + yield* configureSoleTimedMember(fixture, rest.id); + yield* fixture.stack.composition.start; + + const token = yield* signServiceToken(fixture.secret); + const restUrl = (yield* rest.credentials()).url; + if (restUrl === undefined) return yield* Effect.die("Rest URL missing"); + const headers = { authorization: `Bearer ${token}`, apikey: token }; + + const warm = yield* requestWithHeaders(restUrl, headers); + expect(warm.status, warm.body).toBe(200); + expect((yield* rest.status).lifecycle).toBe("running"); + + yield* rest.restart(); + expect((yield* rest.status).wakeEnabled).toBe(true); + + yield* waitForLifecycle(rest, "stopped"); + expect((yield* rest.status).wakeEnabled).toBe(true); + + const rewarm = yield* requestWithHeaders(restUrl, headers); + expect(rewarm.status, rewarm.body).toBe(200); + expect((yield* rest.status).lifecycle).toBe("running"); + + yield* killWorkload(runtime, fixture, rest.id); + const crashed = yield* rest.followStatus.pipe( + Stream.filter((status) => status.lifecycle === "stopped"), + Stream.runHead, + Effect.map(Option.getOrThrow), + ); + expect(crashed.exit !== undefined && Exit.isFailure(crashed.exit)).toBe(true); + expect(crashed.wakeEnabled).toBe(true); + + const rewake = yield* requestWithHeaders(restUrl, headers); + expect(rewake.status, rewake.body).toBe(200); + expect((yield* rest.status).lifecycle).toBe("running"); + }), + ); + +/** Functions must idle-sleep and wake like every other lazy service (no exemption). */ +export const functionsIdleWake = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* wholeStack(runtime); + const functionsInstance = service(fixture, "functions"); + // The default composition policy (not a test override) must give functions the same idle + // timeout as its peers instead of exempting it. + const defaultConfiguration = yield* fixture.stack.composition.describe; + const defaultMember = defaultConfiguration.members.find( + (member) => member.id === functionsInstance.id, + ); + expect(defaultMember?.activation).toBe("lazy"); + expect(defaultMember?.idleMillis).toBe(60_000); + yield* configureSoleTimedMember(fixture, functionsInstance.id); + yield* fixture.stack.composition.start; + yield* sql( + fixture, + "create table if not exists public.whole_stack_items (id text primary key, value text); grant select, insert on public.whole_stack_items to anon, authenticated, service_role;", + ); + + const token = yield* signServiceToken(fixture.secret); + const functionsUrl = (yield* functionsInstance.credentials()).url; + if (functionsUrl === undefined) return yield* Effect.die("Functions URL missing"); + const headers = { + authorization: `Bearer ${token}`, + apikey: token, + "content-type": "application/json", + }; + + const invoke = yield* jsonRequest( + "POST", + `${functionsUrl}/hello`, + { id: "functions-idle-probe" }, + headers, + ); + expect(invoke.status, invoke.body).toBe(200); + expect((yield* functionsInstance.status).lifecycle).toBe("running"); + + yield* waitForLifecycle(functionsInstance, "stopped"); + expect((yield* functionsInstance.status).wakeEnabled).toBe(true); + + const rewake = yield* jsonRequest( + "POST", + `${functionsUrl}/hello`, + { id: "functions-idle-probe" }, + headers, + ); + expect(rewake.status, rewake.body).toBe(200); + expect((yield* functionsInstance.status).lifecycle).toBe("running"); + }), + ); diff --git a/packages/stack/tests/whole-stack/loopback.ts b/packages/stack/tests/whole-stack/loopback.ts new file mode 100644 index 0000000000..26f8d08cbc --- /dev/null +++ b/packages/stack/tests/whole-stack/loopback.ts @@ -0,0 +1,149 @@ +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { captureWorkloads, commandOutput } from "./workloads.ts"; +import type { WholeStack } from "./fixture.ts"; + +export type LoopbackViolation = Readonly<{ + readonly pid: number; + readonly address: string; + readonly command: string; +}>; + +type ProcessEntry = Readonly<{ + readonly pid: number; + readonly ppid: number; + readonly command: string; +}>; + +const processTable = Effect.fn("WholeStack.loopback.processTable")(() => + Effect.gen(function* () { + const output = yield* commandOutput("ps", ["-axo", "pid=,ppid=,command="]); + const entries: Array = []; + for (const line of output.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line); + if (match === null) continue; + entries.push({ pid: Number(match[1]), ppid: Number(match[2]), command: match[3] ?? "" }); + } + return entries; + }), +); + +const rootWorkloadPids = (identities: ReadonlyArray): ReadonlyArray => + identities + .map((identity) => /^\s*(\d+)\s/.exec(identity)?.[1]) + .filter((value): value is string => value !== undefined) + .map(Number); + +/** Walks the host's process table so a workload's descendants are covered: the native launcher + * spawns each workload into its own separate process group, not the launcher's. */ +const descendantsOf = (roots: ReadonlyArray, entries: ReadonlyArray) => { + const byParent = new Map>(); + for (const entry of entries) { + const list = byParent.get(entry.ppid) ?? []; + list.push(entry.pid); + byParent.set(entry.ppid, list); + } + const seen = new Set(roots); + const queue = [...roots]; + while (queue.length > 0) { + const current = queue.shift(); + if (current === undefined) continue; + for (const child of byParent.get(current) ?? []) + if (!seen.has(child)) { + seen.add(child); + queue.push(child); + } + } + return [...seen]; +}; + +type RawSocket = Readonly<{ readonly pid: number; readonly address: string }>; + +const darwinListeners = Effect.fn("WholeStack.loopback.darwinListeners")( + (pids: ReadonlyArray) => + Effect.gen(function* () { + if (pids.length === 0) return [] as ReadonlyArray; + const output = yield* commandOutput("lsof", [ + "-nP", + "-a", + "-iTCP", + "-sTCP:LISTEN", + "-p", + pids.join(","), + ]); + const sockets: Array = []; + for (const line of output.split("\n").slice(1)) { + const trimmed = line.trim(); + if (trimmed === "") continue; + const fields = trimmed.split(/\s+/); + const pid = Number(fields[1]); + const address = fields.at(-2); + if (!Number.isFinite(pid) || address === undefined) continue; + sockets.push({ pid, address }); + } + return sockets; + }), +); + +const linuxListeners = Effect.fn("WholeStack.loopback.linuxListeners")( + (pids: ReadonlySet) => + Effect.gen(function* () { + const output = yield* commandOutput("ss", ["-ltnpH"]); + const sockets: Array = []; + for (const line of output.split("\n")) { + const trimmed = line.trim(); + if (trimmed === "") continue; + const fields = trimmed.split(/\s+/); + const address = fields[3]; + const processField = fields.at(-1); + const pidMatch = processField === undefined ? null : /pid=(\d+)/.exec(processField); + if (address === undefined || pidMatch === null) continue; + const pid = Number(pidMatch[1]); + if (!pids.has(pid)) continue; + sockets.push({ pid, address }); + } + return sockets; + }), +); + +const hostOf = (address: string): string => { + if (address.startsWith("[")) { + const end = address.indexOf("]"); + return end === -1 ? address : address.slice(1, end); + } + const lastColon = address.lastIndexOf(":"); + return lastColon === -1 ? address : address.slice(0, lastColon); +}; + +const isLoopbackAddress = (address: string): boolean => { + const host = hostOf(address); + return host === "127.0.0.1" || host === "::1" || host === "localhost"; +}; + +/** + * Enumerates every listening TCP socket owned by a native stack's workload process trees (the + * launcher and every descendant it spawns) and reports the ones bound outside loopback. Windows' + * native backend is out of scope; callers skip this on `process.platform === "win32"`. + */ +export const captureLoopbackViolations = Effect.fn("WholeStack.loopback.captureLoopbackViolations")( + (fixture: WholeStack) => + Effect.gen(function* () { + const workloads = yield* captureWorkloads("native", fixture); + const roots = rootWorkloadPids(workloads.identities); + expect(roots.length).toBeGreaterThan(0); + const entries = yield* processTable(); + const pids = descendantsOf(roots, entries); + const commands = new Map(entries.map((entry) => [entry.pid, entry.command] as const)); + const sockets = + process.platform === "linux" + ? yield* linuxListeners(new Set(pids)) + : yield* darwinListeners(pids); + return sockets + .filter((socket) => !isLoopbackAddress(socket.address)) + .map((socket): LoopbackViolation => ({ + pid: socket.pid, + address: socket.address, + command: commands.get(socket.pid) ?? `pid ${socket.pid}`, + })); + }), +); diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 0c1e42633b..fb24e1c0e4 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -1,6 +1,7 @@ import { expect } from "@effect/vitest"; -import { Cause, Effect, FileSystem, Ref, Schema } from "effect"; +import { Cause, Config, Effect, FileSystem, Path, Ref, Schema } from "effect"; import { open } from "../../src/effect.ts"; +import { runEngine } from "../docker-fixture.ts"; import { allMembers, clearStackOwner, @@ -27,7 +28,15 @@ import { exerciseAnalytics, queryAnalyticsMarker, stackAnalyticsMarker } from ". import { subscribeRealtime } from "./websocket.ts"; import { watchRecoveryMail } from "./realtime-mail.ts"; import { assertWorkloadsGone, captureWorkloads } from "./workloads.ts"; +import { captureLoopbackViolations } from "./loopback.ts"; import { assertOwnerExited, captureOwnerPid } from "../owner.ts"; +import { + assertConfinedTo, + diffTrees, + proveConfinementDetectsViolations, + snapshotTree, + withSandboxEnvironment, +} from "./confinement.ts"; const withFixture = ( runtime: Runtime, @@ -51,9 +60,6 @@ const assertDefaultPolicy = (fixture: WholeStack) => if (name === "database") { expect(member.activation).toBe("eager"); expect(member.idleMillis).toBeUndefined(); - } else if (name === "functions") { - expect(member.activation).toBe("lazy"); - expect(member.idleMillis).toBeUndefined(); } else { expect(member.activation).toBe("lazy"); expect(member.idleMillis).toBe(name === "studio" ? 300_000 : 60_000); @@ -127,7 +133,7 @@ const assertOwnedPathsGone = Effect.fn("WholeStack.assertOwnedPathsGone")((fixtu expect(yield* fs.exists(databasePath)).toBe(false); expect(yield* fs.exists(functionsPath)).toBe(false); expect(yield* fs.exists(statePath)).toBe(false); - expect(yield* fs.exists(`${fixture.locations.stateRoot}/${fixture.stack.id}`)).toBe(false); + expect((yield* fs.exists(dataRoot)) ? yield* fs.readDirectory(dataRoot) : []).toEqual([]); }), ); @@ -183,7 +189,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( const email = `whole-${phase}-${fixture.stack.id}@example.test`; const password = "whole-stack-password"; const signup = yield* jsonRequest("POST", `${authUrl}/signup`, { email, password }); - expect(signup.status).toBe(200); + expect(signup.status, signup.body).toBe(200); const signupBody = yield* Schema.decodeUnknownEffect( Schema.Struct({ access_token: Schema.String, @@ -209,12 +215,12 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( { id: rowId, owner_id: signupBody.user.id, value: phase }, { ...headers, prefer: "return=representation" }, ); - expect(insert.status).toBe(201); + expect(insert.status, insert.body).toBe(201); const read = yield* requestWithHeaders( `${restUrl}/whole_stack_items?id=eq.${rowId}`, headers, ); - expect(read.status).toBe(200); + expect(read.status, read.body).toBe(200); const rows = yield* Schema.decodeEffect( Schema.fromJsonString( Schema.Array(Schema.Struct({ id: Schema.String, value: Schema.String })), @@ -226,7 +232,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( email: secondEmail, password, }); - expect(secondSignup.status).toBe(200); + expect(secondSignup.status, secondSignup.body).toBe(200); const secondSignupBody = yield* Schema.decodeUnknownEffect( Schema.Struct({ access_token: Schema.String }), )(yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(secondSignup.body)); @@ -234,7 +240,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( authorization: `Bearer ${secondSignupBody.access_token}`, apikey: secondSignupBody.access_token, }); - expect(otherRead.status).toBe(200); + expect(otherRead.status, otherRead.body).toBe(200); expect( yield* Schema.decodeEffect( Schema.fromJsonString(Schema.Array(Schema.Struct({ id: Schema.String }))), @@ -255,7 +261,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( return yield* Effect.die("Mail or Realtime URL missing"); const watcher = yield* watchRecoveryMail(mailUrl, email); const recovery = yield* jsonRequest("POST", `${authUrl}/recover`, { email }); - expect(recovery.status).toBe(200); + expect(recovery.status, recovery.body).toBe(200); const mail = yield* watcher.awaitFullMail; expect(mail.To.map((recipient) => recipient.Address)).toContain(email); const authOrigin = new URL(authUrl).origin; @@ -280,7 +286,7 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( { value: `${phase}-realtime` }, { ...headers, prefer: "return=representation" }, ); - expect(update.status).toBe(200); + expect(update.status, update.body).toBe(200); const change = yield* realtime.nextChange; const event = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -466,7 +472,7 @@ export const parallel = (runtime: Runtime) => authorization: `Bearer ${leftLedger.accessToken}`, apikey: leftLedger.accessToken, }); - expect(crossStack.status).toBe(401); + expect(crossStack.status, crossStack.body).toBe(401); yield* Effect.scoped( Effect.gen(function* () { const rightRealtime = yield* subscribeRealtime( @@ -518,7 +524,7 @@ export const parallel = (runtime: Runtime) => prefer: "return=representation", }, ); - expect(update.status).toBe(200); + expect(update.status, update.body).toBe(200); const change = yield* rightRealtime.nextChange; const event = yield* Schema.decodeUnknownEffect( Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), @@ -532,3 +538,155 @@ export const parallel = (runtime: Runtime) => yield* exerciseStack(right, "parallel-right-after"); }), ); + +/** + * The real environment's own docker endpoint, read here (not through the owner's own pinned-target + * resolution) purely so {@link writeConfinement} can hand the sandboxed owner a `DOCKER_HOST` that + * works without its own `~/.docker` client state. + */ +const resolveRealDockerHost = Effect.gen(function* () { + // oxlint-disable-next-line effecttsgo/process-env-in-effect -- reads the real environment before it is sandboxed below. + const configured = process.env.DOCKER_HOST; + if (configured !== undefined && configured.length > 0) return configured; + const name = (yield* runEngine(["context", "show"])).output.trim(); + const host = (yield* runEngine([ + "context", + "inspect", + name, + "--format", + "{{.Endpoints.docker.Host}}", + ])).output.trim(); + if (host.length === 0) return yield* Effect.die(`Context ${name} has no docker endpoint`); + return host; +}); + +/** + * Rootless Podman runs its engine in-process and keeps image and container storage under the XDG + * data and config roots, so the sandboxed owner must keep the real ones to find them. Like the + * docker CLI's client state, that engine state is outside this test's confinement contract. + */ +const resolveRealPodmanState = Effect.gen(function* () { + const home = yield* Config.string("HOME"); + return { + XDG_DATA_HOME: yield* Config.string("XDG_DATA_HOME").pipe( + Config.withDefault(`${home}/.local/share`), + ), + XDG_CONFIG_HOME: yield* Config.string("XDG_CONFIG_HOME").pipe( + Config.withDefault(`${home}/.config`), + ), + }; +}); + +/** + * Proves a running stack writes only inside its state root and artifact cache (plus the project + * directory it was handed): points HOME, TMPDIR/TMP/TEMP and the XDG roots at fresh, empty + * directories for the stack and its detached owner, then diffs those directories before and after + * a full start/exercise/stop/destroy cycle. The per-user port reservation registry resolves its + * home through the OS passwd database (`getent`/`dscacheutil`), not `$HOME`, so it writes outside + * this sandbox entirely and is not expected to appear in either diff. + */ +export const writeConfinement = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const arena = yield* fs.makeTempDirectoryScoped({ prefix: "stack-confinement-arena-" }); + const home = path.join(arena, "home"); + const tmp = path.join(arena, "tmp"); + // Bun's own transpile cache is a test-harness concern, not a stack confinement one: it gets + // its own allowed root under `tmp` rather than being allow-listed inside `home`. + const bunTranspilerCache = path.join(tmp, "bun-transpiler-cache"); + yield* fs.makeDirectory(home, { recursive: true }); + yield* fs.makeDirectory(bunTranspilerCache, { recursive: true }); + // Resolved against the real environment before HOME below points at the empty sandbox: the + // owner's own `DOCKER_HOST`-pinned resolution then skips the engine's context store + // entirely, so it never needs `~/.docker` under the sandbox. The docker CLI's own client + // state (config, credential helpers) is outside this test's confinement contract; narrowing + // the sandboxed owner to a `DOCKER_HOST` pin instead of its real context is an accepted + // tradeoff for that, not something this test tries to confine. + const dockerHost = runtime === "docker" ? yield* resolveRealDockerHost : undefined; + const podmanState = runtime === "podman" ? yield* resolveRealPodmanState : undefined; + yield* withSandboxEnvironment( + { + HOME: home, + TMPDIR: tmp, + TMP: tmp, + TEMP: tmp, + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_CONFIG_HOME: podmanState?.XDG_CONFIG_HOME ?? path.join(home, ".config"), + XDG_DATA_HOME: podmanState?.XDG_DATA_HOME ?? path.join(home, ".local", "share"), + XDG_STATE_HOME: path.join(home, ".local", "state"), + BUN_RUNTIME_TRANSPILER_CACHE_PATH: bunTranspilerCache, + DOCKER_HOST: dockerHost ?? "", + }, + Effect.gen(function* () { + const beforeHome = yield* snapshotTree(home); + const beforeTmp = yield* snapshotTree(tmp); + const fixture = yield* wholeStack(runtime); + yield* clearIdleTimers(fixture); + yield* fixture.stack.composition.start; + yield* exerciseStack(fixture, "confinement"); + // Podman keeps database files on the host under a subordinate uid the host user cannot + // traverse, so that one directory is recorded but not entered; every other path under + // the roots is still diffed. + const opaque = + runtime === "podman" + ? [ + path.join( + fixture.locations.stateRoot, + fixture.stack.id, + "data", + service(fixture, "database").id, + "data", + ), + ] + : []; + // Allowed roots the named entries below cover; add one line here to extend them. + // Podman's own client state: `podman info` asks the host's rpm which package provides each + // engine binary (rpm's database), a remote machine connection is SSH, and without + // XDG_RUNTIME_DIR (macOS) the client keeps its runtime directory under TMPDIR. + const allowedRoots = [ + fixture.root, + fixture.locations.cacheRoot, + bunTranspilerCache, + ...(runtime === "podman" + ? [ + path.join(home, ".rpmdb"), + path.join(home, ".ssh"), + path.join(tmp, `storage-run-${process.getuid?.()}`), + ] + : []), + ]; + // While the stack is still running, after it has been exercised: a before/after-only + // check misses anything a service writes and removes again before shutdown. + const duringHome = yield* snapshotTree(home); + const duringTmp = yield* snapshotTree(tmp, opaque); + assertConfinedTo(diffTrees(beforeHome, duringHome), allowedRoots); + assertConfinedTo(diffTrees(beforeTmp, duringTmp), allowedRoots); + yield* proveConfinementDetectsViolations(home, duringHome, allowedRoots); + yield* stopWithDiagnostics(fixture); + yield* fixture.stack.destroy; + yield* clearStackOwner(fixture); + assertConfinedTo(diffTrees(beforeHome, yield* snapshotTree(home)), allowedRoots); + assertConfinedTo(diffTrees(beforeTmp, yield* snapshotTree(tmp, opaque)), allowedRoots); + }), + ); + }), + ); + +/** + * Wakes every service eagerly and drives real traffic through each endpoint, then asserts that no + * listening TCP socket owned by a workload's process tree binds outside loopback. Native only: + * container workloads publish their ports to loopback through Docker's own mapping, a different + * mechanism out of this scenario's scope. + */ +export const loopbackOnly = withFixture("native", (fixture) => + Effect.gen(function* () { + yield* setActivation(fixture.stack, "eager"); + yield* fixture.stack.composition.start; + yield* assertLifecycle(fixture, "running"); + yield* exerciseStack(fixture, "loopback"); + const violations = yield* captureLoopbackViolations(fixture); + expect(violations).toEqual([]); + }), +); diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts index da3a1e377d..65c1c442af 100644 --- a/packages/stack/tests/whole-stack/workloads.ts +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -10,7 +10,16 @@ export const WorkloadSnapshot = Schema.Struct({ }); export type WorkloadSnapshot = Schema.Schema.Type; -const commandOutput = Effect.fn("WholeStack.commandOutput")( +const PodmanContainers = Schema.fromJsonString( + Schema.Array( + Schema.Struct({ + Id: Schema.String, + Labels: Schema.Record(Schema.String, Schema.String), + }), + ), +); + +export const commandOutput = Effect.fn("WholeStack.commandOutput")( (command: string, args: ReadonlyArray) => Effect.gen(function* () { const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; @@ -37,22 +46,35 @@ const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: b } satisfies WorkloadSnapshot; } const marker = `com.supabase.stack=${fixture.stack.id}`; - const output = yield* commandOutput("docker", [ - "ps", - ...(includeStopped ? ["--all"] : []), - "--filter", - `label=${marker}`, - "--format", - '{{.ID}}\t{{.Label "com.supabase.instance"}}\t{{.Label "com.supabase.stack-managed"}}', - ]); - const records = output - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.length > 0) - .map((line) => { - const [id, instance, managed] = line.split("\t"); - return { identity: `${id ?? ""} ${instance ?? ""}`.trim(), instance, managed }; - }) + const listArgs = ["ps", ...(includeStopped ? ["--all"] : []), "--filter", `label=${marker}`]; + // Podman's `ps` templates read labels differently across versions; its JSON is stable. + const listed = + runtime === "docker" + ? (yield* commandOutput(runtime, [ + ...listArgs, + "--format", + '{{.ID}}\t{{.Label "com.supabase.instance"}}\t{{.Label "com.supabase.stack-managed"}}', + ])) + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => { + const [id, instance, managed] = line.split("\t"); + return { id, instance, managed }; + }) + : (yield* Schema.decodeEffect(PodmanContainers)( + yield* commandOutput(runtime, [...listArgs, "--format", "json"]), + )).map(({ Id, Labels }) => ({ + id: Id, + instance: Labels["com.supabase.instance"], + managed: Labels["com.supabase.stack-managed"], + })); + const records = listed + .map(({ id, instance, managed }) => ({ + identity: `${id ?? ""} ${instance ?? ""}`.trim(), + instance, + managed, + })) // The stack label alone also tags best-effort containers that carry neither an instance // nor a `stack-managed` identity, such as the host-gateway probe (see `readProbeHosts` in // `Container.ts`); only the two documented identity labels promise a workload. @@ -73,13 +95,43 @@ export const captureWorkloads = Effect.fn("WholeStack.captureWorkloads")( snapshotOutput(runtime, fixture, includeStopped), ); +/** + * Kills one instance's workload the way a crash would: SIGKILL of the native workload process + * (the launcher's child, so the launcher survives to report the exit), or `kill` of its container. + */ +export const killWorkload = Effect.fn("WholeStack.killWorkload")( + (runtime: Runtime, fixture: WholeStack, instanceId: string) => + Effect.gen(function* () { + const { identities } = yield* snapshotOutput(runtime, fixture, false); + if (runtime !== "native") { + const container = identities + .map((identity) => identity.split(" ")) + .find(([, instance]) => instance === instanceId)?.[0]; + if (container === undefined) return yield* Effect.die(`No container for ${instanceId}`); + yield* commandOutput(runtime, ["kill", container]); + return; + } + const launcher = identities + .map((identity) => /^(\d+)\s.*supabase-workload-id=(\S+)$/.exec(identity)) + .find((match) => match?.[2] === instanceId)?.[1]; + if (launcher === undefined) return yield* Effect.die(`No launcher for ${instanceId}`); + const children = (yield* commandOutput("ps", ["-axo", "pid=,ppid="])) + .split("\n") + .map((line) => /^\s*(\d+)\s+(\d+)\s*$/.exec(line)) + .filter((match) => match?.[2] === launcher) + .map((match) => match?.[1] ?? ""); + if (children.length === 0) return yield* Effect.die(`No workload under launcher ${launcher}`); + yield* commandOutput("kill", ["-9", ...children]); + }), +); + export const assertWorkloadsGone = Effect.fn("WholeStack.assertWorkloadsGone")( (runtime: Runtime, fixture: WholeStack, snapshot: WorkloadSnapshot, includeStopped = false) => Effect.gen(function* () { expect(snapshot.identities.length).toBeGreaterThan(0); const current = yield* snapshotOutput(runtime, fixture, includeStopped); - if (runtime === "docker") expect(current.identities).toEqual([]); - if (runtime === "docker" && includeStopped) expect(current.managedIdentities).toEqual([]); + if (runtime !== "native") expect(current.identities).toEqual([]); + if (runtime !== "native" && includeStopped) expect(current.managedIdentities).toEqual([]); for (const marker of snapshot.markers) expect(current.identities.some((identity) => identity.includes(marker))).toBe(false); for (const identity of snapshot.identities)