diff --git a/.github/scripts/sync-workload-catalog.test.ts b/.github/scripts/sync-workload-catalog.test.ts index 4e362c4f35..63419fa8da 100644 --- a/.github/scripts/sync-workload-catalog.test.ts +++ b/.github/scripts/sync-workload-catalog.test.ts @@ -16,32 +16,28 @@ const realCatalog = await Bun.file(CATALOG_PATH).text(); /** A trimmed catalog carrying both entry shapes the script has to rewrite. */ const fixture = `const workloadCatalog = { - "database:database": native( + "database:database": definition( "postgres", "17.6.1.168", "ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}", "bin/supabase-postgres-start", ["bin/supabase-postgres-start"], { - additionalReleases: { "15.14.1.168": "ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}", - }, - containerAlias: "supabase-database", }, ), - "rest:rest": native( + "rest:rest": definition( "postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest", ["bin/postgrest"], - { containerAlias: "supabase-rest" }, ), - "auth:auth": native("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth", [ + "auth:auth": definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth", [ "bin/auth", ]), - "studio:studio": native( + "studio:studio": definition( "studio", "2026.09.04-sha-5a67366", "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:${"c".repeat(64)}", @@ -101,7 +97,7 @@ describe("planCatalogUpdate", () => { expect(plan.previousVersion).toBe("v2.196.0"); expect(plan.target).toBe("default"); expect(plan.source).toContain( - `native("auth", "v2.197.0", "ghcr.io/supabase/cli/auth:v2.197.0@${DIGEST_A}", "bin/auth"`, + `definition("auth", "v2.197.0", "ghcr.io/supabase/cli/auth:v2.197.0@${DIGEST_A}", "bin/auth"`, ); expect(plan.source).not.toContain("v2.196.0"); }); @@ -259,7 +255,7 @@ describe("against the real catalog", () => { // Derived from the catalog, so a new workload is covered without editing this. const services = [ ...new Set( - [...realCatalog.matchAll(/native\(\s*"([a-z][a-z0-9-]*)",/g)].map( + [...realCatalog.matchAll(/definition\(\s*"([a-z][a-z0-9-]*)",/g)].map( (match) => match[1] ?? "", ), ), diff --git a/.github/scripts/sync-workload-catalog.ts b/.github/scripts/sync-workload-catalog.ts index cb00c23bb7..50f5abdf37 100644 --- a/.github/scripts/sync-workload-catalog.ts +++ b/.github/scripts/sync-workload-catalog.ts @@ -1,5 +1,5 @@ /** - * Pins one workload in `packages/stack/src/model/WorkloadCatalog.ts` to a + * Pins one workload in `packages/stack/src/Artifacts.ts` to a * slim-services release, driven by the same `mirror-slim-image` dispatch as the * ECR mirror (`mirror-slim-image.yml`). * @@ -13,7 +13,7 @@ * service or release line is a successful no-op. */ -export const CATALOG_PATH = "packages/stack/src/model/WorkloadCatalog.ts"; +export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; /** Mirrors the payload validation in `mirror-slim-image.yml`. */ const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/; @@ -70,15 +70,15 @@ export function validatePayload(input: { } } -/** `native("", "", ""` — image anchored so postgres != postgrest. */ +/** `definition("", "", ""` — image anchored so postgres != postgrest. */ function defaultEntryPattern(service: string): RegExp { const s = escapeRegExp(service); return new RegExp( - `(native\\(\\s*"${s}",\\s*")([^"]+)("\\s*,\\s*")(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)(")`, + `(definition\\(\\s*"${s}",\\s*")([^"]+)("\\s*,\\s*")(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)(")`, ); } -/** `additionalReleases` entries: `"": ""`. The `:` is what distinguishes them. */ +/** Additional release entries: `"": ""`. The `:` is what distinguishes them. */ function additionalEntryPattern(service: string, version?: string): RegExp { const s = escapeRegExp(service); const key = version === undefined ? `[^"]+` : escapeRegExp(version); diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index b7ce2a3df5..ff359a508f 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -9,6 +9,7 @@ on: pull_request: paths: - ".github/scripts/**" + - "packages/stack/src/Artifacts.ts" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" diff --git a/.github/workflows/sync-stack-workload-catalog.yml b/.github/workflows/sync-stack-workload-catalog.yml index 4ff9f7df1c..8f05a8a19b 100644 --- a/.github/workflows/sync-stack-workload-catalog.yml +++ b/.github/workflows/sync-stack-workload-catalog.yml @@ -1,6 +1,6 @@ name: Sync Stack Workload Catalog -# Pins `packages/stack/src/model/WorkloadCatalog.ts` to slim-services releases, +# Pins `packages/stack/src/Artifacts.ts` to slim-services releases, # off the same `mirror-slim-image` dispatch that drives the ECR mirror. # Dependabot owns the Dockerfile and cannot own this table: these pins carry # image digests, which tag resolution never produces (ADR 0017). @@ -69,7 +69,7 @@ jobs: - name: Check for catalog changes id: check run: | - if git diff --exit-code --quiet packages/stack/src/model/WorkloadCatalog.ts; then + if git diff --exit-code --quiet packages/stack/src/Artifacts.ts; then echo "Workload catalog is already in sync." echo "has_changes=false" >> "$GITHUB_OUTPUT" else @@ -77,7 +77,7 @@ jobs: echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - # Prove the rewritten pin compiles and passes the catalog's own tests. + # Prove the rewritten pin compiles and the affected catalog surfaces pass. - name: Type-check stack if: steps.check.outputs.has_changes == 'true' run: pnpm types:check @@ -85,8 +85,15 @@ jobs: - name: Test catalog if: steps.check.outputs.has_changes == 'true' - run: pnpm run test:unit && pnpm run test:integration - working-directory: packages/stack + run: pnpm run test:github-scripts + + - name: Test affected stack preparation + if: steps.check.outputs.has_changes == 'true' + run: >- + pnpm --filter @supabase/stack test:integration:run + src/preparation/artifacts.integration.test.ts + src/preparation/slim-services.integration.test.ts + --passWithNoTests=false - name: Generate token if: steps.check.outputs.has_changes == 'true' @@ -106,7 +113,7 @@ jobs: commit-message: "chore(stack): pin ${{ github.event.client_payload.service || inputs.service }} to ${{ github.event.client_payload.version || inputs.version }}" title: "chore(stack): pin ${{ github.event.client_payload.service || inputs.service }} to ${{ github.event.client_payload.version || inputs.version }}" body: | - Pins `packages/stack/src/model/WorkloadCatalog.ts` to the + Pins `packages/stack/src/Artifacts.ts` to the `${{ github.event.client_payload.service || inputs.service }}` `${{ github.event.client_payload.version || inputs.version }}` slim-services release and its image digest. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b482ac2fe..9b60e8622d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,12 +148,14 @@ jobs: working-directory: packages/stack run: >- pnpm test:integration - src/state/ports.integration.test.ts - src/state/registry-lease.integration.test.ts - src/state/state-store.integration.test.ts - src/supervisor/host-listener.integration.test.ts - src/supervisor/ingress.integration.test.ts - src/gateway/startup-transports.integration.test.ts + src/Ports.integration.test.ts + src/State.integration.test.ts + src/State.process.integration.test.ts + src/HostProcess.integration.test.ts + src/Network.integration.test.ts + src/Proxy.integration.test.ts + src/HttpProxy.integration.test.ts + --passWithNoTests=false test-summary: if: | @@ -238,7 +240,7 @@ jobs: strategy: fail-fast: false matrix: - runtime: [native, container] + runtime: [native, docker] steps: - name: Checkout uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta @@ -249,9 +251,17 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - name: Run stack end-to-end tests - run: pnpm --filter @supabase/stack test:e2e:run - env: - SUPABASE_STACK_E2E_RUNTIME: ${{ matrix.runtime }} + run: >- + pnpm --filter @supabase/stack test:e2e:run + src/whole-stack.${{ matrix.runtime }}.e2e.test.ts + --passWithNoTests=false + + - name: Run public stack end-to-end tests + if: matrix.runtime == 'native' + run: >- + pnpm --filter @supabase/stack test:e2e:run + src/public.e2e.test.ts + --passWithNoTests=false # Summary job that gates branch protection. It preserves the original check # name and succeeds iff every CLI shard and stack runtime succeeded. diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index a98537aca6..b333547721 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -12,7 +12,7 @@ native runtimes. | `supabase stack prepare` | Download artifacts without starting services. | | `supabase stack start` | Create or resume the project's stack. | | `supabase stack status` | Show identity, readiness, and drift, or export connection variables with `--env`. | -| `supabase stack logs` | Read retained or live stack logs. | +| `supabase stack logs` | Stream live stack logs. | | `supabase stack restart` | Restart an existing stack using its saved effective configuration. | | `supabase stack stop` | Stop a stack while retaining its data. | @@ -38,27 +38,29 @@ supabase stack status --env --output-format json ``` Each example requires the stack backend flag described below. `--env` exports the connection URLs -and credentials of the running stack; text mode emits dotenv assignments, and JSON -or stream-JSON mode emits a variable map. Add `--output-format text` for an explicit dotenv file +and credentials available from the observed composition; text mode emits dotenv assignments, and +JSON or stream-JSON mode emits a variable map. Values that are unavailable because a member is +stopped or unhealthy are omitted. Add `--output-format text` for an explicit dotenv file regardless of automatic agent output detection; this is dotenv data, not a shell script, and values are quoted so that sourcing the file performs no shell expansion. Only this explicit export reveals credentials. Ordinary status remains free of secrets. `--override-name` accepts repeated or comma-separated `EXPORTED_VARIABLE=NAME` entries, requires `--env`, and rejects -unknown variables, invalid names, and collisions. API credentials are omitted when Auth is disabled. +unknown variables, invalid names, and collisions. The DB-derived service-role JWT remains available +when Auth is disabled; unavailable service URLs and credentials are omitted. The stack backend rejects every explicit legacy `-o/--output` value: `env`, `pretty`, `json`, `toml`, `yaml`, `table`, and `csv`. `--output-format text`, `json`, or `stream-json` replace them. `-o env` becomes `--env`. `supabase stack list` reads the global managed-stack registry and reports each readable stack's -project, branch, runtime, and desired lifecycle. Corrupt or unsupported registry entries are -included in a diagnostic section with their full IDs and error reasons, and do not hide readable -entries. The text table shortens readable IDs for scanning; use `--output-format json` or +project, branch, runtime, and owner availability. A corrupt or unsupported registry entry fails the +whole discovery operation with a diagnostic; readable entries are not emitted as a partial list. +The text table shortens readable IDs for scanning; use `--output-format json` or `--output-format stream-json` for the complete structured inventory with full IDs. -Listing is global and has no checkout filter. Desired lifecycle is persisted intent, so `running` -does not prove a live owner exists. Use `supabase stack status` for live state. Registry directories -without a state file are ignored as remnants. +Listing is global and has no checkout filter. Owner availability is not service lifecycle or health; +use `supabase stack status` for live state. Registry directories without a state file are ignored +as remnants. ## Selecting the top-level commands @@ -93,14 +95,35 @@ Without the environment variable, `init` still writes the established template w and without the stack flag. Blank `supabase bootstrap` uses the same scaffold. When the flag is on, `--local` targets of the `db`, `migration`, `test db`, `gen types`, and -`inspect` families use the project stack and provision throwaway shadow Postgres through -`@supabase/stack` (`EphemeralPostgres`). Top-level `supabase pull` uses the same stack shadow +`inspect` families use the project stack and provision a throwaway shadow database owned by the +stack runtime. Top-level `supabase pull` uses the same stack shadow as `db pull`. Linked and `--db-url` targets stay on the Management API for engine selection. A `--db-url` that matches `config.toml` host and port is still rewritten like a published stack target for dump's tool container. Compose names (`supabase_db_*`, `supabase_network_*`, `db:5432`) are not used. The stack backend requires the in-process pg-delta engine; `--use-migra`, `--use-pgadmin`, `--use-pg-schema`, and `db pull --diff-engine migra` are -rejected. The flag does not switch the `functions` command family. +rejected. The flag also routes local `functions serve` through the stack; other `functions` +commands retain their existing behavior. + +`functions serve` requires a running project stack. It attaches to the existing Functions member, +waking it when needed. If Functions was excluded, it creates a temporary standalone Functions +instance on the stack's API listener without changing the saved composition. Ctrl-C destroys that +temporary instance; an existing member remains available. `--no-verify-jwt` and `--env-file` apply +for the serving session and the previous configuration is restored on exit. Ordinary code edits +are picked up by the next invocation without a runtime restart. Import-map and inspector flags +and multiline environment values are currently unsupported on this backend. Normal exit owns cleanup; forcibly killing the CLI +cannot guarantee restoration or removal of its temporary instance. Concurrent override sessions +for the same Functions instance are unsupported. Cleanup waits for submitted lifecycle operations; +additional signals do not force it to abandon cleanup. Restoration restores configuration and keeps +the service running; it does not return it to a sleeping state. + +Stack mode reads the shared `supabase/functions/.env` file when creating Functions; `--env-file` +replaces those custom values for the session. A normal `start` applies changes to the shared file +and restores default JWT verification on the same Functions instance. Per-function +`.env` files are not loaded. Reserved +`SUPABASE_*` values in these files are ignored because the runtime supplies them. With +`--output-format stream-json`, readiness emits a result containing the instance ID and URL, +followed by live log events. `storage ls`/`cp`/`mv`/`rm` and `seed buckets` (including bucket seeding inside `db reset --local`) also consult `experimental.stack`, with the same `SUPABASE_EXPERIMENTAL_STACK` @@ -108,23 +131,26 @@ env-precedence rule as `start`/`stop`/`status`. See [Storage and bucket seeding](#storage-and-bucket-seeding) below. Explicit `--linked`/ `--project-ref` remote targeting for these commands is unaffected by the flag either way. -`db start` brings up a postgres-only project stack on first create. An existing stack resumes -its persisted services (webhooks setup only; no second overlay or migrate-and-seed). +`db start` brings up a postgres-only project stack on first create. An existing stack resumes its +primary database without changing other services (webhooks setup only; no second overlay or +migrate-and-seed). `supabase start` while that postgres-only stack is running stops it and starts the full -configured stack, keeping data. `--from-backup` is not supported on the stack path. `db reset ---local` and declarative `--apply` wipe Postgres through `resetDatabase` and then migrate or -seed on stack credentials. +configured stack, keeping data. `--from-backup` is not supported on the stack path. +`db reset --local` and declarative resets rebuild the existing database while retaining stack +identity, ports, and composition. See +[`db/reset/SIDE_EFFECTS.md`](../src/commands/db/reset/SIDE_EFFECTS.md). `gen types --local` and `inspect db … --local` resolve the project stack through the same `--local` database target as `db dump`. They do not start a stack. -On the stack backend, `db dump --local`, `migration squash`, and `test db` always run catalog -`pg_dump` / `pg_dumpall` / `pg_prove` — native stacks prepend `artifact/bin`, and container -stacks (or platforms with no native artifact: Windows, Intel Mac) run a one-shot of the **same** -catalog Postgres image. There is no host PATH fallback and stack `--local` never uses -`PGHOST=db`. Native `--linked` / `--db-url` keep the resolved host; `--local` native rewrites -loopback to `127.0.0.1`. Missing `pg_prove` fails closed. Install Docker Desktop when a -no-native-artifact platform cannot spawn the one-shot client. +On the stack backend, `db dump --local`, `migration squash`, and `test db --local` run +catalog tools attached to their owning stack. Tools use runtime-facing database addresses; +`test db` preserves the user, password, and database selected by the CLI resolver. + +External `--linked` and `--db-url` tools use CLI-owned adapters without creating a stack. +Supported native platforms run the bundled executable with `artifact/bin` on PATH. Other +platforms use a one-shot container from the catalog Postgres image. Neither path falls back to +host-installed PostgreSQL tools. Missing tools fail closed; container execution requires Docker. `db lint --local` talks to the running stack over SQL. Catalog Postgres includes `plpgsql_check`, so native and container stacks can `CREATE EXTENSION` inside the @@ -132,68 +158,18 @@ lint transaction (always rolled back). It does not launch a client binary. ## Reading stack logs -`supabase stack logs` reads retained logs without starting or stopping the selected stack. Use -`--stack ` or `--stack-id ` to select a stack, `--service ` to filter services, -and `--tail ` to bound retained history (`0` through `1000`, default `100`). `--service` -accepts one capability name; it excludes supervisor and gateway entries, including their startup -diagnostics. Omit it to include all retained sources. Retention is bounded to the newest 1000 -entries or 1 MiB, whichever is reached first. -Add `--follow` (or `-f`) to continue with new entries; `--tail 0` starts with live entries only. -Follow mode leaves the stack running when interrupted. - -The default text output is one ` /: ` line per entry. -`--output-format json` returns one bounded object. A found stack has this shape, with the raw -entry message preserved: - -```json -{ - "found": true, - "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "entries": [ - { - "cursor": { "opaque": "1" }, - "timestamp": "2026-09-08T00:00:00.000Z", - "source": "database", - "stream": "stdout", - "message": "database ready" - } - ], - "cursor": { "opaque": "1" }, - "running": false, - "message": "" -} -``` - -When no default stack exists, JSON output is: - -```json -{ - "found": false, - "entries": [], - "message": "No managed stack found for this context." -} -``` - -`--output-format stream-json` emits one bounded result event for a finite read. With `--follow`, -it emits one `log-entry` event for each history or live entry, with the original message in -`line`. For an absent default stack it emits the standard empty result envelope: - -```json -{ - "type": "result", - "data": { - "found": false, - "entries": [], - "message": "No managed stack found for this context." - }, - "timestamp": "..." -} -``` +`supabase stack logs` streams live stdout/stderr from composition members without +starting an owner or service. Select `--stack ` or `--stack-id `; +`--service ` can include standalone services too. The command +requires a reachable owner and streams until interrupted. Ctrl-C leaves services +running. There is no retained history, `--tail`, or `--follow` flag. -A found stack with no entries emits a result event for a finite stream-json read. Follow mode -emits only log-entry events; a found stack with no retained entries emits no follow events, and a -stopped stack exits successfully. The command is available only while `experimental.stack` is -enabled. +Text uses ` //: ` and strips terminal +control sequences. For automation use `--output-format stream-json`: each +`log-entry` contains `timestamp`, `service`, `instance_id`, `stream`, `line`, and +`source: "live"`. Finite JSON output is not supported. Delivery is best effort; +stdout/stderr and different services may interleave. Missing stacks, unavailable +owners, and unmatched services fail with status 1; interruption exits 130. ## Data and configuration @@ -211,40 +187,38 @@ Host listener assignment for `supabase stack` is documented in [Port intents](./ ## Service selection and shutdown -With the current defaults, lazy REST, Auth, Realtime, Studio, and pooler services stop after 60 seconds without traffic. An -active HTTP request keeps a capability running; an idle HTTP keep-alive socket does not. Open -WebSocket or TCP connections keep a capability running during idle periods. Use `supabase stack start --eager` to activate all enabled capabilities and -disable automatic idle stops. Per-capability `idleTimeoutSeconds` values are available through the -package's Effect API only; the CLI does not expose them as command or project configuration -settings. A request arriving while a capability is stopping waits for cleanup and then wakes it -when the stack still permits activation. Manual `supabase stack stop` prevents wake up until the -stack is started again. A cleanup failure can block new connections until `stop` and `start` -complete recovery; a destroy failure can be retried with `destroy`. +With the current defaults, enabled non-database services with endpoints are lazy and stop after +60 seconds without traffic; Functions has no automatic idle stop. An active HTTP request keeps a +capability running; an idle HTTP keep-alive socket does not. Open WebSocket or TCP connections +keep a capability running during idle periods. Use `supabase stack start --eager` to activate all +enabled capabilities and disable automatic idle stops. A request arriving while a capability is +stopping waits for cleanup and then wakes it when the stack still permits activation. Manual +`supabase stack stop` prevents wake up until the stack is started again. A cleanup failure can +block new connections until `stop` and `start` complete recovery; a destroy failure can be retried +with `destroy`. `supabase stack restart` reuses an existing stack's saved effective configuration. It stops and starts the same stack identity, preserving its data. Normal startup may still download missing -artifacts according to the saved preparation policy. Select a stack with `--stack ` or +artifacts using its current preparation policy. Select a stack with `--stack ` or `--stack-id `. The restart handler does not reload project configuration. Set `SUPABASE_EXPERIMENTAL_STACK=1` when restarting by ID outside the project or with invalid project -configuration, so feature routing does not depend on that configuration. Start flags such as `--exclude`, `--eager`, or -`--preparation` remain in the saved stack configuration; a later normal `start` reloads the project +configuration, so feature routing does not depend on that configuration. Start selection and +activation flags such as `--exclude` and `--eager` remain in the saved stack configuration; +`--preparation` applies only to that invocation. A later normal `start` reloads the project configuration and current flags. -Stacks saved before idle stopping keep it disabled when restarted. To adopt the current defaults, -run `supabase stack stop` followed by `supabase stack start`. Status can report changed effective -defaults even when the project file is unchanged; a stack still marked running must be stopped -before those defaults can be applied. An unconfigured stack must be initialized with `supabase stack start` before it can be restarted. `supabase stack start --exclude studio,analytics -x mail` disables those services in the effective start configuration without changing the project file. Valid names are `rest`, `auth`, `realtime`, `storage`, `functions`, `studio`, `mail`, `analytics`, and `pooler`; the database is required. -Excluding `rest` also disables Studio; excluding `analytics` does not. The effective configuration is +Excluding `rest` while Studio remains selected is rejected; excluding `analytics` does not affect +Studio. The effective configuration is retained in stack state, so starting without `--exclude` restores the project's configured services. -`supabase stack stop --all` stops every readable managed stack while preserving data. It continues -after unreadable entries or individual stop failures, reports a bounded stopped/failed/skipped -summary with per-stack details, and exits nonzero when anything was skipped or failed. Registry-root -enumeration errors remain fatal. +`supabase stack stop --all` stops every managed stack while preserving data. Discovery fails closed +when any registry entry is unreadable, so no partial stop operation is attempted. Individual stop +failures make the command fail and identify the affected stack IDs with their error details; no +success or unavailable summary is emitted when a stop fails. `supabase stack destroy --stack feature-a` permanently removes exactly that stack and its data after confirmation. Use `--yes` for unattended execution. There is no bulk destroy option. @@ -259,11 +233,12 @@ reset --local`). An explicit `--linked`/`--project-ref` remote target for `stora unaffected by the flag either way. Under the stack backend, `--local` Storage operations resolve their endpoint and credential from -the selected stack rather than `[api]`/`[api.tls]`: the endpoint is the stack's API gateway URL -(`status.endpoints.api.url`) plus `/storage/v1/...`, and the credential is the stack's own -service-role JWT read from its credentials. The stack is located by the project root (workdir -realpath) through the `@supabase/stack` API, which reads stack state under `SUPABASE_HOME`. -None of `storage`, `seed buckets`, or `db reset --local` ever creates a stack; the legacy `[api]` +the selected composition rather than `[api]`/`[api.tls]`: the endpoint is the Storage member's +observed HTTP binding plus `/storage/v1/...`, and the service-role JWT is generated from the +primary database member's observed JWT secret. The stack is located by the project root +(workdir realpath) through the `@supabase/stack` API, which reads stack state under +`SUPABASE_HOME`. +Neither `storage` nor `seed buckets` ever creates a stack; the legacy `[api]` port/external-URL/TLS fields, their `SUPABASE_API_*` overrides, `SUPABASE_AUTH_JWT_SECRET`/ `SUPABASE_AUTH_SERVICE_ROLE_KEY`, `SUPABASE_SERVICES_HOSTNAME`, the Docker daemon hostname, and the embedded Kong CA are all legacy-backend only and are not consulted on this path. The @@ -271,57 +246,43 @@ service-role JWT is never printed or logged. Storage's capability state gates these operations: -| Storage capability state | Effect | -| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `disabled` (e.g. `stack start -x storage`) | Fails with `StackStorageCapabilityError` ("Storage is disabled for this stack."), guiding the user to enable Storage, then run `supabase stack stop` followed by `supabase stack start` without `-x storage`. | -| `failed` / `stopped` | Fails with the same `StackStorageCapabilityError` ("Storage failed to start for this stack"/"Storage is stopped for this stack"), guiding the user to `supabase stack restart`. | -| `dormant` / `starting` / `ready` / `stopping` | Proceeds immediately; no client-side wait, everywhere `starting` is checked. A `stopping` capability is woken by the gateway once its cleanup completes. | - -A stack that is not registered, not running, missing its API endpoint or credentials, or whose -stack API is unavailable fails instead with `StackStorageUnavailableError`, guiding the user to -run `supabase stack status`/`supabase stack restart`, or `supabase start` when the stack has never -been configured. Missing API credentials because Auth is disabled instead guides the user to -enable `[auth]` in `supabase/config.toml`, then run `supabase stack stop` followed by -`supabase stack start` without `-x auth`. A stack-gateway 502/503 encountered while Storage -activates is reported as `StackStorageCapabilityError` guiding the user to `supabase stack logs` -then `supabase stack restart`, not as a raw status body — but only for a `--local` target; a -`--linked` failure passes through unchanged. All of these failures exit `1` for +| Storage capability state | Effect | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `disabled` (e.g. `stack start -x storage`) | Fails with `StackStorageCapabilityError` ("Storage is disabled for this stack."), guiding the user to enable Storage and run `supabase stack start` without `-x storage`. | +| `failed` / `stopped` | Fails with the same `StackStorageCapabilityError` ("Storage failed to start for this stack"/"Storage is stopped for this stack"), guiding the user to `supabase stack restart`. | +| `dormant` / `starting` / `ready` | Proceeds immediately; the gateway handles lazy activation without client-side waiting. | +| `stopping` with wake retained (idle stop) | Proceeds immediately; the gateway waits for cleanup and wakes Storage. | +| `stopping` after manual `stack stop` | Fails like `stopped` because manual stop disables wake-up until the stack starts again. | + +A stack that is not registered, not running, has no primary database, or whose stack API is +unavailable fails with `StackStorageUnavailableError`, guiding the user to run `supabase stack +status`/`supabase stack restart`, or `supabase start` when the stack has never been configured. +Missing Storage HTTP endpoints fail with `StackStorageCapabilityError`, as do other Storage +capability failures. A stack-gateway 502/503 encountered while Storage activates is reported as +`StackStorageCapabilityError` guiding the user to `supabase stack logs` then `supabase stack +restart`, not as a raw status body — but only for a `--local` target; a `--linked` failure passes +through unchanged. All of these failures exit `1` for `storage`/`seed buckets`, and no HTTP request is sent when Storage is disabled or the stack is not running. **Lazy activation.** A `dormant` or `starting` Storage capability is not yet listening; the stack gateway activates a lazily-configured Storage on the first request that reaches it and holds that request until it is ready, rather than the CLI polling capability state itself. A request that -arrives while Storage is `stopping` waits for cleanup and then wakes it the same way. `starting` -and `stopping` proceed without waiting everywhere Storage capability is checked, including -`stack start` and `db reset --local`. +arrives while Storage is `stopping` after an idle stop waits for cleanup and then wakes it the same +way. A manually stopped capability keeps wake-up disabled until the stack starts again. These +states are observed directly by Storage operations, including `stack start`. **`stack start` seeds on first configured start.** When a `stack start` (or the top-level `start` -under the flag) invocation runs the stack's first configured start (`desiredLifecycle` was -`unconfigured`, including after `stack prepare`) — never when it resumes an existing one — and +under the flag) invocation runs the stack's first configured start, including after `stack prepare` +— never when it resumes an existing one — and Storage is not `disabled`, it seeds `[storage.buckets]` against the stack before printing status, reusing the `seed buckets` core. A project with no `[storage.buckets]` or `[storage.vector.buckets]` configured resolves no credentials and prints nothing. Auto-confirm is safe here since a first-start stack has no pre-existing buckets. Any other unusable Storage state, -a missing capability/credentials, or a -gateway activation failure prints a stderr warning and skips seeding; any other seeding failure -(e.g. an invalid bucket entry) fails the `start` command (exit `1`) but leaves the stack running. -`start` fails only on a genuine seeding error, never on Storage being unavailable. See +a missing capability/endpoint, a gateway activation failure, or an invalid bucket entry fails the +`start` command; startup does not silently skip the configured seed. See [`stack/start/SIDE_EFFECTS.md`](../src/commands/experimental/stack/start/SIDE_EFFECTS.md). -**`db reset --local` never fails for buckets.** Each reset seeds buckets again after the database -reset, when Storage is `ready`, `dormant`, `starting`, or `stopping`. Any other Storage problem — -an unusable capability state, a missing capability/credentials, a gateway activation failure, or -an invalid bucket config — prints `WARNING: skipped seeding storage buckets: ` -to stderr and the reset still exits `0`. The warning is omitted when the project configures no -buckets, and disabled Storage gets configuration-specific next steps (enable `[storage]`, -`supabase stack stop` then `supabase stack start`, then `supabase seed buckets --local`). When the -underlying stack-storage error carries its own suggestion (a missing API gateway endpoint, Auth -disabled, or a gateway 502/503), that suggestion is the next step instead of the generic -`supabase seed buckets --local` one. This is the -deliberate policy split from `stack start`: the database is already rebuilt by the time buckets -are seeded, so `db reset` never fails the command for a Storage problem, while `start` still fails -on a genuine seeding error. Bucket SQL/schema preparation is the stack runtime's own storage -workload/catalog-setup responsibility; bucket creation and `objects_path` upload from -`[storage.buckets]` remain the CLI's seeding-step responsibility, since the runtime itself never -creates buckets. See [`db/reset/SIDE_EFFECTS.md`](../src/commands/db/reset/SIDE_EFFECTS.md). +**`db reset --local` and declarative reset** reapply the catalog, migrations, and seeds before +resuming the saved composition. Bucket-seeding failures warn after the database reset succeeds. +See [`db/reset/SIDE_EFFECTS.md`](../src/commands/db/reset/SIDE_EFFECTS.md). diff --git a/apps/cli/docs/supabase-home.md b/apps/cli/docs/supabase-home.md index ee15bb4128..4f04baf5f3 100644 --- a/apps/cli/docs/supabase-home.md +++ b/apps/cli/docs/supabase-home.md @@ -45,7 +45,7 @@ Gitignored checkout metadata lives beside the project: checkout-local service-version overrides. Neither file records whether a local stack is running. Stack identity does not write repository or Git markers. -### Global managed runtime +### Global local stack runtime Managed stack documents and runtime artifacts are shared through the global CLI home: @@ -55,27 +55,22 @@ Managed stack documents and runtime artifacts are shared through the global CLI telemetry.json traces/ bin/ - managed/ - stacks/ - / - state.json - data/ - logs/ - runtime/ - control.json + stacks/ + / + state.json + data/ + / + artifacts/ ``` -`state.json` is the single durable managed record. It contains the canonical project root, branch +`state.json` is the single durable stack record. It contains the canonical project root, branch context, and stack name that make up stack identity, together with sticky port intents and -assignments, the desired lifecycle, the runtime selection, and the materialized stack definition -(enabled capabilities with their pinned versions, listeners, and security settings). Runtime-only -service ports are allocated for the -supervisor run and are not persisted as sticky intents. `control.json` contains runtime owner -metadata for the deterministic loopback control endpoint; the ownership protocol is the liveness -authority, and a stale document is reclaimed by a subsequent managed lifecycle operation. - -There is no project-local `stacks/` directory, `state.json`, daemon socket file, or second -StateManager metadata format. +assignments, the runtime selection, the service definitions, and the composition needed to resume +the stack. Runtime-only observations and active owner state are kept in memory. Service data lives +under the stack's `data//` directories, while downloaded native artifacts are shared +under `artifacts/`. + +There is no project-local `stacks/` directory or second state format. ## Service-version inputs diff --git a/apps/cli/scripts/build-binary.ts b/apps/cli/scripts/build-binary.ts index 0a6f25df37..ceda84411a 100644 --- a/apps/cli/scripts/build-binary.ts +++ b/apps/cli/scripts/build-binary.ts @@ -1,4 +1,6 @@ import { bundleServeMainTemplate } from "../src/shared/functions/serve-main-bundler.ts"; +import { bundleStackFunctionsServeMainTemplate } from "../src/command-internal/stack-functions-bundler.ts"; +import { Effect } from "effect"; import { OXFMT_OPTIONAL_PLUGIN_EXTERNALS } from "./bundle-externals.ts"; /** @@ -23,6 +25,9 @@ const result = await Bun.build({ define: { SUPABASE_CLI_VERSION: JSON.stringify(packageJson.version), SUPABASE_FUNCTIONS_SERVE_MAIN_TEMPLATE: JSON.stringify(await bundleServeMainTemplate()), + SUPABASE_STACK_FUNCTIONS_SERVE_MAIN_TEMPLATE: JSON.stringify( + await Effect.runPromise(bundleStackFunctionsServeMainTemplate()), + ), }, }); for (const log of result.logs) { diff --git a/apps/cli/scripts/build.ts b/apps/cli/scripts/build.ts index 87a9e3e3b4..48b3441adb 100644 --- a/apps/cli/scripts/build.ts +++ b/apps/cli/scripts/build.ts @@ -4,7 +4,9 @@ import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import process from "node:process"; import { parseArgs } from "node:util"; +import { Effect } from "effect"; import { bundleServeMainTemplate } from "../src/shared/functions/serve-main-bundler.ts"; +import { bundleStackFunctionsServeMainTemplate } from "../src/command-internal/stack-functions-bundler.ts"; import { OXFMT_OPTIONAL_PLUGIN_EXTERNALS } from "./bundle-externals.ts"; import { darwinBinaries, MACOS_IDENTIFIERS } from "./macos-signing.ts"; @@ -89,6 +91,9 @@ const distDir = path.join(root, "dist"); const goSource = path.resolve(root, "apps/cli-go"); const buildDefines = { SUPABASE_FUNCTIONS_SERVE_MAIN_TEMPLATE: JSON.stringify(await bundleServeMainTemplate()), + SUPABASE_STACK_FUNCTIONS_SERVE_MAIN_TEMPLATE: JSON.stringify( + await Effect.runPromise(bundleStackFunctionsServeMainTemplate()), + ), "process.env.SUPABASE_CLI_POSTHOG_KEY": JSON.stringify(process.env.POSTHOG_API_KEY ?? ""), "process.env.SUPABASE_CLI_POSTHOG_HOST": JSON.stringify(process.env.POSTHOG_ENDPOINT ?? ""), }; diff --git a/apps/cli/src/auth/command-platform-api-factory.layer.ts b/apps/cli/src/auth/command-platform-api-factory.layer.ts index 6c0adad60f..bac5353c7f 100644 --- a/apps/cli/src/auth/command-platform-api-factory.layer.ts +++ b/apps/cli/src/auth/command-platform-api-factory.layer.ts @@ -25,7 +25,7 @@ export const commandPlatformApiFactoryLayer = Layer.effect( make, }); }), -).pipe(Layer.provide(FetchHttpClient.layer), Layer.provide(dohFetchLayer)); +).pipe(Layer.provide(Layer.fresh(FetchHttpClient.layer)), Layer.provide(dohFetchLayer)); /** * Adapts an already-built eager `CommandPlatformApi` into a factory. Use this in diff --git a/apps/cli/src/auth/http-debug.layer.ts b/apps/cli/src/auth/http-debug.layer.ts index 437389ee38..0d10151e3c 100644 --- a/apps/cli/src/auth/http-debug.layer.ts +++ b/apps/cli/src/auth/http-debug.layer.ts @@ -69,4 +69,4 @@ export const httpClientLayer = Layer.effect( logger.http(req.method, redactHttpUrl(req.url)).pipe(Effect.as(req)), ); }), -).pipe(Layer.provide(FetchHttpClient.layer), Layer.provide(dohFetchLayer)); +).pipe(Layer.provide(Layer.fresh(FetchHttpClient.layer)), Layer.provide(dohFetchLayer)); diff --git a/apps/cli/src/command-internal/bundled-postgres-client.integration.test.ts b/apps/cli/src/command-internal/bundled-postgres-client.integration.test.ts new file mode 100644 index 0000000000..dee86dfa13 --- /dev/null +++ b/apps/cli/src/command-internal/bundled-postgres-client.integration.test.ts @@ -0,0 +1,115 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Ref } from "effect"; + +import { + bundledPostgresClientLayer, + BundledPostgresClient, + resolveBundledPostgresRuntime, +} from "./bundled-postgres-client.ts"; +import { dockerRunLayer } from "./docker-run.layer.ts"; +import { ProcessControl } from "../shared/runtime/process-control.service.ts"; +import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import { mockOutput } from "../../tests/helpers/mocks.ts"; +import { mockCommandSettings } from "../../tests/helpers/command-mocks.ts"; + +const processControl = Layer.succeed(ProcessControl, { + awaitSignal: () => Effect.never, + awaitShutdown: Effect.never, + holdSignals: () => Effect.void, + exit: () => Effect.die("unused"), + setExitCode: () => Effect.void, + getExitCode: Effect.succeed(undefined), +}); + +const base = Layer.mergeAll(BunServices.layer, processControl); +const docker = dockerRunLayer.pipe(Layer.provide(base)); +const client = bundledPostgresClientLayer.pipe( + Layer.provide(Layer.mergeAll(base, docker, mockOutput().layer)), +); + +describe("BundledPostgresClient", { timeout: 180_000 }, () => { + it.live("runs the selected runtime with an isolated artifact cache", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const homeDir = yield* fs.makeTempDirectoryScoped({ prefix: "bundled-client-" }); + const runtimeInfo = { + cwd: "/tmp", + platform: process.platform, + arch: process.arch, + homeDir: `${homeDir}/runtime-home`, + execPath: process.execPath, + pid: process.pid, + }; + const runtime = yield* resolveBundledPostgresRuntime( + undefined, + process.platform, + process.arch, + ); + const chunks = yield* Ref.make>([]); + const providedClient = client.pipe( + Layer.provide(Layer.succeed(RuntimeInfo, runtimeInfo)), + Layer.provide(mockCommandSettings({ workdir: "/tmp", supabaseHome: homeDir })), + ); + const result = yield* Effect.gen(function* () { + const client = yield* BundledPostgresClient; + return yield* client.run({ + version: "17", + runtime, + argv: + runtime.kind === "native" + ? ["bash", "-c", "pg_dump --version", "--"] + : ["pg_dump", "--version"], + env: {}, + onStdout: (chunk) => Ref.update(chunks, (current) => [...current, chunk]), + }); + }).pipe(Effect.provide(providedClient)); + expect(result.exitCode).toBe(0); + const output = Uint8Array.from((yield* Ref.get(chunks)).flatMap((chunk) => [...chunk])); + expect(new TextDecoder().decode(output)).toContain("pg_dump"); + if (runtime.kind === "native") { + expect(yield* fs.exists(`${homeDir}/cache/stack`)).toBe(true); + expect(yield* fs.exists(`${homeDir}/runtime-home/cache/stack`)).toBe(false); + } + }), + ).pipe(Effect.provide(base)), + ); + + it.live("runs the Docker runtime", () => + Effect.scoped( + Effect.gen(function* () { + const chunks = yield* Ref.make>([]); + const client = yield* BundledPostgresClient; + const result = yield* client.run({ + version: "17", + runtime: { kind: "container", engine: "docker" }, + argv: ["pg_dump", "--version"], + env: {}, + onStdout: (chunk) => Ref.update(chunks, (current) => [...current, chunk]), + }); + expect(result.exitCode).toBe(0); + const output = Uint8Array.from((yield* Ref.get(chunks)).flatMap((chunk) => [...chunk])); + expect(new TextDecoder().decode(output)).toContain("pg_dump"); + }), + ).pipe( + Effect.provide( + client.pipe( + Layer.provide( + Layer.succeed(RuntimeInfo, { + cwd: "/tmp", + platform: process.platform, + arch: process.arch, + homeDir: "/tmp", + execPath: process.execPath, + pid: process.pid, + }), + ), + Layer.provide( + mockCommandSettings({ workdir: "/tmp", supabaseHome: "/tmp/bundled-client-docker" }), + ), + ), + ), + ), + ); +}); diff --git a/apps/cli/src/command-internal/bundled-postgres-client.ts b/apps/cli/src/command-internal/bundled-postgres-client.ts index 71c75a94c9..78668af596 100644 --- a/apps/cli/src/command-internal/bundled-postgres-client.ts +++ b/apps/cli/src/command-internal/bundled-postgres-client.ts @@ -1,39 +1,73 @@ -import { Context, Crypto, Effect, FileSystem, Layer, Option, Path } from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; import { - ArtifactIntegrityError, - ContainerEngineError, - ContainerEngineResolver, - ContainerPullError, - PostgresClientError, - runPostgresClient, - selectDefaultRuntime, - StackPreparationError, - StackVersionUnsupportedError, - targetForPlatform, - type PostgresClientResult, - type PostgresClientRunError, - type PostgresClientServices, - type RunPostgresClientOptions, - type StackRuntime, - type StackRuntimePreference, -} from "@supabase/stack/effect"; + Context, + Config, + Crypto, + Effect, + FileSystem, + Layer, + Option, + Path, + PlatformError, + Stream, +} from "effect"; +import { FetchHttpClient } from "effect/unstable/http"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { + ArtifactError, + postgresVersion, + prepareNativeArtifact, + resolveArtifact, +} from "@supabase/stack/internal/postgres-artifact"; +import { DockerRun, type DockerRunOpts } from "./docker-run.service.ts"; import { DockerRunError } from "./docker-run.errors.ts"; -import { SUGGEST_DOCKER_INSTALL } from "./docker-suggest.ts"; import { HostPostgresClientError } from "./postgres-client.run.ts"; import { ProcessControl } from "../shared/runtime/process-control.service.ts"; +import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import { Output } from "../shared/output/output.service.ts"; +import { CommandSettings } from "../config/command-settings.service.ts"; const BUNDLED_CLIENT_SUGGESTION = "Re-download the Postgres artifact, or create a new stack that uses a container runtime."; +export type StackRuntimePreference = + | { readonly kind: "native" } + | { readonly kind: "container"; readonly engine: "docker" | "podman" }; +export type StackRuntime = StackRuntimePreference; + +interface PostgresClientMount { + readonly source: string; + readonly target: string; + readonly readOnly?: boolean; +} + +interface PostgresClientResult { + readonly exitCode: number; + readonly stderr: string; +} + +export interface RunPostgresClientOptions { + readonly version?: string; + readonly runtime?: StackRuntimePreference; + readonly argv: ReadonlyArray; + readonly env: Readonly>; + readonly cwd?: string; + readonly mounts?: ReadonlyArray; + readonly network?: "host" | { readonly name: string }; + readonly extraHosts?: ReadonlyArray; + readonly securityOpt?: ReadonlyArray; + readonly projectEnvValues?: Readonly>; + readonly onStdout: (chunk: Uint8Array) => Effect.Effect; + readonly teeStderr?: boolean; + readonly captureStderr?: boolean; +} + +type PostgresClientRunError = DockerRunError; + export interface BundledPostgresClientShape { readonly run: ( options: RunPostgresClientOptions, - ) => Effect.Effect< - PostgresClientResult, - E | PostgresClientRunError | HostPostgresClientError | DockerRunError - >; + ) => Effect.Effect; } export class BundledPostgresClient extends Context.Service< @@ -41,83 +75,150 @@ export class BundledPostgresClient extends Context.Service< BundledPostgresClientShape >()("supabase/cli/BundledPostgresClient") {} -const mapClientFailure = (error: E): E | HostPostgresClientError | DockerRunError => { - if (error instanceof PostgresClientError) { - return new HostPostgresClientError({ - message: error.message, - suggestion: BUNDLED_CLIENT_SUGGESTION, - }); - } - if (error instanceof StackVersionUnsupportedError) { - return new HostPostgresClientError({ message: error.message }); - } - if (error instanceof StackPreparationError || error instanceof ArtifactIntegrityError) { - return new HostPostgresClientError({ - message: error.message, - suggestion: BUNDLED_CLIENT_SUGGESTION, - }); - } - if (error instanceof ContainerPullError) { - return new DockerRunError({ - message: error.message, - reason: "pull", - daemonDown: false, - }); - } - if (error instanceof ContainerEngineError) { - return new DockerRunError({ - message: - error.message.length > 0 - ? error.message - : `failed to run ${error.engine ?? "docker"}. ${SUGGEST_DOCKER_INSTALL}`, - reason: "spawn", - daemonDown: false, - }); - } - return error; -}; - const clientNeedsContainerRuntime = (platform: string, arch?: string): boolean => platform === "win32" || - (arch !== undefined && targetForPlatform({ os: platform, arch }) === undefined); + !( + (platform === "darwin" && arch === "arm64") || + (platform === "linux" && (arch === "x64" || arch === "arm64")) + ); -/** Platforms without a native postgres artifact dump through a one-shot container. */ export const bundledPostgresClientRuntime = ( - stackRuntime: StackRuntime | undefined, + stackRuntime: StackRuntime | "native" | "docker" | "podman" | undefined, platform: string, arch?: string, ): StackRuntimePreference | undefined => { const forceContainer = clientNeedsContainerRuntime(platform, arch); if (stackRuntime !== undefined) { - if (stackRuntime.kind === "native" && forceContainer) { + const selected: StackRuntimePreference = + typeof stackRuntime === "string" + ? stackRuntime === "native" + ? { kind: "native" } + : { kind: "container", engine: stackRuntime } + : stackRuntime; + if (selected.kind === "native" && forceContainer) return { kind: "container", engine: "docker" }; - } - return stackRuntime; + return selected; } - if (forceContainer) return { kind: "container", engine: "docker" }; - return undefined; + return forceContainer ? { kind: "container", engine: "docker" } : { kind: "native" }; }; export const resolveBundledPostgresRuntime = ( - stackRuntime: StackRuntime | undefined, + stackRuntime: StackRuntime | "native" | "docker" | "podman" | undefined, platform: string, arch?: string, -): Effect.Effect< - StackRuntimePreference, - ContainerEngineError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const forced = bundledPostgresClientRuntime(stackRuntime, platform, arch); - if (forced !== undefined) return forced; - const resolver = yield* Effect.serviceOption(ContainerEngineResolver); - return yield* selectDefaultRuntime(Option.getOrUndefined(resolver)); +): Effect.Effect => + Effect.succeed(bundledPostgresClientRuntime(stackRuntime, platform, arch) ?? { kind: "native" }); + +const mapArtifactError = (error: ArtifactError): HostPostgresClientError => + new HostPostgresClientError({ + message: error.message, + ...(error.cause === undefined ? {} : { suggestion: BUNDLED_CLIENT_SUGGESTION }), + }); + +const mapSpawnError = (error: unknown): HostPostgresClientError => + new HostPostgresClientError({ + message: `Unable to run the bundled PostgreSQL client: ${error instanceof Error ? error.message : String(error)}`, + suggestion: BUNDLED_CLIENT_SUGGESTION, + }); + +const nativeRun = Effect.fn("BundledPostgresClient.nativeRun")(function* ( + options: RunPostgresClientOptions, + artifactRoot: string, + path: Path.Path, + spawner: ChildProcessSpawner.ChildProcessSpawner["Service"], + output: Output["Service"], +) { + const [command, ...args] = options.argv; + if (command === undefined) + return yield* new HostPostgresClientError({ message: "PostgreSQL client command is empty." }); + const tool = ["pg_dump", "pg_dumpall", "pg_prove", "psql"].includes(command) + ? path.join(artifactRoot, "bin", command) + : command; + const inheritedPath = yield* Config.option(Config.string("PATH")).pipe( + Effect.mapError(mapSpawnError), + ); + const env = { + ...options.env, + PATH: `${path.join(artifactRoot, "bin")}:${Option.getOrElse(inheritedPath, () => "")}`, + }; + const child = yield* spawner + .spawn( + ChildProcess.make(tool, args, { + cwd: options.cwd, + env, + extendEnv: true, + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + killSignal: command === "pg_prove" ? "SIGKILL" : undefined, + forceKillAfter: "5 seconds", + }), + ) + .pipe(Effect.mapError(mapSpawnError)); + let stderr = ""; + const stderrDecoder = new TextDecoder(); + yield* Effect.all( + [ + Stream.runForEach(child.stdout, (chunk: Uint8Array) => options.onStdout(chunk)), + Stream.runForEach(child.stderr, (chunk: Uint8Array) => + Effect.gen(function* () { + if (options.teeStderr === true) yield* output.rawBytes(chunk, "stderr"); + if (options.captureStderr !== false) + stderr += stderrDecoder.decode(chunk, { stream: true }); + }), + ), + ], + { concurrency: "unbounded" }, + ).pipe( + Effect.mapError((error) => + error instanceof PlatformError.PlatformError ? mapSpawnError(error) : error, + ), + ); + if (options.captureStderr !== false) stderr += stderrDecoder.decode(); + return { exitCode: yield* child.exitCode.pipe(Effect.mapError(mapSpawnError)), stderr }; +}); + +const dockerRun = Effect.fn("BundledPostgresClient.dockerRun")(function* ( + options: RunPostgresClientOptions, + image: string, + docker: DockerRun["Service"], +) { + const dockerOptions: DockerRunOpts = { + image, + cmd: options.argv, + env: options.env, + projectEnvValues: options.projectEnvValues, + binds: (options.mounts ?? []).map( + (mount) => `${mount.source}:${mount.target}${mount.readOnly === true ? ":ro" : ""}`, + ), + workingDir: options.cwd === undefined ? Option.none() : Option.some(options.cwd), + securityOpt: options.securityOpt ?? [], + extraHosts: options.extraHosts ?? [], + network: + options.network === undefined || options.network === "host" + ? { _tag: "host" } + : { _tag: "named", name: options.network.name }, + skipImageResolve: true, + }; + return yield* docker.runStream(dockerOptions, { + onStdout: options.onStdout, + teeStderr: options.teeStderr, + captureStderr: options.captureStderr, }); +}); export const bundledPostgresClientLayer: Layer.Layer< BundledPostgresClient, never, - PostgresClientServices + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | DockerRun + | RuntimeInfo + | ProcessControl + | Output + | CommandSettings > = Layer.effect( BundledPostgresClient, Effect.gen(function* () { @@ -125,23 +226,41 @@ export const bundledPostgresClientLayer: Layer.Layer< const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - return BundledPostgresClient.of({ - run: (options) => - Effect.scoped( - Effect.gen(function* () { - const processControl = yield* Effect.serviceOption(ProcessControl); - if (Option.isSome(processControl)) { - yield* processControl.value.holdSignals(["SIGINT", "SIGTERM", "SIGHUP"]); - } - return yield* runPostgresClient(options); - }), - ).pipe( - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.provideService(Path.Path, path), - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.mapError(mapClientFailure), - ), + const docker = yield* DockerRun; + const runtimeInfo = yield* RuntimeInfo; + const settings = yield* CommandSettings; + const processControl = yield* ProcessControl; + const output = yield* Output; + const run = Effect.fn("BundledPostgresClient.run")(function* ( + options: RunPostgresClientOptions, + ) { + return yield* Effect.scoped( + Effect.gen(function* () { + yield* processControl.holdSignals(["SIGINT", "SIGTERM", "SIGHUP"]); + const runtime = options.runtime ?? { kind: "native" }; + const version = postgresVersion(options.version ?? "17"); + if (runtime.kind === "container") { + const resolved = yield* resolveArtifact({ service: "database", version }).pipe( + Effect.mapError(mapArtifactError), + ); + return yield* dockerRun(options, resolved.image, docker); + } + const prepared = yield* prepareNativeArtifact( + { service: "database", version }, + path.join(settings.supabaseHome, "cache", "stack"), + { os: runtimeInfo.platform, arch: runtimeInfo.arch }, + ).pipe( + Effect.provide(FetchHttpClient.layer), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.mapError(mapArtifactError), + ); + return yield* nativeRun(options, prepared.root, path, spawner, output); + }), + ); }); + return BundledPostgresClient.of({ run }); }), ); diff --git a/apps/cli/src/command-internal/config-validate.parity.unit.test.ts b/apps/cli/src/command-internal/config-validate.parity.unit.test.ts index 836a0a62ca..17b8f86b53 100644 --- a/apps/cli/src/command-internal/config-validate.parity.unit.test.ts +++ b/apps/cli/src/command-internal/config-validate.parity.unit.test.ts @@ -1,3 +1,4 @@ +import { unusedStackServices } from "../../tests/helpers/unused-stack.ts"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -320,6 +321,7 @@ describe("shared api + auth validation branches, cross-caller parity (S vs L)", const exit = yield* resolveStorageCredentials({ projectRef: "", config }).pipe( Effect.provide( Layer.mergeAll( + unusedStackServices, BunServices.layer, runtimeInfoLayer, mockCommandSettings({ workdir: dir }), diff --git a/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts b/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts index e325ec419b..e6c85d019c 100644 --- a/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts +++ b/apps/cli/src/command-internal/db-bootstrap/reset-local-database.ts @@ -9,7 +9,7 @@ * invocation only, emitted by its own handler after calling this function. */ -import { Data, Effect, FileSystem, Option, Path } from "effect"; +import { Data, Effect, FileSystem, Option, Path, Redacted } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { detectGitBranch } from "../../shared/git/git-branch.ts"; @@ -39,11 +39,7 @@ import { buildLocalDbContainerInputs } from "./local-container-inputs.ts"; import { isLocalDbRunning } from "./local-db-running.ts"; import { recreateLocalDatabase } from "./recreate-local-database.ts"; import { currentStackBackend } from "../stack-backend.ts"; -import { - optionalCatalogConfigFromStatus, - stackLocalDatabaseConn, - stackOpenReadyProject, -} from "../stack-local-database.ts"; +import { stackLocalDatabaseConn, stackOpenReadyProject } from "../stack-local-database.ts"; import { classifyStorageCapability, describeStorageCapability, @@ -101,7 +97,7 @@ const suggestionOf = (error: unknown): string | undefined => : undefined; /** Resets the local database in-process. See this module's own header for the full design rationale. */ -export const resetLocalDatabase = Effect.fnUntraced(function* ( +export const resetLocalDatabase = Effect.fn("DbBootstrap.resetLocalDatabase")(function* ( input: ResetLocalDatabaseInput = PLAIN_FULL_RESET, ) { const backend = yield* currentStackBackend; @@ -123,37 +119,56 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( if (backend.kind === "stack") { const opened = yield* stackOpenReadyProject; if (Option.isNone(opened)) - return yield* Effect.fail( - new ResetLocalDbNotRunningError({ message: "The local stack is not running." }), - ); + return yield* new ResetLocalDbNotRunningError({ message: "The local stack is not running." }); const catalog = yield* Effect.serviceOption(StackCatalogSetup); if (Option.isNone(catalog)) return yield* resetFailed("stack catalog setup is unavailable"); - const stackConfig = yield* loadStackConfig(workdir).pipe( - Effect.mapError((cause) => resetFailed(cause.message)), - ); + yield* loadStackConfig(workdir).pipe(Effect.mapError((cause) => resetFailed(cause.message))); const toml = yield* readDbToml(fs, path, workdir); - const runningStatus = yield* opened.value.stack.status.pipe( + const databaseStatus = yield* opened.value.database.status.pipe( Effect.mapError((cause) => resetFailed(`failed to inspect stack: ${cause.message}`)), ); - const optionalConfig = optionalCatalogConfigFromStatus(stackConfig, runningStatus); - yield* output.raw(`Resetting local database${toLogMessage(input.version)}\n`, "stderr"); - yield* opened.value.stack.resetDatabase.pipe( - Effect.catchTag("StackNotRunningError", () => - Effect.fail( - new ResetLocalDbNotRunningError({ message: "The local stack is not running." }), - ), + if (databaseStatus.config.service !== "database") + return yield* resetFailed("the local stack primary service is not a database"); + const composition = yield* opened.value.stack.composition.describe.pipe( + Effect.mapError((cause) => + resetFailed(`failed to inspect stack composition: ${cause.message}`), ), + ); + const members = yield* Effect.forEach(composition.members, ({ id }) => + opened.value.stack.services + .get(id) + .pipe( + Effect.mapError((cause) => + resetFailed(`failed to inspect stack service: ${cause.message}`), + ), + ), + ); + const databaseServices = members.flatMap((member) => + member.service === "auth" || member.service === "storage" || member.service === "realtime" + ? [member.service] + : [], + ); + yield* output.raw(`Resetting local database${toLogMessage(input.version)}\n`, "stderr"); + yield* opened.value.stack.composition.stop.pipe( + Effect.mapError((cause) => resetFailed(`failed to stop local stack: ${cause.message}`)), + ); + yield* opened.value.database.resetData.pipe( Effect.mapError((cause) => resetFailed(`failed to reset local database: ${cause.message}`)), ); + yield* opened.value.database.start.pipe( + Effect.mapError((cause) => resetFailed(`failed to start local database: ${cause.message}`)), + ); + yield* opened.value.database.ready.pipe( + Effect.mapError((cause) => resetFailed(`failed to ready local database: ${cause.message}`)), + ); yield* catalog.value .apply({ target: { - kind: "live", stack: opened.value.stack, - projectRoot: workdir, - config: stackConfig, + database: opened.value.database, + databaseServices, + jwtSecret: Redacted.value(databaseStatus.config.config.jwtSecret), }, - optionalConfig, overlay: { webhooks: "config", webhooksEnabled: toml.webhooksEnabled, @@ -186,9 +201,9 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( }).pipe(Effect.mapError((cause) => resetFailed(cause.message))); }), ); - const status = yield* opened.value.stack.status.pipe( + yield* opened.value.stack.composition.start.pipe( Effect.mapError((cause) => - resetFailed(`failed to inspect stack after reset: ${cause.message}`), + resetFailed(`failed to start local stack services: ${cause.message}`), ), ); // Bucket creation and object seeding are owned by the CLI, not the stack runtime; the @@ -202,12 +217,13 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( `${yellow("WARNING:")} skipped seeding storage buckets: ${reason} ${nextStep}\n`, "stderr", ); - const capability = status.capabilities.find((entry) => entry.name === "storage"); + const storage = members.find((member) => member.service === "storage"); // The database is already rebuilt, so any typed seeding failure only warns; defects and // interruption still propagate. yield* Effect.gen(function* () { const context = yield* loadLocalProjectContext(workdir, (message) => resetFailed(message)); if (!hasConfiguredBuckets(context.config)) return; + const capability = storage === undefined ? undefined : yield* storage.status; const decision = classifyStorageCapability(capability); if (decision !== "proceed") { yield* skipSeeding( @@ -218,7 +234,7 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( ); return; } - const credentials = yield* stackStorageEndpointFor(opened.value.stack, status); + const credentials = yield* stackStorageEndpointFor(opened.value.stack); yield* seedBucketsRun({ projectRef: "", emitSummary: false, @@ -253,7 +269,7 @@ export const resetLocalDatabase = Effect.fnUntraced(function* ( Option.getOrUndefined(cliSettings.projectId), ); if (!running) { - return yield* Effect.fail(notRunning()); + return yield* notRunning(); } // "Resetting local database…" then recreate + migrate + seed. yield* output.raw(`Resetting local database${toLogMessage(input.version)}\n`, "stderr"); diff --git a/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts b/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts index 45042bb087..5caf37116c 100644 --- a/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts +++ b/apps/cli/src/command-internal/db-bootstrap/shadow-cache.ts @@ -148,7 +148,7 @@ export interface ShadowCacheKeyInputs { * PG<=14 setup SQL is excluded because that major is cache-ineligible. */ let shadowBaselineEmbeddedDigestMemo: string | undefined; -export const shadowBaselineEmbeddedDigest = (): string => +const shadowBaselineEmbeddedDigest = (): string => (shadowBaselineEmbeddedDigestMemo ??= createHash("sha256") .update( [ @@ -171,7 +171,7 @@ export const shadowBaselineEmbeddedDigest = (): string => .digest("hex")); /** JSON with recursively key-sorted objects, so `db.settings`' own property order cannot change the key. */ -export function canonicalJson(value: unknown): string { +function canonicalJson(value: unknown): string { if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; const entries = Object.entries(value) @@ -487,10 +487,7 @@ const sweepShadowBaselineRetention = ( }); /** Refresh mtime on a warm hit so frequently used keys survive LRU/TTL. Best-effort. */ -export const touchShadowBaselineTar = ( - fs: FileSystem.FileSystem, - tarPath: string, -): Effect.Effect => +const touchShadowBaselineTar = (fs: FileSystem.FileSystem, tarPath: string): Effect.Effect => Effect.gen(function* () { const now = new Date(yield* Clock.currentTimeMillis); yield* fs.utimes(tarPath, now, now); diff --git a/apps/cli/src/command-internal/db-config.layer.ts b/apps/cli/src/command-internal/db-config.layer.ts index 8659bf3686..de5d68d5da 100644 --- a/apps/cli/src/command-internal/db-config.layer.ts +++ b/apps/cli/src/command-internal/db-config.layer.ts @@ -579,6 +579,7 @@ const dbConfigResolverLayer = Layer.effect( const conn = yield* stackLocalDatabaseConn.pipe( Effect.provideService(CommandSettings, cliSettings), Effect.provideService(StackApi, stackApi), + Effect.provideService(Path.Path, path), ); return { conn, isLocal: true }; } diff --git a/apps/cli/src/command-internal/management-api-runtime.layer.ts b/apps/cli/src/command-internal/management-api-runtime.layer.ts index 7e925a2d7a..ce4c3e6f86 100644 --- a/apps/cli/src/command-internal/management-api-runtime.layer.ts +++ b/apps/cli/src/command-internal/management-api-runtime.layer.ts @@ -54,7 +54,7 @@ export function managementApiRuntimeLayer(subcommand: ReadonlyArray) { const platformApiStack = commandPlatformApiLayer.pipe( Layer.provide(credentials), Layer.provide(cliSettings), - Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.fresh(FetchHttpClient.layer)), Layer.provide(dohFetchLayer), Layer.provide(debugLoggerLayer), Layer.provide(identityStitchLayer), diff --git a/apps/cli/src/command-internal/pg-dump.native.integration.test.ts b/apps/cli/src/command-internal/pg-dump.native.integration.test.ts new file mode 100644 index 0000000000..035705feba --- /dev/null +++ b/apps/cli/src/command-internal/pg-dump.native.integration.test.ts @@ -0,0 +1,124 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { FetchHttpClient } from "effect/unstable/http"; +import { Effect, FileSystem, Layer, Option, Redacted, Ref, Stream } from "effect"; +import { postgres, type ServiceCreation } from "@supabase/stack/effect"; + +import { StackApi, stackApiLayer } from "./stack-api.ts"; +import { streamPgDumpWithClient } from "./pg-dump.run.ts"; +import { NetworkIdFlag } from "./global-flags.ts"; +import { DockerRun } from "./docker-run.service.ts"; +import { BundledPostgresClient } from "./bundled-postgres-client.ts"; +import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import { mockOutput } from "../../tests/helpers/mocks.ts"; + +const runtimes = ["native", "docker"] as const; +const liveStackApi = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + +describe("managed pg_dump against a live stack", { timeout: 180_000 }, () => { + for (const runtime of runtimes) { + it.live(`${runtime} uses runtime credentials and preserves dump filters`, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "cli-pg-dump-" }); + const api = yield* StackApi; + const stack = yield* api.create({ + projectRoot: root, + stateRoot: `${root}/stacks`, + cacheRoot: `${root}/cache`, + runtime, + }); + yield* Effect.addFinalizer(() => + stack.destroy.pipe( + Effect.tapError((error) => Effect.logError(`Failed to destroy test stack: ${error}`)), + Effect.ignore, + ), + ); + const creation: Extract = { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("cli-dump-password"), + jwtSecret: Redacted.make("cli-dump-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }; + const [database] = yield* stack.composition.supabase([creation]); + if (database === undefined) return yield* Effect.die("database was not composed"); + yield* database.start; + yield* database.ready; + const credentials = yield* database.credentials({ from: "runtime" }); + const output = yield* Ref.make([]); + const stderr = yield* Ref.make([]); + const sql = credentials.databaseUrl; + if (sql === undefined) return yield* Effect.die("runtime database URL missing"); + const connection = new URL(sql); + const env = { + PGHOST: connection.hostname, + PGPORT: connection.port, + PGUSER: decodeURIComponent(connection.username), + PGPASSWORD: decodeURIComponent(connection.password), + PGDATABASE: connection.pathname.slice(1), + }; + const setupResult = yield* stack.tools.run(postgres.psql({ major: 17 }), { + args: ["-X", "-v", "ON_ERROR_STOP=1"], + env, + stdin: Stream.make( + new TextEncoder().encode( + "CREATE TABLE public.cli_dump_story(value text); INSERT INTO public.cli_dump_story VALUES ('runtime-row');", + ), + ), + stdout: (bytes) => Ref.update(output, (chunks) => [...chunks, bytes]), + stderr: (bytes) => Ref.update(stderr, (chunks) => [...chunks, bytes]), + }); + expect(setupResult.exitCode).toBe(0); + const dumpEnv = { + ...env, + EXCLUDED_SCHEMAS: "auth|storage", + }; + const chunks = yield* Ref.make([]); + const result = yield* streamPgDumpWithClient({ + image: "unused", + script: "pg_dump --data-only", + env: dumpEnv, + onStdout: (bytes) => Ref.update(chunks, (current) => [...current, bytes]), + client: { kind: "stack", stack, command: "pg_dump", major: 17 }, + }); + expect(result.exitCode).toBe(0); + const dump = new TextDecoder().decode( + Uint8Array.from((yield* Ref.get(chunks)).flatMap((chunk) => [...chunk])), + ); + expect(dump).toContain("cli_dump_story"); + expect(dump).toContain("runtime-row"); + expect(yield* Ref.get(stderr)).toHaveLength(0); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + FetchHttpClient.layer, + liveStackApi, + Layer.succeed(NetworkIdFlag, Option.none()), + Layer.succeed(RuntimeInfo, { + cwd: "/tmp", + platform: process.platform, + arch: process.arch, + homeDir: "/tmp", + execPath: process.execPath, + pid: process.pid, + }), + Layer.succeed(BundledPostgresClient, { run: () => Effect.die("unused") }), + Layer.succeed(DockerRun, { + run: () => Effect.die("unused"), + runCapture: () => Effect.die("unused"), + runStream: () => Effect.die("unused"), + }), + mockOutput().layer, + ), + ), + ), + ); + } +}); diff --git a/apps/cli/src/command-internal/pg-dump.run.integration.test.ts b/apps/cli/src/command-internal/pg-dump.run.integration.test.ts new file mode 100644 index 0000000000..aef3605340 --- /dev/null +++ b/apps/cli/src/command-internal/pg-dump.run.integration.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "@effect/vitest"; +import { BunServices } from "@effect/platform-bun"; +import { Effect, Layer, Option } from "effect"; +import type { Stack } from "@supabase/stack/effect"; +import { streamPgDumpWithClient, transformManagedDumpLine } from "./pg-dump.run.ts"; +import { NetworkIdFlag } from "./global-flags.ts"; +import { DockerRun } from "./docker-run.service.ts"; +import { BundledPostgresClient } from "./bundled-postgres-client.ts"; +import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import { mockOutput } from "../../tests/helpers/mocks.ts"; + +const stackFixture = (output: string) => { + const calls: Array<{ readonly args: ReadonlyArray; readonly command: string }> = []; + const stack = { + tools: { + run: ( + _tool: { readonly command: string }, + options: { + readonly args?: ReadonlyArray; + readonly stdout: (bytes: Uint8Array) => Effect.Effect; + }, + ) => + Effect.gen(function* () { + calls.push({ command: _tool.command, args: options.args ?? [] }); + yield* options.stdout(new TextEncoder().encode(output)); + return { jobId: "job", exitCode: 0 }; + }), + }, + } as unknown as Stack; + return { stack, calls }; +}; + +describe("managed pg_dump runner", () => { + it("keeps the managed dump filters equivalent to the CLI scripts", () => { + const cases = [ + [ + 'CREATE TABLE "public" (id integer);', + "pg_dump --schema-only", + {}, + 'CREATE TABLE IF NOT EXISTS "public" (id integer);', + ], + ["\\unrestrict 123", "pg_dump --data-only", {}, "-- \\unrestrict 123"], + ["--user-data", "pg_dump --data-only", { EXTRA_SED: "/^--/d" }, "--user-data"], + ["\\restrict 123", "pg_dumpall --roles-only", { EXTRA_SED: "/^--/d" }, undefined], + [ + 'CREATE ROLE "postgres";', + "pg_dumpall --roles-only", + { RESERVED_ROLES: "postgres" }, + '-- CREATE ROLE "postgres";', + ], + [ + 'ALTER ROLE "postgres" SET "statement_timeout" TO "5s";', + "pg_dumpall --roles-only", + { ALLOWED_CONFIGS: "statement_timeout" }, + 'ALTER ROLE "postgres" SET "statement_timeout" TO "5s";', + ], + [ + '-- ALTER ROLE "postgres" SET "pgrst.db_schemas" TO "public";', + "pg_dumpall --roles-only", + { ALLOWED_CONFIGS: "pgrst.*", EXTRA_SED: "/^--/d" }, + 'ALTER ROLE "postgres" SET "pgrst.db_schemas" TO "public";', + ], + [ + 'GRANT "member" TO "dashboard_user";', + "pg_dumpall --roles-only", + { RESERVED_ROLES: "dashboard_user", EXTRA_SED: "/^--/d" }, + undefined, + ], + [ + 'CREATE EXTENSION IF NOT EXISTS "pg_tle" WITH SCHEMA "pgtle";', + "pg_dump --schema-only", + {}, + 'CREATE EXTENSION IF NOT EXISTS "pg_tle";', + ], + [ + 'GRANT SELECT ON TABLE "public"."storage" TO "anon";', + "pg_dump --schema-only", + { EXCLUDED_SCHEMAS: "storage" }, + 'GRANT SELECT ON TABLE "public"."storage" TO "anon";', + ], + [ + 'REVOKE SELECT ON TABLE "storage" FROM "anon";', + "pg_dump --schema-only", + { EXCLUDED_SCHEMAS: "storage" }, + '-- REVOKE SELECT ON TABLE "storage" FROM "anon";', + ], + [ + 'GRANT ALL ON FOREIGN DATA WRAPPER "wrapper" TO "authenticated";', + "pg_dump --schema-only", + {}, + 'GRANT ALL ON FOREIGN DATA WRAPPER "wrapper" TO "authenticated";', + ], + [ + 'GRANT ALL ON FOREIGN DATA WRAPPER "wrapper" TO "postgres" WITH GRANT OPTION;', + "pg_dump --schema-only", + {}, + '-- GRANT ALL ON FOREIGN DATA WRAPPER "wrapper" TO "postgres" WITH GRANT OPTION;', + ], + ] as const; + + for (const [input, script, env, expected] of cases) + expect(transformManagedDumpLine(input, script, env)).toBe(expected); + }); + + it.live("keeps data dump filters and quoted extra flags in the Stack tool arguments", () => { + const fixture = stackFixture("\\unrestrict 123\n"); + return streamPgDumpWithClient({ + image: "unused", + script: "pg_dump --data-only", + env: { + PGHOST: "127.0.0.1", + PGPORT: "54322", + PGUSER: "postgres", + PGDATABASE: "postgres", + EXTRA_FLAGS: '--exclude-table "public"."Users"', + }, + onStdout: () => Effect.void, + client: { kind: "stack", stack: fixture.stack, command: "pg_dump", major: 17 }, + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + Layer.succeed(NetworkIdFlag, Option.none()), + Layer.succeed(RuntimeInfo, { + cwd: "/tmp", + platform: "linux", + arch: "x64", + homeDir: "/tmp", + execPath: "/usr/bin/bun", + pid: 1, + }), + Layer.succeed(BundledPostgresClient, { run: () => Effect.die("unused") }), + Layer.succeed(DockerRun, { + run: () => Effect.die("unused"), + runCapture: () => Effect.die("unused"), + runStream: () => Effect.die("unused"), + }), + mockOutput().layer, + ), + ), + Effect.tap(() => + Effect.sync(() => { + expect(fixture.calls[0]?.args).toContain("--exclude-table"); + expect(fixture.calls[0]?.args).toContain('"public"."Users"'); + }), + ), + ); + }); + + it.live("preserves schema filtering while streaming through the Stack tool", () => { + const fixture = stackFixture( + 'CREATE TABLE "public" (id integer);\nGRANT ALL ON TABLE "auth" TO "postgres";\n', + ); + const chunks: Array = []; + expect( + transformManagedDumpLine( + 'GRANT ALL ON TABLE "auth" TO "postgres";', + "pg_dump --schema-only", + { EXCLUDED_SCHEMAS: "auth|storage" }, + ), + ).toContain("-- GRANT"); + return streamPgDumpWithClient({ + image: "unused", + script: "pg_dump --schema-only", + env: { + PGHOST: "127.0.0.1", + PGPORT: "54322", + PGUSER: "postgres", + PGDATABASE: "postgres", + EXCLUDED_SCHEMAS: "auth|storage", + }, + onStdout: (bytes) => + Effect.sync(() => { + chunks.push(bytes); + }), + client: { kind: "stack", stack: fixture.stack, command: "pg_dump", major: 17 }, + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + Layer.succeed(NetworkIdFlag, Option.none()), + Layer.succeed(RuntimeInfo, { + cwd: "/tmp", + platform: "linux", + arch: "x64", + homeDir: "/tmp", + execPath: "/usr/bin/bun", + pid: 1, + }), + Layer.succeed(BundledPostgresClient, { run: () => Effect.die("unused") }), + Layer.succeed(DockerRun, { + run: () => Effect.die("unused"), + runCapture: () => Effect.die("unused"), + runStream: () => Effect.die("unused"), + }), + mockOutput().layer, + ), + ), + Effect.asVoid, + Effect.tap(() => + Effect.sync(() => { + expect(chunks.every((chunk) => chunk.length > 0)).toBe(true); + const output = new TextDecoder().decode( + Uint8Array.from(chunks.flatMap((chunk) => [...chunk])), + ); + expect(output).toContain('CREATE TABLE IF NOT EXISTS "public"'); + expect(output).toContain('-- GRANT ALL ON TABLE "auth"'); + expect(fixture.calls[0]?.args).toContain("--exclude-schema"); + }), + ), + ); + }); +}); diff --git a/apps/cli/src/command-internal/pg-dump.run.ts b/apps/cli/src/command-internal/pg-dump.run.ts index fd735dcf91..ddddeee07f 100644 --- a/apps/cli/src/command-internal/pg-dump.run.ts +++ b/apps/cli/src/command-internal/pg-dump.run.ts @@ -1,5 +1,8 @@ import { Effect, Option } from "effect"; -import type { StackRuntimePreference } from "@supabase/stack/effect"; +import { postgres, type Stack } from "@supabase/stack/effect"; +type StackRuntimePreference = + | { readonly kind: "native" } + | { readonly kind: "container"; readonly engine: "docker" | "podman" }; import { NetworkIdFlag } from "./global-flags.ts"; import { viperEnvStringWithProjectFallback } from "./viper-env.ts"; @@ -8,6 +11,7 @@ import { DockerRun } from "./docker-run.service.ts"; import { DockerRunError } from "./docker-run.errors.ts"; import { getRegistryImageUrl } from "./docker-registry.ts"; import { BundledPostgresClient, resolveBundledPostgresRuntime } from "./bundled-postgres-client.ts"; +import { Output } from "../shared/output/output.service.ts"; /** * Runs a pg_dump/pg_dumpall bash script in a one-shot container, streaming stdout @@ -82,6 +86,12 @@ export const streamPgDump = Effect.fnUntraced(function* (params: { export type PgDumpClient = | { readonly kind: "container" } + | { + readonly kind: "stack"; + readonly stack: Stack; + readonly command: "pg_dump" | "pg_dumpall"; + readonly major: 15 | 17; + } | { readonly kind: "bundled"; readonly command: "pg_dump" | "pg_dumpall"; @@ -90,7 +100,7 @@ export type PgDumpClient = }; export const pgDumpClientExitMessage = (client: PgDumpClient, exitCode: number): string => - client.kind === "bundled" && client.runtime?.kind === "native" + client.kind === "stack" || (client.kind === "bundled" && client.runtime?.kind === "native") ? `error running ${client.command}: exit ${exitCode}` : `error running container: exit ${exitCode}`; @@ -114,8 +124,117 @@ const bundledDumpNetwork = ( return network._tag === "host" ? "host" : { name: network.name }; }; +const transformDumpLine = ( + line: string, + script: string, + env: Readonly>, +): string | undefined => { + const applyExtraSed = (value: string | undefined): string | undefined => + env["EXTRA_SED"] === "/^--/d" && value?.startsWith("--") ? undefined : value; + if (script.includes("--data-only")) { + return line.replace(/^\\(?:un)?restrict /u, "-- $&"); + } + if (script.includes("pg_dumpall")) { + if (/^\\(?:un)?restrict /u.test(line)) return applyExtraSed(`-- ${line}`); + const reservedRoles = env["RESERVED_ROLES"] ?? ""; + const allowedConfigs = env["ALLOWED_CONFIGS"]?.split("|").filter(Boolean) ?? []; + const reservedRole = new RegExp(`^(CREATE|ALTER) ROLE "(${reservedRoles})"`, "u"); + const allowedConfig = new RegExp(`^ALTER ROLE ".*" SET "(${allowedConfigs.join("|")})" `, "u"); + const dropComments = (value: string): string | undefined => applyExtraSed(value); + if (reservedRole.test(line)) + return dropComments(allowedConfig.test(line) ? line : `-- ${line}`); + if (new RegExp(`GRANT ".*" TO "(${reservedRoles})"`, "u").test(line)) + return dropComments(`-- ${line}`); + if (line.startsWith("-- ") && allowedConfig.test(line.slice(3))) + return dropComments(line.slice(3)); + if (line.startsWith("-- ") && line.includes(" GRANT ")) return dropComments(line); + if (line.startsWith("-- ") && line.includes("ALTER ROLE ")) return dropComments(line); + return applyExtraSed(line.replaceAll(/ (NOSUPERUSER|NOREPLICATION)/gu, "")); + } + const replacements: ReadonlyArray = [ + [/^CREATE SCHEMA "/u, 'CREATE SCHEMA IF NOT EXISTS "'], + [/^CREATE TABLE "/u, 'CREATE TABLE IF NOT EXISTS "'], + [/^CREATE SEQUENCE "/u, 'CREATE SEQUENCE IF NOT EXISTS "'], + [/^CREATE VIEW "/u, 'CREATE OR REPLACE VIEW "'], + [/^CREATE FUNCTION "/u, 'CREATE OR REPLACE FUNCTION "'], + [/^CREATE TRIGGER "/u, 'CREATE OR REPLACE TRIGGER "'], + [/^CREATE PUBLICATION "supabase_realtime/u, '-- CREATE PUBLICATION "supabase_realtime'], + [/^CREATE EVENT TRIGGER /u, "-- CREATE EVENT TRIGGER "], + [/^ WHEN TAG IN /u, "-- WHEN TAG IN "], + [/^ EXECUTE FUNCTION /u, "-- EXECUTE FUNCTION "], + [/^ALTER EVENT TRIGGER /u, "-- ALTER EVENT TRIGGER "], + [/^ALTER PUBLICATION "supabase_realtime_/u, '-- ALTER PUBLICATION "supabase_realtime_'], + [/^ALTER FOREIGN DATA WRAPPER (.+) OWNER TO /u, "-- $&"], + [/^ALTER DEFAULT PRIVILEGES FOR ROLE "supabase_admin"/u, "-- $&"], + [/^COMMENT ON EXTENSION /u, "-- $&"], + [/^GRANT ALL ON FOREIGN DATA WRAPPER (.+) TO "postgres" WITH GRANT OPTION/u, "-- $&"], + [/^CREATE POLICY "cron_job_/u, "-- $&"], + [/^ALTER TABLE "cron"/u, "-- $&"], + [/^SET transaction_timeout = 0;/u, "-- SET transaction_timeout = 0;"], + ]; + if (/^\\(?:un)?restrict /u.test(line)) return applyExtraSed(`-- ${line}`); + const excludedSchemas = env["EXCLUDED_SCHEMAS"]?.split("|").filter(Boolean) ?? []; + const excludedSchemaPattern = excludedSchemas.join("|"); + if ( + excludedSchemaPattern.length > 0 && + new RegExp(`^(GRANT|REVOKE) (.+) ON (.+) "(${excludedSchemaPattern})"`, "u").test(line) + ) + return applyExtraSed(`-- ${line}`); + for (const [pattern, replacement] of replacements) + if (pattern.test(line)) return applyExtraSed(line.replace(pattern, replacement)); + if (/^CREATE EXTENSION IF NOT EXISTS "(?:pg_tle|pgsodium|pgmq)"/u.test(line)) + return applyExtraSed( + line.replace(/^(CREATE EXTENSION IF NOT EXISTS "(?:pg_tle|pgsodium|pgmq)").+$/u, "$1;"), + ); + return applyExtraSed(line); +}; + +export const transformManagedDumpLine = transformDumpLine; + +const stackDumpStdout = ( + script: string, + env: Readonly>, + emit: (bytes: Uint8Array) => Effect.Effect, +) => { + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + let carry = ""; + let previous: string | undefined; + const onChunk = (bytes: Uint8Array) => + Effect.suspend(() => { + carry += decoder.decode(bytes, { stream: true }); + const lines = carry.split("\n"); + carry = lines.pop() ?? ""; + return Effect.forEach( + lines, + (line) => { + const transformed = transformDumpLine(line, script, env); + if ( + transformed === undefined || + (script.includes("pg_dumpall") && transformed === previous) + ) + return Effect.void; + previous = transformed; + return emit(encoder.encode(`${transformed}\n`)); + }, + { discard: true }, + ); + }); + const flush = () => + Effect.suspend(() => { + carry += decoder.decode(); + const final = transformDumpLine(carry, script, env); + return final === undefined || + final.length === 0 || + (script.includes("pg_dumpall") && final === previous) + ? Effect.void + : emit(encoder.encode(final)); + }); + return { onChunk, flush }; +}; + /** Compose dump, or catalog `pg_dump`/`pg_dumpall` on the stack backend. */ -export const streamPgDumpWithClient = Effect.fnUntraced(function* (params: { +export const streamPgDumpWithClient = Effect.fn("streamPgDumpWithClient")(function* (params: { readonly image: string; readonly script: string; readonly env: Readonly>; @@ -124,6 +243,78 @@ export const streamPgDumpWithClient = Effect.fnUntraced(function* (params: { readonly client: PgDumpClient; readonly forceHostNetwork?: boolean; }) { + if (params.client.kind === "stack") { + const output = yield* Output; + const flags = + params.client.command === "pg_dumpall" + ? [ + "--roles-only", + "--role", + "postgres", + "--quote-all-identifier", + "--no-role-passwords", + "--no-comments", + ] + : [ + params.script.includes("--data-only") ? "--data-only" : "--schema-only", + "--quote-all-identifier", + "--role", + "postgres", + ]; + const extraFlags = params.env["EXTRA_FLAGS"]?.match(/(?:[^\s"]+|"[^"]*")+/gu) ?? []; + const excludedSchemas = params.env["EXCLUDED_SCHEMAS"]?.split("|").filter(Boolean) ?? []; + const includedSchemas = params.env["INCLUDED_SCHEMAS"]; + const args = [ + ...flags, + ...excludedSchemas.flatMap((schema) => ["--exclude-schema", schema]), + ...(includedSchemas === undefined ? [] : ["--schema", includedSchemas]), + ...(params.script.includes("--data-only") + ? [ + "--exclude-table", + "auth.schema_migrations", + "--exclude-table", + "storage.migrations", + "--exclude-table", + "supabase_functions.migrations", + ] + : []), + ...extraFlags, + "--host", + params.env["PGHOST"] ?? "127.0.0.1", + "--port", + params.env["PGPORT"] ?? "5432", + "--username", + params.env["PGUSER"] ?? "postgres", + ...(params.client.command === "pg_dump" + ? ["--dbname", params.env["PGDATABASE"] ?? "postgres"] + : []), + ]; + let stderr = ""; + const stderrDecoder = new TextDecoder(); + const emit = stackDumpStdout(params.script, params.env, params.onStdout); + if (params.script.includes("--data-only")) + yield* params.onStdout(new TextEncoder().encode("SET session_replication_role = replica;\n")); + const result = yield* params.client.stack.tools.run( + params.client.command === "pg_dump" + ? postgres.pgDump({ major: params.client.major }) + : postgres.pgDumpAll({ major: params.client.major }), + { + args, + env: params.env, + stdout: emit.onChunk, + stderr: (chunk) => + Effect.gen(function* () { + yield* output.rawBytes(chunk, "stderr"); + stderr += stderrDecoder.decode(chunk, { stream: true }); + }), + }, + ); + stderr += stderrDecoder.decode(); + yield* emit.flush(); + if (params.script.includes("--data-only") || params.script.includes("pg_dumpall")) + yield* params.onStdout(new TextEncoder().encode("RESET ALL;\n")); + return { exitCode: result.exitCode, stderr }; + } if (params.client.kind === "bundled") { const bundled = yield* BundledPostgresClient; const runtimeInfo = yield* RuntimeInfo; diff --git a/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts b/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts index af693f1bd3..a80e16ff0e 100644 --- a/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts +++ b/apps/cli/src/command-internal/pgdelta-engine-runtime.layer.ts @@ -16,7 +16,6 @@ import { declarativeSeamLayer } from "../commands/db/shared/pgdelta.seam.layer.t import { localDockerEngineLayer } from "./db-bootstrap/local-db-running.ts"; import { stackApiLayer } from "./stack-api.ts"; import { bundledPostgresClientLayer } from "./bundled-postgres-client.ts"; -import { ephemeralPostgresLayer } from "./stack-shadow.ts"; import { stackCatalogSetupLayer } from "./stack-catalog-setup.ts"; /** The in-process pg-delta engine — the only implementation. */ @@ -50,6 +49,7 @@ export const migraRuntimeLayer = Layer.mergeAll( const httpClient = httpClientLayer.pipe(Layer.provide(debugLoggerLayer)); const localDockerEngine = localDockerEngineLayer.pipe(Layer.provide(debugLoggerLayer)); const seam = declarativeSeamLayer.pipe( + Layer.provide(stackCatalogSetupLayer), Layer.provide(pgDeltaCommandSettingsRuntimeLayer), Layer.provide(dbConnectionLayer), Layer.provide(dockerRunLayer), @@ -58,6 +58,7 @@ const seam = declarativeSeamLayer.pipe( Layer.provide(localDockerEngine), ); const nextShadow = pgDeltaNextShadowLayer.pipe( + Layer.provide(stackApiLayer), Layer.provide(dockerRunLayer), Layer.provide(dbConnectionLayer), Layer.provide(httpClient), @@ -83,7 +84,9 @@ export const pgDeltaCommandRuntimeLayer = Layer.mergeAll( // Exposed for handlers' own direct `isLocalDbRunning` calls (`db diff --use-pgadmin`). localDockerEngine, stackApiLayer, - bundledPostgresClientLayer, - ephemeralPostgresLayer, + bundledPostgresClientLayer.pipe( + Layer.provide(dockerRunLayer), + Layer.provide(pgDeltaCommandSettingsRuntimeLayer), + ), stackCatalogSetupLayer, ); diff --git a/apps/cli/src/command-internal/postgres-client.run.unit.test.ts b/apps/cli/src/command-internal/postgres-client.run.unit.test.ts index 7d893eb833..8850aa96bb 100644 --- a/apps/cli/src/command-internal/postgres-client.run.unit.test.ts +++ b/apps/cli/src/command-internal/postgres-client.run.unit.test.ts @@ -31,7 +31,9 @@ describe("rewriteDumpHostForToolContainer", () => { describe("bundledPostgresClientRuntime", () => { it("keeps a unix native stack on the native client", () => { - expect(bundledPostgresClientRuntime({ kind: "native" }, "darwin")).toEqual({ kind: "native" }); + expect(bundledPostgresClientRuntime({ kind: "native" }, "darwin", "arm64")).toEqual({ + kind: "native", + }); }); it("forces a container client for Windows native stacks", () => { diff --git a/apps/cli/src/command-internal/stack-api.ts b/apps/cli/src/command-internal/stack-api.ts index 09249ac4a3..ed753d6071 100644 --- a/apps/cli/src/command-internal/stack-api.ts +++ b/apps/cli/src/command-internal/stack-api.ts @@ -1,46 +1,35 @@ import { Context, Crypto, Effect, FileSystem, Layer, Path } from "effect"; +import { FetchHttpClient, HttpClient } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { - createStack, - discoverStacks, - findStack, - inspectStack, - openStack, - type StackDiscoveryResult, + create, + discover, + open, + type CreateOptions, + type OpenOptions, + type Stack, } from "@supabase/stack/effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; +import { resolveStackIdentity } from "@supabase/stack/internal/identity"; +type DiscoverResult = Effect.Success>; +type StackError = Effect.Error>; +type IdentityResult = Effect.Success>; +type IdentityError = Effect.Error>; + +/** Operations used by the CLI stack boundary. */ export class StackApi extends Context.Service< StackApi, { - readonly findStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly createStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly openStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly inspectStack: ( - ...args: Parameters - ) => Effect.Effect< - Effect.Success>, - Effect.Error> - >; - readonly discoverStacks: ( - ...args: Parameters - ) => Effect.Effect>>; + readonly create: (options: CreateOptions) => Effect.Effect; + readonly open: (options: OpenOptions) => Effect.Effect; + readonly discover: ( + options: Parameters[0], + ) => Effect.Effect; + readonly resolveIdentity: ( + options: Parameters[0], + ) => Effect.Effect; } ->()("supabase/experimental-stack/StackApi") {} +>()("supabase/stack/StackApi") {} export const stackApiLayer = Layer.effect( StackApi, @@ -49,22 +38,35 @@ export const stackApiLayer = Layer.effect( const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const childProcess = yield* ChildProcessSpawner.ChildProcessSpawner; + const http = yield* HttpClient.HttpClient; const provideServices = (effect: Effect.Effect) => effect.pipe( Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, path), Effect.provideService(Crypto.Crypto, crypto), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcess), + Effect.provideService(HttpClient.HttpClient, http), ); - return { - findStack: (...args: Parameters) => provideServices(findStack(...args)), - createStack: (...args: Parameters) => - provideServices(createStack(...args)), - openStack: (...args: Parameters) => provideServices(openStack(...args)), - inspectStack: (...args: Parameters) => - provideServices(inspectStack(...args)), - discoverStacks: (...args: Parameters) => - provideServices(discoverStacks(...args)), - }; + const createStack = Effect.fn("StackApi.create")((options: CreateOptions) => + provideServices(create(options)), + ); + const openStack = Effect.fn("StackApi.open")((options: OpenOptions) => + provideServices(open(options)), + ); + const discoverStacks = Effect.fn("StackApi.discover")( + (options: Parameters[0]) => provideServices(discover(options)), + ); + const resolveIdentity = Effect.fn("StackApi.resolveIdentity")( + (options: Parameters[0]) => + provideServices(resolveStackIdentity(options)), + ); + return StackApi.of({ + create: createStack, + open: openStack, + discover: discoverStacks, + resolveIdentity, + }); }), -); +).pipe(Layer.provide(FetchHttpClient.layer)); + +export type { Stack }; diff --git a/apps/cli/src/command-internal/stack-auth-config.ts b/apps/cli/src/command-internal/stack-auth-config.ts new file mode 100644 index 0000000000..ba9ba7cae8 --- /dev/null +++ b/apps/cli/src/command-internal/stack-auth-config.ts @@ -0,0 +1,77 @@ +import type { CliConfig } from "@supabase/config"; +import type { ServiceCreation } from "@supabase/stack/effect"; +import { Effect } from "effect"; +import { passwordRequirementsToChar } from "./password-requirements.ts"; + +type AuthConfig = Extract["config"]; + +/** Maps resolved CLI authentication policy to the Auth service. */ +export const resolveAuthConfig = Effect.fn("StackAuthConfig.resolve")( + (auth: CliConfig["auth"], localSmtp: CliConfig["local_smtp"]): Effect.Effect => + Effect.succeed({ + databaseUrl: "postgresql://placeholder", + siteUrl: auth.site_url, + jwtExpiry: auth.jwt_expiry, + disableSignup: !auth.enable_signup, + ...(localSmtp.admin_email === undefined ? {} : { smtpAdminEmail: localSmtp.admin_email }), + ...(localSmtp.sender_name === undefined ? {} : { smtpSenderName: localSmtp.sender_name }), + settings: { + additionalRedirectUrls: auth.additional_redirect_urls.join(","), + enableRefreshTokenRotation: auth.enable_refresh_token_rotation, + refreshTokenReuseInterval: auth.refresh_token_reuse_interval, + enableManualLinking: auth.enable_manual_linking, + enableAnonymousSignIns: auth.enable_anonymous_sign_ins, + minimumPasswordLength: auth.minimum_password_length, + passwordRequirements: passwordRequirementsToChar(auth.password_requirements), + ...(auth.jwt_issuer === undefined ? {} : { jwtIssuer: auth.jwt_issuer }), + rateLimit: auth.rate_limit, + email: { + enable_signup: auth.email.enable_signup, + double_confirm_changes: auth.email.double_confirm_changes, + enable_confirmations: auth.email.enable_confirmations, + secure_password_change: auth.email.secure_password_change, + max_frequency: auth.email.max_frequency, + otp_length: auth.email.otp_length, + otp_expiry: auth.email.otp_expiry, + template: Object.fromEntries( + Object.entries(auth.email.template).map(([name, template]) => [ + name, + { subject: template.subject }, + ]), + ), + notification: Object.fromEntries( + Object.entries(auth.email.notification).map(([name, notification]) => [ + name, + { enabled: notification.enabled, subject: notification.subject }, + ]), + ), + }, + sms: auth.sms, + ...(auth.captcha === undefined ? {} : { captcha: auth.captcha }), + hooks: auth.hook, + mfa: auth.mfa, + ...(auth.sessions === undefined ? {} : { sessions: auth.sessions }), + external: Object.fromEntries( + Object.entries(auth.external).flatMap(([name, provider]) => + provider === undefined ? [] : [[name, provider]], + ), + ), + web3: auth.web3, + oauthServer: auth.oauth_server, + }, + ...(auth.email.smtp?.enabled !== true + ? {} + : { + smtp: { + host: auth.email.smtp.host ?? "", + port: auth.email.smtp.port ?? 587, + user: auth.email.smtp.user ?? "", + pass: auth.email.smtp.pass ?? "", + adminEmail: auth.email.smtp.admin_email ?? "", + ...(auth.email.smtp.sender_name === undefined + ? {} + : { senderName: auth.email.smtp.sender_name }), + }, + }), + }), +); diff --git a/apps/cli/src/command-internal/stack-backend.ts b/apps/cli/src/command-internal/stack-backend.ts index 8a4fd0b329..1e07708224 100644 --- a/apps/cli/src/command-internal/stack-backend.ts +++ b/apps/cli/src/command-internal/stack-backend.ts @@ -27,6 +27,11 @@ const STACK_BACKEND_COMMANDS = new Set([ "seed", ]); +const isFunctionsServePath = (path: ReadonlyArray): boolean => + path.length >= 2 && + ((path[0] === "functions" && path[1] === "serve") || + (path[0] === "help" && path[1] === "functions" && path[2] === "serve")); + export class StackRoutingError extends Data.TaggedError("StackRoutingError")<{ readonly message: string; readonly cause?: unknown; @@ -75,9 +80,13 @@ export const resolveStackBackend = (input: { ? commandPath.slice(1) : commandPath; const command = completePath[0] === "help" ? completePath[1] : completePath[0]; - if (command !== undefined && command !== "stack" && !STACK_BACKEND_COMMANDS.has(command)) { + if ( + command !== undefined && + command !== "stack" && + !STACK_BACKEND_COMMANDS.has(command) && + !isFunctionsServePath(completePath) + ) return "legacy"; - } const enabled = yield* resolveExperimentalFeature({ feature: "stack", diff --git a/apps/cli/src/command-internal/stack-bootstrap.ts b/apps/cli/src/command-internal/stack-bootstrap.ts new file mode 100644 index 0000000000..78dc1b3ee4 --- /dev/null +++ b/apps/cli/src/command-internal/stack-bootstrap.ts @@ -0,0 +1,109 @@ +import { Data, Effect, FileSystem, Path } from "effect"; +import type { DatabaseInstance } from "@supabase/stack/effect"; +import { Output } from "../shared/output/output.service.ts"; +import { applyDatabaseWebhooks } from "./db-bootstrap/db-setup.ts"; +import { parseConnectionString } from "./db-config.parse.ts"; +import { DbConnection, type DbSession } from "./db-connection.service.ts"; +import type { DbTomlValues } from "./db-config.toml-read.ts"; +import { migrateAndSeed } from "./migrate-and-seed.ts"; + +/** Failure while applying the CLI-owned database bootstrap steps. */ +export class StackBootstrapError extends Data.TaggedError("StackBootstrapError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const databaseConn = Effect.fn("StackBootstrap.databaseConnection")(function* ( + database: DatabaseInstance, +) { + const credentials = yield* database.credentials({ from: "host" }); + const conn = parseConnectionString(credentials.databaseUrl ?? ""); + if (conn === undefined) + return yield* new StackBootstrapError({ message: "failed to parse stack database URL" }); + return conn; +}); + +const withDatabaseSession = ( + database: DatabaseInstance, + body: (session: DbSession) => Effect.Effect, + user?: string, +) => + Effect.scoped( + Effect.gen(function* () { + const dbConn = yield* DbConnection; + const conn = yield* databaseConn(database); + const session = yield* dbConn.connect(user === undefined ? conn : { ...conn, user }, { + isLocal: true, + dnsResolver: "native", + }); + return yield* body(session); + }), + ); + +/** Applies only the webhook configuration to an already initialized database. */ +export const applyStackWebhooksOnly = ( + database: DatabaseInstance, + webhooksEnabled: boolean, +): Effect.Effect => + withDatabaseSession(database, (session) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const tmpDir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-webhooks-" }); + yield* applyDatabaseWebhooks(session, fs, path, tmpDir, webhooksEnabled); + }), + ).pipe( + Effect.mapError( + (error) => + new StackBootstrapError({ + message: + typeof error === "object" && error !== null && "message" in error + ? String(error.message) + : String(error), + cause: error, + }), + ), + ); + +/** Applies migrations and seeds after the stack catalog has initialized the database. */ +export const applyStackMigrateAndSeed = ( + database: DatabaseInstance, + workdir: string, + toml: Pick< + DbTomlValues, + "migrationsEnabled" | "seed" | "pgDelta" | "schemaPaths" | "webhooksEnabled" + >, + experimental: boolean, +): Effect.Effect< + void, + StackBootstrapError, + DbConnection | FileSystem.FileSystem | Path.Path | Output +> => + withDatabaseSession( + database, + (session) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* migrateAndSeed(session, fs, path, workdir, "", { + migrationsEnabled: toml.migrationsEnabled, + seed: toml.seed, + experimental, + pgDeltaEnabled: toml.pgDelta.enabled, + schemaPaths: toml.schemaPaths, + localDatabaseWebhooksEnabled: toml.webhooksEnabled, + }); + }), + "postgres", + ).pipe( + Effect.mapError( + (error) => + new StackBootstrapError({ + message: + typeof error === "object" && error !== null && "message" in error + ? String(error.message) + : String(error), + cause: error, + }), + ), + ); diff --git a/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts b/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts new file mode 100644 index 0000000000..531050bfc5 --- /dev/null +++ b/apps/cli/src/command-internal/stack-catalog-setup.integration.test.ts @@ -0,0 +1,144 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Option, Redacted, Stream } from "effect"; +import { FetchHttpClient } from "effect/unstable/http"; +import { tmpdir } from "node:os"; +import { create, postgres } from "@supabase/stack/effect"; +import { mockOutput } from "../../tests/helpers/mocks.ts"; +import { stackCatalogSetupLayer, StackCatalogSetup } from "./stack-catalog-setup.ts"; + +const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const jwtSecret = "stack-catalog-setup-integration-secret"; + +describe("stack catalog setup", { timeout: 180_000 }, () => { + for (const runtime of ["native", "docker"] as const) { + it.live(`initializes selected database services on a stopped ${runtime} composition`, () => { + const buildOutput = mockOutput(); + const callOutput = mockOutput(); + return Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-catalog-${runtime}-` }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/roles.sql`, + "CREATE TABLE IF NOT EXISTS public.catalog_overlay(owner uuid REFERENCES auth.users(id), value text NOT NULL);\n", + ); + const stack = yield* create({ + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot, + runtime, + }); + yield* Effect.acquireUseRelease( + Effect.succeed(stack), + (stack) => + Effect.gen(function* () { + const members = yield* stack.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(jwtSecret), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { + service: "auth", + config: { databaseUrl: "postgresql://placeholder", jwtSecret }, + endpoints: { http: { port: "auto" } }, + }, + { + service: "storage", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret, + filePath: `${root}/unused-storage`, + }, + endpoints: { http: { port: "auto" } }, + }, + { + service: "realtime", + config: { databaseUrl: "postgresql://placeholder", jwtSecret }, + endpoints: { http: { port: "auto" }, rpc: { port: "auto" } }, + }, + ]); + const database = members.find((member) => member.service === "database"); + if (database === undefined) return yield* Effect.die("database member missing"); + yield* database.start; + yield* database.ready; + const runtimeCredentials = yield* database.credentials({ from: "runtime" }); + const databaseUrl = runtimeCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("database URL missing"); + + const setup = yield* Effect.service(StackCatalogSetup); + yield* setup + .apply({ + target: { + stack, + database, + databaseServices: ["auth", "storage", "realtime"], + jwtSecret, + }, + overlay: { + webhooks: "disabled", + webhooksEnabled: false, + apiAutoExposeNewTables: Option.none(), + vault: [], + workdir: root, + announceRoles: true, + }, + }) + .pipe(Effect.provide(callOutput.layer)); + expect(callOutput.stderrText).toContain("Seeding globals from roles.sql..."); + const realtime = members.find((member) => member.service === "realtime"); + if (realtime === undefined) return yield* Effect.die("realtime member missing"); + yield* realtime.start; + yield* realtime.ready; + yield* realtime.stop; + + const rows: Array = []; + const errors: Array = []; + const query = yield* stack.tools.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "-At"], + stdin: Stream.make( + new TextEncoder().encode( + "select coalesce(to_regclass('auth.users')::text,'missing'), coalesce(to_regclass('storage.objects')::text,'missing'), coalesce(to_regclass('realtime.messages')::text,'missing'), coalesce(to_regclass('realtime.subscription')::text,'missing'), coalesce(to_regclass('public.catalog_overlay')::text,'missing');", + ), + ), + stdout: (bytes) => Effect.sync(() => rows.push(new TextDecoder().decode(bytes))), + stderr: (bytes) => + Effect.sync(() => errors.push(new TextDecoder().decode(bytes))), + }); + expect(query.exitCode, errors.join("")).toBe(0); + expect(rows.join("").trim()).toBe( + "users|storage.objects|realtime.messages|realtime.subscription|catalog_overlay", + ); + + const listed = yield* stack.services.list; + expect(listed.map((instance) => instance.id).sort()).toEqual( + members.map((instance) => instance.id).sort(), + ); + for (const instance of listed) { + if (instance.service !== "database") + expect((yield* instance.status).lifecycle).toBe("stopped"); + } + }), + (stack) => stack.destroy, + ); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + FetchHttpClient.layer, + buildOutput.layer, + stackCatalogSetupLayer, + ), + ), + ); + }); + } +}); diff --git a/apps/cli/src/command-internal/stack-catalog-setup.ts b/apps/cli/src/command-internal/stack-catalog-setup.ts index 8c1234079d..830ba8b276 100644 --- a/apps/cli/src/command-internal/stack-catalog-setup.ts +++ b/apps/cli/src/command-internal/stack-catalog-setup.ts @@ -1,12 +1,9 @@ -import { Context, Crypto, Data, Effect, FileSystem, Layer, Path, Redacted } from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; +import { Context, Data, Effect, FileSystem, Layer, Path } from "effect"; import { - schemaInit, - type EffectStack, - type SchemaInitCapabilityName, - type SchemaInitTarget, - type StackConfig, - type StackRuntime, + type DatabaseInstance, + type ServiceCreation, + type ServiceInstance, + type Stack, } from "@supabase/stack/effect"; import { Output } from "../shared/output/output.service.ts"; import { @@ -15,24 +12,30 @@ import { ErrorActionabilityId, } from "../shared/telemetry/error-actionability.ts"; import { parseConnectionString } from "./db-config.parse.ts"; +import type { DbConnectError } from "./db-connection.errors.ts"; import { DbConnection } from "./db-connection.service.ts"; import { dbConnectionLayer } from "./db-connection.layer.ts"; import { applyDatabaseOverlay, type ApplyDatabaseOverlayInput, + type DbSetupError, type SetupDatabaseOptions, } from "./db-bootstrap/db-setup.ts"; -import type { VaultSecret } from "./vault.ts"; - -const PLATFORM_TRIO = [ - "auth", - "storage", - "realtime", -] as const satisfies ReadonlyArray; -const OPTIONAL_CAPS = [ - "analytics", - "pooler", -] as const satisfies ReadonlyArray; +import type { MigrationVaultError, VaultSecret } from "./vault.ts"; + +const DATABASE_SERVICES = ["auth", "storage", "realtime"] as const; +type DatabaseService = (typeof DATABASE_SERVICES)[number]; + +interface ServiceCredentials { + readonly databaseUrl: string; + readonly authDatabaseUrl: string; + readonly storageDatabaseUrl: string; +} + +type TemporaryServiceInstance = + | ServiceInstance<"auth"> + | ServiceInstance<"storage"> + | ServiceInstance<"realtime">; export class StackCatalogSetupError extends Data.TaggedError("StackCatalogSetupError")<{ readonly message: string; @@ -52,180 +55,180 @@ interface StackCatalogOverlay { readonly announceRoles?: boolean; } -interface LiveStackCatalogInput { - readonly kind: "live"; - readonly stack: EffectStack; - readonly projectRoot: string; - readonly config: StackConfig; -} - -interface EphemeralStackCatalogInput { - readonly kind: "ephemeral"; - readonly projectRoot: string; - readonly runtime: StackRuntime; - readonly config: StackConfig; - readonly databaseUrl: string; - readonly databasePassword: Redacted.Redacted; - readonly jwtSecret?: Redacted.Redacted; - readonly networkId?: string; -} - export interface StackCatalogSetupInput { - readonly target: LiveStackCatalogInput | EphemeralStackCatalogInput; + readonly target: { + readonly stack: Stack; + readonly database: DatabaseInstance; + readonly databaseServices: ReadonlyArray; + readonly jwtSecret: string; + }; readonly overlay: StackCatalogOverlay; - /** Used only for analytics/pooler one-shots. Platform trio stays on `target.config`. */ - readonly optionalConfig?: StackConfig; } -const capabilityEnabled = (config: StackConfig, name: SchemaInitCapabilityName): boolean => { - const cap = config.capabilities?.[name]; - return cap === undefined || cap.enabled !== false; +const catalogError = (cause: unknown): StackCatalogSetupError => + cause instanceof StackCatalogSetupError + ? cause + : new StackCatalogSetupError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +type StackCatalogSetupFailure = + | StackCatalogSetupError + | DbSetupError + | MigrationVaultError + | DbConnectError; + +const temporaryServiceError = ( + operation: "start" | "destroy", + instance: TemporaryServiceInstance, + cause: unknown, +): StackCatalogSetupError => + new StackCatalogSetupError({ + message: `temporary ${instance.service} service ${instance.id} failed to ${operation}: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }); + +const credential = ( + credentials: Readonly>, + name: string, +): Effect.Effect => { + const value = credentials[name]; + return value === undefined + ? Effect.fail(new StackCatalogSetupError({ message: `database credential ${name} is missing` })) + : Effect.succeed(value); }; -const jwtSecretFromConfig = (config: StackConfig): Redacted.Redacted | undefined => { - const signing = config.security?.jwt?.signing; - return signing?.kind === "symmetric" ? signing.secret : undefined; +const serviceDefinition = ( + service: DatabaseService, + credentials: ServiceCredentials, + jwtSecret: string, + storagePath: string, +): Extract => { + switch (service) { + case "auth": + return { + service, + config: { + databaseUrl: credentials.authDatabaseUrl, + jwtSecret, + }, + endpoints: {}, + }; + case "storage": + return { + service, + config: { + databaseUrl: credentials.storageDatabaseUrl, + filePath: storagePath, + jwtSecret, + }, + endpoints: {}, + }; + case "realtime": + return { + service, + config: { + databaseUrl: credentials.databaseUrl, + jwtSecret, + }, + endpoints: {}, + }; + } }; -const catalogError = (error: { readonly message: string }): StackCatalogSetupError => - new StackCatalogSetupError({ message: error.message, cause: error }); - -const runSchemaInit = (names: ReadonlyArray, target: SchemaInitTarget) => - names.length === 0 ? Effect.void : schemaInit(names, target).pipe(Effect.mapError(catalogError)); - -const targetConnection = (target: LiveStackCatalogInput | EphemeralStackCatalogInput) => - target.kind === "ephemeral" - ? Effect.succeed({ - databaseUrl: target.databaseUrl, - databasePassword: target.databasePassword, - jwtSecret: target.jwtSecret, - runtime: target.runtime, - }) - : Effect.gen(function* () { - const credentials = yield* target.stack.credentials; - const status = yield* target.stack.status; - return { - databaseUrl: Redacted.value(credentials.database.url), - databasePassword: credentials.database.password, - jwtSecret: jwtSecretFromConfig(target.config), - runtime: status.runtime, - }; - }).pipe(Effect.mapError(catalogError)); - -const applyCatalog = (input: StackCatalogSetupInput) => - Effect.gen(function* () { - const output = yield* Output; - const dbConn = yield* DbConnection; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const connection = yield* targetConnection(input.target); - const schemaTarget: SchemaInitTarget = - input.target.kind === "live" - ? { - kind: "live", - stackId: input.target.stack.id, - projectRoot: input.target.projectRoot, - runtime: connection.runtime, - config: input.target.config, - databaseUrl: connection.databaseUrl, - secrets: { - databasePassword: connection.databasePassword, - ...(connection.jwtSecret === undefined ? {} : { jwtSecret: connection.jwtSecret }), - }, - } - : { - kind: "ephemeral", - projectRoot: input.target.projectRoot, - runtime: connection.runtime, - config: input.target.config, - databaseUrl: connection.databaseUrl, - secrets: { - databasePassword: connection.databasePassword, - ...(connection.jwtSecret === undefined ? {} : { jwtSecret: connection.jwtSecret }), - }, - ...(input.target.networkId === undefined ? {} : { networkId: input.target.networkId }), - }; - const config = input.target.config; - const failClosed = PLATFORM_TRIO.filter((name) => capabilityEnabled(config, name)); - yield* runSchemaInit(failClosed, schemaTarget); - if (input.target.kind === "live") { - const optionalSource = input.optionalConfig ?? config; - const optional = OPTIONAL_CAPS.filter((name) => capabilityEnabled(optionalSource, name)); +const startTemporaryService = ( + instance: TemporaryServiceInstance, +): Effect.Effect => + instance.start.pipe( + Effect.andThen(instance.ready), + Effect.mapError((cause) => temporaryServiceError("start", instance, cause)), + ); + +const destroyTemporaryService = ( + instance: TemporaryServiceInstance, +): Effect.Effect => + instance.destroy.pipe( + Effect.mapError((cause) => temporaryServiceError("destroy", instance, cause)), + ); + +const applyCatalog = Effect.fn("StackCatalogSetup.apply")(function* ( + input: StackCatalogSetupInput, +) { + const dbConn = yield* DbConnection; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const runtimeCredentials = yield* input.target.database + .credentials({ from: "runtime" }) + .pipe(Effect.mapError(catalogError)); + const hostCredentials = yield* input.target.database + .credentials({ from: "host" }) + .pipe(Effect.mapError(catalogError)); + yield* Effect.scoped( + Effect.gen(function* () { + const runtimeDatabaseUrl = yield* credential(runtimeCredentials, "databaseUrl"); + const authDatabaseUrl = yield* credential(runtimeCredentials, "authDatabaseUrl"); + const storageDatabaseUrl = yield* credential(runtimeCredentials, "storageDatabaseUrl"); + const hostDatabaseUrl = yield* credential(hostCredentials, "databaseUrl"); + const serviceCredentials: ServiceCredentials = { + databaseUrl: runtimeDatabaseUrl, + authDatabaseUrl, + storageDatabaseUrl, + }; + const storagePath = yield* fs + .makeTempDirectoryScoped({ prefix: "supabase-stack-catalog-storage-" }) + .pipe(Effect.mapError(catalogError)); + yield* Effect.forEach( - optional, - (name) => - schemaInit([name], schemaTarget).pipe( - Effect.catchTag("RequiresActivatedProcessError", (error) => - output.raw( - `WARNING: skipped ${error.capability} schema init: ${error.message}\n`, - "stderr", - ), - ), - Effect.mapError(catalogError), + input.target.databaseServices, + (service) => + Effect.acquireUseRelease( + input.target.stack.services + .create( + serviceDefinition(service, serviceCredentials, input.target.jwtSecret, storagePath), + ) + .pipe(Effect.mapError(catalogError)), + startTemporaryService, + destroyTemporaryService, ), { discard: true }, ); - } - const conn = parseConnectionString(connection.databaseUrl); - if (conn === undefined) { - return yield* new StackCatalogSetupError({ - message: "failed to parse database URL for catalog overlay", - }); - } - yield* Effect.scoped( - Effect.gen(function* () { - const session = yield* dbConn.connect(conn, { isLocal: true, dnsResolver: "native" }); - yield* applyDatabaseOverlay(session, fs, path, input.overlay.workdir, { - webhooksEnabled: input.overlay.webhooksEnabled, - apiAutoExposeNewTables: input.overlay.apiAutoExposeNewTables, - vault: input.overlay.vault, - webhooks: input.overlay.webhooks, - announceRoles: input.overlay.announceRoles, + + const connection = parseConnectionString(hostDatabaseUrl); + if (connection === undefined) + return yield* new StackCatalogSetupError({ + message: "failed to parse database URL for catalog overlay", }); - }).pipe(Effect.mapError(catalogError)), - ); - }); + const session = yield* dbConn.connect(connection, { + isLocal: true, + dnsResolver: "native", + }); + yield* applyDatabaseOverlay(session, fs, path, input.overlay.workdir, { + webhooksEnabled: input.overlay.webhooksEnabled, + apiAutoExposeNewTables: input.overlay.apiAutoExposeNewTables, + vault: input.overlay.vault, + webhooks: input.overlay.webhooks, + announceRoles: input.overlay.announceRoles, + }); + }), + ); +}); export class StackCatalogSetup extends Context.Service< StackCatalogSetup, { readonly apply: ( input: StackCatalogSetupInput, - ) => Effect.Effect; + ) => Effect.Effect; } >()("supabase/cli/StackCatalogSetup") {} -type CatalogApplyServices = - | DbConnection - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | ChildProcessSpawner.ChildProcessSpawner; - export const stackCatalogSetupLayer = Layer.effect( StackCatalogSetup, Effect.gen(function* () { - const context = yield* Effect.context(); - return { - apply: (input: StackCatalogSetupInput) => - applyCatalog(input).pipe(Effect.provideContext(context), Effect.asVoid), - }; + const dbConn = yield* DbConnection; + return StackCatalogSetup.of({ + apply: (input) => applyCatalog(input).pipe(Effect.provideService(DbConnection, dbConn)), + }); }), ).pipe(Layer.provide(dbConnectionLayer)); - -export const noopStackCatalogSetupLayer = Layer.succeed(StackCatalogSetup, { - apply: () => Effect.void, -}); - -export const recordingStackCatalogSetup = (record: (input: StackCatalogSetupInput) => A) => { - const applied: Array = []; - return { - applied, - layer: Layer.succeed(StackCatalogSetup, { - apply: (input) => - Effect.sync(() => { - applied.push(record(input)); - }), - }), - }; -}; diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index 4c5b36a596..2d61548e7c 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -1,21 +1,19 @@ -import { type CliConfig, validateCliConfig } from "@supabase/config/effect"; -import { - Crypto, - Effect, - Data, - FileSystem, - Option, - Path, - Redacted, - Schema, - SchemaIssue, -} from "effect"; -import { StackConfigSchema, type StackConfig } from "@supabase/stack/effect"; +import { getDefaultCliConfig, type CliConfig } from "@supabase/config"; +import { resolveCliConfigSubtree } from "@supabase/config/internal"; +import { validateCliConfig } from "@supabase/config/effect"; +import { Crypto, Effect, Data, FileSystem, Path, Redacted, SchemaIssue } from "effect"; +import type { ServiceCreation as ServiceCreationType } from "@supabase/stack/effect"; import { loadLocalProjectContext, type LocalProjectContext } from "./local-project-context.ts"; -import { parseDotEnv } from "./dotenv.ts"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import { resolveAuthConfig } from "./stack-auth-config.ts"; +import { parseGoDuration } from "./go-duration.ts"; +import { parseFileSizeLimit } from "./storage-bucket-config.ts"; + +declare const SUPABASE_STACK_FUNCTIONS_SERVE_MAIN_TEMPLATE: string | undefined; import { + decryptAuthSecret, + resolveJwtSecret, envOverride, envOverrideApiMaxRows, envOverrideAuthPasswordRequirements, @@ -45,7 +43,6 @@ import { resolveGotrueWeb3, strToArr, } from "./local-config-values.ts"; -import { collectDotenvPrivateKeys, decryptSecret, isEncryptedSecret } from "./vault-decrypt.ts"; import { actionability, type CliErrorActionabilityDeclaration, @@ -61,16 +58,23 @@ export class StackConfigError extends Data.TaggedError("StackConfigError")<{ } } +interface StackStartConfig { + readonly jwtSecret: Redacted.Redacted; + readonly creations: ( + stackId: string, + options?: { readonly jwtSecret?: Redacted.Redacted }, + ) => Effect.Effect, StackConfigError>; + readonly source: CliConfig; + readonly projectEnvValues: Readonly>; + readonly document?: Record; +} + type StackConfigEffect = Effect.Effect< - StackConfig, + StackStartConfig, StackConfigError, FileSystem.FileSystem | Path.Path | RuntimeInfo | Crypto.Crypto >; -type JwtSigning = - | { readonly kind: "jwks-file"; readonly path: string } - | { readonly kind: "symmetric"; readonly secret: Redacted.Redacted }; - const isRecord = (value: unknown): value is Readonly> => typeof value === "object" && value !== null && !Array.isArray(value); @@ -85,93 +89,6 @@ const withoutUndefined = (value: unknown): unknown => { ); }; -const secret = (value: unknown): Redacted.Redacted | undefined => { - if (Redacted.isRedacted(value)) { - const unwrapped = Redacted.value(value); - return typeof unwrapped === "string" ? Redacted.make(unwrapped) : undefined; - } - return typeof value === "string" ? Redacted.make(value) : undefined; -}; - -const parseEnv = (contents: string): Record> => - Object.fromEntries( - Object.entries(parseDotEnv(contents)).map(([key, value]) => [key, Redacted.make(value)]), - ); - -const envKeyPattern = /^[A-Z_][A-Z0-9_]*$/u; -const defaultStudioApiUrl = "http://127.0.0.1"; - -const validateEnvKeys = ( - values: Readonly>>, - file: string, -) => { - const invalid = Object.keys(values).find((key) => !envKeyPattern.test(key)); - return invalid === undefined - ? Effect.succeed(values) - : Effect.fail( - new StackConfigError({ - message: `Invalid environment variable key ${invalid} in ${file}; use uppercase letters, digits, and underscores.`, - }), - ); -}; - -const readFunctionEnvironments = ( - projectRoot: string, - disabledFunctions: ReadonlySet = new Set(), - skip = false, -): Effect.Effect< - Readonly<{ - readonly shared: Readonly>>; - readonly functions: Readonly< - Record>>> - >; - }>, - StackConfigError, - FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.join(projectRoot, "supabase", "functions"); - if (skip) return { shared: {}, functions: {} }; - const read = (file: string) => - fs.exists(file).pipe( - Effect.flatMap((exists) => - exists - ? fs.readFileString(file).pipe( - Effect.flatMap((contents) => - Effect.try({ - try: () => parseEnv(contents), - // Keep parser diagnostics free of dotenv values, which may contain secrets. - catch: () => - new StackConfigError({ - message: `Failed to parse environment file ${file}`, - }), - }), - ), - Effect.flatMap((values) => validateEnvKeys(values, file)), - ) - : Effect.succeed({}), - ), - ); - const shared = yield* read(path.join(root, ".env")); - const entries = yield* fs.readDirectory(root).pipe(Effect.orElseSucceed(() => [])); - const result: Record>>> = {}; - for (const entry of entries.filter((name) => !name.startsWith(".") && !name.startsWith("_"))) { - if (!/^[A-Za-z0-9_-]+$/u.test(entry)) continue; - const info = yield* fs.stat(path.join(root, entry)).pipe(Effect.option); - if (Option.isNone(info) || info.value.type !== "Directory") continue; - if (disabledFunctions.has(entry)) continue; - const functionEnv = yield* read(path.join(root, entry, ".env")); - result[entry] = { ...shared, ...functionEnv }; - } - return { shared, functions: result }; - }).pipe( - Effect.mapError((cause) => - cause instanceof StackConfigError ? cause : new StackConfigError({ message: String(cause) }), - ), - ); - const section = (document: Readonly> | undefined, name: string) => { const value = document?.[name]; return isRecord(value) ? value : undefined; @@ -186,20 +103,6 @@ const explicitPort = ( return typeof value === "number" ? value : undefined; }; -const pick = ( - value: unknown, - keys: ReadonlyArray, - secretKeys: ReadonlySet = new Set(), -): Record => { - if (!isRecord(value)) return {}; - const result: Record = {}; - for (const key of keys) { - if (!(key in value) || value[key] === undefined) continue; - result[key] = secretKeys.has(key) ? secret(value[key]) : value[key]; - } - return result; -}; - const envPortOrConfigured = ( name: string, document: Readonly> | undefined, @@ -248,365 +151,6 @@ const authProviderNames = [ "zoom", ] as const; -const stackProjectPath = (path: Path.Path, value: string): string => - value.length === 0 || path.isAbsolute(value) - ? value - : `supabase/${value.startsWith("./") ? value.slice(2) : value}`; - -/** Converts a CLI function path (relative to supabase/) to the stack resolver's - * function-directory-relative form. */ -const functionRelativePath = ( - path: Path.Path, - projectRoot: string, - value: string, - name: string, -): string => { - if (value.length === 0) return value; - const functionsRoot = path.join(projectRoot, "supabase", "functions"); - const functionRoot = path.join(functionsRoot, name); - const target = path.isAbsolute(value) - ? path.normalize(value) - : path.normalize( - path.join(projectRoot, "supabase", value.startsWith("./") ? value.slice(2) : value), - ); - return path.relative(functionRoot, target).replaceAll(path.sep, "/"); -}; - -const functionPathError = ( - path: Path.Path, - projectRoot: string, - name: string, - field: string, - value: string, -): string | undefined => { - if (value.length === 0) return undefined; - const functionsRoot = path.join(projectRoot, "supabase", "functions"); - const target = path.isAbsolute(value) - ? path.normalize(value) - : path.normalize( - path.join(projectRoot, "supabase", value.startsWith("./") ? value.slice(2) : value), - ); - const relative = path.relative(functionsRoot, target); - if (path.isAbsolute(relative) || relative === ".." || relative.startsWith(`..${path.sep}`)) - return `functions.${name}.${field} path must be inside supabase/functions`; - return undefined; -}; - -const apiListener = ( - document: Readonly> | undefined, - config: CliConfig, - portOverride?: number, -) => { - const listener = listenerFromSection(document, "api", "port", portOverride, config.api.enabled); - const gatewayEnabled = - config.api.enabled || - config.auth.enabled || - config.realtime.enabled || - config.storage.enabled || - config.edge_runtime.enabled || - config.analytics.enabled; - if (listener === undefined) return listener; - if (gatewayEnabled) { - const port = portOverride ?? explicitPort(document, "api", "port"); - return port === undefined ? {} : { port }; - } - return { ...listener, enabled: false }; -}; - -const listenerFromSection = ( - document: Readonly> | undefined, - sectionName: string, - portKey: string, - portOverride?: number, - enabledOverride?: boolean, -) => { - const raw = section(document, sectionName); - const port = portOverride ?? explicitPort(document, sectionName, portKey); - if (raw === undefined) { - if (enabledOverride === false && port !== undefined) return { enabled: false }; - return port === undefined ? undefined : { port }; - } - const enabled = - enabledOverride ?? (typeof raw["enabled"] === "boolean" ? raw["enabled"] : undefined); - if (enabled === false) return { enabled: false }; - return port === undefined ? undefined : { port }; -}; - -const nestedPort = ( - document: Readonly> | undefined, - sectionName: string, - nestedSection: string, - portKey: string, -): number | undefined => { - const nested = section(section(document, sectionName), nestedSection); - const value = nested?.[portKey]; - return typeof value === "number" ? value : undefined; -}; - -const nestedListener = ( - document: Readonly> | undefined, - sectionName: string, - nestedSection: string, - portKey: string, - portOverride?: number, - enabledOverride?: boolean, -) => { - const parent = section(document, sectionName); - const nested = section(parent, nestedSection); - const port = portOverride ?? nestedPort(document, sectionName, nestedSection, portKey); - if (nested === undefined) { - if (enabledOverride === false && port !== undefined) return { enabled: false }; - return port === undefined ? undefined : { port }; - } - if (enabledOverride === false || (enabledOverride === undefined && nested.enabled === false)) - return { enabled: false }; - return port === undefined ? undefined : { port }; -}; - -const authSettings = ( - auth: CliConfig["auth"], - document: Readonly> | undefined, -) => { - const authDocument = section(document, "auth"); - const resolvedEmail = auth.email; - const resolvedSmtp = section(authDocument && section(authDocument, "email"), "smtp") - ? auth.email.smtp - : undefined; - const resolvedMfa = auth.mfa; - const resolvedSms = auth.sms; - const resolvedExternal = auth.external; - const resolvedHooks = auth.hook; - const resolvedCaptcha = auth.captcha; - const resolvedSessions = auth.sessions; - const resolvedRateLimit = auth.rate_limit; - const resolvedWeb3 = auth.web3; - const resolvedOAuthServer = auth.oauth_server; - const passwordRequirements = auth.password_requirements; - const thirdParty = auth.third_party; - - const hooks: Record< - string, - { enabled: boolean; uri?: string; secrets?: Redacted.Redacted } - > = {}; - const hookDocument = section(authDocument, "hook"); - const hookValues = { - mfa_verification_attempt: resolvedHooks.mfa_verification_attempt, - password_verification_attempt: resolvedHooks.password_verification_attempt, - custom_access_token: resolvedHooks.custom_access_token, - send_sms: resolvedHooks.send_sms, - send_email: resolvedHooks.send_email, - before_user_created: resolvedHooks.before_user_created, - }; - for (const [name, resolved] of Object.entries(hookValues)) { - if (hookDocument?.[name] === undefined) continue; - hooks[name] = { - enabled: resolved.enabled, - ...(resolved.uri === undefined || resolved.uri.length === 0 ? {} : { uri: resolved.uri }), - ...(resolved.secrets === undefined || resolved.secrets.length === 0 - ? {} - : { secrets: secret(resolved.secrets) }), - }; - } - - const external: Record = {}; - const externalDocument = section(authDocument, "external"); - for (const name of authProviderNames) { - const resolved = resolvedExternal[name]; - if ( - resolved === undefined || - auth.external[name] === undefined || - (name !== "apple" && externalDocument?.[name] === undefined) - ) - continue; - external[name] = { - enabled: resolved.enabled, - client_id: resolved.client_id, - secret: secret(resolved.secret), - url: resolved.url, - redirect_uri: resolved.redirect_uri, - skip_nonce_check: resolved.skip_nonce_check, - email_optional: resolved.email_optional, - }; - } - - const sms = { - ...pick(resolvedSms, ["enable_signup", "enable_confirmations", "template", "max_frequency"]), - twilio: { - enabled: resolvedSms.twilio.enabled, - account_sid: resolvedSms.twilio.account_sid, - message_service_sid: resolvedSms.twilio.message_service_sid, - auth_token: secret(resolvedSms.twilio.auth_token), - }, - twilio_verify: { - enabled: resolvedSms.twilio_verify.enabled, - account_sid: resolvedSms.twilio_verify.account_sid, - message_service_sid: resolvedSms.twilio_verify.message_service_sid, - auth_token: secret(resolvedSms.twilio_verify.auth_token), - }, - messagebird: { - enabled: resolvedSms.messagebird.enabled, - originator: resolvedSms.messagebird.originator, - access_key: secret(resolvedSms.messagebird.access_key), - }, - textlocal: { - enabled: resolvedSms.textlocal.enabled, - sender: resolvedSms.textlocal.sender, - api_key: secret(resolvedSms.textlocal.api_key), - }, - vonage: { - enabled: resolvedSms.vonage.enabled, - from: resolvedSms.vonage.from, - api_key: secret(resolvedSms.vonage.api_key), - api_secret: secret(resolvedSms.vonage.api_secret), - }, - ...(auth.sms.test_otp === undefined ? {} : { test_otp: auth.sms.test_otp }), - }; - - const email = { - enable_signup: resolvedEmail.enable_signup, - double_confirm_changes: resolvedEmail.double_confirm_changes, - enable_confirmations: resolvedEmail.enable_confirmations, - secure_password_change: resolvedEmail.secure_password_change, - max_frequency: resolvedEmail.max_frequency, - otp_length: resolvedEmail.otp_length, - otp_expiry: resolvedEmail.otp_expiry, - ...(resolvedSmtp === undefined - ? {} - : { - smtp: { - enabled: resolvedSmtp.enabled, - host: resolvedSmtp.host, - ...(resolvedSmtp.port === 0 ? {} : { port: resolvedSmtp.port }), - user: resolvedSmtp.user, - pass: secret(resolvedSmtp.pass), - admin_email: resolvedSmtp.admin_email, - sender_name: resolvedSmtp.sender_name, - }, - }), - template: Object.fromEntries( - Object.entries(resolvedEmail.template).map(([name, value]) => [ - name, - pick(value, ["subject", "content_path"]), - ]), - ), - notification: Object.fromEntries( - Object.entries(resolvedEmail.notification).map(([name, value]) => [ - name, - pick(value, ["enabled", "subject", "content_path"]), - ]), - ), - }; - - return { - site_url: auth.site_url, - additional_redirect_urls: auth.additional_redirect_urls, - jwt_expiry: auth.jwt_expiry, - jwt_issuer: auth.jwt_issuer, - signing_keys_path: auth.signing_keys_path, - enable_refresh_token_rotation: auth.enable_refresh_token_rotation, - refresh_token_reuse_interval: auth.refresh_token_reuse_interval, - enable_manual_linking: auth.enable_manual_linking, - enable_signup: auth.enable_signup, - enable_anonymous_sign_ins: auth.enable_anonymous_sign_ins, - minimum_password_length: auth.minimum_password_length, - password_requirements: passwordRequirements, - publishable_key: secret(auth.publishable_key), - secret_key: secret(auth.secret_key), - jwt_secret: secret(auth.jwt_secret), - anon_key: secret(auth.anon_key), - service_role_key: secret(auth.service_role_key), - rate_limit: resolvedRateLimit, - ...(resolvedCaptcha === undefined - ? {} - : { - captcha: { - enabled: resolvedCaptcha.enabled, - provider: resolvedCaptcha.provider, - secret: - resolvedCaptcha.secret === undefined - ? undefined - : Redacted.make(resolvedCaptcha.secret), - }, - }), - hook: hooks, - mfa: resolvedMfa, - sessions: resolvedSessions, - email, - sms, - external, - web3: resolvedWeb3, - oauth_server: resolvedOAuthServer, - third_party: thirdParty, - }; -}; - -const functionsSettings = ( - projectRoot: string, - path: Path.Path, - config: CliConfig, - document?: Record, - projectEnvValues: Readonly> = {}, -) => { - const edge = config.edge_runtime; - const edgePolicy = edge.policy; - const edgeDenoVersion = edge.deno_version; - const documentFunctions = section(document, "functions"); - const functions = Object.fromEntries( - Object.entries(config.functions).map(([name, value]) => [ - name, - { - ...(value.enabled === undefined ? {} : { enabled: value.enabled }), - ...(value.verify_jwt === undefined ? {} : { verify_jwt: value.verify_jwt }), - ...(value.import_map === undefined - ? {} - : { import_map: functionRelativePath(path, projectRoot, value.import_map, name) }), - ...(value.entrypoint === undefined - ? {} - : { entrypoint: functionRelativePath(path, projectRoot, value.entrypoint, name) }), - ...(value.static_files === undefined - ? {} - : { - static_files: value.static_files.map((filePath) => - functionRelativePath(path, projectRoot, filePath, name), - ), - }), - env: Object.fromEntries( - Object.entries( - isRecord(documentFunctions?.[name]) && isRecord(documentFunctions[name].env) - ? documentFunctions[name].env - : value.env, - ).flatMap(([key, item]) => { - const resolvedValue = - typeof item === "string" && /^env\([A-Za-z_][A-Za-z0-9_]*\)$/.test(item) - ? projectEnvValues[item.slice(4, -1)] - : item; - const resolved = secret(resolvedValue); - return resolved === undefined ? [] : [[key, resolved]]; - }), - ), - }, - ]), - ); - return { - functions_root: "supabase/functions", - edge_runtime: { - ...(edgePolicy === undefined ? {} : { policy: edgePolicy }), - ...(edgeDenoVersion === undefined ? {} : { deno_version: edgeDenoVersion }), - ...(edge.secrets === undefined - ? {} - : { - secrets: Object.fromEntries( - Object.entries(edge.secrets).flatMap(([key, item]) => { - const resolved = secret(item); - return resolved === undefined ? [] : [[key, resolved]]; - }), - ), - }), - }, - functions, - }; -}; - const resolvedPort = ( name: string, configured: number, @@ -711,6 +255,7 @@ const resolveAuthOverrides = ( admin_email: smtp.adminEmail, sender_name: smtp.senderName, }; + const resolvedSms = resolveAuthSms(authDocument, auth.sms, env); const thirdParty = { firebase: { enabled: envOverrideBool( @@ -849,7 +394,17 @@ const resolveAuthOverrides = ( mfa: resolveAuthMfa(auth.mfa, env), sessions: resolveGotrueSessions(auth.sessions, env), email: { ...resolvedEmail, smtp: resolvedSmtp }, - sms: resolveAuthSms(authDocument, auth.sms, env), + sms: { + ...resolvedSms, + twilio: { + ...resolvedSms.twilio, + content_sid: envOverride( + "SUPABASE_AUTH_SMS_TWILIO_CONTENT_SID", + auth.sms.twilio.content_sid, + env, + ), + }, + }, external: externalResolved, web3: resolveGotrueWeb3(auth.web3, env), oauth_server: resolveGotrueOAuthServer(auth.oauth_server, env), @@ -871,6 +426,14 @@ const resolveEffectiveCliConfig = ( const mail = config.local_smtp; const pooler = db.pooler; const edge = config.edge_runtime; + const experimental = { + ...config.experimental, + orioledb_version: envOverride( + "SUPABASE_EXPERIMENTAL_ORIOLEDB_VERSION", + config.experimental.orioledb_version, + env, + ), + }; const apiSchemasOverride = envOverride("SUPABASE_API_SCHEMAS", undefined, env); const apiExtraSearchPathOverride = envOverride("SUPABASE_API_EXTRA_SEARCH_PATH", undefined, env); const imageDocument = section(section(document, "storage"), "image_transformation"); @@ -1011,6 +574,18 @@ const resolveEffectiveCliConfig = ( "analytics.enabled", env, ), + port: resolvedPort("SUPABASE_ANALYTICS_PORT", analytics.port, "analytics.port", env), + ...(analytics.vector_port === undefined && + envOverride("SUPABASE_ANALYTICS_VECTOR_PORT", undefined, env) === undefined + ? {} + : { + vector_port: resolvedPort( + "SUPABASE_ANALYTICS_VECTOR_PORT", + analytics.vector_port ?? 0, + "analytics.vector_port", + env, + ), + }), backend: envOverrideAnalyticsBackend(analytics.backend, env), gcp_project_id: envOverride("SUPABASE_ANALYTICS_GCP_PROJECT_ID", analytics.gcp_project_id, env), gcp_project_number: envOverride( @@ -1050,7 +625,10 @@ const resolveEffectiveCliConfig = ( config.auth.signing_keys_path, env, ), - jwt_secret: envOverride("SUPABASE_AUTH_JWT_SECRET", config.auth.jwt_secret, env), + jwt_secret: decryptAuthSecret( + envOverride("SUPABASE_AUTH_JWT_SECRET", config.auth.jwt_secret, env), + env, + ), }; return { ...config, @@ -1065,6 +643,7 @@ const resolveEffectiveCliConfig = ( pooler: resolvedPooler, }, edge_runtime: resolvedEdge, + experimental, realtime: { ...realtime, enabled: envOverrideBool( @@ -1083,7 +662,10 @@ const resolveEffectiveCliConfig = ( enabled: envOverrideBool("SUPABASE_STUDIO_ENABLED", studio.enabled, "studio.enabled", env), port: resolvedPort("SUPABASE_STUDIO_PORT", studio.port, "studio.port", env), api_url: envOverride("SUPABASE_STUDIO_API_URL", studio.api_url, env), - openai_api_key: envOverride("SUPABASE_STUDIO_OPENAI_API_KEY", studio.openai_api_key, env), + openai_api_key: decryptAuthSecret( + envOverride("SUPABASE_STUDIO_OPENAI_API_KEY", studio.openai_api_key, env), + env, + ), }, local_smtp: { ...mail, @@ -1112,438 +694,579 @@ const resolveEffectiveCliConfig = ( }; }; -const configInput = ( - projectRoot: string, - path: Path.Path, - config: CliConfig, - document?: Record, - listenerEnvValues: Readonly> = {}, -) => { - // The config has already been resolved and validated. Keep the environment - // separate here only for listener explicitness and deferred function env(). - const db = config.db; - const api = config.api; - const auth = config.auth; - const storage = config.storage; - const realtime = config.realtime; - const studio = config.studio; - const analytics = config.analytics; - const mail = config.local_smtp; - const pooler = db.pooler; - const apiResolved = api; - const dbPort = envPortOrConfigured( - "SUPABASE_DB_PORT", - document, - "db", - "port", - db.port, - listenerEnvValues, - ); - const dbMajorVersion = db.major_version; - const dbSettings = db.settings; - const realtimeResolved = realtime; - const imageTransformationDocument = section(section(document, "storage"), "image_transformation"); - const imageTransformationExplicit = - imageTransformationDocument?.enabled !== undefined || - envOverride("SUPABASE_STORAGE_IMAGE_TRANSFORMATION_ENABLED", undefined, listenerEnvValues) !== - undefined; - const storageResolved = imageTransformationExplicit - ? storage - : { ...storage, image_transformation: undefined }; - const edgeEnabled = config.edge_runtime.enabled; - const analyticsEnabled = analytics.enabled; - const analyticsResolved = analytics; - const authEnabled = auth.enabled; - const studioEnabled = studio.enabled; - const studioPort = envPortOrConfigured( - "SUPABASE_STUDIO_PORT", - document, - "studio", - "port", - studio.port, - listenerEnvValues, - ); - const studioApiUrl = studio.api_url; - const mailEnabled = mail.enabled; - const mailPort = envPortOrConfigured( - "SUPABASE_LOCAL_SMTP_PORT", - document, - "local_smtp", - "port", - mail.port, - listenerEnvValues, - ); - const mailSmtpPort = envPortOrConfigured( - "SUPABASE_LOCAL_SMTP_SMTP_PORT", - document, - "local_smtp", - "smtp_port", - mail.smtp_port ?? 0, - listenerEnvValues, - ); - const mailPop3Port = envPortOrConfigured( - "SUPABASE_LOCAL_SMTP_POP3_PORT", - document, - "local_smtp", - "pop3_port", - mail.pop3_port ?? 0, - listenerEnvValues, - ); - const poolerDocument = section(section(document, "db"), "pooler"); - const poolerEnabledOverride = envOverride( - "SUPABASE_DB_POOLER_ENABLED", - undefined, - listenerEnvValues, - ); - const poolerEnabledExplicit = - poolerDocument?.enabled !== undefined || poolerEnabledOverride !== undefined; - const poolerEnabled = poolerEnabledExplicit ? pooler.enabled : undefined; - const poolerPort = envNestedPortOrConfigured( - "SUPABASE_DB_POOLER_PORT", - document, - "db", - "pooler", - "port", - pooler.port, - listenerEnvValues, - ); - const poolerResolved = pooler; - const signingKeysPath = auth.signing_keys_path; - const authResolvedSettings = authSettings(auth, document); - const jwtIssuer = auth.jwt_issuer; - const jwtSecret = secret(auth.jwt_secret); - const jwtSigning = (): JwtSigning | undefined => { - if (signingKeysPath !== undefined) - return { - kind: "jwks-file", - path: stackProjectPath(path, signingKeysPath), - }; - if (jwtSecret !== undefined) return { kind: "symmetric", secret: jwtSecret }; - return undefined; - }; - const signing = jwtSigning(); - const capability = (enabled: boolean, settings: T) => - enabled ? { settings } : { enabled: false as const }; - return { - capabilities: { - database: { - version: String(dbMajorVersion), - settings: { - health_timeout: db.health_timeout, - settings: dbSettings, - }, - }, - rest: capability(apiResolved.enabled, { - schemas: apiResolved.schemas, - extra_search_path: apiResolved.extra_search_path, - max_rows: apiResolved.max_rows, - external_url: apiResolved.external_url, - }), - auth: capability(authEnabled, { - ...authResolvedSettings, - signing_keys_path: signingKeysPath, - }), - realtime: capability(realtimeResolved.enabled, { - ip_version: realtimeResolved.ip_version, - max_header_length: realtimeResolved.max_header_length, - }), - storage: capability(storageResolved.enabled, { - file_size_limit: storageResolved.file_size_limit, - image_transformation: storageResolved.image_transformation, - buckets: storageResolved.buckets, - s3_protocol: storageResolved.s3_protocol, - vector: storageResolved.vector, - }), - functions: capability( - edgeEnabled, - functionsSettings(projectRoot, path, config, document, listenerEnvValues), - ), - studio: capability(studioEnabled, { - // A host-only default must remain unset so the stack runtime can append - // the allocated API listener port. Explicit URLs remain caller-owned. - api_url: studioApiUrl === defaultStudioApiUrl ? undefined : studioApiUrl, - openai_api_key: secret(studio.openai_api_key), - }), - mail: capability(mailEnabled, { - admin_email: mail.admin_email, - sender_name: mail.sender_name, - }), - analytics: capability(analyticsEnabled, { - backend: analyticsResolved.backend, - gcp_project_id: analyticsResolved.gcp_project_id, - gcp_project_number: analyticsResolved.gcp_project_number, - gcp_jwt_path: analyticsResolved.gcp_jwt_path, - }), - pooler: capability(poolerEnabled !== false, { - pool_mode: poolerResolved.pool_mode, - default_pool_size: poolerResolved.default_pool_size, - max_client_conn: poolerResolved.max_client_conn, - }), - }, - listeners: { - api: apiListener( - document, - config, - envPortOrConfigured( - "SUPABASE_API_PORT", - document, - "api", - "port", - api.port, - listenerEnvValues, - ), - ), - database: listenerFromSection(document, "db", "port", dbPort), - pooler: nestedListener(document, "db", "pooler", "port", poolerPort, poolerEnabled), - studio: listenerFromSection(document, "studio", "port", studioPort, studioEnabled), - mailUi: listenerFromSection(document, "local_smtp", "port", mailPort, mailEnabled), - smtp: listenerFromSection(document, "local_smtp", "smtp_port", mailSmtpPort, mailEnabled), - pop3: listenerFromSection(document, "local_smtp", "pop3_port", mailPop3Port, mailEnabled), - functionsInspector: listenerFromSection( - document, - "edge_runtime", - "inspector_port", - envPortOrConfigured( - "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", - document, - "edge_runtime", - "inspector_port", - config.edge_runtime.inspector_port, - listenerEnvValues, - ), - edgeEnabled, - ), - }, - security: { - jwt: { - ...(jwtIssuer === undefined ? {} : { issuer: jwtIssuer }), - ...(signing === undefined ? {} : { signing }), - }, - }, - }; +const unsupportedConfigPaths = [ + "auth.third_party", + "auth.publishable_key", + "auth.secret_key", + "auth.anon_key", + "auth.service_role_key", + "api.tls", + "analytics.gcp_project_id", + "analytics.gcp_project_number", + "analytics.gcp_jwt_path", + "edge_runtime.deno_version", + "storage.analytics", + "experimental.orioledb_version", + "experimental.s3_host", + "experimental.s3_region", + "experimental.s3_access_key", + "experimental.s3_secret_key", +] as const; + +const pathValue = (value: unknown, path: string): unknown => { + let current = value; + for (const segment of path.split(".")) { + if (!isRecord(current)) return undefined; + current = current[segment]; + } + return current; }; -const decryptConsumedSecrets = ( - value: unknown, - keys: ReadonlyArray, - path = "config", -): Effect.Effect => - Effect.gen(function* () { - if (Redacted.isRedacted(value)) { - const unwrapped = Redacted.value(value); - if (typeof unwrapped !== "string" || !isEncryptedSecret(unwrapped)) return value; - const result = yield* Effect.try({ - try: () => decryptSecret(unwrapped, keys), - catch: () => - new StackConfigError({ - message: `${path} uses an encrypted secret that could not be decrypted`, - }), - }); - if (!result.ok) { - const reason = keys.length === 0 ? "missing private key" : "decryption failed"; - return yield* new StackConfigError({ - message: `${path} uses an encrypted secret that could not be decrypted: ${reason}`, - }); - } - return Redacted.make(result.value); - } - if (Array.isArray(value)) - return yield* Effect.forEach(value, (item, index) => - decryptConsumedSecrets(item, keys, `${path}[${index}]`), - ); - if (!isRecord(value)) return value; - if (value.enabled === false) return value; - const entries = yield* Effect.forEach(Object.entries(value), ([key, item]) => - decryptConsumedSecrets(item, keys, `${path}.${key}`).pipe( - Effect.map((resolved) => [key, resolved] as const), - ), +const valuesEqual = (left: unknown, right: unknown): boolean => { + if ((left === undefined && right === "") || (left === "" && right === undefined)) return true; + if (Redacted.isRedacted(left) || Redacted.isRedacted(right)) { + return ( + Redacted.isRedacted(left) && + Redacted.isRedacted(right) && + Redacted.value(left) === Redacted.value(right) + ); + } + if (Array.isArray(left) || Array.isArray(right)) { + return ( + Array.isArray(left) && + Array.isArray(right) && + left.length === right.length && + left.every((value, index) => valuesEqual(value, right[index])) ); - return Object.fromEntries(entries); - }); + } + if (isRecord(left) || isRecord(right)) { + if (!isRecord(left) || !isRecord(right)) return false; + const keys = new Set([...Object.keys(left), ...Object.keys(right)]); + return [...keys].every((key) => valuesEqual(left[key], right[key])); + } + return Object.is(left, right); +}; -const configValidationError = ( - path: Path.Path, - projectRoot: string, - config: CliConfig, - projectEnvValues: Readonly>, - effectiveEdgeEnabled: boolean, -): string | undefined => { - const figma = config.auth.external.figma; - if (config.auth.enabled && figma?.enabled === true) - return "auth.external.figma is enabled but unsupported by the experimental stack"; - if (!effectiveEdgeEnabled) return undefined; - for (const [name, functionConfig] of Object.entries(config.functions)) { - if (functionConfig.enabled === false) continue; - for (const [field, value] of [ - ["import_map", functionConfig.import_map], - ["entrypoint", functionConfig.entrypoint], - ...functionConfig.static_files.map((path) => ["static_files", path] as const), - ] as const) { - if (typeof value !== "string") continue; - const pathError = functionPathError(path, projectRoot, name, field, value); - if (pathError !== undefined) return pathError; - } - for (const value of Object.values(functionConfig.env)) { - if (typeof value !== "string") continue; - const match = /^env\(([A-Za-z_][A-Za-z0-9_]*)\)$/.exec(value); - const variable = match?.[1]; - if (variable !== undefined && projectEnvValues[variable] === undefined) - return `functions.${name}.env references an unset environment variable ${variable}`; +const firstDifference = (left: unknown, right: unknown, path: string): string | undefined => { + if (valuesEqual(left, right)) return undefined; + if (isRecord(left) && isRecord(right)) { + if (left.enabled === false && right.enabled === false) return undefined; + for (const key of new Set([...Object.keys(left), ...Object.keys(right)])) { + const difference = firstDifference(left[key], right[key], `${path}.${key}`); + if (difference !== undefined) return difference; } + return undefined; } + return path; +}; + +const configValidationError = (config: CliConfig): string | undefined => { + const defaults = getDefaultCliConfig(); + if (config.auth.enabled) { + for (const [name, template] of Object.entries(config.auth.email.template)) + if (template.content_path !== "") + return `auth.email.template.${name}.content_path requires template serving, which is not supported by the experimental stack`; + for (const [name, notification] of Object.entries(config.auth.email.notification)) + if (notification.enabled && notification.content_path !== "") + return `auth.email.notification.${name}.content_path requires template serving, which is not supported by the experimental stack`; + } + if (config.auth.signing_keys_path !== undefined) + return "auth.signing_keys_path is unsupported by the experimental stack"; + for (const path of unsupportedConfigPaths) { + if (path.startsWith("auth.") && !config.auth.enabled) continue; + const value = pathValue(config, path); + const baseline = pathValue(defaults, path); + const difference = firstDifference(value, baseline, path); + if (difference !== undefined) return `${difference} is unsupported by the experimental stack`; + } + if (config.analytics.enabled && config.analytics.backend !== "postgres") + return "analytics.backend must be postgres for the experimental stack"; + if (Object.keys(config.storage.vector.buckets).length > 0) + return "storage.vector settings are unsupported by the experimental stack"; + if (config.db.major_version !== 15 && config.db.major_version !== 17) + return "db.major_version must be 15 or 17 for the experimental stack"; + return undefined; }; +const endpoint = (port: number | undefined): { readonly port: number | "auto" } => ({ + port: port === undefined ? "auto" : port, +}); + /** Loads and translates the effective project config for all stack commands. * Pass `opts.context` to reuse an already-loaded project context. */ -export const loadStackConfig = ( - projectRoot: string, - opts?: { readonly context?: LocalProjectContext }, -): StackConfigEffect => - Effect.gen(function* () { - const path = yield* Path.Path; - const context = - opts?.context ?? - (yield* loadLocalProjectContext(projectRoot, (message) => new StackConfigError({ message }))); - const effectiveInput = yield* Effect.try({ - try: () => - resolveEffectiveCliConfig( - context.config, - context.loaded?.document, - context.projectEnvValues, - ), - catch: (cause) => - new StackConfigError({ - message: cause instanceof Error ? cause.message : "invalid config overrides", - }), - }); - const dotenvPrivateKeys = collectDotenvPrivateKeys(context.projectEnvValues); - const validatedConfig = yield* validateCliConfig(withoutUndefined(effectiveInput)).pipe( - Effect.mapError((cause) => { - const issues = SchemaIssue.makeFormatterStandardSchemaV1({ - leafHook: () => "Invalid value", - checkHook: () => undefined, - })(cause.issue).issues; - const path = issues[0]?.path - ?.map((segment) => String(typeof segment === "object" ? segment.key : segment)) - .join("."); - return new StackConfigError({ - message: path === undefined ? "invalid config" : `invalid config at ${path}`, - }); - }), - ); - const validationError = yield* Effect.try({ - try: () => - configValidationError( - path, +export const loadStackConfig = Effect.fn("StackConfig.load")( + (projectRoot: string, opts?: { readonly context?: LocalProjectContext }): StackConfigEffect => + Effect.gen(function* () { + const context = + opts?.context ?? + (yield* loadLocalProjectContext( projectRoot, - validatedConfig, - context.projectEnvValues, - validatedConfig.edge_runtime.enabled, - ), - catch: (cause) => - new StackConfigError({ - message: cause instanceof Error ? cause.message : "invalid stack config", + (message) => new StackConfigError({ message }), + )); + const effectiveInput = yield* Effect.try({ + try: () => + resolveEffectiveCliConfig( + context.config, + context.loaded?.document, + context.projectEnvValues, + ), + catch: (cause) => + new StackConfigError({ + message: cause instanceof Error ? cause.message : "invalid config overrides", + }), + }); + const validatedConfig = yield* validateCliConfig(withoutUndefined(effectiveInput)).pipe( + Effect.mapError((cause) => { + const issues = SchemaIssue.makeFormatterStandardSchemaV1({ + leafHook: () => "Invalid value", + checkHook: () => undefined, + })(cause.issue).issues; + const path = issues[0]?.path + ?.map((segment) => String(typeof segment === "object" ? segment.key : segment)) + .join("."); + return new StackConfigError({ + message: path === undefined ? "invalid config" : `invalid config at ${path}`, + }); }), - }); - if (validationError !== undefined) - return yield* new StackConfigError({ message: validationError }); + ); + const validationError = configValidationError(validatedConfig); + if (validationError !== undefined) + return yield* new StackConfigError({ message: validationError }); - const environments = yield* readFunctionEnvironments( - projectRoot, - new Set( - Object.entries(validatedConfig.functions) - .filter(([, functionConfig]) => functionConfig.enabled === false) - .map(([name]) => name), - ), - !validatedConfig.edge_runtime.enabled, - ); - const input = yield* Effect.try({ - try: () => - configInput( - projectRoot, - path, - validatedConfig, - context.loaded?.document, - context.projectEnvValues, - ), - catch: (cause) => - new StackConfigError({ - message: - cause instanceof StackConfigError - ? cause.message - : cause instanceof Error - ? cause.message - : "invalid stack config", - }), - }); - const functionSettings: Readonly> = isRecord( - input.capabilities.functions.settings, - ) - ? input.capabilities.functions.settings - : {}; - const functions = isRecord(functionSettings?.functions) ? functionSettings.functions : {}; - const allFunctions = { - ...Object.fromEntries( - Object.keys(environments.functions).map((name) => [ - name, - { env: environments.functions[name] }, - ]), - ), - ...functions, - }; - const mergedInput = - input.capabilities.functions.enabled === false - ? input - : { - ...input, - capabilities: { - ...input.capabilities, - functions: { - ...input.capabilities.functions, - settings: { - ...functionSettings, - edge_runtime: { - ...(isRecord(functionSettings?.edge_runtime) - ? functionSettings.edge_runtime - : {}), - secrets: { - ...environments.shared, - ...(isRecord(functionSettings?.edge_runtime) && - isRecord(functionSettings.edge_runtime.secrets) - ? functionSettings.edge_runtime.secrets - : {}), - }, - }, - functions: Object.fromEntries( - Object.entries(allFunctions).map(([name, value]) => [ - name, - { - ...(isRecord(value) ? value : {}), - env: { - ...environments.functions[name], - ...(isRecord(value) && isRecord(value.env) ? value.env : {}), - }, - }, + const authConfig = yield* resolveAuthConfig(validatedConfig.auth, validatedConfig.local_smtp); + const path = yield* Path.Path; + const functionEnvironments = Object.fromEntries( + yield* Effect.forEach(Object.entries(validatedConfig.functions), ([name, config]) => + resolveCliConfigSubtree( + config.env, + { values: context.projectEnvValues }, + `functions.${name}.env`, + { goViperCompat: true }, + ).pipe( + Effect.map( + (env) => + [ + name, + Object.fromEntries( + Object.entries(env).map(([key, value]) => [ + key, + Redacted.isRedacted(value) ? Redacted.value(value) : value, ]), ), + ] as const, + ), + ), + ), + ); + const functions = yield* Effect.try({ + try: () => + Object.fromEntries( + Object.entries(validatedConfig.functions).map(([name, config]) => { + const resolveFile = (value: string) => { + const resolved = path.resolve(projectRoot, "supabase", value); + const relative = path.relative(projectRoot, resolved); + if ( + relative === ".." || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) + throw new Error( + `functions.${name} paths must remain within the project directory`, + ); + return resolved; + }; + return [ + name, + { + enabled: config.enabled, + verifyJWT: config.verify_jwt, + ...(config.entrypoint === "" + ? {} + : { entrypoint: resolveFile(config.entrypoint) }), + ...(config.import_map === "" + ? {} + : { import_map: resolveFile(config.import_map) }), + static_files: config.static_files.map(resolveFile), + env: functionEnvironments[name], }, + ]; + }), + ), + catch: (cause) => new StackConfigError({ message: String(cause) }), + }); + const functionsEnv = yield* Effect.try({ + try: () => + Object.fromEntries( + Object.entries(validatedConfig.edge_runtime.secrets ?? {}).map(([key, value]) => [ + key, + decryptAuthSecret(value, context.projectEnvValues) ?? "", + ]), + ), + catch: (cause) => new StackConfigError({ message: String(cause) }), + }); + const crypto = yield* Crypto.Crypto; + const storageFileSizeLimit = yield* Effect.try({ + try: () => String(parseFileSizeLimit(validatedConfig.storage.file_size_limit)), + catch: (cause) => + new StackConfigError({ message: `Invalid storage.file_size_limit: ${String(cause)}` }), + }); + const healthTimeoutMs = yield* Effect.try({ + try: () => parseGoDuration(validatedConfig.db.health_timeout) / 1_000_000, + catch: (cause) => + new StackConfigError({ message: `Invalid db.health_timeout: ${String(cause)}` }), + }); + const configuredJwtSecret = yield* Effect.try({ + try: () => + validatedConfig.auth.jwt_secret === undefined + ? undefined + : resolveJwtSecret(validatedConfig.auth.jwt_secret), + catch: (cause) => new StackConfigError({ message: String(cause) }), + }); + const jwtSecret = + configuredJwtSecret === undefined + ? Redacted.make( + yield* crypto.randomUUIDv4.pipe( + Effect.mapError( + (cause) => + new StackConfigError({ message: `Unable to generate JWT secret: ${cause}` }), + ), + ), + ) + : Redacted.make(configuredJwtSecret); + const document = context.loaded?.document; + const rootKey = yield* Effect.try({ + try: () => { + const raw = section(document, "db")?.root_key; + if (raw !== undefined && typeof raw !== "string") + throw new Error("db.root_key must be a string"); + const value = decryptAuthSecret( + envOverride("SUPABASE_DB_ROOT_KEY", raw, context.projectEnvValues), + context.projectEnvValues, + ); + return value === "" ? undefined : value; + }, + catch: (cause) => new StackConfigError({ message: String(cause) }), + }); + const dbPort = envPortOrConfigured( + "SUPABASE_DB_PORT", + document, + "db", + "port", + validatedConfig.db.port, + context.projectEnvValues, + ); + const apiPort = envPortOrConfigured( + "SUPABASE_API_PORT", + document, + "api", + "port", + validatedConfig.api.port, + context.projectEnvValues, + ); + const studioPort = envPortOrConfigured( + "SUPABASE_STUDIO_PORT", + document, + "studio", + "port", + validatedConfig.studio.port, + context.projectEnvValues, + ); + const poolerPort = envNestedPortOrConfigured( + "SUPABASE_DB_POOLER_PORT", + document, + "db", + "pooler", + "port", + validatedConfig.db.pooler.port, + context.projectEnvValues, + ); + const mailPort = envPortOrConfigured( + "SUPABASE_LOCAL_SMTP_PORT", + document, + "local_smtp", + "port", + validatedConfig.local_smtp.port, + context.projectEnvValues, + ); + const mailSmtpPort = envPortOrConfigured( + "SUPABASE_LOCAL_SMTP_SMTP_PORT", + document, + "local_smtp", + "smtp_port", + validatedConfig.local_smtp.smtp_port ?? 0, + context.projectEnvValues, + ); + const mailPop3Port = envPortOrConfigured( + "SUPABASE_LOCAL_SMTP_POP3_PORT", + document, + "local_smtp", + "pop3_port", + validatedConfig.local_smtp.pop3_port ?? 0, + context.projectEnvValues, + ); + const analyticsPort = envPortOrConfigured( + "SUPABASE_ANALYTICS_PORT", + document, + "analytics", + "port", + validatedConfig.analytics.port, + context.projectEnvValues, + ); + const poolMode = + validatedConfig.db.pooler.pool_mode === "session" ? ("session" as const) : "transaction"; + const storagePath = `${projectRoot}/supabase/.temp/stack-uploads`; + const createCreations = ( + stackId: string, + options?: { readonly jwtSecret?: Redacted.Redacted }, + ): Effect.Effect, StackConfigError> => + Effect.gen(function* () { + const effectiveJwtSecret = options?.jwtSecret ?? jwtSecret; + if ( + options?.jwtSecret !== undefined && + configuredJwtSecret !== undefined && + Redacted.value(options.jwtSecret) !== configuredJwtSecret + ) + return yield* new StackConfigError({ + message: "The configured auth.jwt_secret does not match the existing stack", + }); + const bootstrap = validatedConfig.edge_runtime.enabled + ? yield* typeof SUPABASE_STACK_FUNCTIONS_SERVE_MAIN_TEMPLATE === "string" + ? Effect.succeed(SUPABASE_STACK_FUNCTIONS_SERVE_MAIN_TEMPLATE) + : Effect.tryPromise({ + try: () => import("./stack-functions-bundler.ts"), + catch: (cause) => + new StackConfigError({ + message: `Unable to load Functions bundler: ${String(cause)}`, + }), + }).pipe( + Effect.flatMap(({ bundleStackFunctionsServeMainTemplate }) => + bundleStackFunctionsServeMainTemplate().pipe( + Effect.mapError( + (cause) => new StackConfigError({ message: cause.message }), + ), + ), + ), + ) + : undefined; + return [ + { + service: "database", + config: { + version: String(validatedConfig.db.major_version), + databasePassword: Redacted.make("postgres"), + jwtSecret: effectiveJwtSecret, + jwtExpiry: validatedConfig.auth.jwt_expiry, + settings: validatedConfig.db.settings, + healthTimeoutMs, + ...(rootKey === undefined ? {} : { rootKey: Redacted.make(rootKey) }), }, + endpoints: { sql: endpoint(dbPort) }, }, - }; - const decrypted = yield* decryptConsumedSecrets(mergedInput, dotenvPrivateKeys); - const decoded = yield* Schema.decodeUnknownEffect(StackConfigSchema)( - withoutUndefined(decrypted), - { - onExcessProperty: "error", - }, - ).pipe( - Effect.mapError( - (cause) => - new StackConfigError({ - message: `invalid stack config: ${String(cause)}`, - }), - ), - ); - return decoded; - }); + ...(validatedConfig.analytics.enabled + ? [ + { + service: "analytics" as const, + config: { + databaseUrl: "postgresql://placeholder", + backend: "postgres" as const, + apiKey: "api-key", + }, + endpoints: { http: endpoint(analyticsPort) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.db.pooler.enabled + ? [ + { + service: "pooler" as const, + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: Redacted.value(effectiveJwtSecret), + poolMode, + tenant: "pooler-dev", + defaultPoolSize: validatedConfig.db.pooler.default_pool_size, + maxClientConnections: validatedConfig.db.pooler.max_client_conn, + }, + endpoints: { http: endpoint(undefined), sql: endpoint(poolerPort) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.studio.enabled + ? [ + { + service: "pgmeta" as const, + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: endpoint(undefined) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.api.enabled + ? [ + { + service: "rest" as const, + config: { + databaseUrl: "postgresql://placeholder", + schemas: validatedConfig.api.schemas.join(","), + extraSearchPath: validatedConfig.api.extra_search_path.join(","), + maxRows: validatedConfig.api.max_rows, + ...(validatedConfig.api.external_url === undefined + ? {} + : { externalApiUrl: validatedConfig.api.external_url }), + jwtSecret: Redacted.value(effectiveJwtSecret), + }, + endpoints: { http: endpoint(apiPort) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.auth.enabled + ? [ + { + service: "auth" as const, + config: { + ...authConfig, + jwtSecret: Redacted.value(effectiveJwtSecret), + }, + endpoints: { http: endpoint(apiPort) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.realtime.enabled + ? [ + { + service: "realtime" as const, + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: Redacted.value(effectiveJwtSecret), + secretKeyBase: Redacted.value(effectiveJwtSecret), + ipVersion: + validatedConfig.realtime.ip_version === "IPv6" + ? ("IPv6" as const) + : ("IPv4" as const), + maxHeaderLength: validatedConfig.realtime.max_header_length, + }, + endpoints: { + http: endpoint(apiPort), + rpc: endpoint(undefined), + }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.storage.enabled + ? [ + { + service: "storage" as const, + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: Redacted.value(effectiveJwtSecret), + filePath: `${storagePath}/${stackId}`, + fileSizeLimit: storageFileSizeLimit, + s3ProtocolEnabled: validatedConfig.storage.s3_protocol.enabled, + vectorEnabled: validatedConfig.storage.vector.enabled, + vectorMaxBuckets: validatedConfig.storage.vector.max_buckets, + vectorMaxIndexes: validatedConfig.storage.vector.max_indexes, + }, + endpoints: { http: endpoint(apiPort) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.analytics.enabled + ? [ + { + service: "vector" as const, + config: { analyticsUrl: "http://analytics", apiKey: "api-key" }, + endpoints: { + http: endpoint( + envPortOrConfigured( + "SUPABASE_ANALYTICS_VECTOR_PORT", + document, + "analytics", + "vector_port", + validatedConfig.analytics.vector_port ?? 0, + context.projectEnvValues, + ), + ), + }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.storage.image_transformation?.enabled === true + ? [ + { + service: "imgproxy" as const, + config: { filePath: `${storagePath}/${stackId}` }, + endpoints: { http: endpoint(undefined) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.edge_runtime.enabled && bootstrap !== undefined + ? [ + { + service: "functions" as const, + config: { + functionsRoot: `${projectRoot}/supabase/functions`, + filesRoot: projectRoot, + functions, + env: functionsEnv, + bootstrap, + policy: validatedConfig.edge_runtime.policy, + verifyJwt: true, + jwtSecret: Redacted.value(effectiveJwtSecret), + }, + endpoints: { + http: endpoint(apiPort), + inspector: endpoint( + envPortOrConfigured( + "SUPABASE_EDGE_RUNTIME_INSPECTOR_PORT", + document, + "edge_runtime", + "inspector_port", + validatedConfig.edge_runtime.inspector_port, + context.projectEnvValues, + ), + ), + }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.studio.enabled + ? [ + { + service: "studio" as const, + config: { + jwtSecret: Redacted.value(effectiveJwtSecret), + ...(validatedConfig.studio.openai_api_key === undefined + ? {} + : { openaiApiKey: validatedConfig.studio.openai_api_key }), + ...(validatedConfig.studio.api_url === getDefaultCliConfig().studio.api_url + ? {} + : { publicApiUrl: validatedConfig.studio.api_url }), + }, + endpoints: { http: endpoint(studioPort) }, + } satisfies ServiceCreationType, + ] + : []), + ...(validatedConfig.local_smtp.enabled + ? [ + { + service: "mail" as const, + config: {}, + endpoints: { + http: endpoint(mailPort), + smtp: endpoint(mailSmtpPort), + pop3: endpoint(mailPop3Port), + }, + } satisfies ServiceCreationType, + ] + : []), + ]; + }); + return { + jwtSecret, + creations: createCreations, + source: validatedConfig, + projectEnvValues: context.projectEnvValues, + ...(context.loaded?.document === undefined ? {} : { document: context.loaded.document }), + }; + }), +); diff --git a/apps/cli/src/command-internal/stack-functions-bundler.ts b/apps/cli/src/command-internal/stack-functions-bundler.ts new file mode 100644 index 0000000000..e2a44e6b21 --- /dev/null +++ b/apps/cli/src/command-internal/stack-functions-bundler.ts @@ -0,0 +1,42 @@ +import { fileURLToPath } from "node:url"; + +import { build } from "esbuild"; +import { Data, Effect } from "effect"; + +export class StackFunctionsBundleError extends Data.TaggedError("StackFunctionsBundleError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +/** Bundles the stack runtime's Functions entrypoint for an offline native stack. */ +export const bundleStackFunctionsServeMainTemplate = Effect.fn( + "StackFunctionsBundler.bundleServeMainTemplate", +)(function* () { + const entrypoint = fileURLToPath( + import.meta.resolve("@supabase/stack/internal/functions/serve-main"), + ); + const result = yield* Effect.tryPromise({ + try: () => + build({ + entryPoints: [entrypoint], + bundle: true, + format: "esm", + platform: "browser", + minify: true, + write: false, + legalComments: "none", + logLevel: "silent", + }), + catch: (cause) => + new StackFunctionsBundleError({ + message: "Unable to bundle the stack Functions runtime", + cause, + }), + }); + const output = result.outputFiles[0]?.text; + if (output === undefined) + return yield* new StackFunctionsBundleError({ + message: "esbuild produced no stack Functions template", + }); + return output; +}); diff --git a/apps/cli/src/command-internal/stack-functions-env.ts b/apps/cli/src/command-internal/stack-functions-env.ts new file mode 100644 index 0000000000..427306d46e --- /dev/null +++ b/apps/cli/src/command-internal/stack-functions-env.ts @@ -0,0 +1,61 @@ +import { Data, Effect, FileSystem, Predicate } from "effect"; +import { parseDotEnv } from "./dotenv.ts"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../shared/telemetry/error-actionability.ts"; + +export class StackFunctionsEnvError extends Data.TaggedError("StackFunctionsEnvError")<{ + readonly message: string; + readonly path: string; + readonly cause?: unknown; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.invalidConfig; + } +} + +/** Reads a Functions dotenv file and removes reserved Stack-provided variables. */ +export const readStackFunctionsEnv = Effect.fn("StackFunctionsEnv.read")(function* ( + filePath: string, + optional: boolean, +) { + const fs = yield* FileSystem.FileSystem; + const contents = yield* fs.readFileString(filePath).pipe( + Effect.catchTag("PlatformError", (error) => + optional && Predicate.isTagged(error.reason, "NotFound") + ? Effect.succeed("") + : Effect.fail( + new StackFunctionsEnvError({ + path: filePath, + message: `Unable to load environment file ${filePath}: ${error.message}`, + cause: error, + }), + ), + ), + ); + const parsed = yield* Effect.try({ + try: () => parseDotEnv(contents), + catch: (cause) => + new StackFunctionsEnvError({ + path: filePath, + message: `Unable to parse environment file ${filePath}: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + }); + const env = Object.fromEntries( + Object.entries(parsed).filter(([key]) => !key.startsWith("SUPABASE_")), + ); + if (Object.keys(env).some((key) => !/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key))) + return yield* new StackFunctionsEnvError({ + path: filePath, + message: `Environment names in ${filePath} must start with a letter or underscore and contain only letters, digits, and underscores.`, + }); + if (Object.values(env).some((value) => /[\0\r\n]/u.test(value))) + return yield* new StackFunctionsEnvError({ + path: filePath, + message: `Multiline environment values in ${filePath} are not supported by the stack backend.`, + }); + return env; +}); diff --git a/apps/cli/src/command-internal/stack-local-database.ts b/apps/cli/src/command-internal/stack-local-database.ts index df84baadf7..75fd661b34 100644 --- a/apps/cli/src/command-internal/stack-local-database.ts +++ b/apps/cli/src/command-internal/stack-local-database.ts @@ -1,196 +1,111 @@ import { Data, Effect, FileSystem, Option, Path, Redacted } from "effect"; -import { Output } from "../shared/output/output.service.ts"; +import type { DatabaseInstance, ServiceCreation, Stack } from "@supabase/stack/effect"; import { actionability, type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../shared/telemetry/error-actionability.ts"; -import { - CAPABILITY_NAMES, - excludeStackCapabilities, - type EffectStack, - type StackConfig, - type StackRuntime, - type StackStatus, -} from "@supabase/stack/effect"; import { parseConnectionString } from "./db-config.parse.ts"; -import { DbConnection, type DbSession, type PgConnInput } from "./db-connection.service.ts"; import { CommandSettings } from "../config/command-settings.service.ts"; import { LocalDbRunningError } from "./db-bootstrap/local-db-running.ts"; -import { applyDatabaseWebhooks } from "./db-bootstrap/db-setup.ts"; import { currentStackBackend } from "./stack-backend.ts"; import { StackApi } from "./stack-api.ts"; import { loadStackConfig } from "./stack-config.ts"; -import { readDbToml, type DbTomlValues } from "./db-config.toml-read.ts"; +import { readDbToml } from "./db-config.toml-read.ts"; import { StackCatalogSetup } from "./stack-catalog-setup.ts"; import { resolveExperimentalWithProjectEnv } from "./global-flags.ts"; -import { migrateAndSeed } from "./migrate-and-seed.ts"; - -const stackDatabaseConn = (stack: EffectStack) => - Effect.gen(function* () { - const credentials = yield* stack.credentials; - const conn = parseConnectionString(Redacted.value(credentials.database.url)); - if (conn === undefined) - return yield* Effect.fail({ message: "failed to parse stack database URL" }); - return conn; - }); - -const withStackDatabaseSession = ( - stack: EffectStack, - body: (session: DbSession) => Effect.Effect, -) => - Effect.scoped( - Effect.gen(function* () { - const dbConn = yield* DbConnection; - const conn = yield* stackDatabaseConn(stack); - const session = yield* dbConn.connect(conn, { isLocal: true, dnsResolver: "native" }); - return yield* body(session); - }), - ); - -/** Webhooks-only setup for an existing cluster. */ -export const applyStackWebhooksOnly = ( - stack: EffectStack, - webhooksEnabled: boolean, -): Effect.Effect< - void, - { readonly message: string }, - DbConnection | FileSystem.FileSystem | Path.Path -> => - withStackDatabaseSession(stack, (session) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const tmpDir = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-webhooks-" }); - yield* applyDatabaseWebhooks(session, fs, path, tmpDir, webhooksEnabled); - }), - ).pipe( - Effect.mapError((error) => - typeof error === "object" && error !== null && "message" in error - ? { message: String(error.message) } - : { message: String(error) }, - ), - ); +import { applyStackMigrateAndSeed, applyStackWebhooksOnly } from "./stack-bootstrap.ts"; +import { defaultStackRuntime } from "./stack-runtime.ts"; +import { Output } from "../shared/output/output.service.ts"; +import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import type { PgConnInput } from "./db-connection.service.ts"; -/** First-create migrate and seed after schema init and catalog overlay. */ -export const applyStackMigrateAndSeed = ( - stack: EffectStack, - workdir: string, - toml: Pick< - DbTomlValues, - "migrationsEnabled" | "seed" | "pgDelta" | "schemaPaths" | "webhooksEnabled" - >, - experimental: boolean, -): Effect.Effect< - void, - { readonly message: string }, - DbConnection | FileSystem.FileSystem | Path.Path | Output -> => - withStackDatabaseSession(stack, (session) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* migrateAndSeed(session, fs, path, workdir, "", { - migrationsEnabled: toml.migrationsEnabled, - seed: toml.seed, - experimental, - pgDeltaEnabled: toml.pgDelta.enabled, - schemaPaths: toml.schemaPaths, - localDatabaseWebhooksEnabled: toml.webhooksEnabled, - }); - }), - ).pipe( - Effect.mapError((error) => - typeof error === "object" && error !== null && "message" in error - ? { message: String(error.message) } - : { message: String(error) }, - ), - ); +type Settings = CommandSettings["Service"]; +const stateRoot = (settings: Settings, path: Path.Path) => + path.join(settings.supabaseHome, "stacks"); +const cacheRoot = (settings: Settings, path: Path.Path) => + path.join(settings.supabaseHome, "cache", "stack"); const notRunning = (message = "The local stack is not running.") => new LocalDbRunningError({ message }); - const startFailed = (cause: { readonly message: string }) => - new LocalDbRunningError({ - message: `failed to start local database: ${cause.message}`, - }); + new LocalDbRunningError({ message: `failed to start local database: ${cause.message}` }); -const startFailedAfterEngine = (cause: { readonly message: string }) => - startFailed({ - message: `${cause.message}. The stack is running; recover with db reset.`, - }); +type StackInstances = Effect.Success; -/** Capabilities `db start` and `stack start --exclude` can leave disabled. */ -export const STACK_START_EXCLUDABLE_CAPABILITIES = CAPABILITY_NAMES.filter( - (name) => name !== "database", -); +const databaseFor = (instances: StackInstances): DatabaseInstance | undefined => + instances.find((instance): instance is DatabaseInstance => instance.service === "database"); -const postgresOnlyStackStartConfig = (config: StackConfig): StackConfig => - excludeStackCapabilities(config, STACK_START_EXCLUDABLE_CAPABILITIES); +const stackForProject = Effect.fn("StackLocalDatabase.findProject")(function* ( + api: StackApi["Service"], + settings: Settings, + path: Path.Path, +) { + const identity = yield* api.resolveIdentity({ projectRoot: settings.workdir }); + const discovered = yield* api.discover({ stateRoot: stateRoot(settings, path) }); + return discovered.find( + ({ definition }) => + definition.identity.projectRoot === identity.projectRoot && + definition.identity.branchContext === identity.branchContext && + definition.identity.stackName === identity.stackName, + ); +}); -/** Optional catalog jobs follow the running definition, not full TOML. */ -export const optionalCatalogConfigFromStatus = ( - config: StackConfig, - status: Pick, -): StackConfig => - excludeStackCapabilities( - config, - STACK_START_EXCLUDABLE_CAPABILITIES.filter( - (name) => - status.capabilities.find((capability) => capability.name === name)?.state === "disabled", - ), +const openProjectStack = Effect.fn("StackLocalDatabase.openProject")(function* () { + const api = yield* StackApi; + const settings = yield* CommandSettings; + const path = yield* Path.Path; + const found = yield* stackForProject(api, settings, path); + if (found === undefined) return Option.none(); + return Option.some( + yield* api.open({ + id: found.definition.id, + stateRoot: stateRoot(settings, path), + cacheRoot: cacheRoot(settings, path), + }), ); +}); -const databaseReady = (stack: EffectStack) => - Effect.gen(function* () { - const status = yield* stack.status.pipe(Effect.mapError((cause) => notRunning(cause.message))); - const database = status.capabilities.find((capability) => capability.name === "database"); - if (status.lifecycle !== "running" || database?.state !== "ready") return Option.none(); - return Option.some({ stack, runtime: status.runtime }); - }); +const databaseFromStack = Effect.fn("StackLocalDatabase.database")(function* (stack: Stack) { + const composition = yield* stack.composition.describe; + const members = yield* Effect.forEach(composition.members, ({ id }) => stack.services.get(id)); + return databaseFor(members); +}); -/** - * Finds and opens the registered project stack, or `None` when the stack API is unavailable or - * no stack is registered for this project. `onFailure` maps the underlying find/open failure to - * the caller's own error type. - */ -export const stackOpenProjectBy = ( - onFailure: (cause: { readonly message: string }) => E, -): Effect.Effect, E, CommandSettings> => - Effect.gen(function* () { - const api = yield* Effect.serviceOption(StackApi); - if (Option.isNone(api)) return Option.none(); - const cliSettings = yield* CommandSettings; - const descriptor = yield* api.value - .findStack({ projectRoot: cliSettings.workdir }) - .pipe(Effect.mapError((cause) => onFailure(cause))); - if (Option.isNone(descriptor)) return Option.none(); - const stack = yield* api.value - .openStack(descriptor.value.id) - .pipe(Effect.mapError((cause) => onFailure(cause))); - return Option.some(stack); - }); +const databaseReady = Effect.fn("StackLocalDatabase.ready")(function* (stack: Stack) { + const database = yield* databaseFromStack(stack); + if (database === undefined) + return Option.none<{ readonly stack: Stack; readonly database: DatabaseInstance }>(); + const observation = yield* database.status.pipe( + Effect.map(Option.some), + Effect.catchTag("StackError", () => Effect.succeed(Option.none())), + ); + if ( + Option.isNone(observation) || + observation.value.lifecycle !== "running" || + observation.value.health !== "healthy" + ) + return Option.none(); + return Option.some({ stack, database }); +}); + +/** Finds and opens the registered project stack without starting its owner. */ +export const stackOpenProjectBy = (onFailure: (cause: { readonly message: string }) => E) => + openProjectStack().pipe(Effect.mapError(onFailure)); -/** Ready project stack, or none when the stack is missing or the database is not ready. */ +/** Returns the project stack only when its primary database is healthy. */ export const stackOpenReadyProject = stackOpenProjectBy((cause) => notRunning(cause.message)).pipe( Effect.flatMap((opened) => Option.isNone(opened) ? Effect.succeed(Option.none()) : databaseReady(opened.value), ), ); -export const stackProjectRuntime: Effect.Effect = - Effect.gen(function* () { - const api = yield* Effect.serviceOption(StackApi); - if (Option.isNone(api)) return undefined; - const cliSettings = yield* CommandSettings; - const descriptor = yield* api.value - .findStack({ projectRoot: cliSettings.workdir }) - .pipe(Effect.orElseSucceed(() => Option.none())); - return Option.match(descriptor, { - onNone: () => undefined, - onSome: (value) => value.runtime, - }); - }); +export const stackProjectRuntime = Effect.gen(function* () { + const api = yield* StackApi; + const settings = yield* CommandSettings; + const path = yield* Path.Path; + return (yield* stackForProject(api, settings, path))?.definition.runtime; +}); export class StackRuntimeUnavailableError extends Data.TaggedError("StackRuntimeUnavailableError")<{ readonly message: string; @@ -206,55 +121,43 @@ const RUNTIME_UNAVAILABLE = new StackRuntimeUnavailableError({ suggestion: "Start the stack, or start with --runtime docker.", }); -/** Fail instead of treating an unknown engine as Docker. */ export const stackRequireProjectRuntime: Effect.Effect< - StackRuntime, + "native" | "docker" | "podman", StackRuntimeUnavailableError, - CommandSettings -> = Effect.gen(function* () { - const api = yield* Effect.serviceOption(StackApi); - if (Option.isNone(api)) return yield* RUNTIME_UNAVAILABLE; - const cliSettings = yield* CommandSettings; - const descriptor = yield* api.value.findStack({ projectRoot: cliSettings.workdir }).pipe( - Effect.mapError( - (cause) => - new StackRuntimeUnavailableError({ - message: cause.message, - suggestion: RUNTIME_UNAVAILABLE.suggestion, - }), - ), - ); - if (Option.isNone(descriptor)) return yield* RUNTIME_UNAVAILABLE; - return descriptor.value.runtime; -}); + CommandSettings | StackApi | Path.Path +> = stackProjectRuntime.pipe( + Effect.flatMap((runtime) => + runtime === undefined ? Effect.fail(RUNTIME_UNAVAILABLE) : Effect.succeed(runtime), + ), + Effect.mapError( + (cause) => + new StackRuntimeUnavailableError({ + message: cause.message, + suggestion: RUNTIME_UNAVAILABLE.suggestion, + }), + ), +); -/** Catalog pin from `status().versions.database`, when the project stack is available. */ +/** Reads the configured database version from the saved database definition. */ export const stackProjectDatabaseVersion: Effect.Effect< string | undefined, never, - CommandSettings + CommandSettings | StackApi | Path.Path > = Effect.gen(function* () { - const api = yield* Effect.serviceOption(StackApi); - if (Option.isNone(api)) return undefined; - const cliSettings = yield* CommandSettings; - const descriptor = yield* api.value - .findStack({ projectRoot: cliSettings.workdir }) - .pipe(Effect.orElseSucceed(() => Option.none())); - if (Option.isNone(descriptor)) return undefined; - const stack = yield* api.value - .openStack(descriptor.value.id) - .pipe(Effect.orElseSucceed(() => undefined)); - if (stack === undefined) return undefined; - const status = yield* stack.status.pipe(Effect.orElseSucceed(() => undefined)); - if (status === undefined || typeof status.versions.database !== "string") return undefined; - return status.versions.database; + const opened = yield* openProjectStack().pipe( + Effect.mapError(() => undefined), + Effect.orElseSucceed(() => Option.none()), + ); + if (Option.isNone(opened)) return undefined; + const database = yield* databaseFromStack(opened.value).pipe( + Effect.option, + Effect.map(Option.getOrUndefined), + ); + if (database === undefined) return undefined; + const status = yield* database.status.pipe(Effect.option, Effect.map(Option.getOrUndefined)); + return status?.config.service === "database" ? status.config.config.version : undefined; }); -const STACK_NATIVE_ENGINE_MESSAGE = "The stack backend only supports the pg-delta engine."; - -const stackNativeEngineAdvice = (flag: string) => - `${STACK_NATIVE_ENGINE_MESSAGE} Do not pass ${flag}; set SUPABASE_EXPERIMENTAL_STACK=0.`; - export class StackNativeEngineError extends Data.TaggedError("StackNativeEngineError")<{ readonly message: string; }> { @@ -267,101 +170,150 @@ export const stackRejectNativeDockerDiffEngine = ( flag: string, ): Effect.Effect => Effect.gen(function* () { - const backend = yield* currentStackBackend; - if (backend.kind !== "stack") return; - return yield* new StackNativeEngineError({ message: stackNativeEngineAdvice(flag) }); + if ((yield* currentStackBackend).kind === "stack") + return yield* new StackNativeEngineError({ + message: `The stack backend only supports the pg-delta engine. Do not pass ${flag}; set SUPABASE_EXPERIMENTAL_STACK=0.`, + }); }); -const stackLocalDatabaseUrl: Effect.Effect = - Effect.gen(function* () { - const opened = yield* stackOpenReadyProject; - if (Option.isNone(opened)) return yield* notRunning(); - const credentials = yield* opened.value.stack.credentials.pipe( - Effect.mapError((cause) => notRunning(cause.message)), - ); - return Redacted.value(credentials.database.url); - }); +const stackLocalDatabaseUrl: Effect.Effect< + string, + LocalDbRunningError, + CommandSettings | StackApi | Path.Path +> = Effect.gen(function* () { + const opened = yield* stackOpenReadyProject.pipe( + Effect.mapError((cause) => notRunning(cause.message)), + ); + if (Option.isNone(opened)) return yield* notRunning(); + const status = yield* opened.value.database.status.pipe( + Effect.mapError((cause) => notRunning(cause.message)), + ); + const endpoint = status.endpoints.find(({ name }) => name === "sql"); + if (status.config.service !== "database") + return yield* notRunning("The local stack primary service is not a database."); + if (endpoint === undefined) + return yield* notRunning("The local stack database SQL endpoint is unavailable."); + const password = Redacted.value(status.config.config.databasePassword); + const host = endpoint.host.includes(":") ? `[${endpoint.host}]` : endpoint.host; + return `postgresql://postgres:${encodeURIComponent(password)}@${host}:${endpoint.port}/postgres`; +}); export const stackLocalDatabaseConn: Effect.Effect< PgConnInput, LocalDbRunningError, - CommandSettings -> = Effect.gen(function* () { - const url = yield* stackLocalDatabaseUrl; - const conn = parseConnectionString(url); - if (conn === undefined) { - return yield* notRunning(`failed to parse stack database URL`); - } - return conn; -}); + CommandSettings | StackApi | Path.Path +> = stackLocalDatabaseUrl.pipe( + Effect.flatMap((url) => { + const conn = parseConnectionString(url); + return conn === undefined + ? Effect.fail(notRunning("failed to parse stack database URL")) + : Effect.succeed(conn); + }), + Effect.mapError((cause) => notRunning(cause.message)), +); -/** Postgres-only `db start` / resume path. */ -export const stackEnsurePostgresOnlyStarted = Effect.gen(function* () { - const api = yield* Effect.serviceOption(StackApi); - if (Option.isNone(api)) return yield* startFailed({ message: "stack API is unavailable" }); - const cliSettings = yield* CommandSettings; - const output = yield* Output; - const fs = yield* FileSystem.FileSystem; +/** Starts or resumes the primary database, initializing the catalog on first creation. */ +export const stackEnsurePostgresOnlyStarted = Effect.fn( + "StackLocalDatabase.ensurePostgresOnlyStarted", +)(function* () { + const api = yield* StackApi; + const settings = yield* CommandSettings; const path = yield* Path.Path; - const config = yield* loadStackConfig(cliSettings.workdir).pipe(Effect.mapError(startFailed)); - const toml = yield* readDbToml(fs, path, cliSettings.workdir).pipe(Effect.mapError(startFailed)); + const runtime = yield* RuntimeInfo; + const fs = yield* FileSystem.FileSystem; + const output = yield* Output; + const config = yield* loadStackConfig(settings.workdir).pipe(Effect.mapError(startFailed)); + const toml = yield* readDbToml(fs, path, settings.workdir).pipe(Effect.mapError(startFailed)); const experimental = yield* resolveExperimentalWithProjectEnv({ ...toml.projectEnv }); - const applyCatalog = (stack: EffectStack) => - Effect.gen(function* () { - const catalog = yield* Effect.serviceOption(StackCatalogSetup); - if (Option.isNone(catalog)) - return yield* startFailedAfterEngine({ message: "stack catalog setup is unavailable" }); - yield* catalog.value - .apply({ - target: { - kind: "live", - stack, - projectRoot: cliSettings.workdir, - config, - }, - optionalConfig: postgresOnlyStackStartConfig(config), - overlay: { - webhooks: "config", - webhooksEnabled: toml.webhooksEnabled, - apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, - vault: toml.vault, - workdir: cliSettings.workdir, - }, - }) - .pipe(Effect.mapError(startFailedAfterEngine)); - }); - const existing = yield* api.value - .findStack({ projectRoot: cliSettings.workdir }) - .pipe(Effect.mapError(startFailed)); - if (Option.isNone(existing) || existing.value.desiredLifecycle === "unconfigured") { - const stack = Option.isNone(existing) - ? yield* api.value - .createStack({ projectRoot: cliSettings.workdir }) + const existing = yield* stackForProject(api, settings, path).pipe(Effect.mapError(startFailed)); + const stack = + existing === undefined + ? yield* api + .create({ + projectRoot: settings.workdir, + stateRoot: stateRoot(settings, path), + cacheRoot: cacheRoot(settings, path), + runtime: defaultStackRuntime(runtime), + }) .pipe(Effect.mapError(startFailed)) - : yield* api.value.openStack(existing.value.id).pipe(Effect.mapError(startFailed)); - if (stack.dockerFallbackNotice !== undefined) - yield* output.raw(`${stack.dockerFallbackNotice}\n`, "stderr"); - yield* stack - .start({ config: postgresOnlyStackStartConfig(config) }) - .pipe(Effect.mapError(startFailed)); - yield* applyCatalog(stack); - yield* applyStackMigrateAndSeed(stack, cliSettings.workdir, toml, experimental).pipe( - Effect.mapError(startFailedAfterEngine), + : yield* api + .open({ + id: existing.definition.id, + stateRoot: stateRoot(settings, path), + cacheRoot: cacheRoot(settings, path), + }) + .pipe(Effect.mapError(startFailed)); + const currentDatabase = yield* databaseFromStack(stack).pipe(Effect.mapError(startFailed)); + if (currentDatabase !== undefined) { + if (Option.isSome(yield* databaseReady(stack).pipe(Effect.mapError(startFailed)))) { + yield* applyStackWebhooksOnly(currentDatabase, toml.webhooksEnabled).pipe( + Effect.mapError(startFailed), + ); + return "already-running" as const; + } + yield* currentDatabase.start.pipe(Effect.mapError(startFailed)); + yield* currentDatabase.ready.pipe(Effect.mapError(startFailed)); + yield* applyStackWebhooksOnly(currentDatabase, toml.webhooksEnabled).pipe( + Effect.mapError(startFailed), ); - return "started"; + return "started" as const; } - const stack = yield* api.value.openStack(existing.value.id).pipe(Effect.mapError(startFailed)); - const status = yield* stack.status.pipe(Effect.mapError(startFailed)); - const database = status.capabilities.find((capability) => capability.name === "database"); - if (status.lifecycle === "running" && database?.state === "ready") { - yield* applyStackWebhooksOnly(stack, toml.webhooksEnabled).pipe( - Effect.mapError(startFailedAfterEngine), + const existingDatabase = yield* stack.services.list.pipe(Effect.mapError(startFailed)); + if (databaseFor(existingDatabase) !== undefined) + return yield* startFailed({ + message: + "A standalone database exists outside the saved stack composition. Destroy the standalone database before starting this stack.", + }); + const creations = yield* config.creations(stack.id).pipe(Effect.mapError(startFailed)); + const databaseCreation = creations.find( + (creation): creation is Extract => + creation.service === "database", + ); + if (databaseCreation === undefined) + return yield* startFailed({ message: "database is disabled in the local configuration" }); + const [database] = yield* stack.composition + .supabase([databaseCreation]) + .pipe(Effect.mapError(startFailed)); + if (database === undefined || database.service !== "database") + return yield* startFailed({ message: "stack did not create a database instance" }); + return yield* Effect.gen(function* () { + yield* database.start.pipe(Effect.mapError(startFailed)); + yield* database.ready.pipe(Effect.mapError(startFailed)); + const catalog = yield* StackCatalogSetup; + const databaseServices = creations.flatMap((creation) => + creation.service === "auth" || + creation.service === "storage" || + creation.service === "realtime" + ? [creation.service] + : [], ); - return "already-running"; - } - yield* stack.start().pipe(Effect.mapError(startFailed)); - yield* applyStackWebhooksOnly(stack, toml.webhooksEnabled).pipe( - Effect.mapError(startFailedAfterEngine), + yield* catalog + .apply({ + target: { stack, database, databaseServices, jwtSecret: Redacted.value(config.jwtSecret) }, + overlay: { + webhooks: "config", + webhooksEnabled: toml.webhooksEnabled, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + workdir: settings.workdir, + }, + }) + .pipe(Effect.mapError(startFailed)); + yield* applyStackMigrateAndSeed(database, settings.workdir, toml, experimental).pipe( + Effect.mapError(startFailed), + ); + return "started" as const; + }).pipe( + Effect.onError(() => + database.destroy.pipe( + Effect.tapError((cause) => + output.raw( + `Failed to destroy newly created database ${database.id}: ${cause.message}. Run supabase stack destroy to remove the incomplete stack before retrying.\n`, + "stderr", + ), + ), + Effect.ignore, + ), + ), ); - return "started"; }); diff --git a/apps/cli/src/command-internal/stack-runtime.ts b/apps/cli/src/command-internal/stack-runtime.ts new file mode 100644 index 0000000000..05c4cb6c29 --- /dev/null +++ b/apps/cli/src/command-internal/stack-runtime.ts @@ -0,0 +1,9 @@ +/** Selects native execution where the catalog publishes portable artifacts. */ +export const defaultStackRuntime = (runtime: { + readonly platform: string; + readonly arch: string; +}): "native" | "docker" => + (runtime.platform === "linux" && (runtime.arch === "x64" || runtime.arch === "arm64")) || + (runtime.platform === "darwin" && runtime.arch === "arm64") + ? "native" + : "docker"; diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 6cf98d8816..4c538f9e3d 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -1,96 +1,52 @@ -import { CliConfigSchema, type CliConfig } from "@supabase/config"; +import { StackError, type Stack } from "@supabase/stack/effect"; +import { CliConfigSchema } from "@supabase/config"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Redacted, Schema } from "effect"; -import { - ContainerEngineResolver, - EphemeralPostgresError, - databaseBootstrapIdentity, - schemaInitArtifactIdentity, - type CreateEphemeralPostgresOptions, - type EffectEphemeralPostgres, - type StackConfig, -} from "@supabase/stack/effect"; +import { Effect, FileSystem, Layer, Option, Path, Schema } from "effect"; import { mockOutput } from "../../tests/helpers/mocks.ts"; +import { mockCommandSettings } from "../../tests/helpers/command-mocks.ts"; import { runtimeInfoLayer } from "../shared/runtime/runtime-info.layer.ts"; -import { - mockCommandSettings, - useTempWorkdir, - withEnvVar, -} from "../../tests/helpers/command-mocks.ts"; -import { SHADOW_CACHE_ENV } from "./db-bootstrap/shadow-cache.ts"; import { DbConnection } from "./db-connection.service.ts"; -import { loadLocalProjectContext } from "./local-project-context.ts"; -import { stackBackendLayer } from "./stack-backend.ts"; +import { dbConnectionLayer } from "./db-connection.layer.ts"; +import { StackApi, stackApiLayer } from "./stack-api.ts"; +import { stackCatalogSetupLayer } from "./stack-catalog-setup.ts"; +import { parseConnectionString } from "./db-config.parse.ts"; import { - StackEphemeralPostgres, stackAcquireShadowDatabase, - stackShadowBaselineTarFileName, - stackShadowCacheKey, + stackMigrateShadow, + stackWithShadowDatabase, } from "./stack-shadow.ts"; import type { ShadowSetupInput } from "./db-bootstrap/shadow-database.ts"; -import { - noopStackCatalogSetupLayer, - recordingStackCatalogSetup, - StackCatalogSetup, -} from "./stack-catalog-setup.ts"; - -const tmp = useTempWorkdir("stack-shadow-"); -const defaultConfig: CliConfig = Schema.decodeSync(CliConfigSchema)({}); -const nativeRuntime = { kind: "native" as const }; -const nativeAcquire = { runtime: nativeRuntime }; - -const mockEphemeral = () => { - const restores: Array = []; - const exports: Array = []; - const runtimes: Array = []; - const create = ( - options: CreateEphemeralPostgresOptions, - ): Effect.Effect => - Effect.sync(() => { - restores.push(options.restoreFrom); - runtimes.push(options.runtime); - return { - host: "127.0.0.1", - port: 59999, - version: "17.6.1", - runtime: { kind: "native" as const }, - artifactIdentity: "native:17.6.1", - url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), - start: Effect.void, - stop: Effect.void, - exportPgData: (tarPath: string) => - Effect.gen(function* () { - exports.push(tarPath); - const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString(tarPath, "pgdata").pipe(Effect.ignore); - }), - }; - }); - return { - restores, - exports, - runtimes, - layer: Layer.succeed(StackEphemeralPostgres, { - create, - resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), - }), - }; -}; -const db = Layer.succeed(DbConnection, { - connect: () => - Effect.succeed({ - exec: () => Effect.void, - query: () => Effect.succeed([]), - execBatch: () => Effect.void, - extensionExists: () => Effect.succeed(false), - copyToCsv: () => Effect.succeed(new Uint8Array()), - queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), +const defaultConfig = Schema.decodeUnknownSync(CliConfigSchema)({}); + +const recordingApi = (roots: string[], handles: Stack[]) => + Layer.effect( + StackApi, + Effect.gen(function* () { + const api = yield* StackApi; + return StackApi.of({ + ...api, + create: Effect.fn("ShadowTest.create")((options) => + Effect.sync(() => { + roots.push(options.projectRoot); + }).pipe( + Effect.andThen(api.create(options)), + Effect.tap((stack) => + Effect.sync(() => { + handles.push(stack); + }), + ), + ), + ), + }); }), -}); - -const input = (fs: FileSystem.FileSystem, path: Path.Path): ShadowSetupInput => ({ + ).pipe(Layer.provide(stackApiLayer), Layer.provide(BunServices.layer)); +const input = ( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, +): ShadowSetupInput => ({ db: { major_version: 17, settings: {} }, experimental: defaultConfig.experimental, jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", @@ -102,12 +58,12 @@ const input = (fs: FileSystem.FileSystem, path: Path.Path): ShadowSetupInput( - home: string, - value: string, - body: Effect.Effect, -): Effect.Effect => - withEnvVar("SUPABASE_HOME", home, withEnvVar(SHADOW_CACHE_ENV, value, body)); - -const expectedCacheKey = ( - overrides: Partial[0]> = {}, -): string => - stackShadowCacheKey({ - artifactIdentity: "native:17.6.1", - majorVersion: 17, - runtimeKind: "native", - jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", - jwtExpiry: 3600, - dbPassword: "postgres", - dbSettings: {}, - rolesSql: "", - bootstrapIdentity: databaseBootstrapIdentity, - webhooksEnabled: false, - apiGrantsKept: true, - vault: [], - jwks: "", - storageTargetMigration: "", - authEnabled: false, - storageEnabled: false, - realtimeEnabled: false, - authArtifact: "", - storageArtifact: "", - realtimeArtifact: "", - ...overrides, - }); - -const catalogAuthEnabled = (config: StackConfig): boolean => { - const cap = config.capabilities?.auth; - return cap === undefined || cap.enabled !== false; -}; - -const engineResolver = (installed: boolean) => - Layer.succeed(ContainerEngineResolver, { - isInstalled: () => Effect.succeed(installed), - resolve: () => Effect.die("unused"), - }); +const query = Effect.fn("ShadowTest.query")(function* (url: string, sql: string) { + const connection = parseConnectionString(url); + if (connection === undefined) return yield* Effect.die("invalid shadow URL"); + const db = yield* DbConnection; + const session = yield* db.connect(connection, { isLocal: true, dnsResolver: "native" }); + return yield* session.query(sql); +}); -describe("stackAcquireShadowDatabase", () => { +describe("stack shadow databases", () => { it.live( - "exports a stack-shadow-baseline tar on a cold miss and restores it on a warm hit", - () => { - const ephemeral = mockEphemeral(); - const out = mockOutput(); - const catalog = recordingStackCatalogSetup((input) => input.target.kind); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - const first = yield* stackAcquireShadowDatabase(input(fs, path), nativeAcquire); - expect(first.baselinePresent).toBe(false); - expect(catalog.applied).toEqual(["ephemeral"]); - expect(first.artifactIdentity).toBe("native:17.6.1"); - expect(ephemeral.restores).toEqual([undefined]); - expect(ephemeral.exports).toHaveLength(1); - expect(ephemeral.exports[0]?.endsWith(`.${String(process.pid)}.partial`)).toBe(true); - const names = (yield* fs.readDirectory( - path.join(home, "cache", "shadow-baseline"), - )).filter((name) => name.endsWith(".tar") && !name.includes(".partial")); - expect(names).toHaveLength(1); - const info = yield* fs.stat(path.join(home, "cache", "shadow-baseline", names[0]!)); - expect((Number(info.mode) & 0o777).toString(8)).toBe("600"); - expect(names[0]?.startsWith("stack-shadow-baseline-")).toBe(true); - expect(names[0]).toBe(stackShadowBaselineTarFileName(expectedCacheKey())); - - const warm = yield* stackAcquireShadowDatabase(input(fs, path), nativeAcquire); - expect(warm.baselinePresent).toBe(true); - expect(ephemeral.restores[1]?.endsWith(names[0] ?? "")).toBe(true); - expect(catalog.applied).toEqual(["ephemeral"]); - }), - ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral.layer, - catalog.layer, - ), - ), - ); - }, - ); - - it.live("skips the cache when SUPABASE_SHADOW_CACHE is 0", () => { - const ephemeral = mockEphemeral(); - const out = mockOutput(); - return Effect.scoped( + "isolates two fresh databases, replays migrations, and removes each owned namespace", + () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - return yield* withShadowCacheHome( - home, - "0", - Effect.gen(function* () { - const handle = yield* stackAcquireShadowDatabase(input(fs, path), nativeAcquire); - expect(handle.baselinePresent).toBe(false); - expect(ephemeral.exports).toHaveLength(0); - const names = yield* fs - .readDirectory(path.join(home, "cache", "shadow-baseline")) - .pipe(Effect.orElseSucceed(() => [])); - expect(names.filter((name) => name.endsWith(".tar"))).toEqual([]); - }), + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-test-" }); + yield* fs.makeDirectory(path.join(root, "supabase", "migrations"), { recursive: true }); + yield* fs.writeFileString( + path.join(root, "supabase", "migrations", "20260919000000_shadow.sql"), + "CREATE TABLE public.shadow_probe(value text); INSERT INTO public.shadow_probe VALUES ('migrated');", ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, + const output = mockOutput(); + const roots: string[] = []; + const handles: Stack[] = []; + const layers = Layer.mergeAll( + recordingApi(roots, handles), + stackCatalogSetupLayer, + dbConnectionLayer, runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral.layer, - noopStackCatalogSetupLayer, - ), - ), - ); - }); - - it.live("keeps the cluster uncached when the baseline export fails", () => { - const restores: Array = []; - const out = mockOutput(); - const layer = Layer.succeed(StackEphemeralPostgres, { - create: (options) => - Effect.sync(() => { - restores.push(options.restoreFrom); - return { - host: "127.0.0.1", - port: 59999, - version: "17.6.1", - runtime: { kind: "native" as const }, - artifactIdentity: "native:17.6.1", - url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), - start: Effect.void, - stop: Effect.void, - exportPgData: () => - Effect.fail( - new EphemeralPostgresError({ message: "export failed", reason: "snapshot" }), - ), - }; - }), - resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), - }); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - const handle = yield* stackAcquireShadowDatabase(input(fs, path), nativeAcquire); - expect(handle.baselinePresent).toBe(false); - expect(handle.snapshotKey).toBeUndefined(); - expect(restores).toEqual([undefined]); - expect(out.stderrText).toContain("Warning: shadow baseline not cached:"); - const names = yield* fs - .readDirectory(path.join(home, "cache", "shadow-baseline")) - .pipe(Effect.orElseSucceed(() => [])); - expect( - names.filter((name) => name.endsWith(".tar") && !name.includes(".partial")), - ).toEqual([]); - }), + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - layer, - noopStackCatalogSetupLayer, - ), - ), - ); - }); - - it.live("warns and cold-provisions when a cached baseline restore fails", () => { - const restores: Array = []; - const out = mockOutput(); - const layer = Layer.succeed(StackEphemeralPostgres, { - create: (options) => { - restores.push(options.restoreFrom); - if (options.restoreFrom !== undefined) - return Effect.fail( - new EphemeralPostgresError({ - message: "restore failed", - reason: "restore-mismatch", + yield* Effect.gen(function* () { + const setup = input(fs, path, root); + yield* Effect.scoped( + Effect.gen(function* () { + const [first, second] = yield* Effect.all( + [ + stackAcquireShadowDatabase(setup, { runtime: "native" }), + stackAcquireShadowDatabase(setup, { runtime: "native" }), + ], + { concurrency: "unbounded" }, + ); + expect(first.stack.id).not.toBe(second.stack.id); + expect(first.database.id).not.toBe(second.database.id); + expect(first.port).not.toBe(second.port); + + yield* stackMigrateShadow(first, setup); + expect(yield* query(first.url, "SELECT value FROM public.shadow_probe")).toEqual([ + { value: "migrated" }, + ]); + expect( + yield* query(second.url, "SELECT to_regclass('public.shadow_probe') AS table_name"), + ).toEqual([{ table_name: null }]); }), ); - return Effect.succeed({ - host: "127.0.0.1", - port: 59999, - version: "17.6.1", - runtime: { kind: "native" as const }, - artifactIdentity: "native:17.6.1", - url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), - start: Effect.void, - stop: Effect.void, - exportPgData: () => Effect.void, - }); - }, - resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), - }); - return Effect.scoped( + for (const shadowRoot of roots) expect(yield* fs.exists(shadowRoot)).toBe(false); + for (const handle of handles) + expect(yield* Effect.flip(handle.services.list)).toBeInstanceOf(StackError); + expect(output.stderrText).not.toContain("Failed to destroy"); + const api = yield* StackApi; + expect(yield* api.discover({ stateRoot: path.join(root, "stacks") })).toEqual([]); + }).pipe(Effect.provide(layers)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 120_000, + ); + + it.live( + "destroys a shadow when its consumer fails", + () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - const cacheDir = path.join(home, "cache", "shadow-baseline"); - yield* fs.makeDirectory(cacheDir, { recursive: true }); - const tarName = stackShadowBaselineTarFileName(expectedCacheKey()); - yield* fs.writeFileString(path.join(cacheDir, tarName), "corrupt"); - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - const handle = yield* stackAcquireShadowDatabase(input(fs, path), nativeAcquire); - expect(handle.baselinePresent).toBe(false); - expect(restores).toHaveLength(2); - expect(restores[0]?.endsWith(tarName)).toBe(true); - expect(restores[1]).toBeUndefined(); - expect(out.stderrText).toContain("Warning: shadow baseline not cached: restore failed"); - }), - ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - layer, - noopStackCatalogSetupLayer, - ), - ), - ); - }); - - it.live("stops the cluster when acquire is interrupted during catalog overlay", () => { - const out = mockOutput(); - return Effect.scoped( - Effect.gen(function* () { - const started = yield* Deferred.make(); - let stopped = false; - const ephemeral = Layer.succeed(StackEphemeralPostgres, { - create: () => - Effect.sync(() => ({ - host: "127.0.0.1", - port: 59999, - version: "17.6.1", - runtime: { kind: "native" as const }, - artifactIdentity: "native:17.6.1", - url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:59999/postgres"), - start: Effect.void, - stop: Effect.sync(() => { - stopped = true; - }), - exportPgData: () => Effect.die("export should not run"), - })), - resolveRelease: () => Effect.succeed({ version: "17.6.1", image: "postgres:17.6.1" }), - }); - const catalog = Layer.succeed(StackCatalogSetup, { - apply: () => + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-failure-" }); + const output = mockOutput(); + const roots: string[] = []; + const handles: Stack[] = []; + const result = yield* stackWithShadowDatabase( + input(fs, path, root), + (_handle) => Effect.gen(function* () { - yield* Deferred.succeed(started, undefined); - return yield* Effect.never; + return yield* Effect.fail("consumer failed"); }), - }); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - const fiber = yield* Effect.forkChild( - withShadowCacheHome( - home, - "0", - stackAcquireShadowDatabase(input(fs, path), nativeAcquire), - ).pipe( - Effect.scoped, - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral, - catalog, - ), + { runtime: "native" }, + ).pipe( + Effect.provide( + Layer.mergeAll( + recordingApi(roots, handles), + stackCatalogSetupLayer, + dbConnectionLayer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, ), ), + Effect.flip, ); - yield* Deferred.await(started); - yield* Fiber.interrupt(fiber); - expect(stopped).toBe(true); - }), - ).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("hashes and creates a docker runtime when Docker is installed", () => { - const ephemeral = mockEphemeral(); - const out = mockOutput(); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - yield* stackAcquireShadowDatabase(input(fs, path)); - expect(ephemeral.runtimes[0]).toEqual({ kind: "container", engine: "docker" }); - const names = (yield* fs.readDirectory( - path.join(home, "cache", "shadow-baseline"), - )).filter((name) => name.endsWith(".tar") && !name.includes(".partial")); - expect(names).toEqual([ - stackShadowBaselineTarFileName( - expectedCacheKey({ - artifactIdentity: "container:docker:postgres:17.6.1", - runtimeKind: "container:docker", - }), - ), - ]); - }), - ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral.layer, - noopStackCatalogSetupLayer, - engineResolver(true), - ), - ), - ); - }); - - it.live("hashes and creates a native runtime when Docker is not installed", () => { - const ephemeral = mockEphemeral(); - const out = mockOutput(); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - yield* stackAcquireShadowDatabase(input(fs, path)); - expect(ephemeral.runtimes[0]).toEqual({ kind: "native" }); - const names = (yield* fs.readDirectory( - path.join(home, "cache", "shadow-baseline"), - )).filter((name) => name.endsWith(".tar") && !name.includes(".partial")); - expect(names).toEqual([stackShadowBaselineTarFileName(expectedCacheKey())]); - }), - ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral.layer, - noopStackCatalogSetupLayer, - engineResolver(false), - ), - ), - ); - }); - - it.live("overlays remotes-disabled auth onto ephemeral catalog and the cache key", () => { - const ephemeral = mockEphemeral(); - const out = mockOutput(); - const catalog = recordingStackCatalogSetup((applied) => applied.target.config); - const remoteRef = "abcdefghijklmnopqrst"; - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - const scratch = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(path.join(scratch, "supabase"), { recursive: true }); - yield* fs.writeFileString( - path.join(scratch, "supabase", "config.toml"), - [ - 'project_id = "stack-shadow-remotes"', - "[auth]", - "enabled = true", - 'jwt_secret = "super-secret-jwt-token-with-at-least-32-characters-long"', - "", - "[remotes.prod]", - `project_id = "${remoteRef}"`, - "[remotes.prod.auth]", - "enabled = false", - "", - ].join("\n"), - ); - const context = yield* loadLocalProjectContext( - scratch, - (message) => new Error(message), - remoteRef, - ); - const setup = input(fs, path); - const remotesInput = { - ...setup, - workdir: scratch, - context, - setup: { ...setup.setup, authEnabledForSetup: false }, - }; - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - yield* stackAcquireShadowDatabase(remotesInput, nativeAcquire); - expect(catalog.applied).toHaveLength(1); - const applied = catalog.applied[0]; - expect(applied).toBeDefined(); - if (applied === undefined) return; - expect(catalogAuthEnabled(applied)).toBe(false); - const names = (yield* fs.readDirectory( - path.join(home, "cache", "shadow-baseline"), - )).filter((name) => name.endsWith(".tar") && !name.includes(".partial")); - expect(names).toEqual([ - stackShadowBaselineTarFileName( - expectedCacheKey({ authEnabled: catalogAuthEnabled(applied) }), - ), - ]); - }), - ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral.layer, - catalog.layer, - ), - ), - ); - }); - - it.live( - "still schema-inits auth when remotes enable it and SUPABASE_AUTH_ENABLED is false", - () => { - const ephemeral = mockEphemeral(); - const out = mockOutput(); - const catalog = recordingStackCatalogSetup((applied) => applied.target.config); - const remoteRef = "abcdefghijklmnopqrst"; - const authArtifact = schemaInitArtifactIdentity("auth") ?? "missing"; - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped(); - const scratch = yield* fs.makeTempDirectoryScoped(); - yield* fs.makeDirectory(path.join(scratch, "supabase"), { recursive: true }); - yield* fs.writeFileString( - path.join(scratch, "supabase", "config.toml"), - [ - 'project_id = "stack-shadow-remotes-on"', - "[auth]", - "enabled = true", - 'jwt_secret = "super-secret-jwt-token-with-at-least-32-characters-long"', - "", - "[remotes.prod]", - `project_id = "${remoteRef}"`, - "[remotes.prod.auth]", - "enabled = true", - "", - ].join("\n"), - ); - yield* fs.writeFileString( - path.join(scratch, "supabase", ".env"), - "SUPABASE_AUTH_ENABLED=false\n", - ); - const context = yield* loadLocalProjectContext( - scratch, - (message) => new Error(message), - remoteRef, - ); - const setup = input(fs, path); - const remotesInput = { - ...setup, - workdir: scratch, - context, - setup: { ...setup.setup, authEnabledForSetup: true }, - }; - return yield* withShadowCacheHome( - home, - "1", - Effect.gen(function* () { - yield* stackAcquireShadowDatabase(remotesInput, nativeAcquire); - expect(catalog.applied).toHaveLength(1); - const applied = catalog.applied[0]; - expect(applied).toBeDefined(); - if (applied === undefined) return; - expect(catalogAuthEnabled(applied)).toBe(true); - const names = (yield* fs.readDirectory( - path.join(home, "cache", "shadow-baseline"), - )).filter((name) => name.endsWith(".tar") && !name.includes(".partial")); - expect(names).toEqual([ - stackShadowBaselineTarFileName( - expectedCacheKey({ - authEnabled: catalogAuthEnabled(applied), - authArtifact, - }), - ), - ]); - }), - ); - }), - ).pipe( - Effect.provide( - Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - out.layer, - db, - mockCommandSettings({ workdir: tmp.current }), - stackBackendLayer("stack"), - ephemeral.layer, - catalog.layer, - ), - ), - ); - }, + expect(result).toBe("consumer failed"); + for (const handle of handles) + expect(yield* Effect.flip(handle.services.list)).toBeInstanceOf(StackError); + expect(output.stderrText).not.toContain("Failed to destroy"); + expect(roots).toHaveLength(1); + for (const root of roots) expect(yield* fs.exists(root)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 120_000, ); }); diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index ed0efb4158..a26d324cde 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -1,762 +1,191 @@ -import { scryptSync } from "node:crypto"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- pid scopes the exclusive temp name across processes. -import process from "node:process"; -import { - Clock, - Context, - Crypto, - Effect, - FileSystem, - Layer, - Option, - Path, - Predicate, - Redacted, - Result, - Scope, - Semaphore, -} from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import { - ContainerEngineResolver, - createEphemeralPostgres, - databaseBootstrapIdentity, - resolveEphemeralPostgresRelease, - schemaInitArtifactIdentity, - selectDefaultRuntimeSelection, - type CreateEphemeralPostgresOptions, - type EffectEphemeralPostgres, - type EphemeralPostgresRelease, - type EphemeralPostgresSettings, - type SchemaInitCapabilityName, - type StackConfig, - type StackRuntime, - type StackRuntimePreference, - type StackVersionUnsupportedError, -} from "@supabase/stack/effect"; +import { Effect, FileSystem, Path, Redacted, Schema } from "effect"; +import type { DatabaseInstance, Stack } from "@supabase/stack/effect"; +import { CommandSettings } from "../config/command-settings.service.ts"; import { Output } from "../shared/output/output.service.ts"; import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; -import { CommandSettings } from "../config/command-settings.service.ts"; -import { DbConnection } from "./db-connection.service.ts"; -import { shadowBaselineCacheDir } from "./pgdelta.paths.ts"; -import { - SHADOW_BASELINE_KEEP, - SHADOW_BASELINE_MAX_AGE_MS, - SHADOW_CACHE_ENV, - canonicalJson, - shadowBaselineEmbeddedDigest, - shadowBaselineTarsToEvict, - touchShadowBaselineTar, -} from "./db-bootstrap/shadow-cache.ts"; -import { viperEnvBoolWithProjectFallback } from "./viper-env.ts"; +import { StackApi } from "./stack-api.ts"; +import { StackCatalogSetup } from "./stack-catalog-setup.ts"; +import { stackProjectRuntime } from "./stack-local-database.ts"; +import { defaultStackRuntime } from "./stack-runtime.ts"; +import { parseConnectionString } from "./db-config.parse.ts"; +import { toPostgresURL } from "./postgres-url.ts"; import { connectShadowDatabase, ShadowDbError, type ShadowSetupInput, - type ShadowSourceResult, } from "./db-bootstrap/shadow-database.ts"; +import type { SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; -import { stackProjectRuntime } from "./stack-local-database.ts"; -import { loadStackConfig } from "./stack-config.ts"; -import { StackCatalogSetup } from "./stack-catalog-setup.ts"; -import { resolveSetupWebhooksEnabled, type SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; -import type { VaultSecret } from "./vault.ts"; - -/** Injectable ephemeral-cluster factory for tests. */ -export class StackEphemeralPostgres extends Context.Service< - StackEphemeralPostgres, - { - readonly create: typeof createEphemeralPostgres; - readonly resolveRelease: typeof resolveEphemeralPostgresRelease; - } ->()("supabase/experimental-stack/EphemeralPostgres") {} - -export const ephemeralPostgresLayer = Layer.succeed(StackEphemeralPostgres, { - create: createEphemeralPostgres, - resolveRelease: resolveEphemeralPostgresRelease, -}); -const TAR_PREFIX = "stack-shadow-baseline-"; - -const ensurePrivateCacheDir = (fs: FileSystem.FileSystem, cacheDir: string) => - Effect.gen(function* () { - yield* fs.makeDirectory(cacheDir, { recursive: true, mode: 0o700 }).pipe( - Effect.mapError( - (cause) => - new ShadowDbError({ - message: `failed to create ${cacheDir}: ${cause.message}`, - reason: "filesystem", - }), - ), - ); - yield* fs.chmod(cacheDir, 0o700).pipe( - Effect.mapError( - (cause) => - new ShadowDbError({ - message: `failed to set permissions on ${cacheDir}: ${cause.message}`, - reason: "filesystem", - }), - ), - ); - }); - -/** A partial older than 5 minutes is abandoned; a live export finishes in seconds. */ -const STACK_SHADOW_PARTIAL_ABANDON_MS = 5 * 60 * 1000; - -export const stackShadowBaselineTarFileName = (key: string): string => `${TAR_PREFIX}${key}.tar`; - -const isStackShadowBaselineTar = (fileName: string): boolean => - /^stack-shadow-baseline-[0-9a-f]{16}\.tar$/u.test(fileName); - -export function isStackShadowBaselinePartial(fileName: string): boolean { - return /^stack-shadow-baseline-[0-9a-f]{16}\.tar\.\d+\.partial$/u.test(fileName); -} - -const stackShadowExportMutex = Semaphore.makeUnsafe(1); - -export interface StackShadowCacheKeyInputs { - readonly artifactIdentity: string; - readonly majorVersion: number; - readonly runtimeKind: string; - readonly jwtSecret: string; - readonly jwtExpiry: number; - readonly dbPassword: string; - readonly dbSettings: unknown; - readonly rolesSql: string; - readonly bootstrapIdentity: string; - readonly webhooksEnabled: boolean; - readonly apiGrantsKept: boolean; - readonly vault: ReadonlyArray; - readonly jwks: string; - readonly storageTargetMigration: string; - readonly authEnabled: boolean; - readonly storageEnabled: boolean; - readonly realtimeEnabled: boolean; - readonly authArtifact: string; - readonly storageArtifact: string; - readonly realtimeArtifact: string; -} - -export const stackShadowCacheKey = (inputs: StackShadowCacheKeyInputs): string => { - const quoted = (value: string) => JSON.stringify(value); - const lines: Array = [ - `artifact=${quoted(inputs.artifactIdentity)}`, - `major_version=${inputs.majorVersion}`, - `runtime=${quoted(inputs.runtimeKind)}`, - `jwt_secret=${quoted(inputs.jwtSecret)}`, - `jwt_expiry=${inputs.jwtExpiry}`, - `db_password=${quoted(inputs.dbPassword)}`, - `db_settings=${canonicalJson(inputs.dbSettings ?? {})}`, - `bootstrap=${quoted(inputs.bootstrapIdentity)}`, - `api_grants_kept=${inputs.apiGrantsKept}`, - `webhooks_enabled=${inputs.webhooksEnabled}`, - `baseline_embedded_digest=${shadowBaselineEmbeddedDigest()}`, - `schema_init=auth=${inputs.authEnabled},storage=${inputs.storageEnabled},realtime=${inputs.realtimeEnabled}`, - inputs.authEnabled ? `auth_artifact=${quoted(inputs.authArtifact)}` : "auth_artifact=excluded", - inputs.storageEnabled - ? `storage_artifact=${quoted(inputs.storageArtifact)}` - : "storage_artifact=excluded", - inputs.realtimeEnabled - ? `realtime_artifact=${quoted(inputs.realtimeArtifact)}` - : "realtime_artifact=excluded", - inputs.realtimeEnabled && inputs.majorVersion >= 15 - ? `realtime_jwks=${quoted(inputs.jwks)}` - : "realtime_jwks=excluded", - inputs.storageEnabled && inputs.majorVersion >= 15 - ? `storage_target_migration=${quoted(inputs.storageTargetMigration)}` - : "storage_target_migration=excluded", - ]; - for (const secret of inputs.vault - .filter((secret) => secret.resolved) - .sort((left, right) => (left.name < right.name ? -1 : left.name > right.name ? 1 : 0))) { - lines.push(`vault=${JSON.stringify([secret.name, secret.value])}`); - } - return scryptSync( - `${lines.join("\n")}\nroles_sql=\n${inputs.rolesSql}`, - "supabase-stack-shadow-cache-key", - 32, - ) - .toString("hex") - .slice(0, 16); -}; - -const capabilityPinVersion = ( - cap: { readonly enabled?: boolean; readonly version?: string } | undefined, -): string | undefined => cap?.version; - -const trioSchemaInitArtifact = ( - enabled: boolean, - name: Extract, - config: StackConfig | undefined, -): string => { - if (!enabled) return ""; - return ( - schemaInitArtifactIdentity(name, capabilityPinVersion(config?.capabilities?.[name])) ?? - "missing" - ); -}; +type Runtime = "native" | "docker" | "podman"; export interface StackShadowAcquiredHandle { + readonly stack: Stack; + readonly database: DatabaseInstance; readonly url: string; readonly host: string; readonly port: number; - readonly artifactIdentity: string; - readonly runtime: StackRuntime; - readonly baselinePresent: boolean; - readonly snapshotKey?: string; - readonly ephemeral: EffectEphemeralPostgres; + readonly runtime: Runtime; + readonly version: string; } -export interface StackShadowAcquireOpts { - readonly bypassCache?: boolean; +interface ShadowOptions { readonly port?: number; - readonly runtime?: StackRuntimePreference; + readonly runtime?: Runtime; readonly webhooks?: SetupDatabaseOptions["webhooks"]; } -const cacheEnabled = (projectEnv: Record | undefined, bypass: boolean): boolean => - !bypass && - viperEnvBoolWithProjectFallback(SHADOW_CACHE_ENV, projectEnv ?? {}, { - whenUnset: true, +const shadowError = (cause: { readonly message: string }) => + cause instanceof ShadowDbError + ? cause + : new ShadowDbError({ message: cause.message, reason: "database" }); + +const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (opts: ShadowOptions) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const api = yield* StackApi; + const settings = yield* CommandSettings; + const runtimeInfo = yield* RuntimeInfo; + const runtime = opts.runtime ?? (yield* stackProjectRuntime) ?? defaultStackRuntime(runtimeInfo); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-shadow-" }); + const stack = yield* api.create({ + projectRoot: root, + stateRoot: path.join(settings.supabaseHome, "stacks"), + cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), + runtime, }); - -const readRolesSql = ( - fs: FileSystem.FileSystem, - path: Path.Path, - workdir: string, -): Effect.Effect => - fs.readFileString(path.join(workdir, "supabase", "roles.sql")).pipe( - Effect.catchTag("PlatformError", (error) => - Predicate.isTagged(error.reason, "NotFound") - ? Effect.succeed("") - : Effect.fail( - new ShadowDbError({ - message: `failed to read supabase/roles.sql: ${error.message}`, - reason: "filesystem", - }), - ), - ), - ); - -const runtimePreference = ( - runtime: StackRuntime | undefined, - override?: StackRuntimePreference, -): StackRuntimePreference | undefined => { - if (override !== undefined) return override; - if (runtime === undefined) return undefined; - return runtime.kind === "native" - ? { kind: "native" } - : { kind: "container", engine: runtime.engine }; -}; - -const postgresSettings = (value: unknown): EphemeralPostgresSettings | undefined => { - if (value === undefined || value === null || typeof value !== "object" || Array.isArray(value)) - return undefined; - return Object.fromEntries( - Object.entries(value).filter( - (entry): entry is [string, string | number | boolean] => - typeof entry[1] === "string" || - typeof entry[1] === "number" || - typeof entry[1] === "boolean", - ), - ); -}; - -// A disabled capability carries no nested pins, so re-enabling one for setup compiles defaults. -const overlaySetupEnabled = ( - current: C, - enabled: boolean, -): C | { readonly enabled: false } | { readonly enabled: true } => { - if (!enabled) return { enabled: false }; - if (current === undefined) return { enabled: true }; - return { ...current, enabled: true }; -}; - -const overlaySetupTrio = ( - config: StackConfig, - setup: { - readonly authEnabledForSetup: boolean; - readonly storageEnabledForSetup: boolean; - readonly realtimeEnabledForSetup: boolean; - }, -): StackConfig => ({ - ...config, - capabilities: { - ...config.capabilities, - auth: overlaySetupEnabled(config.capabilities?.auth, setup.authEnabledForSetup), - storage: overlaySetupEnabled(config.capabilities?.storage, setup.storageEnabledForSetup), - realtime: overlaySetupEnabled(config.capabilities?.realtime, setup.realtimeEnabledForSetup), - }, -}); - -const loadEphemeralCatalogConfig = ( - input: ShadowSetupInput, -): Effect.Effect< - StackConfig, - ShadowDbError, - FileSystem.FileSystem | Path.Path | RuntimeInfo | Crypto.Crypto -> => - loadStackConfig(input.workdir, { context: input.context }).pipe( - Effect.map((config) => overlaySetupTrio(config, input.setup)), - Effect.mapError((cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" })), - ); - -const createOptions = ( - input: ShadowSetupInput, - runtime: StackRuntime, - restoreFrom: string | undefined, - port: number | undefined, - snapshotKey?: string, -): CreateEphemeralPostgresOptions => ({ - databasePassword: Redacted.make(input.password), - jwtSecret: Redacted.make(input.jwtSecret), - jwtExpiry: input.jwtExpiry, - postgresSettings: postgresSettings(input.db.settings), - healthTimeout: `${String(input.healthTimeoutSeconds)}s`, - version: String(input.setup.majorVersion), - runtime, - ...(port === undefined ? {} : { port }), - ...(restoreFrom === undefined ? {} : { restoreFrom }), - ...(snapshotKey === undefined ? {} : { snapshotKey }), -}); - -const connFrom = (handle: EffectEphemeralPostgres, password: string) => ({ - host: handle.host, - port: handle.port, - user: "postgres", - password, - database: "postgres", + return { stack, runtime }; }); -const applyColdCatalog = ( - handle: EffectEphemeralPostgres, +const initialize = Effect.fn("StackShadow.initialize")(function* ( + stack: Stack, + runtime: Runtime, input: ShadowSetupInput, - webhooks: SetupDatabaseOptions["webhooks"], -): Effect.Effect< - void, - ShadowDbError, - FileSystem.FileSystem | Path.Path | Output | RuntimeInfo | Crypto.Crypto -> => - Effect.gen(function* () { - const catalog = yield* Effect.serviceOption(StackCatalogSetup); - if (Option.isNone(catalog)) - return yield* new ShadowDbError({ - message: "stack catalog setup is unavailable", - reason: "database", - }); - const config = yield* loadEphemeralCatalogConfig(input); - yield* catalog.value - .apply({ - target: { - kind: "ephemeral", - projectRoot: input.workdir, - runtime: handle.runtime, - config, - databaseUrl: Redacted.value(handle.url), - databasePassword: Redacted.make(input.password), - jwtSecret: Redacted.make(input.jwtSecret), - ...(handle.networkId === undefined ? {} : { networkId: handle.networkId }), - }, - overlay: { - webhooks, - webhooksEnabled: input.setup.webhooksEnabled, - apiAutoExposeNewTables: input.setup.apiAutoExposeNewTables, - vault: input.setup.vault, - workdir: input.workdir, - announceRoles: false, - }, - }) - .pipe( - Effect.mapError( - (cause) => new ShadowDbError({ message: cause.message, reason: "database" }), - ), - ); - }); - -const artifactIdentityFor = (runtime: StackRuntime, version: string, image: string): string => - runtime.kind === "container" ? `container:${runtime.engine}:${image}` : `native:${version}`; - -const sweepAbandonedPartials = ( - fs: FileSystem.FileSystem, - path: Path.Path, - cacheDir: string, -): Effect.Effect => - Effect.gen(function* () { - const names = yield* fs.readDirectory(cacheDir).pipe(Effect.orElseSucceed(() => [])); - const now = yield* Clock.currentTimeMillis; - yield* Effect.forEach( - names.filter(isStackShadowBaselinePartial), - (fileName) => - Effect.gen(function* () { - const filePath = path.join(cacheDir, fileName); - const info = yield* fs.stat(filePath); - const mtime = Option.getOrUndefined(info.mtime); - if (mtime !== undefined && now - mtime.getTime() > STACK_SHADOW_PARTIAL_ABANDON_MS) { - yield* fs.remove(filePath).pipe(Effect.ignore); - } - }).pipe(Effect.ignore), - { discard: true }, - ); - }); - -const sweepCache = ( - fs: FileSystem.FileSystem, - path: Path.Path, - cacheDir: string, - keepName: string | undefined, -): Effect.Effect => - Effect.gen(function* () { - const now = yield* Clock.currentTimeMillis; - const names = yield* fs.readDirectory(cacheDir).pipe(Effect.orElseSucceed(() => [])); - const entries: Array<{ readonly fileName: string; readonly mtimeMs: number }> = []; - for (const fileName of names) { - if (!isStackShadowBaselineTar(fileName)) continue; - const info = yield* fs.stat(path.join(cacheDir, fileName)).pipe(Effect.option); - if (Option.isNone(info) || Option.isNone(info.value.mtime)) continue; - entries.push({ fileName, mtimeMs: info.value.mtime.value.getTime() }); - } - yield* Effect.forEach( - shadowBaselineTarsToEvict(entries, now, { - keep: SHADOW_BASELINE_KEEP, - maxAgeMs: SHADOW_BASELINE_MAX_AGE_MS, - retainFileName: keepName, - isPublishedTar: isStackShadowBaselineTar, - }), - (fileName) => fs.remove(path.join(cacheDir, fileName)).pipe(Effect.ignore), - { discard: true }, - ); - }); - -const writeStackShadowBaselineTar = ( - fs: FileSystem.FileSystem, - path: Path.Path, - cacheDir: string, - tarPath: string, - exportPgData: (tempPath: string) => Effect.Effect, - skipIfPublished: boolean, -): Effect.Effect => - stackShadowExportMutex.withPermit( - Effect.gen(function* () { - if (skipIfPublished) { - const published = yield* fs.exists(tarPath).pipe(Effect.orElseSucceed(() => false)); - if (published) return; - } - yield* ensurePrivateCacheDir(fs, cacheDir); - yield* sweepAbandonedPartials(fs, path, cacheDir); - const tempPath = `${tarPath}.${String(process.pid)}.partial`; - yield* fs.remove(tempPath).pipe(Effect.ignore); - yield* Effect.gen(function* () { - yield* Effect.scoped( - fs.open(tempPath, { flag: "wx", mode: 0o600 }).pipe( - Effect.mapError( - (cause) => - new ShadowDbError({ - message: `failed to create ${tempPath}: ${cause.message}`, - reason: "filesystem", - }), - ), - Effect.asVoid, - ), - ); - yield* exportPgData(tempPath); - yield* fs.chmod(tempPath, 0o600).pipe( - Effect.mapError( - (cause) => - new ShadowDbError({ - message: `failed to restrict ${tempPath}: ${cause.message}`, - reason: "filesystem", - }), - ), - ); - yield* fs.rename(tempPath, tarPath).pipe( - Effect.mapError( - (cause) => - new ShadowDbError({ - message: `failed to publish ${tarPath}: ${cause.message}`, - reason: "filesystem", - }), - ), - ); - }).pipe(Effect.onError(() => fs.remove(tempPath).pipe(Effect.ignore))); - yield* sweepCache(fs, path, cacheDir, path.basename(tarPath)); - }), - ); - -const mapCreateError = (cause: unknown): ShadowDbError => - new ShadowDbError({ - message: - typeof cause === "object" && cause !== null && "message" in cause - ? String(Reflect.get(cause, "message")) - : String(cause), - reason: "database", - }); - -const runtimeKindFor = (runtime: StackRuntime): string => - runtime.kind === "container" ? `container:${runtime.engine}` : "native"; - -const ephemeralApis = (): Effect.Effect<{ - readonly create: typeof createEphemeralPostgres; - readonly resolveRelease: ( - version?: string, - ) => Effect.Effect; -}> => - Effect.serviceOption(StackEphemeralPostgres).pipe( - Effect.map((value) => - Option.getOrElse(value, () => ({ - create: createEphemeralPostgres, - resolveRelease: resolveEphemeralPostgresRelease, - })), + opts: ShadowOptions, +) { + const settings = yield* Schema.decodeEffect( + Schema.Record(Schema.String, Schema.Union([Schema.String, Schema.Finite, Schema.Boolean])), + )( + Object.fromEntries( + Object.entries(input.db.settings ?? {}).filter(([, value]) => value !== undefined), ), - ); - -const ownCluster = (ephemeral: EffectEphemeralPostgres) => - Effect.addFinalizer(() => ephemeral.stop.pipe(Effect.ignore)); - -export const stackAcquireShadowDatabase = ( - input: ShadowSetupInput, - opts: StackShadowAcquireOpts = {}, -): Effect.Effect< - StackShadowAcquiredHandle, - ShadowDbError | E, - | Output - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | RuntimeInfo - | ChildProcessSpawner.ChildProcessSpawner - | Scope.Scope - | CommandSettings -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const apis = yield* ephemeralApis(); - const output = yield* Output; - const projectRuntime = yield* stackProjectRuntime; - const preference = runtimePreference(projectRuntime, opts.runtime); - const selected: { - readonly runtime: StackRuntime; - readonly dockerFallbackNotice?: string; - } = - preference === undefined - ? yield* selectDefaultRuntimeSelection( - Option.getOrUndefined(yield* Effect.serviceOption(ContainerEngineResolver)), - ).pipe(Effect.mapError(mapCreateError)) - : { - runtime: - preference.kind === "container" - ? { kind: "container", engine: preference.engine ?? "docker" } - : { kind: "native" }, - }; - if (selected.dockerFallbackNotice !== undefined) - yield* output.raw(`${selected.dockerFallbackNotice}\n`, "stderr"); - const runtime = selected.runtime; - const rolesSql = yield* readRolesSql(input.fs, input.path, input.workdir); - const cacheOn = cacheEnabled(input.setup.projectEnvValues, opts.bypassCache === true); - const cacheDir = shadowBaselineCacheDir(path); - const webhooks = opts.webhooks; - yield* ensurePrivateCacheDir(fs, cacheDir); - - const startEmpty = () => - apis - .create(createOptions(input, runtime, undefined, opts.port)) - .pipe(Effect.mapError(mapCreateError)); - - if (!cacheOn) { - const ephemeral = yield* startEmpty(); - yield* ownCluster(ephemeral); - yield* applyColdCatalog(ephemeral, input, webhooks); - return { - url: Redacted.value(ephemeral.url), - host: ephemeral.host, - port: ephemeral.port, - artifactIdentity: ephemeral.artifactIdentity, - runtime: ephemeral.runtime, - baselinePresent: false, - ephemeral, - }; - } - - const jwks = - input.setup.realtimeEnabledForSetup && input.setup.majorVersion >= 15 - ? yield* input.setup.jwks - : ""; - const release = yield* apis - .resolveRelease(String(input.setup.majorVersion)) - .pipe(Effect.mapError(mapCreateError)); - const identity = artifactIdentityFor(runtime, release.version, release.image); - const trioEnabled = - input.setup.authEnabledForSetup || - input.setup.storageEnabledForSetup || - input.setup.realtimeEnabledForSetup; - const stackConfig = trioEnabled ? yield* loadEphemeralCatalogConfig(input) : undefined; - const key = stackShadowCacheKey({ - artifactIdentity: identity, - majorVersion: input.setup.majorVersion, - runtimeKind: runtimeKindFor(runtime), - jwtSecret: input.jwtSecret, + ).pipe(Effect.mapError(shadowError)); + const database = yield* stack.services.create({ + service: "database", + config: { + version: String(input.setup.majorVersion), + databasePassword: Redacted.make(input.password), + jwtSecret: Redacted.make(input.jwtSecret), jwtExpiry: input.jwtExpiry, - dbPassword: input.password, - dbSettings: input.db.settings, - rolesSql, - bootstrapIdentity: databaseBootstrapIdentity, - webhooksEnabled: resolveSetupWebhooksEnabled(webhooks, input.setup.webhooksEnabled), - apiGrantsKept: Option.getOrElse(input.setup.apiAutoExposeNewTables, () => true), + healthTimeoutMs: input.healthTimeoutSeconds * 1_000, + ...(input.rootKey === undefined ? {} : { rootKey: Redacted.make(input.rootKey) }), + settings, + }, + endpoints: { sql: { port: opts.port ?? "auto" } }, + }); + yield* database.start; + yield* database.ready; + const catalog = yield* StackCatalogSetup; + yield* catalog.apply({ + target: { + stack, + database, + databaseServices: [ + ...(input.setup.authEnabledForSetup ? ["auth" as const] : []), + ...(input.setup.storageEnabledForSetup ? ["storage" as const] : []), + ...(input.setup.realtimeEnabledForSetup ? ["realtime" as const] : []), + ], + jwtSecret: input.jwtSecret, + }, + overlay: { + ...(opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }), + webhooksEnabled: input.setup.webhooksEnabled, + apiAutoExposeNewTables: input.setup.apiAutoExposeNewTables, vault: input.setup.vault, - jwks, - storageTargetMigration: input.setup.storageTargetMigration, - authEnabled: input.setup.authEnabledForSetup, - storageEnabled: input.setup.storageEnabledForSetup, - realtimeEnabled: input.setup.realtimeEnabledForSetup, - authArtifact: trioSchemaInitArtifact(input.setup.authEnabledForSetup, "auth", stackConfig), - storageArtifact: trioSchemaInitArtifact( - input.setup.storageEnabledForSetup, - "storage", - stackConfig, - ), - realtimeArtifact: trioSchemaInitArtifact( - input.setup.realtimeEnabledForSetup, - "realtime", - stackConfig, - ), - }); - const tarName = stackShadowBaselineTarFileName(key); - const tarPath = path.join(cacheDir, tarName); - const cached = yield* fs.exists(tarPath).pipe(Effect.orElseSucceed(() => false)); - yield* sweepAbandonedPartials(fs, path, cacheDir); - yield* sweepCache(fs, path, cacheDir, tarName); - - if (cached) { - const restored = yield* Effect.result( - apis.create(createOptions(input, runtime, tarPath, opts.port, key)), - ); - if (Result.isSuccess(restored)) { - yield* ownCluster(restored.success); - yield* touchShadowBaselineTar(fs, tarPath); - return { - url: Redacted.value(restored.success.url), - host: restored.success.host, - port: restored.success.port, - artifactIdentity: restored.success.artifactIdentity, - runtime: restored.success.runtime, - baselinePresent: true, - snapshotKey: key, - ephemeral: restored.success, - }; - } - const output = yield* Output; - yield* output.raw( - `Warning: shadow baseline not cached: ${restored.failure.message}\n`, - "stderr", - ); - } - - const probe = yield* startEmpty(); - yield* ownCluster(probe); - yield* applyColdCatalog(probe, input, webhooks); - const exported = yield* Effect.result( - Effect.gen(function* () { - const rolesSqlNow = yield* readRolesSql(input.fs, input.path, input.workdir); - if (rolesSqlNow !== rolesSql) { - return yield* new ShadowDbError({ - message: "supabase/roles.sql changed during provisioning", - reason: "filesystem", - }); - } - yield* probe.stop.pipe(Effect.mapError(mapCreateError)); - yield* writeStackShadowBaselineTar( - fs, - path, - cacheDir, - tarPath, - (tempPath) => probe.exportPgData(tempPath, key).pipe(Effect.mapError(mapCreateError)), - !cached, - ); - }), - ); - yield* probe.start.pipe(Effect.mapError(mapCreateError)); - if (Result.isFailure(exported)) { - const output = yield* Output; - yield* output.raw( - `Warning: shadow baseline not cached: ${exported.failure.message}\n`, - "stderr", - ); - } - return { - url: Redacted.value(probe.url), - host: probe.host, - port: probe.port, - artifactIdentity: probe.artifactIdentity, - runtime: probe.runtime, - baselinePresent: false, - snapshotKey: Result.isSuccess(exported) ? key : undefined, - ephemeral: probe, - }; + workdir: input.workdir, + announceRoles: false, + }, }); + const credentials = yield* database.credentials({ from: "host" }); + const credentialUrl = credentials.databaseUrl; + const credentialsConn = + credentialUrl === undefined ? undefined : parseConnectionString(credentialUrl); + if (credentialsConn === undefined || credentialUrl === undefined) + return yield* new ShadowDbError({ + message: "Shadow database URL is unavailable", + reason: "connect", + }); + const url = toPostgresURL({ ...credentialsConn, user: "postgres" }); + const conn = credentialsConn; + return { + stack, + database, + url, + host: conn.host, + port: conn.port, + runtime, + version: String(input.setup.majorVersion), + } satisfies StackShadowAcquiredHandle; +}); -export const stackWithShadowDatabase = ( - input: ShadowSetupInput, - use: (handle: StackShadowAcquiredHandle) => Effect.Effect, - opts: StackShadowAcquireOpts = {}, -): Effect.Effect< - A, - E2 | ShadowDbError | E, - | R2 - | Output - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | RuntimeInfo - | ChildProcessSpawner.ChildProcessSpawner - | CommandSettings -> => - Effect.scoped( - Effect.gen(function* () { - const handle = yield* stackAcquireShadowDatabase(input, opts); - return yield* use(handle); - }), - ); - -export const stackPrepareShadowSource = ( - handle: StackShadowAcquiredHandle, +/** Acquires a fresh shadow for callers whose enclosing scope owns its lifetime. */ +export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(function* ( input: ShadowSetupInput, -): Effect.Effect< - Pick, - ShadowDbError, - DbConnection | Output | Scope.Scope | FileSystem.FileSystem | Path.Path -> => - stackMigrateShadow(handle, input).pipe( - Effect.as({ sourceUrl: handle.url, targetUrlOverride: undefined }), + opts: ShadowOptions = {}, +) { + const output = yield* Output; + const namespace = yield* Effect.acquireRelease(acquireNamespace(opts), ({ stack }) => + stack.destroy.pipe( + Effect.catch((cause) => + output.raw( + `Failed to destroy shadow stack ${stack.id}: ${cause.message}. Run supabase stack destroy --stack-id ${stack.id} to remove it.\n`, + "stderr", + ), + ), + ), ); - -export const stackMigrateShadow = ( + return yield* initialize(namespace.stack, namespace.runtime, input, opts); +}, Effect.mapError(shadowError)); + +/** Runs a command against a fresh shadow, then destroys its owned namespace. */ +export const stackWithShadowDatabase = Effect.fn("StackShadow.withDatabase")( + ( + input: ShadowSetupInput, + use: (handle: StackShadowAcquiredHandle) => Effect.Effect, + opts: ShadowOptions = {}, + ) => Effect.scoped(stackAcquireShadowDatabase(input, opts).pipe(Effect.flatMap(use))), +); + +export const stackMigrateShadow = Effect.fn("StackShadow.migrate")(function* ( handle: StackShadowAcquiredHandle, input: ShadowSetupInput, -): Effect.Effect< - void, - ShadowDbError, - DbConnection | Output | Scope.Scope | FileSystem.FileSystem | Path.Path -> => - Effect.scoped( - Effect.gen(function* () { - const migrationsDir = input.path.join(input.workdir, "supabase", "migrations"); - const pending = yield* listLocalMigrationPaths(input.fs, input.path, migrationsDir).pipe( - Effect.mapError( - (cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" }), - ), - ); - const session = yield* connectShadowDatabase(connFrom(handle.ephemeral, input.password)); - yield* applyMigrations( - session, - input.fs, - input.path, - pending, - (message) => new ShadowDbError({ message, reason: "database" }), - ).pipe( - Effect.catchTag("DbConnectError", (cause) => - Effect.fail(new ShadowDbError({ message: cause.message, reason: "connect" })), - ), - ); - }), +) { + const migrationsDir = input.path.join(input.workdir, "supabase", "migrations"); + const pending = yield* listLocalMigrationPaths(input.fs, input.path, migrationsDir).pipe( + Effect.mapError((cause) => new ShadowDbError({ message: cause.message, reason: "filesystem" })), ); + const conn = parseConnectionString(handle.url); + if (conn === undefined) + return yield* new ShadowDbError({ message: "Invalid shadow database URL", reason: "connect" }); + const session = yield* connectShadowDatabase(conn); + yield* applyMigrations( + session, + input.fs, + input.path, + pending, + (message) => new ShadowDbError({ message, reason: "database" }), + ).pipe( + Effect.catchTag("DbConnectError", (cause) => + Effect.fail(new ShadowDbError({ message: cause.message, reason: "connect" })), + ), + Effect.mapError(shadowError), + ); +}); + +export const stackPrepareShadowSource = Effect.fn("StackShadow.prepareSource")( + (handle: StackShadowAcquiredHandle, input: ShadowSetupInput) => + Effect.scoped(stackMigrateShadow(handle, input)).pipe( + Effect.as({ sourceUrl: handle.url, targetUrlOverride: undefined }), + ), +); diff --git a/apps/cli/src/command-internal/stack-shadow.unit.test.ts b/apps/cli/src/command-internal/stack-shadow.unit.test.ts deleted file mode 100644 index e7c94a00ef..0000000000 --- a/apps/cli/src/command-internal/stack-shadow.unit.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { databaseBootstrapIdentity } from "@supabase/stack/effect"; -import { - stackShadowBaselineTarFileName, - stackShadowCacheKey, - isStackShadowBaselinePartial, -} from "./stack-shadow.ts"; - -const overlay = { - webhooksEnabled: false, - apiGrantsKept: true, - vault: [] as const, - jwks: "", - storageTargetMigration: "", - authEnabled: false, - storageEnabled: false, - realtimeEnabled: false, - authArtifact: "", - storageArtifact: "", - realtimeArtifact: "", -}; - -const base = { - artifactIdentity: "native:17.6.1", - majorVersion: 17, - runtimeKind: "native", - jwtSecret: "jwt", - jwtExpiry: 3600, - dbPassword: "postgres", - dbSettings: {}, - rolesSql: "", - bootstrapIdentity: databaseBootstrapIdentity, - ...overlay, -}; - -describe("stackShadowCacheKey", () => { - it("changes when the artifact identity or runtime kind changes", () => { - const native = stackShadowCacheKey(base); - const otherArtifact = stackShadowCacheKey({ - ...base, - artifactIdentity: "native:17.6.2", - }); - const container = stackShadowCacheKey({ - ...base, - artifactIdentity: "container:docker:example", - runtimeKind: "container:docker", - }); - expect(native).toMatch(/^[0-9a-f]{16}$/u); - expect(native).not.toBe(otherArtifact); - expect(native).not.toBe(container); - expect(stackShadowBaselineTarFileName(native)).toBe(`stack-shadow-baseline-${native}.tar`); - }); - - it("recognizes only this module's own partial temp files as abandoned-sweep candidates", () => { - const key = "0123456789abcdef"; - expect(isStackShadowBaselinePartial(`stack-shadow-baseline-${key}.tar.4242.partial`)).toBe( - true, - ); - for (const other of [ - stackShadowBaselineTarFileName(key), - `shadow-baseline-${key}.tar.4242.partial`, - `stack-shadow-baseline-${key}.tar.partial`, - `stack-shadow-baseline-${key}.tar.4242.partial.bak`, - "catalog-local-migrations-abc-123.json", - ]) { - expect(isStackShadowBaselinePartial(other), other).toBe(false); - } - }); - - it("changes when roles.sql, db settings, or bootstrap identity change", () => { - const withRoles = stackShadowCacheKey({ ...base, rolesSql: "create role x;" }); - expect(stackShadowCacheKey(base)).not.toBe(withRoles); - expect(stackShadowCacheKey({ ...base, dbSettings: { max_connections: 20 } })).not.toBe( - stackShadowCacheKey(base), - ); - expect(stackShadowCacheKey({ ...base, bootstrapIdentity: "bootstrap-v2" })).not.toBe( - stackShadowCacheKey(base), - ); - }); - - it("changes when overlay or schema-init membership changes", () => { - expect(stackShadowCacheKey({ ...base, webhooksEnabled: true })).not.toBe( - stackShadowCacheKey(base), - ); - expect(stackShadowCacheKey({ ...base, apiGrantsKept: false })).not.toBe( - stackShadowCacheKey(base), - ); - expect( - stackShadowCacheKey({ - ...base, - vault: [{ name: "a", value: "secret", resolved: true }], - }), - ).not.toBe(stackShadowCacheKey(base)); - expect(stackShadowCacheKey({ ...base, authEnabled: true })).not.toBe(stackShadowCacheKey(base)); - expect( - stackShadowCacheKey({ - ...base, - realtimeEnabled: true, - jwks: '{"keys":[]}', - }), - ).not.toBe(stackShadowCacheKey({ ...base, realtimeEnabled: true, jwks: "{}" })); - expect( - stackShadowCacheKey({ - ...base, - storageEnabled: true, - storageTargetMigration: "20240101000000", - }), - ).not.toBe(stackShadowCacheKey({ ...base, storageEnabled: true })); - expect( - stackShadowCacheKey({ - ...base, - authEnabled: true, - authArtifact: "v2.196.0:ghcr.io/supabase/cli/auth:v2.196.0", - }), - ).not.toBe( - stackShadowCacheKey({ - ...base, - authEnabled: true, - authArtifact: "v2.197.0:ghcr.io/supabase/cli/auth:v2.197.0", - }), - ); - }); -}); diff --git a/apps/cli/src/command-internal/stack-storage.native.integration.test.ts b/apps/cli/src/command-internal/stack-storage.native.integration.test.ts new file mode 100644 index 0000000000..c4992bb0b7 --- /dev/null +++ b/apps/cli/src/command-internal/stack-storage.native.integration.test.ts @@ -0,0 +1,193 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Option, Path, Stream } from "effect"; +import { FetchHttpClient } from "effect/unstable/http"; + +import { CommandPlatformApiFactory } from "../auth/command-platform-api-factory.service.ts"; +import { dbConnectionLayer } from "./db-connection.layer.ts"; +import { stackCatalogSetupLayer } from "./stack-catalog-setup.ts"; +import { + classifyStorageCapability, + describeStorageCapability, + stackStorageEndpointFor, +} from "./stack-storage.ts"; +import { makeStorageGateway } from "./storage-gateway.ts"; +import { + StackApi, + stackApiLayer, + stackTargetResolverLayer, +} from "../commands/experimental/stack/stack.shared.ts"; +import { + mockCommandSettings, + mockTelemetryStateTracked, +} from "../../tests/helpers/command-mocks.ts"; +import { mockOutput, mockTty } from "../../tests/helpers/mocks.ts"; +import { CliArgs } from "../shared/cli/cli-args.service.ts"; +import { stdinLayer } from "../shared/runtime/stdin.layer.ts"; +import { runtimeInfoLayer } from "../shared/runtime/runtime-info.layer.ts"; +import { ExperimentalFlag, YesFlag } from "./global-flags.ts"; +import { stackStart } from "../commands/experimental/stack/start/start.handler.ts"; + +const projectConfig = ` +project_id = "stack-storage-native-integration" + +[api] +enabled = true + +[auth] +enabled = false + +[realtime] +enabled = false + +[storage] +enabled = true + +[edge_runtime] +enabled = false + +[studio] +enabled = false + +[analytics] +enabled = false + +[db.pooler] +enabled = false + +[local_smtp] +enabled = false +`; + +const startFlags = () => ({ + exclude: [], + stack: Option.none(), + stackId: Option.none(), + runtime: "native" as const, + preparation: "on-demand" as const, + eager: false, +}); + +const makeLayers = (root: string) => { + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); + const liveStackApi = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + const resolver = stackTargetResolverLayer.pipe( + Layer.provide(Layer.mergeAll(BunServices.layer, settings, liveStackApi)), + ); + const output = mockOutput(); + const telemetry = mockTelemetryStateTracked(); + const tty = mockTty({ stdinIsTty: false, stdoutIsTty: false }); + return Layer.mergeAll( + BunServices.layer, + FetchHttpClient.layer, + runtimeInfoLayer, + settings, + liveStackApi, + resolver, + output.layer, + telemetry.layer, + stackCatalogSetupLayer, + dbConnectionLayer, + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(YesFlag, false), + Layer.succeed(CliArgs, { args: ["stack", "start"] }), + Layer.succeed(CommandPlatformApiFactory, { make: Effect.die("unused") }), + stdinLayer.pipe(Layer.provide(tty)), + tty, + ); +}; + +describe("native stack Storage gateway", () => { + it.live( + "serves lazy Storage without Auth through bucket and object lifecycle operations", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-storage-native-" }); + yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); + yield* fs.writeFileString(path.join(root, "supabase", "config.toml"), projectConfig); + const layers = makeLayers(root); + yield* Effect.ensuring( + Effect.gen(function* () { + const stackId = yield* stackStart(startFlags()); + const api = yield* StackApi; + const stack = yield* api.open({ + id: stackId, + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(root, "cache"), + }); + const composition = yield* stack.composition.describe; + const members = yield* Effect.forEach(composition.members, (member) => + stack.services.get(member.id).pipe(Effect.map((instance) => ({ member, instance }))), + ); + const storageMember = members.find(({ instance }) => instance.service === "storage"); + expect(storageMember?.member.activation).toBe("lazy"); + expect(members.some(({ instance }) => instance.service === "auth")).toBe(false); + + const credentials = yield* stackStorageEndpointFor(stack); + const gateway = yield* makeStorageGateway({ + baseUrl: credentials.baseUrl, + apiKey: credentials.apiKey, + userAgent: "stack-storage-native-integration", + }); + const bucket = `native-${crypto.randomUUID().slice(0, 8)}`; + const local = path.join(root, "payload.txt"); + yield* fs.writeFileString(local, "native storage payload"); + + yield* gateway.createBucket(bucket, { + public: false, + fileSizeLimit: 0, + allowedMimeTypes: [], + }); + expect((yield* gateway.listBuckets()).some(({ name }) => name === bucket)).toBe(true); + + yield* gateway.uploadObject(`${bucket}/source.txt`, local, { + contentType: "text/plain", + cacheControl: "max-age=60", + overwrite: false, + }); + expect((yield* gateway.listObjects(bucket, "", 0)).map(({ name }) => name)).toContain( + "source.txt", + ); + const downloaded = yield* gateway + .downloadObject(`${bucket}/source.txt`) + .pipe(Stream.runCollect); + expect( + new TextDecoder().decode(Uint8Array.from(downloaded.flatMap((chunk) => [...chunk]))), + ).toBe("native storage payload"); + + yield* gateway.moveObject(bucket, "source.txt", "moved.txt"); + expect((yield* gateway.listObjects(bucket, "", 0)).map(({ name }) => name)).toContain( + "moved.txt", + ); + yield* gateway.deleteObjects(bucket, ["moved.txt"]); + expect(yield* gateway.listObjects(bucket, "", 0)).toEqual([]); + yield* gateway.deleteBucket(bucket); + + if (storageMember === undefined) return yield* Effect.die("Storage member missing"); + yield* stack.composition.stop; + const stopped = yield* storageMember.instance.status; + expect(stopped.lifecycle).toBe("stopped"); + expect(classifyStorageCapability(stopped)).toBe("unusable"); + expect(describeStorageCapability(stopped)).toBe("Storage is stopped for this stack."); + }), + Effect.gen(function* () { + const api = yield* StackApi; + const stateRoot = path.join(root, "stacks"); + const registered = yield* api.discover({ stateRoot }); + yield* Effect.forEach(registered, ({ definition }) => + api + .open({ + id: definition.id, + stateRoot, + cacheRoot: path.join(root, "cache"), + }) + .pipe(Effect.flatMap((stack) => stack.destroy)), + ); + }).pipe(Effect.catch((cause) => Effect.die(cause))), + ).pipe(Effect.provide(layers)); + }).pipe(Effect.provide(BunServices.layer)), + { timeout: 180_000 }, + ); +}); diff --git a/apps/cli/src/command-internal/stack-storage.ts b/apps/cli/src/command-internal/stack-storage.ts index adee1cf95e..f8ecb809b1 100644 --- a/apps/cli/src/command-internal/stack-storage.ts +++ b/apps/cli/src/command-internal/stack-storage.ts @@ -1,12 +1,10 @@ /** - * Resolves Storage REST credentials from a managed stack. The endpoint is the stack's `api` - * gateway URL; Storage is lazy-activated by the gateway itself on the first request - * (`packages/stack/src/gateway/HttpGateway.ts`), so this module never polls for readiness and - * never falls back to the legacy Kong-derived endpoint. + * Resolves Storage REST credentials from a managed stack. The endpoint is a composition HTTP + * endpoint; lazy Storage members may be woken by the first gateway request. */ -import { Data, Effect, Match, Option, Redacted } from "effect"; -import type { CapabilityStatus, EffectStack, StackStatus } from "@supabase/stack/effect"; +import { Data, Effect, Match, Option, Path, Redacted } from "effect"; +import type { Observation, ServiceInstance, Stack } from "@supabase/stack/effect"; import { actionability, type CliErrorActionabilityDeclaration, @@ -19,6 +17,7 @@ import { stackOpenProjectBy } from "./stack-local-database.ts"; import { sanitizeInlineName } from "./http-errors.ts"; import { StorageGatewayStatusError } from "./storage-gateway.errors.ts"; import type { StorageCredentials } from "./storage-credentials.ts"; +import { generateGoJwt } from "./go-jwt.ts"; const INSPECT_OR_RESTART_SUGGESTION = "Run supabase stack status to inspect the stack, or supabase stack restart."; @@ -38,7 +37,6 @@ export class StackStorageCapabilityError extends Data.TaggedError("StackStorageC readonly message: string; readonly suggestion?: string; readonly disabled?: boolean; - /** The original error, preserved for `--debug` (e.g. a `StorageGatewayStatusError`). */ readonly cause?: unknown; }> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { @@ -46,152 +44,177 @@ export class StackStorageCapabilityError extends Data.TaggedError("StackStorageC } } -/** - * Classifies a stack's Storage capability for reporting purposes: `"proceed"` when the gateway - * can serve requests (including the states it activates or wakes on demand), `"disabled"` when - * Storage was excluded from the stack, and `"unusable"` when it stopped, failed, or is absent. - */ +const capabilityState = (observation: Observation | undefined) => { + if (observation === undefined) return undefined; + if (observation.error !== undefined) return "failed" as const; + if (observation.health === "unhealthy") return "failed" as const; + if (observation.lifecycle === "stopped" && observation.wakeEnabled) return "dormant" as const; + if (observation.lifecycle === "stopping" && observation.wakeEnabled) return "dormant" as const; + if (observation.lifecycle === "running" && observation.health === "healthy") + return "ready" as const; + if (observation.lifecycle === "running") return "starting" as const; + return observation.lifecycle; +}; + +/** Resolves bucket-seeding credentials from a configured Storage instance. */ +export const stackStorageCredentialsFor = ( + storage: ServiceInstance<"storage">, + jwtSecret: string, +): Effect.Effect => + storage.status.pipe( + Effect.mapError((cause) => new StackStorageUnavailableError({ message: cause.message })), + Effect.flatMap( + ( + observation, + ): Effect.Effect< + StorageCredentials, + StackStorageUnavailableError | StackStorageCapabilityError + > => { + if (classifyStorageCapability(observation) !== "proceed") + return Effect.fail( + new StackStorageUnavailableError({ message: describeStorageCapability(observation) }), + ); + return storageCredentialsFrom(observation, jwtSecret); + }, + ), + ); + +/** Classifies the composition Storage observation without starting it. */ export const classifyStorageCapability = ( - capability: CapabilityStatus | undefined, + observation: Observation | undefined, ): "proceed" | "disabled" | "unusable" => { - if (capability === undefined) return "unusable"; - return Match.value(capability.state).pipe( - Match.when("disabled", () => "disabled" as const), - Match.whenOr("dormant", "starting", "ready", "stopping", () => "proceed" as const), - Match.whenOr("stopped", "failed", () => "unusable" as const), + const state = capabilityState(observation); + if (state === undefined) return "disabled"; + return Match.value(state).pipe( + Match.when("dormant", () => "proceed" as const), + Match.when("starting", () => "proceed" as const), + Match.when("ready", () => "proceed" as const), + Match.whenOr("stopped", "stopping", "failed", () => "unusable" as const), Match.exhaustive, ); }; -// Callers concatenate this with a following sentence (a next-step suggestion), so the -// sanitized error always needs its own terminal punctuation. const withTerminalPunctuation = (text: string): string => (/[.!?]$/.test(text) ? text : `${text}.`); -/** - * Renders the current Storage capability state in the same wording `stackStorageEndpointFor` - * fails with, so pre-check warnings (`db reset`, `stack start`) and the endpoint resolver never - * drift apart. - */ -export const describeStorageCapability = (capability: CapabilityStatus | undefined): string => - Match.value(classifyStorageCapability(capability)).pipe( - Match.when("disabled", () => "Storage is disabled for this stack."), - Match.when("proceed", () => "Storage is available for this stack."), - Match.when("unusable", () => - capability === undefined - ? "The stack reports no Storage capability." - : capability.state === "failed" - ? `Storage failed to start for this stack${ - capability.error === undefined - ? "." - : `: ${withTerminalPunctuation(sanitizeInlineName(capability.error))}` - }` - : "Storage is stopped for this stack.", - ), - Match.exhaustive, - ); +/** Renders the current Storage observation for command diagnostics. */ +export const describeStorageCapability = (observation: Observation | undefined): string => { + const state = capabilityState(observation); + if (state === undefined) return "Storage is disabled for this stack."; + if (state === "failed") { + const detail = observation?.error?.message; + return `Storage failed to start for this stack${ + detail === undefined ? "." : `: ${withTerminalPunctuation(sanitizeInlineName(detail))}` + }`; + } + if (state === "stopped" || state === "stopping") return "Storage is stopped for this stack."; + return "Storage is available for this stack."; +}; -const notRunningSuggestion = (lifecycle: Exclude): string => +const notRunningSuggestion = (lifecycle: "starting" | "stopping" | "stopped"): string => Match.value(lifecycle).pipe( Match.when("starting", () => "The stack is still starting; retry shortly."), Match.when( "stopping", () => "The stack is shutting down; run supabase start once it has stopped.", ), - Match.when( - "destroying", - () => "The stack is being destroyed; run supabase start to create a new one.", - ), - Match.whenOr("stopped", "unconfigured", () => "Run supabase start."), + Match.when("stopped", () => "Run supabase start."), Match.exhaustive, ); -/** Storage REST credentials resolved from an already-open stack handle and its current status. */ +const storageCredentialsFrom = ( + observation: Observation, + jwtSecret: string, +): Effect.Effect => { + const endpoint = observation.endpoints.find(({ name }) => name === "http"); + return endpoint === undefined + ? Effect.fail( + new StackStorageCapabilityError({ message: "The stack exposes no Storage endpoint." }), + ) + : Effect.succeed({ + baseUrl: `http://${endpoint.host}:${endpoint.port}`, + apiKey: generateGoJwt(jwtSecret, "service_role"), + localKongCa: undefined, + }); +}; + +/** Resolves the composition Storage endpoint without launching the owner or any member. */ export const stackStorageEndpointFor = ( - stack: EffectStack, - status: StackStatus, + stack: Stack, ): Effect.Effect => Effect.gen(function* () { - if (status.lifecycle !== "running") { - return yield* new StackStorageUnavailableError({ - message: - status.lifecycle === "unconfigured" - ? "The stack has not been started yet." - : `The stack is ${status.lifecycle}, not running.`, - suggestion: notRunningSuggestion(status.lifecycle), - }); - } - const apiEndpoint = status.endpoints.api; - if (apiEndpoint === undefined) { - return yield* new StackStorageUnavailableError({ - message: "The stack exposes no API gateway endpoint.", - suggestion: INSPECT_OR_RESTART_SUGGESTION, - }); - } - const capability = status.capabilities.find((entry) => entry.name === "storage"); - const classification = classifyStorageCapability(capability); - if (classification === "disabled") { + const composition = yield* stack.composition.describe.pipe( + Effect.mapError((cause) => new StackStorageUnavailableError({ message: cause.message })), + ); + const members = yield* Effect.forEach(composition.members, ({ id }) => + stack.services + .get(id) + .pipe( + Effect.mapError((cause) => new StackStorageUnavailableError({ message: cause.message })), + ), + ); + const storage = members.find((instance) => instance.service === "storage"); + if (storage === undefined) return yield* new StackStorageCapabilityError({ - message: describeStorageCapability(capability), + message: "Storage is disabled for this stack.", suggestion: - "Set [storage] enabled = true in supabase/config.toml, then run supabase stack stop followed by supabase stack start without -x storage, and retry.", + "Set [storage] enabled = true in supabase/config.toml, then run supabase start without --exclude storage.", disabled: true, }); - } - if (classification === "unusable") { - return yield* new StackStorageCapabilityError({ - message: describeStorageCapability(capability), - suggestion: - capability === undefined - ? INSPECT_OR_RESTART_SUGGESTION - : "Run supabase stack restart, then retry.", + const database = members.find((instance) => instance.service === "database"); + if (database === undefined) + return yield* new StackStorageUnavailableError({ + message: "The stack has no primary database composition member.", + suggestion: INSPECT_OR_RESTART_SUGGESTION, }); - } - const credentials = yield* stack.credentials.pipe( + const databaseObservation = yield* database.status.pipe( Effect.mapError((cause) => new StackStorageUnavailableError({ message: cause.message })), ); - if (credentials.api === undefined) { + if (databaseObservation.lifecycle !== "running" || databaseObservation.health !== "healthy") return yield* new StackStorageUnavailableError({ - message: "The stack exposes no API credentials because Auth is disabled.", + message: "The primary database is not ready for Storage requests.", suggestion: - "Set [auth] enabled = true in supabase/config.toml, then run supabase stack stop followed by supabase stack start without -x auth, and retry.", + databaseObservation.health === "starting" + ? notRunningSuggestion("starting") + : databaseObservation.lifecycle === "starting" || + databaseObservation.lifecycle === "stopping" || + databaseObservation.lifecycle === "stopped" + ? notRunningSuggestion(databaseObservation.lifecycle) + : INSPECT_OR_RESTART_SUGGESTION, + }); + const storageObservation = yield* storage.status.pipe( + Effect.mapError((cause) => new StackStorageCapabilityError({ message: cause.message })), + ); + if (classifyStorageCapability(storageObservation) !== "proceed") + return yield* new StackStorageCapabilityError({ + message: describeStorageCapability(storageObservation), + suggestion: "Run supabase stack restart, then retry.", }); - } - return { - baseUrl: apiEndpoint.url, - apiKey: Redacted.value(credentials.api.serviceRoleJwt), - localKongCa: undefined, - } satisfies StorageCredentials; + if (databaseObservation.config.service !== "database") + return yield* new StackStorageUnavailableError({ + message: "The primary database configuration is unavailable.", + suggestion: INSPECT_OR_RESTART_SUGGESTION, + }); + return yield* storageCredentialsFrom( + storageObservation, + Redacted.value(databaseObservation.config.config.jwtSecret), + ); }); -/** Storage REST credentials for the project stack, opening it from the command's workdir. */ +/** Storage credentials for the project stack, opening it without launching the owner. */ export const stackStorageEndpoint: Effect.Effect< StorageCredentials, StackStorageUnavailableError | StackStorageCapabilityError, - CommandSettings + CommandSettings | StackApi | Path.Path > = Effect.gen(function* () { - const api = yield* Effect.serviceOption(StackApi); - if (Option.isNone(api)) { - return yield* new StackStorageUnavailableError({ - message: "The stack API is unavailable for this command.", - suggestion: "Re-run with --debug and report this at https://github.com/supabase/cli/issues.", - }); - } const opened = yield* stackOpenProjectBy( - (cause) => - new StackStorageUnavailableError({ - message: cause.message, - }), + (cause) => new StackStorageUnavailableError({ message: cause.message }), ); - if (Option.isNone(opened)) { + if (Option.isNone(opened)) return yield* new StackStorageUnavailableError({ message: "No stack is registered for this project.", suggestion: "Run supabase start to create it.", }); - } - const status = yield* opened.value.status.pipe( - Effect.mapError((cause) => new StackStorageUnavailableError({ message: cause.message })), - ); - return yield* stackStorageEndpointFor(opened.value, status); + return yield* stackStorageEndpointFor(opened.value); }); /** diff --git a/apps/cli/src/command-internal/test-db.handler.ts b/apps/cli/src/command-internal/test-db.handler.ts index 81da628cb6..7769dc7ce1 100644 --- a/apps/cli/src/command-internal/test-db.handler.ts +++ b/apps/cli/src/command-internal/test-db.handler.ts @@ -1,12 +1,14 @@ import * as nodePath from "node:path"; import { Effect, FileSystem, Option, Path } from "effect"; +import { postgres, type DatabaseInstance, type Stack } from "@supabase/stack/effect"; import { CliArgs } from "../shared/cli/cli-args.service.ts"; import { CommandSettings } from "../config/command-settings.service.ts"; import { TelemetryState } from "../telemetry/telemetry-state.service.ts"; import { DbConfigResolver } from "./db-config.service.ts"; import { readDbToml } from "./db-config.toml-read.ts"; -import { DbConnection } from "./db-connection.service.ts"; +import { DbConnection, type PgConnInput } from "./db-connection.service.ts"; +import { LocalDbRunningError } from "./db-bootstrap/local-db-running.ts"; import { DockerRun } from "./docker-run.service.ts"; import { resolveDbTargetFlags } from "./db-target-flags.ts"; import { DebugFlag, DnsResolverFlag, NetworkIdFlag } from "./global-flags.ts"; @@ -21,13 +23,14 @@ import { } from "./test-db.errors.ts"; import { buildPgProveArgs } from "./test-db.pg-prove-args.ts"; import { currentStackBackend } from "./stack-backend.ts"; -import { stackProjectDatabaseVersion, stackRequireProjectRuntime } from "./stack-local-database.ts"; import { rewriteDumpHostForToolContainer, toolContainerUsesHostNetwork, } from "./postgres-client.run.ts"; import { isBitbucketPipeline } from "./bitbucket-pipeline.ts"; import { BundledPostgresClient, resolveBundledPostgresRuntime } from "./bundled-postgres-client.ts"; +import { parseConnectionString } from "./db-config.parse.ts"; +import { stackOpenReadyProject } from "./stack-local-database.ts"; const ENABLE_PGTAP = "create extension if not exists pgtap with schema extensions"; const DISABLE_PGTAP = "drop extension if exists pgtap"; @@ -46,6 +49,66 @@ const VERDICT_PREFIX = "Result: "; const NO_TESTS_VERDICT = "Result: NOTESTS"; const FILES_SUMMARY = /^Files=(\d+),/; +type ManagedStack = { + readonly stack: Stack; + readonly database: DatabaseInstance; + readonly connection: PgConnInput; + readonly major: 15 | 17; +}; + +const managedStackFor = Effect.fn("test.db.managedStack")(function* () { + const opened = yield* stackOpenReadyProject; + if (Option.isNone(opened)) + return yield* new LocalDbRunningError({ + message: + "The local stack database is unavailable or not running; start it with `supabase start`.", + }); + const { stack, database } = opened.value; + const status = yield* database.status.pipe( + Effect.mapError( + (cause) => + new LocalDbRunningError({ + message: `unable to inspect the local database: ${cause.message}`, + }), + ), + ); + if (status.lifecycle !== "running") + return yield* new LocalDbRunningError({ message: "The local stack database is not running." }); + if (status.config.service !== "database") + return yield* new LocalDbRunningError({ + message: "The local stack primary service is not a database.", + }); + const major = + status.config.config.version === "15" + ? 15 + : status.config.config.version === "17" + ? 17 + : undefined; + if (major === undefined) + return yield* new LocalDbRunningError({ + message: `The local database major version ${status.config.config.version} is not supported by pg_prove.`, + }); + const credentials = yield* database.credentials({ from: "runtime" }).pipe( + Effect.mapError( + (cause) => + new LocalDbRunningError({ + message: `unable to read local database credentials: ${cause.message}`, + }), + ), + ); + const databaseUrl = credentials.databaseUrl; + if (databaseUrl === undefined) + return yield* new LocalDbRunningError({ + message: "The local database did not provide runtime credentials.", + }); + const connection = parseConnectionString(databaseUrl); + if (connection === undefined) + return yield* new LocalDbRunningError({ + message: "The local database returned malformed runtime credentials.", + }); + return { stack, database, connection, major } satisfies ManagedStack; +}); + function sanitizeProjectId(src: string): string { return src .replace(/[^a-zA-Z0-9_.-]+/g, "_") @@ -109,36 +172,43 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { }); const backend = yield* currentStackBackend; - const stackRuntime = - backend.kind === "stack" && connType === "local" - ? yield* stackRequireProjectRuntime - : undefined; + const managedStack = + backend.kind === "stack" && connType === "local" ? yield* managedStackFor() : undefined; const proveRuntime = - backend.kind === "stack" - ? yield* resolveBundledPostgresRuntime(stackRuntime, runtimeInfo.platform, runtimeInfo.arch) + backend.kind === "stack" && managedStack === undefined + ? yield* resolveBundledPostgresRuntime(undefined, runtimeInfo.platform, runtimeInfo.arch) : undefined; const useNativeProve = proveRuntime?.kind === "native"; const stackPublishedProve = backend.kind === "stack" && !useNativeProve; const networkId = Option.getOrUndefined(networkIdFlag); const dumpUsesHostNetwork = toolContainerUsesHostNetwork(networkId); + const proveConnection = managedStack?.connection ?? conn; const runEnv = { - PGHOST: useNativeProve - ? connType === "local" - ? "127.0.0.1" - : conn.host - : stackPublishedProve - ? rewriteDumpHostForToolContainer(conn.host, { - platform: runtimeInfo.platform, - usesHostNetwork: dumpUsesHostNetwork, - }) - : isLocal - ? "db" - : conn.host, - PGPORT: isLocal && backend.kind !== "stack" ? "5432" : String(conn.port), - PGUSER: conn.user, - PGPASSWORD: conn.password, - PGDATABASE: conn.database, + PGHOST: + managedStack !== undefined + ? proveConnection.host + : useNativeProve + ? connType === "local" + ? "127.0.0.1" + : conn.host + : stackPublishedProve + ? rewriteDumpHostForToolContainer(conn.host, { + platform: runtimeInfo.platform, + usesHostNetwork: dumpUsesHostNetwork, + }) + : isLocal + ? "db" + : conn.host, + PGPORT: + managedStack !== undefined + ? String(proveConnection.port) + : isLocal && backend.kind !== "stack" + ? "5432" + : String(conn.port), + PGUSER: managedStack === undefined ? proveConnection.user : conn.user, + PGPASSWORD: managedStack === undefined ? proveConnection.password : conn.password, + PGDATABASE: managedStack === undefined ? proveConnection.database : conn.database, }; // A non-empty `--network-id` overrides everything (even host mode); @@ -218,12 +288,36 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { } return output.rawBytes(chunk, "stdout"); }); + if (managedStack !== undefined) { + const hostPath = args.hostPaths[0]; + const hostWorkingDir = + hostPath === undefined + ? undefined + : nodePath.extname(hostPath) !== "" + ? nodePath.dirname(hostPath) + : hostPath; + return yield* managedStack.stack.tools + .run(postgres.pgProve({ major: managedStack.major }), { + args: args.cmd.slice(1), + env: runEnv, + pgProve: { + mounts: args.mounts, + ...(hostWorkingDir === undefined ? {} : { cwd: hostWorkingDir }), + ...(Option.isSome(args.workingDir) ? { workingDir: args.workingDir.value } : {}), + }, + stdout: onStdout, + stderr: (bytes) => output.rawBytes(bytes, "stderr"), + }) + .pipe( + Effect.mapError( + (cause) => new TestDbRunError({ message: `pg_prove run failed: ${cause.message}` }), + ), + ); + } if (backend.kind === "stack") { const bundled = yield* BundledPostgresClient; const toml = yield* readDbToml(fs, path, cliSettings.workdir); - const version = - (connType === "local" ? yield* stackProjectDatabaseVersion : undefined) ?? - String(toml.majorVersion); + const version = String(toml.majorVersion); const hostPath = args.hostPaths[0]; const hostWorkingDir = hostPath === undefined @@ -277,7 +371,7 @@ export const testDb = Effect.fn("test.db")(function* (flags: TestDbFlags) { if (exitCode !== 0) { return yield* Effect.fail( new TestDbRunError({ - message: `error running ${useNativeProve ? "pg_prove" : "container"}: exit ${exitCode}`, + message: `error running ${managedStack !== undefined || useNativeProve ? "pg_prove" : "container"}: exit ${exitCode}`, }), ); } diff --git a/apps/cli/src/command-internal/test-db.integration.test.ts b/apps/cli/src/command-internal/test-db.integration.test.ts index f6c3defac2..4f69a6dadd 100644 --- a/apps/cli/src/command-internal/test-db.integration.test.ts +++ b/apps/cli/src/command-internal/test-db.integration.test.ts @@ -2,7 +2,7 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { ConfigProvider, Effect, Exit, Layer, Option, Stream } from "effect"; +import { ConfigProvider, Effect, Exit, Layer, Option } from "effect"; import { mockOutput } from "../../tests/helpers/mocks.ts"; import { @@ -18,13 +18,7 @@ import { DbConnectError, DbExecError } from "./db-connection.errors.ts"; import { DbConnection, type DbSession, type PgConnInput } from "./db-connection.service.ts"; import { DockerRunError } from "./docker-run.errors.ts"; import { DockerRun, type DockerRunOpts } from "./docker-run.service.ts"; -import { BundledPostgresClient } from "./bundled-postgres-client.ts"; -import { - ContainerEngineResolver, - StackIdSchema, - type EffectStack, - type RunPostgresClientOptions, -} from "@supabase/stack/effect"; +import { BundledPostgresClient, type RunPostgresClientOptions } from "./bundled-postgres-client.ts"; import { testDb } from "./test-db.handler.ts"; import { stackBackendLayer } from "./stack-backend.ts"; import { StackApi } from "./stack-api.ts"; @@ -44,6 +38,13 @@ const REMOTE_CONN: PgConnInput = { database: "postgres", }; +const stackApiStub = Layer.succeed(StackApi, { + create: () => Effect.die("stack API unused"), + open: () => Effect.die("stack API unused"), + discover: () => Effect.die("stack API unused"), + resolveIdentity: () => Effect.die("stack API unused"), +}); + function mockResolver(opts: { conn?: PgConnInput; isLocal?: boolean } = {}) { const calls: Array<{ readonly connType: string; @@ -216,12 +217,6 @@ function mockBundledPostgresClient(opts: { }; } -const containerEngineResolver = (installed: boolean) => - Layer.succeed(ContainerEngineResolver, { - isInstalled: () => Effect.succeed(installed), - resolve: () => Effect.die("unused"), - }); - const runtimeInfoLayer = (platform: NodeJS.Platform, arch?: NodeJS.Architecture) => Layer.succeed(RuntimeInfo, { cwd: "/work/project", @@ -267,7 +262,7 @@ function setup(opts: SetupOpts = {}) { connection.layer, docker.layer, bundled.layer, - containerEngineResolver(opts.dockerInstalled ?? true), + stackApiStub, mockCommandSettings({ workdir: opts.workdir ?? "/work/project", projectId: Option.none() }), telemetry.layer, runtimeInfoLayer(opts.platform ?? "linux", opts.arch), @@ -420,7 +415,7 @@ describe("test db integration", () => { }), ); expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "container", engine: "docker" }); + expect(bundled.lastOpts?.runtime).toEqual({ kind: "native" }); expect(bundled.lastOpts?.network).toBe("host"); expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("127.0.0.1"); expect(bundled.lastOpts?.env?.["PGPORT"]).toBe("54322"); @@ -428,127 +423,6 @@ describe("test db integration", () => { }).pipe(Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack")))); }); - const PROVE_STACK_ID = StackIdSchema.make("e".repeat(64)); - const unusedProve = () => Effect.die("unused"); - const proveStackApi = ( - runtime: { kind: "native" } | { kind: "container"; engine: "docker" }, - databaseVersion = "17.6.1.168", - ) => { - const stack: EffectStack = { - id: PROVE_STACK_ID, - status: Effect.succeed({ - id: PROVE_STACK_ID, - lifecycle: "running", - desiredLifecycle: "running", - runtime, - endpoints: {}, - versions: { database: databaseVersion }, - capabilities: [], - artifacts: [], - }), - credentials: Effect.die("unused"), - prepare: unusedProve, - start: unusedProve, - stop: Effect.die("unused"), - destroy: Effect.die("unused"), - resetDatabase: Effect.die("unused"), - logs: unusedProve, - followLogs: () => Stream.empty, - }; - return Layer.succeed(StackApi, { - createStack: unusedProve, - findStack: () => - Effect.succeed( - Option.some({ - id: PROVE_STACK_ID, - projectRoot: "/work/project", - name: "default", - branchContext: "main", - runtime, - desiredLifecycle: "running", - }), - ), - discoverStacks: unusedProve, - openStack: () => Effect.succeed(stack), - inspectStack: unusedProve, - }); - }; - - it.live("stack --local native uses bundled pg_prove, not supabase/pg_prove", () => { - const { layer, docker, bundled } = setup({ isLocal: true }); - return Effect.gen(function* () { - yield* testDb(flags({ local: true })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "native" }); - expect(bundled.lastOpts?.argv.slice(0, 5)).toEqual([ - "pg_prove", - "--ext", - ".pg", - "--ext", - ".sql", - ]); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("127.0.0.1"); - }).pipe( - Effect.provide( - Layer.mergeAll(layer, stackBackendLayer("stack"), proveStackApi({ kind: "native" })), - ), - ); - }); - - it.live("stack --local docker uses catalog pg_prove, not supabase/pg_prove", () => { - const { layer, docker, bundled } = setup({ isLocal: true }); - return Effect.gen(function* () { - yield* testDb(flags({ local: true })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "container", engine: "docker" }); - expect(bundled.lastOpts?.argv.slice(0, 5)).toEqual([ - "pg_prove", - "--ext", - ".pg", - "--ext", - ".sql", - ]); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("127.0.0.1"); - expect(bundled.lastOpts?.mounts).toEqual([ - { - source: "/work/project/supabase/tests", - target: "/work/project/supabase/tests", - readOnly: true, - }, - ]); - expect(bundled.lastOpts?.cwd).toBe("/work/project/supabase/tests"); - expect(bundled.lastOpts?.extraHosts).toEqual(["host.docker.internal:host-gateway"]); - expect(bundled.lastOpts?.securityOpt).toEqual(["label:disable"]); - }).pipe( - Effect.provide( - Layer.mergeAll( - layer, - stackBackendLayer("stack"), - proveStackApi({ kind: "container", engine: "docker" }), - ), - ), - ); - }); - - it.live( - "stack --local on Windows uses catalog container pg_prove against host.docker.internal", - () => { - const { layer, docker, bundled } = setup({ isLocal: true, platform: "win32" }); - return Effect.gen(function* () { - yield* testDb(flags({ local: true })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "container", engine: "docker" }); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("host.docker.internal"); - expect(bundled.lastOpts?.network).toBe("host"); - expect(bundled.lastOpts?.extraHosts).toEqual([]); - }).pipe( - Effect.provide( - Layer.mergeAll(layer, stackBackendLayer("stack"), proveStackApi({ kind: "native" })), - ), - ); - }, - ); - it.live("stack --db-url native keeps PGHOST when isLocal is true", () => { const { layer, docker, bundled } = setup({ conn: { ...LOCAL_CONN, host: "db.internal" }, diff --git a/apps/cli/src/command-internal/test-db.layers.ts b/apps/cli/src/command-internal/test-db.layers.ts index 6595bf5bcc..1da3775110 100644 --- a/apps/cli/src/command-internal/test-db.layers.ts +++ b/apps/cli/src/command-internal/test-db.layers.ts @@ -47,6 +47,6 @@ export const testDbRuntimeLayer = (commandPath: ReadonlyArray) => identityStitchLayer, telemetryStateLayer, stackApiLayer, - bundledPostgresClientLayer, + bundledPostgresClientLayer.pipe(Layer.provide(dockerRunLayer), Layer.provide(cliSettings)), commandRuntimeLayer(commandPath), ); diff --git a/apps/cli/src/command-internal/test-db.layers.unit.test.ts b/apps/cli/src/command-internal/test-db.layers.unit.test.ts index c9ed4b7832..b1038fa719 100644 --- a/apps/cli/src/command-internal/test-db.layers.unit.test.ts +++ b/apps/cli/src/command-internal/test-db.layers.unit.test.ts @@ -20,7 +20,6 @@ import { } from "../../tests/helpers/mocks.ts"; import { isolatedHomeLayer, - mockCommandSettings, mockTelemetryStateLayer, useTempWorkdir, } from "../../tests/helpers/command-mocks.ts"; @@ -76,8 +75,6 @@ function ambientStubs() { return Layer.mergeAll( BunServices.layer, - // The runtime layer under test builds the real commandSettingsLayer against - // the real filesystem — see isolatedHomeLayer's docs. isolatedHomeLayer(tempRoot.current), mockTty(), mockProcessControl().layer, @@ -85,7 +82,6 @@ function ambientStubs() { mockTelemetryRuntime(), out.layer, flagLayers, - mockCommandSettings({ workdir: "/tmp/test-db-layers-test" }), mockTelemetryStateLayer, heavyServiceStubs, ); diff --git a/apps/cli/src/command-internal/test-db.native.integration.test.ts b/apps/cli/src/command-internal/test-db.native.integration.test.ts new file mode 100644 index 0000000000..300cbf8ae1 --- /dev/null +++ b/apps/cli/src/command-internal/test-db.native.integration.test.ts @@ -0,0 +1,207 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { FetchHttpClient } from "effect/unstable/http"; +import { Cause, Effect, Exit, FileSystem, Layer, Option, Redacted } from "effect"; +import { Analytics } from "../shared/telemetry/analytics.service.ts"; +import { CommandRuntime } from "../shared/runtime/command-runtime.service.ts"; +import { mockOutput, mockProcessControl } from "../../tests/helpers/mocks.ts"; +import { + mockCommandSettings, + mockTelemetryStateTracked, +} from "../../tests/helpers/command-mocks.ts"; +import { CliArgs } from "../shared/cli/cli-args.service.ts"; +import { RuntimeInfo } from "../shared/runtime/runtime-info.service.ts"; +import { DbConfigResolver } from "./db-config.service.ts"; +import { BundledPostgresClient } from "./bundled-postgres-client.ts"; +import { dbConnectionLayer } from "./db-connection.layer.ts"; +import { DbConnection } from "./db-connection.service.ts"; +import { parseConnectionString } from "./db-config.parse.ts"; +import { DebugFlag, DnsResolverFlag, NetworkIdFlag } from "./global-flags.ts"; +import { StackApi, stackApiLayer } from "./stack-api.ts"; +import { stackBackendLayer } from "./stack-backend.ts"; +import { testDb } from "./test-db.handler.ts"; +import { runTestDbCommand } from "./test-db.command-handler.ts"; +import { DockerRun } from "./docker-run.service.ts"; +import { StackError } from "@supabase/stack/effect"; + +const runtimes = ["native", "docker"] as const; +const liveStackApi = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + +const databaseCreation = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("cli-pgtap-password"), + jwtSecret: Redacted.make("cli-pgtap-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, +}; + +const flags = (path: string) => ({ + paths: [path], + dbUrl: Option.none(), + linked: false, + local: true, + projectRef: Option.none(), +}); + +describe("managed test db pgTAP", { timeout: 180_000 }, () => { + for (const runtime of runtimes) { + it.live(`${runtime} streams passing and failing pgTAP through test db`, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "cli-test-db-" }); + const tests = `${root}/tests`; + yield* fs.makeDirectory(tests); + const passing = `${tests}/passing.sql`; + const failing = `${tests}/failing.sql`; + yield* fs.writeFileString( + passing, + "SELECT plan(1); SELECT is(current_user::text, 'postgres', 'runs as postgres'); SELECT * FROM finish();\n", + ); + yield* fs.writeFileString( + failing, + "SELECT plan(1); SELECT fail('managed pgTAP failure'); SELECT * FROM finish();\n", + ); + + const api = yield* StackApi; + const stack = yield* Effect.acquireRelease( + api.create({ + projectRoot: root, + stateRoot: `${root}/stacks`, + cacheRoot: `${root}/cache`, + runtime, + }), + (stack) => stack.destroy.pipe(Effect.catch((cause) => Effect.die(cause))), + ); + const [database] = yield* stack.composition.supabase([databaseCreation]); + if (database === undefined) return yield* Effect.die("database was not composed"); + yield* database.start; + yield* database.ready; + const hostCredentials = yield* database.credentials({ from: "host" }); + const databaseUrl = hostCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("host database URL missing"); + const hostConnection = parseConnectionString(databaseUrl); + if (hostConnection === undefined) return yield* Effect.die("invalid host database URL"); + const connection = { ...hostConnection, user: "postgres" }; + + const output = mockOutput(); + const settings = mockCommandSettings({ + workdir: root, + supabaseHome: root, + }); + const services = Layer.mergeAll( + settings, + output.layer, + mockTelemetryStateTracked().layer, + Layer.succeed(DbConfigResolver, { + resolve: () => Effect.succeed({ conn: connection, isLocal: true }), + resolvePoolerFallback: () => Effect.succeed(Option.none()), + }), + Layer.succeed(RuntimeInfo, { + cwd: root, + platform: process.platform, + arch: process.arch, + homeDir: root, + execPath: process.execPath, + pid: process.pid, + }), + Layer.succeed(DebugFlag, true), + Layer.succeed(DnsResolverFlag, "native" as const), + Layer.succeed(NetworkIdFlag, Option.none()), + Layer.succeed(CliArgs, { args: [] }), + Layer.succeed(BundledPostgresClient, { run: () => Effect.die("unused") }), + Layer.succeed(DockerRun, { + run: () => Effect.die("unused"), + runCapture: () => Effect.die("unused"), + runStream: () => Effect.die("unused"), + }), + stackBackendLayer("stack"), + dbConnectionLayer, + ); + const assertPgtapDropped = Effect.gen(function* () { + const dbConnection = yield* DbConnection; + const session = yield* dbConnection.connect(connection, { + isLocal: true, + dnsResolver: "native", + }); + expect(yield* session.extensionExists("pgtap")).toBe(false); + }); + + yield* testDb(flags(passing)).pipe(Effect.provide(services)); + expect(output.stdoutText).toContain("Result: PASS"); + yield* assertPgtapDropped.pipe(Effect.provide(services)); + + const failedOutput = mockOutput(); + const failedExit = yield* Effect.exit( + testDb(flags(failing)).pipe(Effect.provide(Layer.merge(services, failedOutput.layer))), + ); + expect(Exit.isFailure(failedExit)).toBe(true); + if (Exit.isFailure(failedExit)) { + const failure = Cause.findErrorOption(failedExit.cause); + expect(Option.isSome(failure)).toBe(true); + if (Option.isSome(failure)) { + expect(failure.value).toMatchObject({ + _tag: "TestDbRunError", + }); + expect(failure.value.message).toMatch(/^error running pg_prove:/); + } + } + expect(failedOutput.stdoutText).toContain("not ok"); + expect(failedOutput.stderrText).toContain("Connecting to local database"); + yield* assertPgtapDropped.pipe(Effect.provide(services)); + + if (runtime !== "native") return; + const partialTap = "not ok 1 - managed tool failed after streaming\n"; + const jsonOutput = mockOutput({ format: "json" }); + const processControl = mockProcessControl(); + const failingStackApi = Layer.succeed(StackApi, { + ...api, + open: () => + Effect.succeed({ + ...stack, + tools: { + ...stack.tools, + run: (_tool, options) => + Effect.gen(function* () { + yield* options.stdout(new TextEncoder().encode(partialTap)); + return yield* Effect.fail( + new StackError({ operation: "tool", message: "pg_prove unavailable" }), + ); + }), + }, + }), + }); + yield* runTestDbCommand(flags(failing)).pipe( + Effect.provide( + Layer.mergeAll( + services, + jsonOutput.layer, + processControl.layer, + failingStackApi, + Layer.succeed(CommandRuntime, { + commandPath: ["test", "db"], + commandRunId: "test-db-native", + }), + Layer.succeed(Analytics, { + capture: () => Effect.void, + identify: () => Effect.void, + alias: () => Effect.void, + groupIdentify: () => Effect.void, + }), + ), + ), + ); + expect(jsonOutput.stdoutText).toBe(partialTap); + expect(jsonOutput.stderrText).toContain("pg_prove run failed: pg_prove unavailable"); + expect(processControl.exitCode).toBe(1); + yield* assertPgtapDropped.pipe(Effect.provide(services)); + }), + ).pipe( + Effect.provide(Layer.mergeAll(BunServices.layer, FetchHttpClient.layer, liveStackApi)), + ), + ); + } +}); diff --git a/apps/cli/src/command-internal/test-db.pg-prove-args.ts b/apps/cli/src/command-internal/test-db.pg-prove-args.ts index 382eee6b18..95e8c232dc 100644 --- a/apps/cli/src/command-internal/test-db.pg-prove-args.ts +++ b/apps/cli/src/command-internal/test-db.pg-prove-args.ts @@ -1,9 +1,13 @@ import * as nodePath from "node:path"; import { Option } from "effect"; -import type { PostgresClientMount } from "@supabase/stack/effect"; - import { toDockerMountPath } from "./docker-path.ts"; +interface PostgresClientMount { + readonly source: string; + readonly target: string; + readonly readOnly: boolean; +} + export interface PgProveArgs { /** Full `pg_prove` argv (without the leading binary, which the image provides). */ readonly cmd: ReadonlyArray; diff --git a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md index 62b4391641..df124dc67b 100644 --- a/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/diff/SIDE_EFFECTS.md @@ -8,9 +8,13 @@ edge-runtime involved). `--use-pg-schema` is the CLI's sole remaining Go delegation on this command — a documented keep-in-Go exception (CLI-1960), not a pending port. -When `[experimental].stack` is on, the shadow is `EphemeralPostgres` under -`$SUPABASE_HOME/managed/ephemeral-postgres//` (`~/.supabase/managed/…` by default). Migra, pgAdmin, and `--use-pg-schema` are -rejected because that shadow is always stack. +When `[experimental].stack` is on, each shadow is a fresh database in an invocation-owned, +unique temporary stack namespace. The command applies the catalog and project migrations as needed, +then destroys its namespace when the Effect scope closes. Shadow baselines are not cached; +the snapshot cache described below belongs to the legacy container backend. Native artifacts +are shared through `$SUPABASE_HOME/cache/stack`; shadow state and data use the normal stack registry, so `stack list` and `stack destroy` can +find a shadow left by an abrupt CLI exit. Each shadow owns a unique temporary project root +and uses an automatically assigned port; `db.shadow_port` applies only to the legacy backend. Migra, pgAdmin, and `--use-pg-schema` are rejected in stack mode. Pg-delta runs in-process. Coverage gaps warn, while `--strict-coverage` makes them fatal, and `PGDELTA_DEBUG` writes diagnostic JSON under @@ -30,26 +34,22 @@ it, and JSON `null` disables formatting without disabling safe compaction. | `/supabase/migrations/*.sql` | SQL | shadow provisioning (applied to the shadow source) — `--use-pgadmin` too, via the SAME `migrateShadowDatabase` | | `/supabase/roles.sql` | SQL | shadow provisioning, PG14 and PG15 alike (unlike `db reset`'s PG15-only local path); also hashed into the shadow-baseline cache key on every cache-eligible acquire, warm hits included (where no baseline is applied at all); missing file tolerated | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | warm shadow-cache hit — the matching snapshot is streamed into the fresh shadow; every cache-eligible acquire (warm hit and successful cold export) also enumerates and `stat`s every `shadow-baseline-*.tar` for LRU keep-3 + 2-day mtime TTL and may delete other keys (`SUPABASE_HOME` overrides the `~/.supabase` root) | -| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar` | tar | `[experimental].stack` only — slim-init + stack bootstrap baseline; key includes artifact identity and runtime kind (native vs container). Never mixed with `shadow-baseline-*.tar` | | `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | abandoned-partial sweep on every cache-eligible acquire (warm hit and cold export) — enumerated and `stat`ed, and removed when older than 5 minutes (a crashed/SIGKILLed earlier export's leftover) | -| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar..partial` | tar | `[experimental].stack` abandoned-partial sweep — same 5-minute TTL, stack prefix only (legacy `shadow-baseline-*.partial` names are not candidates) | | `[db.migrations].schema_paths` globs / `/supabase/database/**` / `/supabase/schemas/**` | SQL | migra engine only, for the local-target declarative-schema fallback; pg-delta always compares the migrations baseline directly to the live target | | `~/.supabase/access-token` | plain text | `--linked` / `--db-url` with no `SUPABASE_ACCESS_TOKEN` | | `/supabase/.temp/project-ref` | plain text | `--linked` ref resolution — skipped when `--project-ref` (or `SUPABASE_PROJECT_ID`) is set | ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/migrations/_.sql` | SQL | non-empty `--file` diff; bundled pg-delta may emit ordered transaction-aware files, while pgAdmin always emits one | -| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output`; flattened review representation, not a portable apply script | -| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot (native diff targets + the explicit `--from/--to migrations` shadow; never `--use-pgadmin`/`--use-pg-schema`); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | -| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar` | tar | `[experimental].stack` COLD export of an `EphemeralPostgres` cluster; same LRU keep-3 + 2-day TTL, separate glob so keys cannot collide with legacy SQL-template baselines | -| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | -| `~/.supabase/cache/shadow-baseline/stack-shadow-baseline-.tar..partial` | tar | `[experimental].stack` cold export temp file — pid-scoped, `chmod` 0600, `rename`d into the stack tar above; abandoned leftovers older than 5 minutes are swept on later acquires | -| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | -| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| Path | Format | When | +| --------------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/migrations/_.sql` | SQL | non-empty `--file` diff; bundled pg-delta may emit ordered transaction-aware files, while pgAdmin always emits one | +| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output`; flattened review representation, not a portable apply script | +| `/supabase/.temp/pgdelta/v2/debug//*.json` | JSON | bundled engine with `PGDELTA_DEBUG` | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar` | tar | cache-enabled COLD shadow provision creates the current key's snapshot (native diff targets + the explicit `--from/--to migrations` shadow; never `--use-pgadmin`/`--use-pg-schema`); a warm hit `touch`es its mtime (LRU); every cache-eligible acquire may delete other keys under LRU keep-3 + 2-day mtime TTL — ~90MB (`SUPABASE_HOME` overrides the root) | +| `~/.supabase/cache/shadow-baseline/shadow-baseline-.tar..partial` | tar | during a cold export — the in-flight temp file, `rename`d into the tar above on success and removed on failure; only a crash/SIGKILL leaves it behind, and later cold exports / warm hits sweep leftovers older than 5 minutes | +| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | +| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | ## Docker @@ -66,8 +66,7 @@ it, and JSON `null` disables formatting without disabling safe compaction. no `targetUrlOverride`. - When `[experimental].stack` / `SUPABASE_EXPERIMENTAL_STACK=1` is on, `--local` inspects the project stack (`findStack` + `status`, database ready) instead of `supabase_db_`, - and shadows are `@supabase/stack` `EphemeralPostgres` clusters (slim-init baseline, cache - prefix `stack-shadow-baseline-`). Every stack runtime rejects `--use-migra` / `--use-pgadmin` / + and shadows are temporary databases owned by the stack runtime (slim-init baseline, cache `--use-pg-schema`. - `supabase/migra` container — the migra OOM bash fallback only. - **Differ container** (`--use-pgadmin`, CLI-1968) — `supabase/pgadmin-schema-diff:cli-0.0.5` diff --git a/apps/cli/src/commands/db/diff/diff.integration.test.ts b/apps/cli/src/commands/db/diff/diff.integration.test.ts index 13ffebec97..6231973e44 100644 --- a/apps/cli/src/commands/db/diff/diff.integration.test.ts +++ b/apps/cli/src/commands/db/diff/diff.integration.test.ts @@ -1,3 +1,4 @@ +import { unusedStackServices } from "../../../../tests/helpers/unused-stack.ts"; import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, join } from "node:path"; import { BunServices } from "@effect/platform-bun"; @@ -382,6 +383,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { }); const baseLayer = Layer.mergeAll( + unusedStackServices, // Listed first so the fake service layers below (`Layer.mergeAll` is last-wins) // override its real implementations, matching `start.integration.test.ts`. BunServices.layer, diff --git a/apps/cli/src/commands/db/diff/diff.stack.e2e.test.ts b/apps/cli/src/commands/db/diff/diff.stack.e2e.test.ts deleted file mode 100644 index fd25a31d9a..0000000000 --- a/apps/cli/src/commands/db/diff/diff.stack.e2e.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { describe, expect, test } from "vitest"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- e2e fixture appends experimental.stack to project config -import { appendFile, readdir } from "node:fs/promises"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- e2e fixture joins project and cache paths -import path from "node:path"; - -import { makeTempHome, makeTempStackProject, runSupabase } from "../../../../tests/helpers/cli.ts"; - -const DB_START_COMMAND_TIMEOUT_MS = 480_000; -const DB_START_CLEANUP_TIMEOUT_MS = 120_000; -const STACK_DB_AUX_TIMEOUT_MS = 180_000; -const STACK_DIFF_TIMEOUT_MS = 180_000; -const STACK_DIFF_TEST_TIMEOUT_MS = - DB_START_COMMAND_TIMEOUT_MS + - STACK_DB_AUX_TIMEOUT_MS + - STACK_DIFF_TIMEOUT_MS + - DB_START_CLEANUP_TIMEOUT_MS; - -const STACK_BASELINE_TAR = /^stack-shadow-baseline-[0-9a-f]{16}\.tar$/u; -const COMPOSE_BASELINE_TAR = /^shadow-baseline-[0-9a-f]{16}\.tar$/u; -const PROBE_FN_SQL = - /CREATE(?:\s+OR\s+REPLACE)?\s+FUNCTION\s+"?public"?\s*\.\s*"?probe_fn"?\s*\(\)/i; - -describe("supabase db diff (e2e, stack shadow)", () => { - test( - "stack db diff --local publishes a stack-shadow-baseline tar", - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-db-diff-stack-e2e-"); - await appendFile( - path.join(project.dir, "supabase", "config.toml"), - "\n[experimental]\nstack = true\n", - ); - try { - const started = await runSupabase(["db", "start"], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: DB_START_COMMAND_TIMEOUT_MS, - }); - expect(started.exitCode, started.stderr).toBe(0); - - const createFunction = await runSupabase( - [ - "db", - "query", - `create function public.probe_fn() -returns void -language sql -as $$ select 1; $$;`, - "--local", - ], - { cwd: project.dir, home: home.dir, exitTimeoutMs: STACK_DB_AUX_TIMEOUT_MS }, - ); - expect(createFunction.exitCode, createFunction.stderr).toBe(0); - - const diff = await runSupabase(["db", "diff", "--local", "--use-pg-delta"], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: STACK_DIFF_TIMEOUT_MS, - env: { SUPABASE_SHADOW_CACHE: undefined }, - }); - expect(diff.exitCode, `${diff.stdout}\n${diff.stderr}`).toBe(0); - expect(diff.stdout).toMatch(PROBE_FN_SQL); - - const cacheDir = path.join(home.dir, "cache", "shadow-baseline"); - const entries = await readdir(cacheDir).catch(() => [] as Array); - expect(entries.filter((entry) => STACK_BASELINE_TAR.test(entry))).toHaveLength(1); - expect(entries.filter((entry) => COMPOSE_BASELINE_TAR.test(entry))).toHaveLength(0); - expect(entries.filter((entry) => entry.endsWith(".partial"))).toHaveLength(0); - } finally { - await runSupabase(["stack", "destroy", "--yes"], { - cwd: project.dir, - home: home.dir, - env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, - exitTimeoutMs: DB_START_CLEANUP_TIMEOUT_MS, - }).catch(() => undefined); - } - }, - STACK_DIFF_TEST_TIMEOUT_MS, - ); -}); diff --git a/apps/cli/src/commands/db/dump/dump.handler.ts b/apps/cli/src/commands/db/dump/dump.handler.ts index 8ee425d6f1..5ca6439b09 100644 --- a/apps/cli/src/commands/db/dump/dump.handler.ts +++ b/apps/cli/src/commands/db/dump/dump.handler.ts @@ -1,4 +1,5 @@ import { Effect, FileSystem, Option, Path, Predicate } from "effect"; +import type { Stack } from "@supabase/stack/effect"; import { CommandSettings } from "../../../config/command-settings.service.ts"; import { ProjectRefResolver } from "../../../config/project-ref.service.ts"; @@ -7,6 +8,7 @@ import { TelemetryState } from "../../../telemetry/telemetry-state.service.ts"; import { DbConfigResolver } from "../../../command-internal/db-config.service.ts"; import type { DbConnType } from "../../../command-internal/db-target-flags.ts"; import { loadProjectEnv, readDbToml } from "../../../command-internal/db-config.toml-read.ts"; +import { parseConnectionString } from "../../../command-internal/db-config.parse.ts"; import { resolveDbImage } from "../../../command-internal/db-image.ts"; import { ipv6Suggestion, @@ -43,6 +45,7 @@ import { import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; import { stackProjectDatabaseVersion, + stackOpenReadyProject, stackRequireProjectRuntime, } from "../../../command-internal/stack-local-database.ts"; import { resolveBundledPostgresRuntime } from "../../../command-internal/bundled-postgres-client.ts"; @@ -194,12 +197,20 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { const tomlValues = yield* readDbToml(fs, path, cliSettings.workdir, linkedRef); const backend = yield* currentStackBackend; + const managedProject = + backend.kind === "stack" && connType === "local" + ? yield* stackOpenReadyProject.pipe( + Effect.mapError((cause) => new DbDumpRunError({ message: cause.message })), + Effect.map(Option.getOrUndefined), + ) + : undefined; + const managedStack: Stack | undefined = managedProject?.stack; const stackRuntime = backend.kind === "stack" && connType === "local" ? yield* stackRequireProjectRuntime : undefined; const bundledRuntime = - backend.kind === "stack" + backend.kind === "stack" && managedStack === undefined ? yield* resolveBundledPostgresRuntime(stackRuntime, runtimeInfo.platform, runtimeInfo.arch) : undefined; const useNativeClient = bundledRuntime?.kind === "native"; @@ -215,7 +226,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { ? undefined : envNetworkId, ); - const stackPublishedTarget = backend.kind === "stack" && isLocal; + const stackPublishedTarget = backend.kind === "stack" && isLocal && managedStack === undefined; const dumpConn = useNativeClient ? connType === "local" ? dumpConnForHostClient(conn) @@ -235,14 +246,21 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { : String(tomlValues.majorVersion); const dumpMajor = tomlValues.majorVersion; const dumpClient = - backend.kind === "stack" + managedStack !== undefined ? { - kind: "bundled" as const, + kind: "stack" as const, + stack: managedStack, command: roleOnly ? ("pg_dumpall" as const) : ("pg_dump" as const), - version: catalogVersion, - runtime: bundledRuntime, + major: Number(catalogVersion.split(".")[0]) === 15 ? (15 as const) : (17 as const), } - : { kind: "container" as const }; + : backend.kind === "stack" + ? { + kind: "bundled" as const, + command: roleOnly ? ("pg_dumpall" as const) : ("pg_dump" as const), + version: catalogVersion, + runtime: bundledRuntime, + } + : { kind: "container" as const }; // 4. Pick the mode-specific script + env. --schema/-s and --exclude/-x arrive here // already CSV-parsed by `parseSchemaFlags`. @@ -267,7 +285,20 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { script: dumpSchemaScript, buildEnv: buildSchemaDumpEnv, } as const); - const modeEnv = mode.buildEnv(dumpConn, opt); + const managedDumpConn = + managedProject === undefined + ? undefined + : parseConnectionString( + yield* managedProject.database.credentials({ from: "runtime" }).pipe( + Effect.mapError((cause) => new DbDumpRunError({ message: cause.message })), + Effect.map((credentials) => credentials.databaseUrl ?? ""), + ), + ); + if (managedProject !== undefined && managedDumpConn === undefined) + return yield* new DbDumpRunError({ + message: "The local stack runtime database URL is unavailable.", + }); + const modeEnv = mode.buildEnv(managedDumpConn ?? dumpConn, opt); // Keys off `path.length > 0`, not flag presence: `--file ""` means stdout, no // file opened. @@ -406,7 +437,7 @@ export const dbDump = Effect.fn("db.dump")(function* (flags: DbDumpFlags) { Effect.catchIf( (error): error is DockerRunError => Predicate.isTagged(error, "DockerRunError") && - stackRuntime?.kind === "native" && + stackRuntime === "native" && runtimeInfo.platform === "win32", (error) => Effect.fail( diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index 895c87b95a..5c95a3b6aa 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -4,7 +4,7 @@ import process from "node:process"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option, Stream } from "effect"; +import { Cause, Effect, Exit, Layer, Option, Redacted, Stream } from "effect"; import { mockOutput, mockTty, processEnvLayer } from "../../../../tests/helpers/mocks.ts"; import { @@ -33,12 +33,12 @@ import { DbConfigConnectTempRoleError } from "../../../command-internal/db-confi import { DockerRunError } from "../../../command-internal/docker-run.errors.ts"; import { DockerRun, type DockerRunOpts } from "../../../command-internal/docker-run.service.ts"; import { BundledPostgresClient } from "../../../command-internal/bundled-postgres-client.ts"; -import { ContainerEngineResolver, type RunPostgresClientOptions } from "@supabase/stack/effect"; +import type { RunPostgresClientOptions } from "../../../command-internal/bundled-postgres-client.ts"; +import type { DatabaseInstance, ServiceCreation, Stack } from "@supabase/stack/effect"; import type { DbDumpFlags } from "./dump.command.ts"; import { dbDump } from "./dump.handler.ts"; import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; import { StackApi } from "../../../command-internal/stack-api.ts"; -import { StackIdSchema, type EffectStack } from "@supabase/stack/effect"; const LOCAL_CONN: PgConnInput = { host: "127.0.0.1", @@ -55,6 +55,113 @@ const REMOTE_CONN: PgConnInput = { database: "postgres", }; +const managedDumpStackApi = (runtime: "native" | "docker") => { + const id = "m".repeat(64); + const creation: Extract = { + service: "database", + config: { + version: "17.6.1.173", + databasePassword: Redacted.make("secret"), + jwtSecret: Redacted.make("secret"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: 54322 } }, + }; + const database: DatabaseInstance = { + id, + service: "database", + start: Effect.void, + ready: Effect.void, + stop: Effect.void, + restart: () => Effect.void, + destroy: Effect.void, + prepare: Effect.void, + status: Effect.succeed({ + id, + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: 54322 }], + config: creation, + lifecycle: "running", + health: "healthy", + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: true, + registered: true, + }), + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => + Effect.succeed({ databaseUrl: "postgresql://postgres:secret@127.0.0.1:54322/postgres" }), + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: Effect.die("unused"), + }; + const stack = { + id, + services: { + create: () => Effect.die("unused"), + get: () => Effect.succeed(database), + list: Effect.succeed([database]), + }, + composition: { + describe: Effect.succeed({ + members: [{ id, activation: "eager" as const }], + dependencies: [], + }), + supabase: () => Effect.succeed([database]), + configure: () => Effect.void, + start: Effect.succeed([]), + stop: Effect.succeed([]), + restart: Effect.succeed([]), + }, + stop: Effect.void, + destroy: Effect.void, + tools: { + run: ( + _tool: { readonly command: string; readonly major: number }, + options: { + readonly stdout: (bytes: Uint8Array) => Effect.Effect; + readonly stderr: (bytes: Uint8Array) => Effect.Effect; + }, + ): Effect.Effect<{ readonly jobId: string; readonly exitCode: number }, E, R> => + Effect.gen(function* () { + yield* options.stdout(new TextEncoder().encode('CREATE TABLE "public" (id integer);\n')); + return { jobId: "job", exitCode: 0 }; + }), + }, + } satisfies Stack; + return Layer.succeed(StackApi, { + create: () => Effect.succeed(stack), + open: () => Effect.succeed(stack), + discover: () => + Effect.succeed([ + { + definition: { + id, + identity: { projectRoot: "/work/project", branchContext: "main", stackName: "default" }, + runtime, + instances: [], + composition: { members: [{ id, activation: "eager" as const }], dependencies: [] }, + ports: [], + }, + host: undefined, + }, + ]), + resolveIdentity: () => + Effect.succeed({ projectRoot: "/work/project", branchContext: "main", stackName: "default" }), + }); +}; + +const unusedStackApi = Layer.succeed(StackApi, { + create: () => Effect.die("unused"), + open: () => Effect.die("unused"), + discover: () => Effect.die("unused"), + resolveIdentity: () => Effect.die("unused"), +}); + function mockResolver(opts: { conn?: PgConnInput; isLocal?: boolean; @@ -251,12 +358,6 @@ function mockBundledPostgresClient(opts: { }; } -const containerEngineResolver = (installed: boolean) => - Layer.succeed(ContainerEngineResolver, { - isInstalled: () => Effect.succeed(installed), - resolve: () => Effect.die("unused"), - }); - const runtimeInfoLayer = (platform: NodeJS.Platform, arch?: NodeJS.Architecture) => Layer.succeed(RuntimeInfo, { cwd: "/work/project", @@ -316,7 +417,7 @@ function setup(opts: SetupOpts = {}) { projectRef.layer, docker.layer, bundled.layer, - containerEngineResolver(opts.dockerInstalled ?? true), + unusedStackApi, mockCommandSettings({ workdir: opts.workdir ?? "/work/project", projectId: opts.projectId ?? Option.none(), @@ -654,6 +755,34 @@ describe("db dump integration", () => { }).pipe(Effect.provide(layer)); }); + it.live("keeps an explicit local db-url on the bundled client path", () => { + const { layer, bundled } = setup({ + conn: { + host: "127.0.0.1", + port: 54322, + user: "custom", + password: "secret", + database: "custom", + }, + isLocal: true, + stdout: "", + }); + return Effect.gen(function* () { + yield* dbDump( + flags({ + dbUrl: Option.some("postgresql://custom:secret@127.0.0.1:54322/custom"), + }), + ); + expect(bundled.lastOpts?.env).toMatchObject({ + PGHOST: "127.0.0.1", + PGPORT: "54322", + PGUSER: "custom", + PGPASSWORD: "secret", + PGDATABASE: "custom", + }); + }).pipe(Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack")))); + }); + it.live("caches the linked project even when connection resolution fails (Go PostRun)", () => { // The project ref is resolved before the connection is built, and the // linked-project cache is refreshed unconditionally afterward. So an @@ -1029,304 +1158,19 @@ describe("db dump integration", () => { }); } - const DUMP_STACK_ID = StackIdSchema.make("d".repeat(64)); - const unusedDump = () => Effect.die("unused"); - const unusedDumpEffect = Effect.die("unused"); - const dumpStackApi = ( - runtime: { kind: "native" } | { kind: "container"; engine: "docker" }, - databaseVersion = "17.6.1.168", - ) => { - const stack: EffectStack = { - id: DUMP_STACK_ID, - status: Effect.succeed({ - id: DUMP_STACK_ID, - lifecycle: "running", - desiredLifecycle: "running", - runtime, - endpoints: {}, - versions: { database: databaseVersion }, - capabilities: [], - artifacts: [], - }), - credentials: unusedDumpEffect, - prepare: unusedDump, - start: unusedDump, - stop: unusedDumpEffect, - destroy: unusedDumpEffect, - resetDatabase: unusedDumpEffect, - logs: unusedDump, - followLogs: () => Stream.empty, - }; - return Layer.succeed(StackApi, { - createStack: unusedDump, - findStack: () => - Effect.succeed( - Option.some({ - id: DUMP_STACK_ID, - projectRoot: "/work/project", - name: "default", - branchContext: "main", - runtime, - desiredLifecycle: "running", - }), - ), - discoverStacks: unusedDump, - openStack: () => Effect.succeed(stack), - inspectStack: unusedDump, - }); - }; - - it.live( - "dump --local on the stack backend fails instead of using Docker when no stack exists", - () => { - const { layer, docker } = setup({ isLocal: true, stdout: "-- schema\n" }); - return Effect.gen(function* () { - const exit = yield* Effect.exit(dbDump(flags({ local: Option.some(true) }))); - expect(Exit.isFailure(exit)).toBe(true); - expect(failMessage(exit)).toContain("Could not determine the stack runtime"); - expect(docker.lastOpts).toBeUndefined(); - }).pipe( - Effect.provide( - Layer.mergeAll( - layer, - stackBackendLayer("stack"), - Layer.succeed(StackApi, { - createStack: unusedDump, - findStack: () => Effect.succeed(Option.none()), - discoverStacks: unusedDump, - openStack: unusedDump, - inspectStack: unusedDump, - }), + describe("managed stack dump", () => { + for (const runtime of ["native", "docker"] as const) { + it.live(`streams schema output through the ${runtime} stack tool`, () => { + const { layer, out } = setup({ isLocal: true, stdout: "" }); + return Effect.gen(function* () { + yield* dbDump(flags({ local: Option.some(true) })); + expect(out.stdoutText).toContain('CREATE TABLE IF NOT EXISTS "public"'); + }).pipe( + Effect.provide( + Layer.mergeAll(layer, stackBackendLayer("stack"), managedDumpStackApi(runtime)), ), - ), - ); - }, - ); - - it.live( - "dump --db-url on the stack backend does not require a project stack even when isLocal is true", - () => { - const { layer, docker, resolver, bundled } = setup({ - isLocal: true, - stdout: "-- schema\n", - platform: "darwin", - }); - return Effect.gen(function* () { - yield* dbDump( - flags({ - dbUrl: Option.some("postgresql://postgres:review@127.0.0.1:54322/postgres"), - }), ); - expect(resolver.calls[0]).toMatchObject({ connType: "db-url" }); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("host.docker.internal"); - expect(bundled.lastOpts?.network).toBe("host"); - }).pipe( - Effect.provide( - Layer.mergeAll( - layer, - stackBackendLayer("stack"), - Layer.succeed(StackApi, { - createStack: unusedDump, - findStack: () => Effect.succeed(Option.none()), - discoverStacks: unusedDump, - openStack: unusedDump, - inspectStack: unusedDump, - }), - ), - ), - ); - }, - ); - - it.live("dump --local on a docker stack ignores compose SUPABASE_NETWORK_ID", () => { - const { layer, docker, bundled } = setup({ - isLocal: true, - stdout: "-- schema\n", - platform: "darwin", - env: { SUPABASE_NETWORK_ID: "supabase_network_test" }, - }); - return Effect.gen(function* () { - yield* dbDump(flags({ local: Option.some(true) })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.network).toBe("host"); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("host.docker.internal"); - }).pipe( - Effect.provide( - Layer.mergeAll( - layer, - stackBackendLayer("stack"), - dumpStackApi({ kind: "container", engine: "docker" }), - ), - ), - ); - }); - - it.live("dump --db-url native keeps the resolved host even when isLocal is true", () => { - const { layer, docker, bundled } = setup({ - conn: { ...LOCAL_CONN, host: "db.internal" }, - isLocal: true, - stdout: "-- schema\n", - dockerInstalled: false, - platform: "darwin", - }); - return Effect.gen(function* () { - yield* dbDump( - flags({ - dbUrl: Option.some("postgresql://postgres:postgres@db.internal:54322/postgres"), - }), - ); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "native" }); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("db.internal"); - }).pipe(Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack")))); - }); - - it.live("dump --linked on the stack backend keeps the remote host for a native client", () => { - const { layer, docker, bundled } = setup({ - conn: REMOTE_CONN, - isLocal: false, - stdout: "-- schema\n", - dockerInstalled: false, - platform: "darwin", - }); - return Effect.gen(function* () { - yield* dbDump(flags({ linked: Option.some(true) })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "native" }); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe(REMOTE_CONN.host); - }).pipe(Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack")))); - }); - - it.live("dump --linked on a native-less platform uses a Docker pg_dump client", () => { - const { layer, docker, bundled } = setup({ - conn: REMOTE_CONN, - isLocal: false, - stdout: "-- schema\n", - dockerInstalled: false, - platform: "darwin", - arch: "x64", - }); - return Effect.gen(function* () { - yield* dbDump(flags({ linked: Option.some(true) })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "container", engine: "docker" }); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe(REMOTE_CONN.host); - }).pipe(Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack")))); - }); - - it.live("dump --local on a native stack uses bundled pg_dump, not a tool container", () => { - mkdirSync(join(tmp.current, "supabase"), { recursive: true }); - writeFileSync( - join(tmp.current, "supabase", "config.toml"), - 'project_id = "test"\n[db]\nmajor_version = 17\n', - ); - const { layer, docker, bundled, out } = setup({ - isLocal: true, - workdir: tmp.current, - stdout: "-- schema\n", - }); - return Effect.gen(function* () { - yield* dbDump(flags({ local: Option.some(true) })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "native" }); - expect(bundled.lastOpts?.argv).toEqual(["bash", "-c", expect.any(String), "--"]); - expect(out.stdoutText).toContain("-- schema"); - }).pipe( - Effect.provide( - Layer.mergeAll(layer, stackBackendLayer("stack"), dumpStackApi({ kind: "native" })), - ), - ); - }); - - it.live("dump --local on a native stack reports bundled pg_dump exit, not a container", () => { - mkdirSync(join(tmp.current, "supabase"), { recursive: true }); - writeFileSync( - join(tmp.current, "supabase", "config.toml"), - 'project_id = "test"\n[db]\nmajor_version = 17\n', - ); - const { layer, docker } = setup({ - isLocal: true, - workdir: tmp.current, - exitCode: 1, - }); - return Effect.gen(function* () { - const exit = yield* dbDump(flags({ local: Option.some(true) })).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - expect(failMessage(exit)).toBe("error running pg_dump: exit 1"); - expect(docker.lastOpts).toBeUndefined(); - }).pipe( - Effect.provide( - Layer.mergeAll(layer, stackBackendLayer("stack"), dumpStackApi({ kind: "native" })), - ), - ); - }); - - it.live("dump --local on a Windows native stack uses a Docker pg_dump client", () => { - const { layer, docker, bundled } = setup({ - isLocal: true, - stdout: "-- schema\n", - platform: "win32", - }); - return Effect.gen(function* () { - yield* dbDump(flags({ local: Option.some(true) })); - expect(docker.lastOpts).toBeUndefined(); - expect(bundled.lastOpts?.runtime).toEqual({ kind: "container", engine: "docker" }); - expect(bundled.lastOpts?.env?.["PGHOST"]).toBe("host.docker.internal"); - expect(bundled.lastOpts?.network).toBe("host"); - }).pipe( - Effect.provide( - Layer.mergeAll(layer, stackBackendLayer("stack"), dumpStackApi({ kind: "native" })), - ), - ); - }); - - it.live( - "dump --local on a Windows native stack asks for Docker Desktop when Docker is missing", - () => { - const { layer } = setup({ - isLocal: true, - platform: "win32", - runFails: true, }); - return Effect.gen(function* () { - const exit = yield* dbDump(flags({ local: Option.some(true) })).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - expect(failMessage(exit)).toContain("failed to run docker"); - expect(failSuggestion(exit)).toContain("Docker Desktop"); - }).pipe( - Effect.provide( - Layer.mergeAll(layer, stackBackendLayer("stack"), dumpStackApi({ kind: "native" })), - ), - ); - }, - ); - - it.live( - "dump --local on the stack backend uses status().versions.database for the catalog pin", - () => { - mkdirSync(join(tmp.current, "supabase"), { recursive: true }); - writeFileSync( - join(tmp.current, "supabase", "config.toml"), - 'project_id = "test"\n[db]\nmajor_version = 17\n', - ); - const { layer, bundled } = setup({ - isLocal: true, - workdir: tmp.current, - stdout: "-- schema\n", - }); - return Effect.gen(function* () { - yield* dbDump(flags({ local: Option.some(true) })); - expect(bundled.lastOpts?.version).toBe("15.14.1.168"); - }).pipe( - Effect.provide( - Layer.mergeAll( - layer, - stackBackendLayer("stack"), - dumpStackApi({ kind: "native" }, "15.14.1.168"), - ), - ), - ); - }, - ); + } + }); }); diff --git a/apps/cli/src/commands/db/dump/dump.layers.ts b/apps/cli/src/commands/db/dump/dump.layers.ts index 32458dd429..0bf46f8693 100644 --- a/apps/cli/src/commands/db/dump/dump.layers.ts +++ b/apps/cli/src/commands/db/dump/dump.layers.ts @@ -77,6 +77,6 @@ export const dbDumpRuntimeLayer = Layer.mergeAll( identityStitchLayer, telemetryStateLayer, stackApiLayer, - bundledPostgresClientLayer, + bundledPostgresClientLayer.pipe(Layer.provide(dockerRunLayer), Layer.provide(cliSettings)), commandRuntimeLayer(["db", "dump"]), ); diff --git a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md index 023cb1c078..de70163154 100644 --- a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md @@ -14,9 +14,13 @@ runs the same in-process declarative export (`supabase/schemas` plus `--declarative`. `--experimental --declarative` does not print that line: `--declarative` already selected the export. -When `[experimental].stack` is on, the shadow is `EphemeralPostgres` under -`$SUPABASE_HOME/managed/ephemeral-postgres//` (`~/.supabase/managed/…` by default). Migra (`--diff-engine migra`) is rejected -because that shadow is always stack. +When `[experimental].stack` is on, each shadow is a fresh database in an invocation-owned, +unique temporary stack namespace. The command applies the catalog and project migrations as needed, +then destroys its namespace when the Effect scope closes. Shadow baselines are not cached; +the snapshot cache described below belongs to the legacy container backend. Native artifacts +are shared through `$SUPABASE_HOME/cache/stack`; shadow state and data use the normal stack registry, so `stack list` and `stack destroy` can +find a shadow left by an abrupt CLI exit. Each shadow owns a unique temporary project root +and uses an automatically assigned port; `db.shadow_port` applies only to the legacy backend. Migra (`--diff-engine migra`) is rejected in stack mode. Pg-delta runs in-process. Coverage gaps warn; `--strict-coverage` makes them fatal, while `PGDELTA_DEBUG` writes diagnostic JSON under diff --git a/apps/cli/src/commands/db/pull/pull.integration.test.ts b/apps/cli/src/commands/db/pull/pull.integration.test.ts index 7138fe8c2d..23a22e1781 100644 --- a/apps/cli/src/commands/db/pull/pull.integration.test.ts +++ b/apps/cli/src/commands/db/pull/pull.integration.test.ts @@ -1,3 +1,4 @@ +import { unusedStackServices } from "../../../../tests/helpers/unused-stack.ts"; import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, join } from "node:path"; import { BunServices } from "@effect/platform-bun"; @@ -421,6 +422,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { }); const baseLayer = Layer.mergeAll( + unusedStackServices, // Listed first so the fake service layers below (`Layer.mergeAll` is last-wins) // override its real implementations, matching `start.integration.test.ts`. BunServices.layer, diff --git a/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md b/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md index aee64ebd74..770677e98d 100644 --- a/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md @@ -30,13 +30,15 @@ removed `DeclarativeSeam.execInherit` seam — see those commands' own When the `experimental.stack` feature flag is on (`SUPABASE_EXPERIMENTAL_STACK=1|0` env precedence, same rules as [`docs/stack-commands.md`](../../../../docs/stack-commands.md)), the -local path calls `resetDatabase` on the project stack instead of the container recreate -described above. After the reset, buckets are seeded — reusing the `seed buckets` local path — -when Storage is `ready`, `dormant`, `starting`, or `stopping`; the gateway holds requests during -lazy activation and wakes a stopping capability after cleanup, so neither state waits. The command -never fails the reset for a Storage problem: an unusable capability state -(`disabled`/`failed`/`stopped`), a missing capability/credentials, a stack-gateway activation -failure, or an invalid bucket config all print `WARNING: skipped seeding storage buckets: +local path stops the saved composition, resets the registered database instance's data, starts +and readies that instance, reapplies the catalog overlay and migrations, then starts the saved +composition again instead of recreating containers. The reset retains the saved database +configuration and composition members; it does not rebuild the stack from current config. After +the reset, buckets are seeded — reusing the `seed buckets` local path — when Storage is running +(healthy or still starting), starting, or stopping/stopped with wake enabled; an unhealthy or +errored Storage is skipped. The gateway handles lazy activation. The command never fails the +reset for a Storage problem: an unavailable service, missing credentials, a stack-gateway +activation failure, or an invalid bucket config all print `WARNING: skipped seeding storage buckets: ` to stderr and exit `0`, where the next step is `Run supabase seed buckets --local once Storage is available.` (or, for disabled Storage, to enable `[storage]`, run `supabase stack stop` then `supabase stack start`, @@ -44,14 +46,16 @@ then `supabase seed buckets --local`), unless the underlying stack-storage error suggestion (a missing API gateway endpoint, Auth disabled, or a gateway 502/503), in which case that suggestion is printed as the next step instead. The warning is omitted when the project configures no `[storage.buckets]` or vector buckets, since there is nothing to seed. The -Storage gateway URL is the selected stack's API gateway URL (`status.endpoints.api.url`), not -`[api].port`/`[api.tls]`, and the credential is the stack's service-role JWT — never `[api]`/ +Storage gateway URL and service state come from the saved composition Storage member, not the +current `[api]` config, and the credential is derived from the retained database JWT — never `[api]`/ `SUPABASE_API_*`/`SUPABASE_AUTH_{JWT_SECRET,SERVICE_ROLE_KEY}`. Bucket SQL/schema preparation is the stack runtime's own storage workload/catalog setup responsibility; bucket creation and `objects_path` upload from `[storage.buckets]` remain this command's (via `seed buckets`) responsibility — the runtime never creates buckets itself. Durable stack state lives under -`$SUPABASE_HOME/managed/stacks//`. A missing stack reports "The local stack is not -running." Config, including `functions/.env`, is validated before the wipe. +`$SUPABASE_HOME/stacks//`. A missing stack reports "The local stack is not +running." The database and stack config are validated before the wipe. If catalog preparation +or migrations fail, companion services remain stopped. Fix the reported error and rerun +`supabase db reset --local` to complete the reset and resume them. ## Files Read diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index d61178b048..87e16dd2f2 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -47,16 +47,21 @@ import type { OutputFormat } from "../../../shared/output/types.ts"; import { dockerRunLayer } from "../../../command-internal/docker-run.layer.ts"; import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; import { StackApi } from "../../../command-internal/stack-api.ts"; -import { recordingStackCatalogSetup } from "../../../command-internal/stack-catalog-setup.ts"; import { - CAPABILITY_NAMES, - StackIdSchema, - type CapabilityState, - type EffectStack, + StackCatalogSetup, + type StackCatalogSetupInput, +} from "../../../command-internal/stack-catalog-setup.ts"; +import type { + ServiceCreation, + ServiceInstance, + ServiceInstances, + Stack, + Observation, } from "@supabase/stack/effect"; import { DbConfigResolver } from "../../../command-internal/db-config.service.ts"; import type { DbConfigFlags, ResolvedDbConfig } from "../../../command-internal/db-config.types.ts"; import { DbConfigConnectTempRoleError } from "../../../command-internal/db-config.errors.ts"; +import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; import { LocalDockerEngine } from "../../../command-internal/db-bootstrap/local-db-running.ts"; import { DbExecError } from "../../../command-internal/db-connection.errors.ts"; import { @@ -476,116 +481,319 @@ function recordingStackStorageHttpClientBucketListTransportFails() { return { layer, requests }; } -const RESET_STACK_ID = StackIdSchema.make("c".repeat(64)); +const RESET_STACK_ID = "c".repeat(64); +const RESET_JWT = "stack-jwt-secret-with-at-least-32-characters"; + +type ResetServiceKind = "database" | "auth" | "storage" | "realtime" | "pooler"; +type StackService = ServiceInstances[ResetServiceKind]; + +const serviceCreation = ( + service: ResetServiceKind, + databaseUrl = "postgresql://postgres:postgres@127.0.0.1:5432/postgres", +): Extract => { + switch (service) { + case "database": + return { + service, + config: { + version: "15", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(RESET_JWT), + jwtExpiry: 3600, + }, + endpoints: {}, + }; + case "auth": + return { service, config: { databaseUrl, jwtSecret: RESET_JWT }, endpoints: {} }; + case "storage": + return { + service, + config: { databaseUrl, filePath: "/tmp/storage", jwtSecret: RESET_JWT }, + endpoints: {}, + }; + case "realtime": + return { service, config: { databaseUrl, jwtSecret: RESET_JWT }, endpoints: {} }; + case "pooler": + return { service, config: { databaseUrl, jwtSecret: RESET_JWT }, endpoints: {} }; + } +}; + +const stackService = ( + id: string, + creation: Extract, + observation: Effect.Effect, + overrides: { + readonly start?: Effect.Effect; + readonly ready?: Effect.Effect; + readonly stop?: Effect.Effect; + readonly resetData?: Effect.Effect; + } = {}, +): StackService => { + const base = { + id, + start: overrides.start ?? Effect.void, + ready: overrides.ready ?? Effect.void, + stop: overrides.stop ?? Effect.void, + restart: () => Effect.void, + destroy: Effect.void, + prepare: Effect.void, + status: observation, + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => Effect.succeed({}), + } satisfies Omit; + switch (creation.service) { + case "database": + return { + ...base, + service: "database", + credentials: () => + Effect.succeed({ databaseUrl: "postgresql://postgres:postgres@127.0.0.1:5432/postgres" }), + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: overrides.resetData ?? Effect.die("unused"), + }; + case "auth": + return { ...base, service: "auth" }; + case "storage": + return { ...base, service: "storage" }; + case "realtime": + return { ...base, service: "realtime" }; + case "pooler": + return { ...base, service: "pooler" }; + } +}; + +function makeStackObservation( + id: string, + creation: Extract, + opts: { + lifecycle?: "stopped" | "starting" | "running" | "stopping"; + health?: "starting" | "healthy" | "unhealthy"; + wakeEnabled?: boolean; + endpoints?: ReadonlyArray<{ + name: string; + protocol: "tcp" | "http"; + host: string; + port: number; + }>; + error?: { readonly _tag: "ServiceError"; readonly operation: string; readonly message: string }; + } = {}, +): Observation { + return { + id, + endpoints: opts.endpoints ?? [], + config: creation, + lifecycle: opts.lifecycle ?? "stopped", + health: opts.health, + error: opts.error, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: opts.wakeEnabled ?? false, + registered: true, + }; +} function mockResetStackApi(opts: { readonly workdir: string; readonly ready: boolean; readonly storageReady?: boolean; - /** Overrides the derived storage capability state (default: `ready` when `storageReady`, else `stopped`). */ - readonly storageState?: CapabilityState; + /** Overrides the derived storage observation state (default: `ready` when `storageReady`, else `stopped`). */ + readonly storageState?: + | "ready" + | "dormant" + | "starting" + | "stopping" + | "stopped" + | "failed" + | "disabled"; readonly storageError?: string; readonly apiEndpoint?: { readonly url: string; readonly port: number }; - readonly serviceRoleJwt?: string; - readonly capabilityStates?: Partial>; }) { let resetCalls = 0; - const unused = Effect.die("unused"); - const unusedFn = () => unused; - const stack: EffectStack = { - id: RESET_STACK_ID, - status: Effect.succeed({ - id: RESET_STACK_ID, - lifecycle: opts.ready ? "running" : "stopped", - desiredLifecycle: opts.ready ? "running" : "stopped", - runtime: { kind: "native" }, - endpoints: - opts.apiEndpoint === undefined - ? {} - : { - api: { - protocol: "http" as const, - address: "127.0.0.1", - port: opts.apiEndpoint.port, - url: opts.apiEndpoint.url, - }, - }, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "database" ? "eager" : "lazy", - state: - opts.capabilityStates?.[name] ?? - (name === "database" - ? opts.ready - ? "ready" - : "stopped" - : name === "storage" - ? (opts.storageState ?? (opts.storageReady === true ? "ready" : "stopped")) - : "stopped"), - ...(name === "storage" && opts.storageError !== undefined - ? { error: opts.storageError } - : {}), - })), - artifacts: [], + let stopCalls = 0; + let startCalls = 0; + let databaseRunning = opts.ready; + const database = serviceCreation("database"); + const storage = serviceCreation("storage"); + const auth = serviceCreation("auth"); + const realtime = serviceCreation("realtime"); + const pooler = serviceCreation("pooler"); + const endpoint = + opts.apiEndpoint === undefined + ? [] + : [ + { + name: "http", + protocol: "http" as const, + host: "127.0.0.1", + port: opts.apiEndpoint.port, + }, + ]; + const storageState = opts.storageState ?? (opts.storageReady === true ? "ready" : "stopped"); + const storageMember = + storageState === "disabled" + ? undefined + : stackService( + STORAGE_ID, + storage, + Effect.succeed( + makeStackObservation(STORAGE_ID, storage, { + lifecycle: + storageState === "dormant" + ? "stopped" + : storageState === "stopping" + ? "stopping" + : storageState === "failed" + ? "running" + : storageState === "starting" + ? "running" + : storageState === "ready" + ? "running" + : "stopped", + health: + storageState === "ready" + ? "healthy" + : storageState === "starting" + ? "starting" + : storageState === "failed" + ? "unhealthy" + : undefined, + wakeEnabled: + storageState === "dormant" || + storageState === "starting" || + storageState === "stopping", + endpoints: endpoint, + ...(opts.storageError === undefined + ? {} + : { + error: { + _tag: "ServiceError" as const, + operation: "start", + message: opts.storageError, + }, + }), + }), + ), + ); + const dbStatus = Effect.sync(() => + makeStackObservation(DB_ID, database, { + lifecycle: databaseRunning ? "running" : "stopped", + health: databaseRunning ? "healthy" : undefined, + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: 54329 }], }), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:54329/postgres"), - password: Redacted.make("postgres"), - }, - api: { - publishableKey: "anon", - secretKey: Redacted.make("service"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make(opts.serviceRoleJwt ?? "service"), - }, + ); + const db = stackService(DB_ID, database, dbStatus, { + resetData: Effect.suspend(() => + databaseRunning + ? Effect.die("database reset while running") + : Effect.sync(() => { + resetCalls++; + }), + ), + start: Effect.sync(() => { + databaseRunning = true; + startCalls++; }), - prepare: unusedFn, - start: unusedFn, - stop: unused, - destroy: unused, - resetDatabase: Effect.sync(() => { - resetCalls++; - return { - id: RESET_STACK_ID, - lifecycle: "running" as const, - desiredLifecycle: "running" as const, - runtime: { kind: "native" as const }, - endpoints: {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "database" ? ("eager" as const) : ("lazy" as const), - state: name === "database" ? ("ready" as const) : ("dormant" as const), + ready: Effect.void, + }); + const members: Array = [ + db, + stackService( + "supabase_auth_test", + auth, + Effect.succeed(makeStackObservation("supabase_auth_test", auth)), + ), + ...(storageMember === undefined ? [] : [storageMember]), + stackService( + "supabase_realtime_test", + realtime, + Effect.succeed(makeStackObservation("supabase_realtime_test", realtime)), + ), + stackService( + "supabase_pooler_test", + pooler, + Effect.succeed(makeStackObservation("supabase_pooler_test", pooler)), + ), + ]; + const stack: Stack = { + id: RESET_STACK_ID, + services: { + create: () => Effect.die("unused"), + get: (id) => { + const found = members.find((member) => member.id === id); + return found === undefined ? Effect.die(`missing service ${id}`) : Effect.succeed(found); + }, + list: Effect.succeed(members), + }, + composition: { + describe: Effect.succeed({ + members: members.map(({ id }, index) => ({ + id, + activation: index === 0 ? ("eager" as const) : ("lazy" as const), })), - artifacts: [], - }; - }), - logs: unusedFn, - followLogs: () => Stream.empty, + dependencies: [], + }), + supabase: () => Effect.die("unused"), + configure: () => Effect.die("unused"), + stop: Effect.sync(() => { + stopCalls++; + databaseRunning = false; + return members.map((member) => + makeStackObservation(member.id, serviceCreation(member.service), { + lifecycle: "stopped", + }), + ); + }), + start: Effect.sync(() => { + startCalls++; + databaseRunning = true; + return members.map((member) => + makeStackObservation(member.id, serviceCreation(member.service), { + lifecycle: "running", + health: "healthy", + }), + ); + }), + restart: Effect.die("unused"), + }, + stop: Effect.die("unused"), + destroy: Effect.die("unused"), + tools: { run: () => Effect.die("unused") }, }; return { layer: Layer.succeed(StackApi, { - createStack: unusedFn, - findStack: () => - Effect.succeed( - Option.some({ - id: RESET_STACK_ID, - projectRoot: opts.workdir, - name: "default", - branchContext: "main", - runtime: { kind: "native" as const }, - desiredLifecycle: "running" as const, - }), - ), - discoverStacks: unusedFn, - openStack: () => Effect.succeed(stack), - inspectStack: unusedFn, + create: () => Effect.die("unused"), + resolveIdentity: () => + Effect.succeed({ projectRoot: opts.workdir, branchContext: "main", stackName: "default" }), + discover: () => + Effect.succeed([ + { + definition: { + id: RESET_STACK_ID, + identity: { projectRoot: opts.workdir, branchContext: "main", stackName: "default" }, + runtime: "native" as const, + instances: members.map((member) => ({ id: member.id, creation: {} })), + composition: { members: [], dependencies: [] }, + ports: [], + }, + host: undefined, + }, + ]), + open: () => Effect.succeed(stack), }), get resetCalls() { return resetCalls; }, + get stopCalls() { + return stopCalls; + }, + get startCalls() { + return startCalls; + }, }; } @@ -634,11 +842,16 @@ function setup( stackBackend?: boolean; stackDatabaseReady?: boolean; stackStorageReady?: boolean; - stackStorageState?: CapabilityState; + stackStorageState?: + | "ready" + | "dormant" + | "starting" + | "stopping" + | "stopped" + | "failed" + | "disabled"; stackStorageError?: string; stackApiEndpoint?: { readonly url: string; readonly port: number }; - stackServiceRoleJwt?: string; - stackCapabilityStates?: Partial>; httpClient?: Layer.Layer; }, ) { @@ -674,15 +887,18 @@ function setup( storageState: opts.stackStorageState, storageError: opts.stackStorageError, apiEndpoint: opts.stackApiEndpoint, - serviceRoleJwt: opts.stackServiceRoleJwt, - capabilityStates: opts.stackCapabilityStates, }); const catalog = opts.stackBackend === true - ? recordingStackCatalogSetup((input) => ({ - kind: input.target.kind, - analytics: input.optionalConfig?.capabilities?.analytics?.enabled, - })) + ? (() => { + const applied: Array = []; + return { + layer: Layer.succeed(StackCatalogSetup, { + apply: (input) => Effect.sync(() => void applied.push(input)), + }), + applied, + }; + })() : undefined; const requests: Array<{ method: string; url: string; body: unknown }> = []; const storageRoutes = opts.storageRoutes; @@ -949,7 +1165,10 @@ describe("db reset", () => { return Effect.gen(function* () { yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer)); expect(stackApi.resetCalls).toBe(1); - expect(catalogApplied).toEqual([{ kind: "live", analytics: undefined }]); + expect(stackApi.stopCalls).toBe(1); + expect(stackApi.startCalls).toBe(2); + expect(catalogApplied[0]?.target.databaseServices).toEqual(["auth", "storage", "realtime"]); + expect(catalogApplied[0]?.target.jwtSecret).toBe(RESET_JWT); expect(child.spawned.some((s) => s.args[0] === "container" && s.args[1] === "rm")).toBe( false, ); @@ -959,31 +1178,33 @@ describe("db reset", () => { }); }); - it.live("skips optional analytics catalog when the running stack has it disabled", () => { + it.live("preserves the existing stack composition while rebuilding the database", () => { const { layer, catalogApplied } = setup(tmp.current, { toml: 'project_id = "test"\n', args: ["db", "reset", "--local"], isLocal: true, stackBackend: true, - stackCapabilityStates: { analytics: "disabled" }, }); return Effect.gen(function* () { yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer)); - expect(catalogApplied).toEqual([{ kind: "live", analytics: false }]); + expect(catalogApplied[0]?.target.databaseServices).toEqual(["auth", "storage", "realtime"]); }); }); - it.live("re-inits optional analytics catalog when the running stack has it ready", () => { - const { layer, catalogApplied } = setup(tmp.current, { + it.live("leaves the stopped composition after a migration failure", () => { + const { layer, stackApi } = setup(tmp.current, { toml: 'project_id = "test"\n', + files: { ...migrationFile("20240101000000", "create table stack_reset_failure ();") }, + execFailsOn: "create table stack_reset_failure", args: ["db", "reset", "--local"], isLocal: true, stackBackend: true, - stackCapabilityStates: { analytics: "ready" }, }); return Effect.gen(function* () { - yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer)); - expect(catalogApplied).toEqual([{ kind: "live", analytics: undefined }]); + const exit = yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer), Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(stackApi.stopCalls).toBe(1); + expect(stackApi.startCalls).toBe(1); }); }); @@ -1005,7 +1226,6 @@ describe("db reset", () => { stackBackend: true, stackStorageReady: true, stackApiEndpoint: { url: "http://127.0.0.1:55421", port: 55421 }, - stackServiceRoleJwt: "stack-service-role-jwt-not-from-config", httpClient: Layer.succeed( HttpClient.HttpClient, HttpClient.make((request) => { @@ -1033,8 +1253,9 @@ describe("db reset", () => { ); expect(requests.some((request) => request.url.includes("127.0.0.1:54321"))).toBe(false); expect( - requests.some((request) => - request.authorization.includes("stack-service-role-jwt-not-from-config"), + requests.some( + (request) => + request.authorization === `Bearer ${generateGoJwt(RESET_JWT, "service_role")}`, ), ).toBe(true); }); @@ -1070,7 +1291,6 @@ describe("db reset", () => { stackBackend: true, stackStorageState: "dormant", stackApiEndpoint: { url: "http://127.0.0.1:55422", port: 55422 }, - stackServiceRoleJwt: "dormant-stack-jwt", httpClient: client.layer, }); return Effect.gen(function* () { @@ -1078,9 +1298,11 @@ describe("db reset", () => { expect(client.requests.some((r) => r.url.includes("127.0.0.1:55422/storage/v1"))).toBe( true, ); - expect(client.requests.some((r) => r.authorization.includes("dormant-stack-jwt"))).toBe( - true, - ); + expect( + client.requests.some( + (r) => r.authorization === `Bearer ${generateGoJwt(RESET_JWT, "service_role")}`, + ), + ).toBe(true); }); }); @@ -1167,7 +1389,6 @@ describe("db reset", () => { stackBackend: true, stackStorageState: "starting", stackApiEndpoint: { url: "http://127.0.0.1:55425", port: 55425 }, - stackServiceRoleJwt: "starting-stack-jwt", httpClient: client.layer, }); return Effect.gen(function* () { @@ -1175,9 +1396,11 @@ describe("db reset", () => { expect(client.requests.some((r) => r.url.includes("127.0.0.1:55425/storage/v1"))).toBe( true, ); - expect(client.requests.some((r) => r.authorization.includes("starting-stack-jwt"))).toBe( - true, - ); + expect( + client.requests.some( + (r) => r.authorization === `Bearer ${generateGoJwt(RESET_JWT, "service_role")}`, + ), + ).toBe(true); }); }, ); @@ -1192,7 +1415,6 @@ describe("db reset", () => { stackBackend: true, stackStorageState: "stopping", stackApiEndpoint: { url: "http://127.0.0.1:55430", port: 55430 }, - stackServiceRoleJwt: "stopping-stack-jwt", httpClient: client.layer, }); return Effect.gen(function* () { @@ -1200,9 +1422,11 @@ describe("db reset", () => { expect(client.requests.some((r) => r.url.includes("127.0.0.1:55430/storage/v1"))).toBe( true, ); - expect(client.requests.some((r) => r.authorization.includes("stopping-stack-jwt"))).toBe( - true, - ); + expect( + client.requests.some( + (r) => r.authorization === `Bearer ${generateGoJwt(RESET_JWT, "service_role")}`, + ), + ).toBe(true); expect(out.stderrText).not.toContain("skipped seeding storage buckets"); }); }); @@ -1296,12 +1520,9 @@ describe("db reset", () => { const exit = yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer), Effect.exit); expect(Exit.isSuccess(exit)).toBe(true); expect(out.stderrText).toContain( - "WARNING: skipped seeding storage buckets: The stack exposes no API gateway endpoint.", + "WARNING: skipped seeding storage buckets: The stack exposes no Storage endpoint.", ); expect(out.stderrText).toContain( - "Run supabase stack status to inspect the stack, or supabase stack restart.", - ); - expect(out.stderrText).not.toContain( "Run supabase seed buckets --local once Storage is available.", ); }); @@ -1330,22 +1551,6 @@ describe("db reset", () => { }, ); - it.live("fails stack reset on a bad functions/.env before wiping", () => { - const { layer, stackApi } = setup(tmp.current, { - toml: 'project_id = "test"\n', - files: { "supabase/functions/.env": "lowercase=value\nSECRET=value\n" }, - args: ["db", "reset", "--local"], - isLocal: true, - stackBackend: true, - }); - return Effect.gen(function* () { - const exit = yield* dbReset(DEFAULT_FLAGS).pipe(Effect.provide(layer), Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) expect(JSON.stringify(exit.cause)).toContain("functions/.env"); - expect(stackApi.resetCalls).toBe(0); - }); - }); - it.live( "fails a local reset before the destructive recreate on a malformed config.toml", () => { diff --git a/apps/cli/src/commands/db/reset/reset.layers.ts b/apps/cli/src/commands/db/reset/reset.layers.ts index ab0007dcf4..7a2db4e800 100644 --- a/apps/cli/src/commands/db/reset/reset.layers.ts +++ b/apps/cli/src/commands/db/reset/reset.layers.ts @@ -80,7 +80,7 @@ export const dbResetRuntimeLayer = Layer.mergeAll( dockerRunLayer, // Backs `isLocalDbRunning`'s direct Engine-API probe (+ its `--debug` trace). localDockerEngineLayer.pipe(Layer.provide(debugLoggerLayer)), - // Exposed so `db reset --local` can open the project stack and call `resetDatabase`. + // Exposed so `db reset --local` can open the project stack and reset its database instance. stackApiLayer, stackCatalogSetupLayer, commandRuntimeLayer(["db", "reset"]), diff --git a/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts b/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts new file mode 100644 index 0000000000..d4a3445d04 --- /dev/null +++ b/apps/cli/src/commands/db/reset/reset.stack.e2e.test.ts @@ -0,0 +1,270 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Data, Effect, FileSystem, Layer, Path, Redacted } from "effect"; +import { FetchHttpClient, HttpClient } from "effect/unstable/http"; +import { PgClient } from "@effect/sql-pg"; +import { create as createStack } from "@supabase/stack/effect"; +import { tmpdir } from "node:os"; + +import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; + +const COMMAND_TIMEOUT_MS = 8 * 60_000; +const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS + 2 * 60_000; +const JWT_SECRET = "db-reset-stack-e2e-jwt-secret-with-32-chars"; +const nativeSupported = + (process.platform === "linux" && (process.arch === "x64" || process.arch === "arm64")) || + (process.platform === "darwin" && process.arch === "arm64"); + +const projectConfig = `project_id = "db-reset-stack-e2e" + +[experimental] +stack = true + +[api] +enabled = true + +[db] +major_version = 17 + +[db.seed] +enabled = true +sql_paths = ["seed.sql"] + +[auth] +enabled = true + +[realtime] +enabled = false + +[storage] +enabled = false + +[edge_runtime] +enabled = false + +[studio] +enabled = false + +[analytics] +enabled = false + +[db.pooler] +enabled = false + +[local_smtp] +enabled = false +`; + +class DbResetStackE2eError extends Data.TaggedError("DbResetStackE2eError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const writeFixture = Effect.fn("DbResetStackE2e.writeFixture")(function* ( + root: string, + fs: FileSystem.FileSystem, + path: Path.Path, +) { + const migrations = path.join(root, "supabase", "migrations"); + yield* fs.makeDirectory(migrations, { recursive: true }); + yield* fs.writeFileString(path.join(root, "supabase", "config.toml"), projectConfig); + yield* fs.writeFileString( + path.join(migrations, "20260920000000_reset_story.sql"), + `create table public.reset_story_marker (value text primary key); +create table public.reset_story_user_link ( + user_id uuid references auth.users(id), + value text not null +); +`, + ); + yield* fs.writeFileString( + path.join(root, "supabase", "seed.sql"), + "insert into public.reset_story_marker(value) values ('seeded');\n", + ); +}); + +const query = Effect.fn("DbResetStackE2e.query")(function* (url: string, sql: string) { + return yield* Effect.scoped( + Effect.gen(function* () { + const services = yield* Layer.build(PgClient.layer({ url: Redacted.make(url) })); + return yield* Context.get(services, PgClient.PgClient).unsafe(sql); + }), + ).pipe( + Effect.mapError( + (cause) => new DbResetStackE2eError({ message: `query failed: ${sql}`, cause }), + ), + ); +}); + +const responseStatus = Effect.fn("DbResetStackE2e.responseStatus")(function* (url: string) { + const http = yield* HttpClient.HttpClient; + const response = yield* http + .get(url) + .pipe( + Effect.mapError( + (cause) => new DbResetStackE2eError({ message: `request failed: ${url}`, cause }), + ), + ); + yield* response.arrayBuffer; + return response.status; +}); + +const composeStack = Effect.fn("DbResetStackE2e.composeStack")(function* ( + root: string, + home: string, + runtime: "native" | "docker", +) { + const stack = yield* createStack({ + projectRoot: root, + stateRoot: `${home}/stacks`, + cacheRoot: `${home}/cache/stack`, + runtime, + }); + yield* Effect.addFinalizer(() => + stack.destroy.pipe( + Effect.catch((cause) => + Effect.die(new DbResetStackE2eError({ message: "stack cleanup failed", cause })), + ), + ), + ); + yield* stack.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(JWT_SECRET), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { + service: "rest", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: JWT_SECRET, + }, + endpoints: { http: { port: "auto" } }, + }, + { + service: "auth", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: JWT_SECRET, + }, + endpoints: { http: { port: "auto" } }, + }, + ]); + yield* stack.composition.start; + return stack; +}); + +describe("supabase db reset (stack e2e)", () => { + for (const runtime of ["native", "docker"] as const) { + if (runtime === "native" && !nativeSupported) continue; + + it.live( + `resets the same ${runtime} database instance and resumes its lazy composition`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: `db-reset-${runtime}-` }); + const home = yield* fs.makeTempDirectoryScoped({ prefix: `db-reset-home-${runtime}-` }); + yield* writeFixture(root, fs, path); + yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); + yield* fs.makeDirectory(path.join(tmpdir(), "supabase-stack-artifacts"), { + recursive: true, + }); + yield* fs.symlink( + path.join(tmpdir(), "supabase-stack-artifacts"), + path.join(home, "cache", "stack"), + ); + + const stack = yield* composeStack(root, home, runtime); + + const beforeComposition = yield* stack.composition.describe; + const beforeServices = yield* stack.services.list; + const database = beforeServices.find((service) => service.service === "database"); + const rest = beforeServices.find((service) => service.service === "rest"); + const auth = beforeServices.find((service) => service.service === "auth"); + if ( + database?.service !== "database" || + rest?.service !== "rest" || + auth?.service !== "auth" + ) + return yield* Effect.die("database, REST, and Auth instances are missing"); + const beforeCredentials = yield* database.credentials(); + const databaseUrl = beforeCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("database URL is missing"); + const beforeStatus = yield* database.status; + const beforeSqlPort = beforeStatus.endpoints.find(({ name }) => name === "sql")?.port; + if (beforeSqlPort === undefined) return yield* Effect.die("database SQL port is missing"); + + yield* query( + databaseUrl, + `create table public.reset_story_stale (value text); + insert into public.reset_story_stale values ('before-reset'); + create role reset_story_stale_role login password 'stale-password';`, + ); + yield* query(databaseUrl, "create database reset_story_stale_database;"); + + const reset = yield* runSupabaseEffect(["db", "reset", "--local"], { + cwd: root, + home, + env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, + exitTimeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(reset.exitCode, `stdout:\n${reset.stdout}\nstderr:\n${reset.stderr}`).toBe(0); + + const afterComposition = yield* stack.composition.describe; + expect(afterComposition).toEqual(beforeComposition); + const afterServices = yield* stack.services.list; + const afterDatabase = afterServices.find((service) => service.service === "database"); + const afterRest = afterServices.find((service) => service.service === "rest"); + const afterAuth = afterServices.find((service) => service.service === "auth"); + if ( + afterDatabase?.service !== "database" || + afterRest?.service !== "rest" || + afterAuth?.service !== "auth" + ) + return yield* Effect.die("database, REST, and Auth instances disappeared"); + expect(afterDatabase.id).toBe(database.id); + expect((yield* afterDatabase.credentials()).databaseUrl).toBe(databaseUrl); + expect( + (yield* afterDatabase.status).endpoints.find(({ name }) => name === "sql")?.port, + ).toBe(beforeSqlPort); + + const rows = yield* query( + databaseUrl, + `select + (select count(*) from supabase_migrations.schema_migrations where version = '20260920000000') as migration_count, + (select count(*) from public.reset_story_marker where value = 'seeded') as seed_count, + to_regclass('public.reset_story_stale') as stale_table, + to_regclass('auth.users') is not null as auth_users, + exists (select 1 from pg_roles where rolname = 'reset_story_stale_role') as stale_role, + exists (select 1 from pg_database where datname = 'reset_story_stale_database') as stale_database;`, + ); + expect(rows).toEqual([ + { + migration_count: "1", + seed_count: "1", + stale_table: null, + auth_users: true, + stale_role: false, + stale_database: false, + }, + ]); + + const restStatus = yield* afterRest.status; + expect(restStatus.lifecycle).toBe("stopped"); + expect(restStatus.wakeEnabled).toBe(true); + const restCredentials = yield* afterRest.credentials(); + const restUrl = restCredentials.url; + if (restUrl === undefined) return yield* Effect.die("REST URL is missing"); + expect(yield* responseStatus(restUrl)).toBe(200); + }).pipe(Effect.provide([BunServices.layer, FetchHttpClient.layer])), + { timeout: TEST_TIMEOUT_MS }, + ); + } +}); diff --git a/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md index db631bd37a..86ad6ef40a 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md @@ -6,9 +6,9 @@ platform view. Pg-delta runs in-process. Coverage gaps warn; `--strict-coverage` makes them fatal, and `PGDELTA_DEBUG` writes diagnostic JSON under `supabase/.temp/pgdelta/v2/debug//`. -When `[experimental].stack` is on, `--local` uses the project stack. Shadow work from the -sibling commands lives under `$SUPABASE_HOME/managed/ephemeral-postgres//` -(`~/.supabase/managed/…` by default). +When `[experimental].stack` is on, `--local` connects to the running primary database in the +project stack's composition. It does not launch a stopped database or choose a standalone +shadow database. `--no-cache` (a flag shared across the `declarative` group) has no effect on `generate` — the export connects directly to the target and provisions no diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index bf68e2a5fb..c50002d8af 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -3,7 +3,8 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option } from "effect"; +import { Cause, Effect, Exit, Layer, Option, Redacted, Stream } from "effect"; +import { StackError, type DatabaseInstance, type Stack } from "@supabase/stack/effect"; import { stripAnsi } from "../../../../../../tests/helpers/ansi.ts"; import { @@ -42,6 +43,7 @@ import { CommandPlatformApi } from "../../../../../auth/command-platform-api.ser import { CommandPlatformApiFactory } from "../../../../../auth/command-platform-api-factory.service.ts"; import { dockerRunLayer } from "../../../../../command-internal/docker-run.layer.ts"; import { stackBackendLayer } from "../../../../../command-internal/stack-backend.ts"; +import { StackApi } from "../../../../../command-internal/stack-api.ts"; import { DbConfigResolver } from "../../../../../command-internal/db-config.service.ts"; import { type DbSession, @@ -82,6 +84,99 @@ interface EngineExportCall { readonly strictCoverage: boolean; } +const GENERATE_STACK_ID = "e".repeat(64); +const unusedStack = Effect.die("unused"); +const unusedStackFn = () => unusedStack; +const STACK_GENERATE_PORT = 54329; + +function generateStackApi(workdir: string) { + const database: DatabaseInstance = { + id: "primary", + service: "database", + start: unusedStack, + ready: unusedStack, + stop: unusedStack, + restart: unusedStackFn, + destroy: unusedStack, + prepare: unusedStack, + credentials: unusedStackFn, + exportSnapshot: unusedStackFn, + restoreSnapshot: unusedStackFn, + resetData: unusedStack, + logs: Stream.empty, + followStatus: Stream.empty, + status: Effect.succeed({ + id: "primary", + config: { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make("secret"), + jwtExpiry: 3600, + }, + }, + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: STACK_GENERATE_PORT }], + lifecycle: "running", + health: "healthy", + registered: true, + wakeEnabled: true, + intentRevision: 0, + currentOperation: undefined, + launchId: undefined, + exit: undefined, + error: undefined, + cleanupError: undefined, + }), + }; + const composition = { + members: [{ id: database.id, activation: "eager" as const }], + dependencies: [], + }; + const stack: Stack = { + id: GENERATE_STACK_ID, + services: { + create: unusedStackFn, + list: Effect.succeed([database]), + get: (id) => + id === database.id + ? Effect.succeed(database) + : Effect.fail(new StackError({ operation: "get", message: "unknown instance" })), + }, + composition: { + describe: Effect.succeed(composition), + supabase: unusedStackFn, + configure: unusedStackFn, + start: unusedStack, + stop: unusedStack, + restart: unusedStack, + }, + stop: unusedStack, + destroy: unusedStack, + tools: { run: unusedStackFn }, + }; + const identity = { projectRoot: workdir, branchContext: "main", stackName: "default" }; + return Layer.succeed(StackApi, { + create: unusedStackFn, + open: () => Effect.succeed(stack), + resolveIdentity: () => Effect.succeed(identity), + discover: () => + Effect.succeed([ + { + definition: { + id: stack.id, + identity, + runtime: "native", + instances: [], + composition, + ports: [], + }, + host: undefined, + }, + ]), + }); +} + function setup(workdir: string, opts: SetupOpts = {}) { const out = mockOutput({ promptConfirmResponses: opts.promptConfirmResponses, @@ -248,7 +343,9 @@ function setup(workdir: string, opts: SetupOpts = {}) { processControl.layer, alwaysReadyHttpClientLayer, dockerRun, - ...(opts.stackBackend === true ? [stackBackendLayer("stack")] : []), + ...(opts.stackBackend === true + ? [stackBackendLayer("stack"), generateStackApi(workdir)] + : [generateStackApi(workdir)]), ); return { layer, diff --git a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md index 9623bfa722..155c9b64ed 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md @@ -3,8 +3,12 @@ Diffs local migrations state against declarative schema files and writes the delta as a new timestamped migration. -When `[experimental].stack` is on, shadows are `EphemeralPostgres` clusters under -`$SUPABASE_HOME/managed/ephemeral-postgres//` (`~/.supabase/managed/…` by default). +When `[experimental].stack` is on, each shadow uses a fresh, invocation-owned stack namespace +with an automatically assigned port. State and data live under `$SUPABASE_HOME/stacks`, and +native artifacts are shared through `$SUPABASE_HOME/cache/stack`. The command destroys its +shadow namespaces when its Effect scope closes. An abrupt process exit can leave a namespace +visible to `stack list` for manual `stack destroy` cleanup. Stack shadows do not use the +baseline snapshot cache described below; `--no-cache` affects only the legacy backend. Pg-delta runs in-process and uses two scoped shadow databases. Coverage gaps warn; `--strict-coverage` makes @@ -137,7 +141,7 @@ existing SQL or creates an export manifest. with the same flag. A real version/tag mismatch still suggests `supabase stop --all --no-backup` then `supabase start`. -### Shadow baseline cache (`SUPABASE_SHADOW_CACHE`, default ON) +### Legacy shadow baseline cache (`SUPABASE_SHADOW_CACHE`, default ON) The bundled (pg-delta next) engine provisions both plan shadows through `acquireShadowDatabase` (`pgdelta-next-shadow.layer.ts`): on by default, off when diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index 93ae6241b6..8b1740bdba 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -41,7 +41,7 @@ import { CommandPlatformApiFactory } from "../../../../../auth/command-platform- import { dockerRunLayer } from "../../../../../command-internal/docker-run.layer.ts"; import { stackBackendLayer } from "../../../../../command-internal/stack-backend.ts"; import { StackApi } from "../../../../../command-internal/stack-api.ts"; -import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; +import { StackError, type DatabaseInstance, type Stack } from "@supabase/stack/effect"; import { DbConfigResolver } from "../../../../../command-internal/db-config.service.ts"; import { type DbBatchStatement, @@ -85,64 +85,96 @@ interface SetupOpts { stackBackend?: boolean; } -const SYNC_STACK_ID = StackIdSchema.make("e".repeat(64)); +const SYNC_STACK_ID = "e".repeat(64); const unusedSync = Effect.die("unused"); const unusedSyncFn = () => unusedSync; const STACK_APPLY_PORT = 54329; function syncStackApi(workdir: string, port: number) { - const stack: EffectStack = { - id: SYNC_STACK_ID, + const database: DatabaseInstance = { + id: "primary", + service: "database", + start: unusedSync, + ready: unusedSync, + stop: unusedSync, + restart: unusedSyncFn, + destroy: unusedSync, + prepare: unusedSync, + credentials: unusedSyncFn, + exportSnapshot: unusedSyncFn, + restoreSnapshot: unusedSyncFn, + resetData: unusedSync, + logs: Stream.empty, + followStatus: Stream.empty, status: Effect.succeed({ - id: SYNC_STACK_ID, - lifecycle: "running", - desiredLifecycle: "running", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "database" ? "eager" : "lazy", - state: name === "database" ? "ready" : "dormant", - })), - artifacts: [], - }), - credentials: Effect.succeed({ - database: { - url: Redacted.make(`postgresql://postgres:postgres@127.0.0.1:${port}/postgres`), - password: Redacted.make("postgres"), - }, - api: { - publishableKey: "anon", - secretKey: Redacted.make("service"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service"), + id: "primary", + config: { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make("secret"), + jwtExpiry: 3600, + }, }, + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port }], + lifecycle: "running", + health: "healthy", + registered: true, + wakeEnabled: true, + intentRevision: 0, + currentOperation: undefined, + launchId: undefined, + exit: undefined, + error: undefined, + cleanupError: undefined, }), - prepare: unusedSyncFn, - start: unusedSyncFn, + }; + const composition = { + members: [{ id: database.id, activation: "eager" as const }], + dependencies: [], + }; + const stack: Stack = { + id: SYNC_STACK_ID, + services: { + create: unusedSyncFn, + list: Effect.succeed([database]), + get: (id) => + id === database.id + ? Effect.succeed(database) + : Effect.fail(new StackError({ operation: "get", message: "unknown instance" })), + }, + composition: { + describe: Effect.succeed(composition), + supabase: unusedSyncFn, + configure: unusedSyncFn, + start: unusedSync, + stop: unusedSync, + restart: unusedSync, + }, stop: unusedSync, destroy: unusedSync, - resetDatabase: unusedSync, - logs: unusedSyncFn, - followLogs: () => Stream.empty, + tools: { run: unusedSyncFn }, }; + const identity = { projectRoot: workdir, branchContext: "main", stackName: "default" }; return Layer.succeed(StackApi, { - createStack: unusedSyncFn, - findStack: () => - Effect.succeed( - Option.some({ - id: SYNC_STACK_ID, - projectRoot: workdir, - name: "default", - branchContext: "main", - runtime: { kind: "native" as const }, - desiredLifecycle: "running", - }), - ), - discoverStacks: unusedSyncFn, - openStack: () => Effect.succeed(stack), - inspectStack: unusedSyncFn, + create: unusedSyncFn, + open: () => Effect.succeed(stack), + resolveIdentity: () => Effect.succeed(identity), + discover: () => + Effect.succeed([ + { + definition: { + id: stack.id, + identity, + runtime: "native", + instances: [], + composition, + ports: [], + }, + host: undefined, + }, + ]), }); } diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts index f88c638d98..6e08a7823a 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.layer.ts @@ -54,6 +54,7 @@ import { stackAcquireShadowDatabase, stackMigrateShadow, } from "../../../command-internal/stack-shadow.ts"; +import { StackApi } from "../../../command-internal/stack-api.ts"; import { stackCatalogSetupLayer } from "../../../command-internal/stack-catalog-setup.ts"; const allocateFreeHostPort = Effect.callback>((resume) => { @@ -145,6 +146,7 @@ export const pgDeltaNextShadowLayer = Layer.effect( const dbConnection = yield* DbConnection; const httpClient = yield* HttpClient.HttpClient; const cliSettings = yield* CommandSettings; + const stackApi = yield* StackApi; const runtimeWith = (outputService: typeof Output.Service) => { const deps = Layer.mergeAll( @@ -162,6 +164,7 @@ export const pgDeltaNextShadowLayer = Layer.effect( Layer.succeed(HttpClient.HttpClient, httpClient), Layer.succeed(Crypto.Crypto, crypto), Layer.succeed(CommandSettings, cliSettings), + Layer.succeed(StackApi, stackApi), Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), ); return Layer.mergeAll(deps, stackCatalogSetupLayer.pipe(Layer.provide(deps))); @@ -290,19 +293,18 @@ export const pgDeltaNextShadowLayer = Layer.effect( }).pipe(Effect.provide(runtimeWith(outputService)), Effect.mapError(nextShadowError)); const stackAcquire = (input: NativeShadowInput, opts: ShadowCacheOpts) => - stackAcquireShadowDatabase(input.base, { - ...(opts.bypassCache === true ? { bypassCache: true } : {}), - port: input.base.shadowPort, - ...(opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }), - }); + stackAcquireShadowDatabase( + input.base, + opts.webhooks === undefined ? {} : { webhooks: opts.webhooks }, + ); const stackProvisionMigrations = (input: NativeShadowInput, opts: ShadowCacheOpts) => Effect.gen(function* () { const handle = yield* stackAcquire(input, opts); - yield* stackMigrateShadow(handle, input.base); + yield* Effect.scoped(stackMigrateShadow(handle, input.base)); return { migrationsUrl: handle.url, - snapshotKey: handle.snapshotKey, + snapshotKey: undefined, } satisfies ProvisionedMigrationsShadow; }).pipe(Effect.provide(runtime), Effect.mapError(nextShadowError)); @@ -311,8 +313,8 @@ export const pgDeltaNextShadowLayer = Layer.effect( const handle = yield* stackAcquire(input, opts); return { declarativeUrl: handle.url, - restoredFromPgDataSnapshot: handle.baselinePresent, - snapshotKey: handle.snapshotKey, + restoredFromPgDataSnapshot: false, + snapshotKey: undefined, } satisfies ProvisionedDeclarativeShadow; }).pipe(Effect.provide(runtime), Effect.mapError(nextShadowError)); @@ -327,22 +329,22 @@ export const pgDeltaNextShadowLayer = Layer.effect( return PgDeltaNextShadow.of({ provisionMigrations: (opts) => Effect.gen(function* () { - const port = yield* nextPort(); + const backend = yield* currentStackBackend; + const port = backend.kind === "stack" ? 0 : yield* nextPort(); const built = yield* buildNativeBase(opts); const input = buildNativeInput(opts, built, port); - const backend = yield* currentStackBackend; return backend.kind === "stack" ? yield* stackProvisionMigrations(input, cacheOpts(opts, "config")) : yield* provisionMigrations(input, cacheOpts(opts, "config")); }).pipe(Effect.mapError(nextShadowError)), provisionPlan: (opts) => Effect.gen(function* () { - const migrationsPort = yield* nextPort(); - const declarativePort = yield* nextPort(migrationsPort); + const backend = yield* currentStackBackend; + const migrationsPort = backend.kind === "stack" ? 0 : yield* nextPort(); + const declarativePort = backend.kind === "stack" ? 0 : yield* nextPort(migrationsPort); const built = yield* buildNativeBase(opts); const migrationsInput = buildNativeInput(opts, built, migrationsPort); const declarativeInput = buildNativeInput(opts, built, declarativePort); - const backend = yield* currentStackBackend; if (backend.kind === "stack") { const migrations = yield* stackProvisionMigrations( migrationsInput, diff --git a/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts new file mode 100644 index 0000000000..ff51128750 --- /dev/null +++ b/apps/cli/src/commands/db/shared/pgdelta-next-shadow.stack.integration.test.ts @@ -0,0 +1,166 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { FetchHttpClient } from "effect/unstable/http"; +import { Effect, FileSystem, Layer, Option } from "effect"; + +import { mockCommandSettings } from "../../../../tests/helpers/command-mocks.ts"; +import { mockOutput } from "../../../../tests/helpers/mocks.ts"; +import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; +import { runtimeInfoLayer } from "../../../shared/runtime/runtime-info.layer.ts"; +import { + DebugFlag, + ExperimentalFlag, + NetworkIdFlag, +} from "../../../command-internal/global-flags.ts"; +import { DbConnection } from "../../../command-internal/db-connection.service.ts"; +import { DockerRun } from "../../../command-internal/docker-run.service.ts"; +import { dbConnectionLayer } from "../../../command-internal/db-connection.layer.ts"; +import { parseConnectionString } from "../../../command-internal/db-config.parse.ts"; +import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { pgDeltaNextShadowLayer } from "./pgdelta-next-shadow.layer.ts"; +import { PgDeltaNextShadow } from "./pgdelta-next-shadow.service.ts"; +import type { DbTomlValues } from "../../../command-internal/db-config.toml-read.ts"; + +const toml = { + projectEnv: {}, + envLookup: () => undefined, + apiSchemas: ["public", "graphql_public"], + port: 54321, + shadowPort: 54320, + password: "postgres", + poolerConnectionString: Option.none(), + projectId: Option.some("test"), + majorVersion: 17, + orioledbVersion: Option.none(), + denoVersion: 2, + pgDelta: { + enabled: true, + declarativeSchemaPath: Option.none(), + formatOptions: Option.none(), + }, + webhooksEnabled: false, + baseline: { + authEnabled: false, + storageEnabled: false, + realtimeEnabled: false, + apiAutoExposeNewTables: Option.none(), + vaultNames: [], + }, + migrationsEnabled: true, + schemaPaths: [], + schemaPathPatterns: [], + seed: { enabled: false, sqlPaths: [] }, + vault: [], + appliedRemote: undefined, + remoteOverrideKeys: new Set(), +} satisfies DbTomlValues; + +const fakeDocker = Layer.succeed(DockerRun, { + run: () => Effect.die("Docker is not used by the native stack backend"), + runCapture: () => Effect.die("Docker is not used by the native stack backend"), + runStream: () => Effect.die("Docker is not used by the native stack backend"), +}); + +const query = Effect.fn("PgDeltaNextShadowStackTest.query")(function* ( + db: DbConnection["Service"], + url: string, + sql: string, +) { + const connection = parseConnectionString(url); + if (connection === undefined) return yield* Effect.die("invalid shadow URL"); + const session = yield* db.connect(connection, { isLocal: true, dnsResolver: "native" }); + return yield* session.query(sql); +}); + +describe("pg-delta next stack shadow provisioning", () => { + it.live( + "keeps migrations on the migration shadow and destroys both shadows with the caller scope", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "pgdelta-next-stack-" }); + yield* fs.makeDirectory(`${root}/supabase/migrations`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "test"\n[db]\nmajor_version = 17\n[auth]\nenabled = false\n[storage]\nenabled = false\n[realtime]\nenabled = false\n', + ); + yield* fs.writeFileString( + `${root}/supabase/migrations/20260919000000_shadow.sql`, + "CREATE TABLE public.pgdelta_next_probe(value text); INSERT INTO public.pgdelta_next_probe VALUES ('migration');\n", + ); + + const stateRoot = `${root}/stacks`; + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); + const output = mockOutput().layer; + const apiLayer = stackApiLayer.pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(BunServices.layer), + ); + const shadowLayer = pgDeltaNextShadowLayer.pipe( + Layer.provide(apiLayer), + Layer.provide(dbConnectionLayer), + Layer.provide(fakeDocker), + Layer.provide(FetchHttpClient.layer), + Layer.provide(settings), + Layer.provide(runtimeInfoLayer), + Layer.provide(output), + Layer.provide(Layer.succeed(DebugFlag, false)), + Layer.provide(Layer.succeed(ExperimentalFlag, false)), + Layer.provide(Layer.succeed(NetworkIdFlag, Option.none())), + Layer.provide(Layer.succeed(CliArgs, { args: [] })), + Layer.provide(BunServices.layer), + ); + const services = Layer.mergeAll( + BunServices.layer, + FetchHttpClient.layer, + stackBackendLayer("stack"), + apiLayer, + dbConnectionLayer, + fakeDocker, + settings, + output, + Layer.succeed(DebugFlag, false), + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(NetworkIdFlag, Option.none()), + Layer.succeed(CliArgs, { args: [] }), + shadowLayer, + ); + + const urls = yield* Effect.scoped( + Effect.gen(function* () { + const db = yield* DbConnection; + const shadows = yield* PgDeltaNextShadow; + const plan = yield* shadows.provisionPlan({ + context: { projectId: "test", cwd: root, denoVersion: 2, projectEnv: {} }, + toml, + projectRef: "test", + bypassCache: true, + }); + expect(plan.migrationsUrl).not.toBe(plan.declarativeUrl); + expect( + yield* query(db, plan.migrationsUrl, "SELECT value FROM public.pgdelta_next_probe"), + ).toEqual([{ value: "migration" }]); + expect( + yield* query( + db, + plan.declarativeUrl, + "SELECT to_regclass('public.pgdelta_next_probe') AS table_name", + ), + ).toEqual([{ table_name: null }]); + const api = yield* StackApi; + expect(yield* api.discover({ stateRoot })).toHaveLength(2); + return plan; + }).pipe(Effect.provide(services)), + ); + + expect(urls.migrationsUrl).toMatch(/^postgresql:/); + const api = yield* Effect.gen(function* () { + const api = yield* StackApi; + return yield* api.discover({ stateRoot }); + }).pipe(Effect.provide(services)); + expect(api).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + 180_000, + ); +}); diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts index 63a11c21fc..9c6f0d14af 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.integration.test.ts @@ -15,6 +15,7 @@ import { useShadowCacheDisabled, } from "../../../../tests/helpers/command-mocks.ts"; import { mockOutput, mockRuntimeInfo } from "../../../../tests/helpers/mocks.ts"; +import { unusedStackServices } from "../../../../tests/helpers/unused-stack.ts"; import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; import { DebugFlag, @@ -119,6 +120,7 @@ function setup( // satisfy as it's applied, so `BunServices.layer` only ever fills in `FileSystem`/`Path`. Layer.provide(shadowSpawner.layer), Layer.provide(mockLocalDockerEngineUnavailableLayer), + Layer.provide(unusedStackServices), Layer.provide(BunServices.layer), ); diff --git a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts index d41159905c..b49c5771aa 100644 --- a/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts +++ b/apps/cli/src/commands/db/shared/pgdelta.seam.layer.ts @@ -18,6 +18,7 @@ import { DeclarativeSeam } from "./pgdelta.seam.service.ts"; import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; import { stackEnsurePostgresOnlyStarted } from "../../../command-internal/stack-local-database.ts"; +import { StackCatalogSetup } from "../../../command-internal/stack-catalog-setup.ts"; const shadowDockerCause = (stderr: string): { readonly docker: "daemon" } | Record => isDockerDaemonUnreachable(stderr) ? { docker: "daemon" } : {}; @@ -77,6 +78,7 @@ export const declarativeSeamLayer = Layer.effect( // hand-enumerating every transitive dependency. const experimentalFlag = yield* ExperimentalFlag; const cliArgs = yield* CliArgs; + const stackCatalogSetup = yield* StackCatalogSetup; const context = yield* Effect.context(); return DeclarativeSeam.of({ @@ -84,12 +86,13 @@ export const declarativeSeamLayer = Layer.effect( Effect.gen(function* () { const backend = yield* currentStackBackend; if (backend.kind === "stack") { - return yield* stackEnsurePostgresOnlyStarted.pipe( + return yield* stackEnsurePostgresOnlyStarted().pipe( Effect.asVoid, Effect.provideContext(context), Effect.provideService(StackApi, stackApi), Effect.provideService(ExperimentalFlag, experimentalFlag), Effect.provideService(CliArgs, cliArgs), + Effect.provideService(StackCatalogSetup, stackCatalogSetup), Effect.mapError( (cause) => new DeclarativeShadowDbError({ diff --git a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md index feb465d504..dacc88ac69 100644 --- a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md @@ -84,12 +84,13 @@ volume was confirmed fresh this run). ## Files Written -| Path | Format | When | -| --------------------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------- | -| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | -| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | -| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | -| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | +| Path | Format | When | +| --------------------------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------- | +| `/supabase/.branches/_current_branch` | text | only if absent — writes `"main"` (see the step-by-step sequence above for exactly when) | +| local Docker volume `supabase_db_` | — | the Postgres data volume, created on first start (or first `--from-backup` restore) | +| local Docker network `supabase_network_` (or `--network-id`) | — | created if it doesn't already exist | +| `$SUPABASE_HOME/stacks//` | JSON | the stack definition, service identities, selected runtime, and service state on the stack backend | +| `~/.supabase/telemetry.json` | JSON | always — telemetry flush (`Effect.ensuring(telemetryState.flush)`), success and failure | ## Subprocesses @@ -184,9 +185,15 @@ Same result object as the terminal `result` event; progress on stderr. When `[experimental].stack` is on, this command creates or resumes a postgres-only project stack instead of a Compose container. First create runs schema init, overlay, and -migrate-and-seed. An existing cluster applies webhooks only. Durable state lives under -`$SUPABASE_HOME/managed/stacks//`. Postgres-only first create skips analytics and -pooler artifact downloads. +migrate-and-seed. It briefly starts configured Auth, Storage, and Realtime instances while +applying the database catalog, then destroys those temporary instances. An existing cluster +applies webhooks only. Durable state lives under `$SUPABASE_HOME/stacks//`, including +the selected runtime and service identities. Postgres-only first create skips analytics and +pooler artifact downloads. If first initialization fails after the database is created, the +new database is destroyed so the command can be retried after fixing the cause. If cleanup itself +fails, stderr identifies the incomplete instance and recommends `supabase stack destroy`. +The resident owner process manages the database beyond the CLI invocation. Stack mode rejects +`--from-backup` with exit code 1. ## Notes diff --git a/apps/cli/src/commands/db/start/start.e2e.test.ts b/apps/cli/src/commands/db/start/start.e2e.test.ts index 9dbb397a6a..7fcafa59b4 100644 --- a/apps/cli/src/commands/db/start/start.e2e.test.ts +++ b/apps/cli/src/commands/db/start/start.e2e.test.ts @@ -1,17 +1,10 @@ import { describe, expect, test } from "vitest"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- e2e fixture appends experimental.stack to project config -import { appendFile } from "node:fs/promises"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- e2e fixture joins project paths -import path from "node:path"; import { makeTempHome, makeTempStackProject, runSupabase } from "../../../../tests/helpers/cli.ts"; const DB_START_COMMAND_TIMEOUT_MS = 480_000; const DB_START_CLEANUP_TIMEOUT_MS = 120_000; const DB_START_TEST_TIMEOUT_MS = DB_START_COMMAND_TIMEOUT_MS + DB_START_CLEANUP_TIMEOUT_MS; -const STACK_DB_AUX_TIMEOUT_MS = 180_000; -const STACK_DB_TEST_TIMEOUT_MS = - DB_START_COMMAND_TIMEOUT_MS * 2 + STACK_DB_AUX_TIMEOUT_MS + DB_START_CLEANUP_TIMEOUT_MS; describe("supabase db start (e2e)", () => { test( @@ -39,54 +32,4 @@ describe("supabase db start (e2e)", () => { }, DB_START_TEST_TIMEOUT_MS, ); - - test( - "stack db start serves --local consumers and rejects migra", - async () => { - const home = makeTempHome(); - const project = await makeTempStackProject("supabase-db-start-stack-e2e-"); - await appendFile( - path.join(project.dir, "supabase", "config.toml"), - "\n[experimental]\nstack = true\n", - ); - try { - const started = await runSupabase(["db", "start"], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: DB_START_COMMAND_TIMEOUT_MS, - }); - expect(started.exitCode, started.stderr).toBe(0); - - const types = await runSupabase(["gen", "types", "--local"], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: STACK_DB_AUX_TIMEOUT_MS, - }); - expect(types.exitCode, types.stderr).toBe(0); - - const reset = await runSupabase(["db", "reset", "--local", "--no-seed", "--yes"], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: DB_START_COMMAND_TIMEOUT_MS, - }); - expect(reset.exitCode, reset.stderr).toBe(0); - - const diff = await runSupabase(["db", "diff", "--local", "--use-migra"], { - cwd: project.dir, - home: home.dir, - exitTimeoutMs: STACK_DB_AUX_TIMEOUT_MS, - }); - expect(diff.exitCode, diff.stderr).not.toBe(0); - expect(`${diff.stdout}${diff.stderr}`).toContain("pg-delta engine"); - } finally { - await runSupabase(["stack", "destroy", "--yes"], { - cwd: project.dir, - home: home.dir, - env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, - exitTimeoutMs: DB_START_CLEANUP_TIMEOUT_MS, - }).catch(() => undefined); - } - }, - STACK_DB_TEST_TIMEOUT_MS, - ); }); diff --git a/apps/cli/src/commands/db/start/start.handler.ts b/apps/cli/src/commands/db/start/start.handler.ts index c3185eedbb..6bfd0d0b1a 100644 --- a/apps/cli/src/commands/db/start/start.handler.ts +++ b/apps/cli/src/commands/db/start/start.handler.ts @@ -35,7 +35,7 @@ export const dbStart = Effect.fn("db.start")(function* (flags: DbStartFlags) { }), ); } - const result = yield* stackEnsurePostgresOnlyStarted; + const result = yield* stackEnsurePostgresOnlyStarted(); if (result === "already-running") { if (output.format === "text") { yield* output.raw("Postgres database is already running.\n", "stderr"); diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index c9424cdd71..79efc86553 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -34,13 +34,19 @@ import { DbConnection, type DbSession } from "../../../command-internal/db-conne import { dockerRunLayer } from "../../../command-internal/docker-run.layer.ts"; import { dbStart } from "./start.handler.ts"; import type { DbStartFlags } from "./start.command.ts"; +import { stackLocalDatabaseConn } from "../../../command-internal/stack-local-database.ts"; import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; import { StackApi } from "../../../command-internal/stack-api.ts"; import { - noopStackCatalogSetupLayer, - recordingStackCatalogSetup, + StackCatalogSetup, + StackCatalogSetupError, } from "../../../command-internal/stack-catalog-setup.ts"; -import { CAPABILITY_NAMES, StackIdSchema, type EffectStack } from "@supabase/stack/effect"; +import { + StackError, + type DatabaseInstance, + type ServiceCreation, + type Stack, +} from "@supabase/stack/effect"; const DEFAULT_FLAGS: DbStartFlags = { fromBackup: Option.none() }; const PG_NET_CREATE_FINGERPRINT = "create extension if not exists pg_net schema extensions"; @@ -284,16 +290,12 @@ interface SetupOpts { readonly connectFailuresRetryable?: boolean; /** Record catalog apply targets instead of the default noop. */ readonly recordCatalog?: boolean; + /** Fail catalog setup after the database has been created. */ + readonly catalogFails?: boolean; } function setup(opts: SetupOpts = {}) { - const catalog = - opts.recordCatalog === true - ? recordingStackCatalogSetup((input) => ({ - kind: input.target.kind, - analytics: input.optionalConfig?.capabilities?.analytics?.enabled, - })) - : undefined; + const catalogApplied: Array<{ readonly serviceCount: number }> = []; const workdir = opts.workdir ?? tempRoot.current; if (opts.skipConfig !== true) { writeConfig(workdir, opts.configContents ?? 'project_id = "test"\n'); @@ -333,6 +335,13 @@ function setup(opts: SetupOpts = {}) { }), }); + const unusedStackApi = Layer.succeed(StackApi, { + create: () => Effect.die("unused"), + open: () => Effect.die("unused"), + discover: () => Effect.die("unused"), + resolveIdentity: () => Effect.die("unused"), + }); + const layer = Layer.mergeAll( BunServices.layer, out.layer, @@ -342,6 +351,7 @@ function setup(opts: SetupOpts = {}) { mockLocalDockerEngineUnavailableLayer, alwaysReadyHttpClientLayer, dbConnection, + unusedStackApi, dockerRunLayer.pipe(Layer.provide(child.layer), Layer.provide(mockProcessControl().layer)), mockProcessControl().layer, mockRuntimeInfo({ platform: opts.platform ?? "linux", cwd: opts.cwd ?? workdir }), @@ -352,7 +362,14 @@ function setup(opts: SetupOpts = {}) { Layer.succeed(CliArgs, { args: ["db", "start"] }), Layer.succeed(ExperimentalFlag, opts.experimental ?? false), Layer.succeed(DebugFlag, opts.debug ?? false), - catalog?.layer ?? noopStackCatalogSetupLayer, + Layer.succeed(StackCatalogSetup, { + apply: (input) => + opts.catalogFails === true + ? Effect.fail(new StackCatalogSetupError({ message: "catalog setup failed" })) + : Effect.sync(() => { + catalogApplied.push({ serviceCount: input.target.databaseServices.length }); + }), + }), ); return { layer, @@ -360,7 +377,7 @@ function setup(opts: SetupOpts = {}) { telemetry, child, dbSession, - catalogApplied: catalog?.applied ?? [], + catalogApplied, get connectAttempts() { return connectAttempts; }, @@ -1538,212 +1555,283 @@ describe("db start", () => { }); describe("db start stack backend", () => { - const STACK_ID = StackIdSchema.make("b".repeat(64)); - const unused = Effect.die("unused"); - const unusedFn = () => unused; - - function mockStackApi(opts: { - readonly existing?: boolean; - readonly unconfigured?: boolean; - readonly databaseReady?: boolean; - }) { - const startConfigs: Array = []; - const stack: EffectStack = { - id: STACK_ID, - status: Effect.succeed({ - id: STACK_ID, - lifecycle: opts.databaseReady === true ? "running" : "stopped", - desiredLifecycle: opts.databaseReady === true ? "running" : "stopped", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "database" ? "eager" : "lazy", - state: name === "database" && opts.databaseReady === true ? "ready" : "stopped", - })), - artifacts: [], - }), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), - password: Redacted.make("secret"), - }, - api: { - publishableKey: "anon", - secretKey: Redacted.make("service"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service"), - }, + const stackId = "b".repeat(64); + const databaseCreation: Extract = { + service: "database", + config: { + version: "17.6.1.173", + databasePassword: Redacted.make("secret"), + jwtSecret: Redacted.make("secret"), + jwtExpiry: 3600, + }, + endpoints: {}, + }; + + const databaseInstance = ( + state: { + running: boolean; + destroyed: boolean; + credentialsCalled?: boolean; + }, + onDestroy: () => void = () => {}, + ): DatabaseInstance => ({ + id: "database-primary", + service: "database", + start: Effect.sync(() => { + state.running = true; + }), + ready: Effect.void, + stop: Effect.sync(() => { + state.running = false; + }), + restart: () => Effect.void, + destroy: Effect.sync(() => { + state.destroyed = true; + onDestroy(); + }), + prepare: Effect.void, + status: Effect.sync(() => ({ + id: "database-primary", + endpoints: state.running + ? [{ name: "sql", protocol: "tcp" as const, host: "127.0.0.1", port: 54329 }] + : [], + config: databaseCreation, + lifecycle: state.running ? ("running" as const) : ("stopped" as const), + health: state.running ? ("healthy" as const) : undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: state.running, + registered: true, + })), + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => + Effect.sync(() => { + state.credentialsCalled = true; + return { databaseUrl: "postgresql://postgres:secret@127.0.0.1:54329/postgres" }; }), - prepare: unusedFn, - start: (startOpts) => - Effect.sync(() => { - startConfigs.push(startOpts?.config); - return { - id: STACK_ID, - lifecycle: "running" as const, - desiredLifecycle: "running" as const, - runtime: { kind: "native" as const }, - endpoints: {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "database" ? ("eager" as const) : ("lazy" as const), - state: name === "database" ? ("ready" as const) : ("dormant" as const), - })), - artifacts: [], - }; - }), - stop: unused, - destroy: unused, - resetDatabase: unused, - logs: unusedFn, - followLogs: () => Stream.empty, + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: Effect.die("unused"), + }); + + const stackFixture = (existing: boolean, running = false, standalone = false) => { + const state: { running: boolean; destroyed: boolean; credentialsCalled?: boolean } = { + running, + destroyed: false, + }; + let members: ReadonlyArray = []; + let registered: ReadonlyArray = []; + const database = databaseInstance(state, () => { + members = []; + registered = []; + }); + members = existing && !standalone ? [database] : []; + registered = existing ? [database] : []; + const stack: Stack = { + id: stackId, + services: { + create: () => Effect.die("unused: composition factory creates database"), + get: (id: string) => + id === database.id + ? Effect.succeed(database) + : Effect.fail(new StackError({ operation: "get", message: `unknown ${id}` })), + list: Effect.sync(() => registered), + }, + composition: { + describe: Effect.sync(() => ({ + members: members.map(({ id }) => ({ id, activation: "eager" as const })), + dependencies: [], + })), + supabase: (creations: ReadonlyArray) => + Effect.sync(() => { + members = creations.map(() => database); + registered = [database]; + return members; + }), + configure: () => Effect.void, + start: Effect.succeed([]), + stop: Effect.succeed([]), + restart: Effect.succeed([]), + }, + stop: Effect.void, + destroy: Effect.void, + tools: { run: () => Effect.die("unused") }, }; - const api = Layer.succeed(StackApi, { - createStack: () => Effect.succeed(stack), - findStack: () => + return { stack, state }; + }; + + const stackLayer = (root: string, fixture: ReturnType, existing: boolean) => + Layer.succeed(StackApi, { + create: () => Effect.succeed(fixture.stack), + open: () => Effect.succeed(fixture.stack), + discover: () => Effect.succeed( - opts.existing === true - ? Option.some({ - id: STACK_ID, - projectRoot: tempRoot.current, - name: "default", - branchContext: "main", - runtime: { kind: "native" as const }, - desiredLifecycle: - opts.unconfigured === true ? ("unconfigured" as const) : ("stopped" as const), - }) - : Option.none(), + existing + ? [ + { + definition: { + id: stackId, + identity: { projectRoot: root, branchContext: "main", stackName: "default" }, + runtime: "native", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }, + host: undefined, + }, + ] + : [], ), - discoverStacks: unusedFn, - openStack: () => Effect.succeed(stack), - inspectStack: unusedFn, + resolveIdentity: () => + Effect.succeed({ projectRoot: root, branchContext: "main", stackName: "default" }), }); - return { api, startConfigs }; - } - it.live("starts a postgres-only stack when none exists", () => { - const { layer, catalogApplied, out } = setup({ recordCatalog: true }); - mkdirSync(join(tempRoot.current, "supabase", "migrations"), { recursive: true }); - writeFileSync( - join(tempRoot.current, "supabase", "migrations", "20240101000000_dogfood.sql"), - "create table public.dogfood ();\n", - ); - const stack = mockStackApi({}); + it.live("creates and starts only the primary database", () => { + const { layer, catalogApplied } = setup({ recordCatalog: true }); + const fixture = stackFixture(false); return Effect.gen(function* () { yield* dbStart(DEFAULT_FLAGS).pipe( - Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, false), + ), + ), ); - expect(stack.startConfigs).toHaveLength(1); - expect(stack.startConfigs[0]).toMatchObject({ - capabilities: { - rest: { enabled: false }, - }, - }); - expect(catalogApplied).toEqual([{ kind: "live", analytics: false }]); - expect(out.stderrText).toContain("Applying migration 20240101000000_dogfood.sql"); + expect(fixture.state.running).toBe(true); + expect(catalogApplied).toEqual([{ serviceCount: 3 }]); }); }); - it.live("honors SUPABASE_EXPERIMENTAL from project .env on first-create migrate", () => { - const previous = process.env["SUPABASE_EXPERIMENTAL"]; - delete process.env["SUPABASE_EXPERIMENTAL"]; - const { layer, out } = setup({ - recordCatalog: true, - projectEnvContents: "SUPABASE_EXPERIMENTAL=true\n", - }); - mkdirSync(join(tempRoot.current, "supabase", "migrations"), { recursive: true }); - writeFileSync( - join(tempRoot.current, "supabase", "migrations", "20240101000000_dogfood.sql"), - "create table public.dogfood ();\n", - ); - const stack = mockStackApi({}); + it.live("destroys a newly created database when catalog setup fails", () => { + const { layer } = setup({ catalogFails: true }); + const fixture = stackFixture(false); return Effect.gen(function* () { - yield* dbStart(DEFAULT_FLAGS).pipe( - Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + const exit = yield* dbStart(DEFAULT_FLAGS).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, false), + ), + ), + Effect.exit, ); - expect(out.stderrText).not.toContain("Applying migration 20240101000000_dogfood.sql"); - }).pipe( - Effect.ensuring( - Effect.sync(() => { - if (previous === undefined) delete process.env["SUPABASE_EXPERIMENTAL"]; - else process.env["SUPABASE_EXPERIMENTAL"] = previous; - }), - ), - ); + expect(Exit.isFailure(exit)).toBe(true); + expect(fixture.state.destroyed).toBe(true); + expect((yield* fixture.stack.composition.describe).members).toHaveLength(0); + }); }); - it.live("does not persist exclusions when a stack already exists", () => { - const { layer, catalogApplied, out } = setup({ recordCatalog: true }); - mkdirSync(join(tempRoot.current, "supabase", "migrations"), { recursive: true }); - writeFileSync( - join(tempRoot.current, "supabase", "migrations", "20240101000000_dogfood.sql"), - "create table public.dogfood ();\n", - ); - const stack = mockStackApi({ existing: true }); + it.live("resumes the existing database without replacing its composition", () => { + const { layer, out, catalogApplied } = setup({ recordCatalog: true }); + const fixture = stackFixture(true); return Effect.gen(function* () { yield* dbStart(DEFAULT_FLAGS).pipe( - Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, true), + ), + ), ); - expect(stack.startConfigs).toEqual([undefined]); + expect(fixture.state.running).toBe(true); expect(catalogApplied).toEqual([]); - expect(out.stderrText).not.toContain("Applying migration"); + expect(out.stderrText).not.toContain("already running"); }); }); - it.live("applies the postgres-only overlay when an unconfigured identity already exists", () => { - const { layer, catalogApplied, out } = setup({ recordCatalog: true }); - mkdirSync(join(tempRoot.current, "supabase", "migrations"), { recursive: true }); - writeFileSync( - join(tempRoot.current, "supabase", "migrations", "20240101000000_dogfood.sql"), - "create table public.dogfood ();\n", - ); - const stack = mockStackApi({ existing: true, unconfigured: true }); + it.live("rejects a standalone database outside the saved composition", () => { + const { layer, catalogApplied } = setup({ recordCatalog: true }); + const fixture = stackFixture(true, false, true); return Effect.gen(function* () { - yield* dbStart(DEFAULT_FLAGS).pipe( - Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + const exit = yield* dbStart(DEFAULT_FLAGS).pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, true), + ), + ), + Effect.exit, ); - expect(stack.startConfigs).toHaveLength(1); - expect(stack.startConfigs[0]).toMatchObject({ - capabilities: { - rest: { enabled: false }, - }, - }); - expect(catalogApplied).toEqual([{ kind: "live", analytics: false }]); - expect(out.stderrText).toContain("Applying migration 20240101000000_dogfood.sql"); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(JSON.stringify(exit.cause)).toContain("standalone database"); + expect(JSON.stringify(exit.cause)).toContain("Destroy"); + } + expect(fixture.state.running).toBe(false); + expect(fixture.state.destroyed).toBe(false); + expect(catalogApplied).toEqual([]); + expect(yield* fixture.stack.composition.describe).toMatchObject({ members: [] }); + expect((yield* fixture.stack.services.list).map(({ id }) => id)).toEqual([ + "database-primary", + ]); }); }); - it.live("reports an already-running stack database without starting", () => { - const { layer, out, catalogApplied } = setup({ recordCatalog: true }); - const stack = mockStackApi({ existing: true, databaseReady: true }); + it.live("does not launch a stopped stack while resolving local credentials", () => { + const { layer } = setup(); + const fixture = stackFixture(true, false); return Effect.gen(function* () { - yield* dbStart(DEFAULT_FLAGS).pipe( - Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + const exit = yield* stackLocalDatabaseConn.pipe( + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, true), + ), + ), + Effect.exit, ); - expect(out.stderrText).toContain("Postgres database is already running."); - expect(stack.startConfigs).toEqual([]); - expect(catalogApplied).toEqual([]); + expect(Exit.isFailure(exit)).toBe(true); + expect(fixture.state.credentialsCalled).toBeUndefined(); + }); + }); + + it.live("uses the running primary's observed connection without launching its owner", () => { + const { layer, out } = setup(); + const fixture = stackFixture(true, true); + const provided = Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, true), + ); + return Effect.gen(function* () { + const conn = yield* stackLocalDatabaseConn.pipe(Effect.provide(provided)); + expect(conn.host).toBe("127.0.0.1"); + expect(conn.port).toBe(54329); + expect(conn.password).toBe("secret"); + expect(fixture.state.credentialsCalled).toBeUndefined(); + yield* dbStart(DEFAULT_FLAGS).pipe(Effect.provide(provided)); + expect(out.stderrText).toContain("already running"); + expect(fixture.state.destroyed).toBe(false); }); }); it.live("refuses --from-backup", () => { const { layer } = setup(); - const stack = mockStackApi({}); + const fixture = stackFixture(false); return Effect.gen(function* () { const exit = yield* dbStart(flags("backup.sql")).pipe( - Effect.provide(Layer.mergeAll(layer, stackBackendLayer("stack"), stack.api)), + Effect.provide( + Layer.mergeAll( + layer, + stackBackendLayer("stack"), + stackLayer(tempRoot.current, fixture, false), + ), + ), Effect.exit, ); expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(JSON.stringify(exit.cause)).toContain( - "db start --from-backup is not supported when the stack backend is enabled.", - ); - } - expect(stack.startConfigs).toEqual([]); + expect(fixture.state.running).toBe(false); }); }); }); diff --git a/apps/cli/src/commands/db/start/start.roles.e2e.test.ts b/apps/cli/src/commands/db/start/start.roles.e2e.test.ts new file mode 100644 index 0000000000..39b9c5a4c5 --- /dev/null +++ b/apps/cli/src/commands/db/start/start.roles.e2e.test.ts @@ -0,0 +1,178 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Path } from "effect"; +import { tmpdir } from "node:os"; + +import { runSupabaseEffect } from "../../../../tests/helpers/cli.ts"; + +const COMMAND_TIMEOUT_MS = 8 * 60_000; +const AUX_TIMEOUT_MS = 3 * 60_000; +const TEST_TIMEOUT_MS = COMMAND_TIMEOUT_MS + AUX_TIMEOUT_MS * 5 + 2 * 60_000; + +describe("supabase db start (e2e, role alignment)", () => { + it.live( + "keeps CLI migration and shadow consumers on postgres", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-db-start-roles-" }); + const home = yield* fs.makeTempDirectoryScoped({ + prefix: "supabase-db-start-roles-home-", + }); + const configPath = path.join(root, "supabase", "config.toml"); + const migrations = path.join(root, "supabase", "migrations"); + yield* fs.makeDirectory(migrations, { recursive: true }); + yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); + const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); + yield* fs.makeDirectory(artifacts, { recursive: true }); + yield* fs.symlink(artifacts, path.join(home, "cache", "stack")); + yield* fs.writeFileString( + configPath, + `project_id = "db-start-roles-e2e" + +[experimental] +stack = true +[experimental.pgdelta] +enabled = true +[db] +major_version = 17 +[api] +enabled = false +[auth] +enabled = false +[realtime] +enabled = false +[storage] +enabled = false +[edge_runtime] +enabled = false +[studio] +enabled = false +[analytics] +enabled = false +[db.pooler] +enabled = false +[local_smtp] +enabled = false +`, + ); + yield* fs.writeFileString( + path.join(migrations, "20260920000000_role_story.sql"), + "create table public.role_story (id integer primary key);\n", + ); + yield* Effect.addFinalizer(() => + runSupabaseEffect(["stack", "destroy", "--yes"], { + cwd: root, + home, + env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, + exitTimeoutMs: AUX_TIMEOUT_MS, + }).pipe( + Effect.flatMap((result) => + result.exitCode === 0 + ? Effect.void + : Effect.die(`stack cleanup failed: ${result.stderr}`), + ), + Effect.catch((error) => Effect.die(`stack cleanup failed: ${String(error)}`)), + ), + ); + + const started = yield* runSupabaseEffect(["db", "start"], { + cwd: root, + home, + cleanupProcessGroupOnClose: false, + exitTimeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(started.exitCode, `${started.stdout}\n${started.stderr}`).toBe(0); + + const types = yield* runSupabaseEffect(["gen", "types", "--local"], { + cwd: root, + home, + exitTimeoutMs: AUX_TIMEOUT_MS, + }); + expect(types.exitCode, types.stderr).toBe(0); + expect(types.stdout).toContain("role_story"); + + const migra = yield* runSupabaseEffect(["db", "diff", "--local", "--use-migra"], { + cwd: root, + home, + exitTimeoutMs: AUX_TIMEOUT_MS, + }); + expect(migra.exitCode, migra.stderr).not.toBe(0); + expect(`${migra.stdout}${migra.stderr}`).toContain("pg-delta engine"); + + const historyOwner = yield* runSupabaseEffect( + [ + "db", + "query", + "select tableowner from pg_tables where schemaname = 'supabase_migrations' and tablename = 'schema_migrations'", + "--local", + "--output", + "json", + ], + { cwd: root, home, exitTimeoutMs: AUX_TIMEOUT_MS }, + ); + expect(historyOwner.exitCode, historyOwner.stderr).toBe(0); + expect(JSON.parse(historyOwner.stdout)).toEqual([{ tableowner: "postgres" }]); + + const alter = yield* runSupabaseEffect( + ["db", "query", "alter table public.role_story add column note text", "--local"], + { cwd: root, home, exitTimeoutMs: AUX_TIMEOUT_MS }, + ); + expect(alter.exitCode, alter.stderr).toBe(0); + + const insert = yield* runSupabaseEffect( + ["db", "query", "insert into public.role_story values (1, 'kept')", "--local"], + { cwd: root, home, exitTimeoutMs: AUX_TIMEOUT_MS }, + ); + expect(insert.exitCode, insert.stderr).toBe(0); + + const diff = yield* runSupabaseEffect(["db", "diff", "--local", "--use-pg-delta"], { + cwd: root, + home, + exitTimeoutMs: AUX_TIMEOUT_MS, + }); + expect(diff.exitCode, `${diff.stdout}\n${diff.stderr}`).toBe(0); + expect(diff.stdout).toMatch(/ALTER TABLE[\s\S]*role_story[\s\S]*ADD COLUMN[\s\S]*note/i); + expect(diff.stdout).not.toMatch(/CREATE TABLE[\s\S]*role_story/i); + + const localRow = yield* runSupabaseEffect( + [ + "db", + "query", + "select id, note from public.role_story", + "--local", + "--output", + "json", + ], + { cwd: root, home, exitTimeoutMs: AUX_TIMEOUT_MS }, + ); + expect(localRow.exitCode, localRow.stderr).toBe(0); + expect(JSON.parse(localRow.stdout)).toEqual([{ id: 1, note: "kept" }]); + + const generate = yield* runSupabaseEffect( + [ + "db", + "schema", + "declarative", + "generate", + "--local", + "--schema", + "public", + "--overwrite", + ], + { cwd: root, home, exitTimeoutMs: AUX_TIMEOUT_MS }, + ); + expect(generate.exitCode, generate.stderr).toBe(0); + + const sync = yield* runSupabaseEffect( + ["db", "schema", "declarative", "sync", "--no-apply"], + { cwd: root, home, exitTimeoutMs: AUX_TIMEOUT_MS }, + ); + expect(sync.exitCode, sync.stderr).toBe(0); + }).pipe(Effect.provide(BunServices.layer)), + ), + { timeout: TEST_TIMEOUT_MS }, + ); +}); diff --git a/apps/cli/src/commands/db/test/test.integration.test.ts b/apps/cli/src/commands/db/test/test.integration.test.ts index d388262b43..f3e5e33f01 100644 --- a/apps/cli/src/commands/db/test/test.integration.test.ts +++ b/apps/cli/src/commands/db/test/test.integration.test.ts @@ -54,6 +54,7 @@ import { PgDeltaSslProbe } from "../../../command-internal/pgdelta-ssl-probe.ser import { runTestDbCommand } from "../../../command-internal/test-db.command-handler.ts"; import { GoProxy } from "../../../command-internal/go-proxy.service.ts"; import { dbCommand } from "../db.command.ts"; +import { unusedStackServices } from "../../../../tests/helpers/unused-stack.ts"; const LOCAL_CONN: PgConnInput = { host: "127.0.0.1", @@ -165,6 +166,7 @@ function setup(opts: SetupOpts = {}) { const docker = mockDockerRun({ exitCode: opts.exitCode, stdout: opts.stdout }); const args = ["db", "test"]; const layer = Layer.mergeAll( + unusedStackServices, out.layer, processControl.layer, analytics.layer, @@ -217,6 +219,7 @@ describe("db test (alias) integration", () => { const args = ["db", "test", "--local", "--linked"]; const analytics = mockContextualAnalytics(); const layer = Layer.mergeAll( + unusedStackServices, BunServices.layer, mockRuntimeInfo(), mockTty(), diff --git a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md index 831ba4c188..672f1c7f07 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/destroy/SIDE_EFFECTS.md @@ -1,25 +1,40 @@ # `supabase stack destroy` -The command is available only when the `experimental.stack` feature flag is enabled. - -Permanently stops and removes one managed stack, including its persisted data. - -The command targets the current project stack by default, or an explicit `--stack` name or -`--stack-id`. It requires an interactive text terminal on both stdout and stdin for confirmation; -`--yes` is required for redirected, non-interactive, and machine-readable invocations. -`SUPABASE_YES` participates in the existing confirmation setting; -an explicit `--yes=false` overrides it. It never accepts `--all`. - -The stack package reads the selected descriptor and removes resources and state under -`${SUPABASE_HOME:-~/.supabase}/managed/stacks/`. It owns stopping the Supervisor, removing -native processes or containers, and deleting persistent stack data. The CLI does not delete paths -or Docker resources itself and makes no Management API calls. Project files are retained. - -The confirmation prompt identifies the stack name, project directory, and immutable stack ID. -Rejection or missing noninteractive confirmation performs no destructive operation. Text output -reports the destroyed stack ID. JSON returns -`{ "destroyed": true, "id": "...", "message": "" }`; stream-JSON wraps the same payload in -the standard result event. Success exits `0`; invalid targets, confirmation refusal, and -destruction failures exit `1`; interruption follows the command runtime's interruption exit. -Standard command instrumentation records command metadata without exporting credentials, and -telemetry state flushes after both successful and failed runs. +Permanently removes the selected managed namespace, its service registrations, +owned database data, port claims, and attached jobs. Caller-owned Storage upload +files remain. The experimental feature flag controls command registration. + +## Selection and confirmation + +Select the current project/branch/name, `--stack `, or `--stack-id `. +The selectors are mutually exclusive; a missing target fails. Explicit legacy +`-o/--output` is rejected in favor of `--output-format`. + +Interactive text mode asks for confirmation and states that Storage uploads are +preserved. Non-interactive and machine-output runs require `--yes`. Rejection or +cancellation does not open or destroy a stack. Discovery may create/chmod the +registry directory to 0700 but does not launch an owner. + +After confirmation the command opens the selected handle and destroys its entire +namespace. Destruction may start an owner to clean up a stopped namespace. +Cleanup failures remain errors; the command does not claim success on failure. +A failed destroy may leave its owner running; retry destruction or use +`stack stop` to shut down that owner. + +## Files and network + +Reads identity/state under `/stacks//` and, for +implicit selection, the project's canonical Git context. Removes only the +selected namespace's owned state/data; other stack namespaces and caller-owned +uploads are preserved. The independent artifact cache is retained. Communicates +with the local owner and selected runtime, with no hosted API calls. Shared +routing/settings can read project configuration and profiles. + +## Output, exit codes and telemetry + +Text prints `Stack destroyed.`; JSON and stream-json success data contain +`destroyed: true` and `id`. Exit 0 on destruction, 1 on invalid flags, missing +selection, rejected/cancelled confirmation or cleanup failure, and 130 on +interruption. Standard command telemetry is unchanged, with no custom events. +Telemetry flushes on success and failure to +`/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.command.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.command.ts index 65f186c126..3a05bced3d 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.command.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.command.ts @@ -21,7 +21,9 @@ const config = { export type StackDestroyFlags = CliCommand.Command.Config.Infer; export const stackDestroyCommand = Command.make("destroy", config).pipe( - Command.withDescription("Permanently destroy a managed local Supabase stack and its data."), + Command.withDescription( + "Permanently destroy a managed local stack and its owned data, preserving Storage uploads.", + ), Command.withShortDescription("Destroy a managed local stack"), Command.withExamples([ { diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts index 09972c87b4..9bdae4642a 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.handler.ts @@ -1,5 +1,5 @@ -import { Effect, Match, Option } from "effect"; -import { isStackError, StackIdSchema } from "@supabase/stack/effect"; +import { Cause, Effect, Exit, Option, Path } from "effect"; +import type { StackError } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag, resolveYes } from "../../../../command-internal/global-flags.ts"; import { promptYesNo } from "../../../../command-internal/prompt-yes-no.ts"; @@ -10,7 +10,7 @@ import { StackApi, StackTargetError, rejectStackOutput, - validateStackId, + StackTargetResolver, validateStackTarget, } from "../stack.shared.ts"; import type { StackDestroyFlags } from "./destroy.command.ts"; @@ -24,46 +24,12 @@ const mapTargetError = (error: StackTargetError) => cause: error, }); -const destroyError = (error: unknown): StackCommandDestroyError => { - const stackError = isStackError(error) ? error : undefined; - const classification = - stackError === undefined - ? { reason: "unknown" as const } - : Match.value(stackError).pipe( - Match.tag("StackNotFoundError", "InvalidStackIdentityError", () => ({ - reason: "flags" as const, - })), - Match.tag("ContainerEngineError", () => ({ - reason: "runtime" as const, - suggestion: - "Check that the selected container engine is installed and its daemon is running, then retry the command.", - })), - Match.tag( - "StackOwnershipConflictError", - "StackNotRunningError", - "StackMustBeStoppedError", - "StackLifecycleConflictError", - "StackRuntimeError", - "StackCleanupError", - "StackDestructionError", - "StackUpgradeRequiredError", - () => ({ reason: "lifecycle" as const }), - ), - Match.tag( - "InvalidStackConfigError", - "StackStateFormatUnsupportedError", - "InvalidProjectRootError", - "StackStateInvalidError", - () => ({ reason: "invalid-config" as const }), - ), - Match.orElse(() => ({ reason: "unknown" as const })), - ); - return new StackCommandDestroyError({ - ...classification, - message: stackError?.message ?? String(error), - cause: error, +const destroyError = (cause: StackError) => + new StackCommandDestroyError({ + reason: "unknown", + message: cause.message, + cause, }); -}; export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( flags: StackDestroyFlags, @@ -80,29 +46,24 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( stackId: Option.getOrUndefined(flags.stackId), }).pipe(Effect.mapError(mapTargetError)); - const target = yield* Effect.gen(function* () { - if (Option.isSome(flags.stackId)) { - const id = yield* validateStackId(flags.stackId.value).pipe( - Effect.mapError(mapTargetError), - ); - return yield* api.inspectStack(id).pipe( - Effect.map(({ descriptor }) => descriptor), - Effect.mapError(destroyError), - ); - } - const found = yield* api - .findStack({ - projectRoot: settings.workdir, - ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), - }) - .pipe(Effect.mapError(destroyError)); - if (Option.isSome(found)) return found.value; + const resolver = yield* StackTargetResolver; + const path = yield* Path.Path; + const target = yield* resolver + .resolve({ + projectRoot: settings.workdir, + ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), + ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), + runtime: "auto", + }) + .pipe(Effect.mapError(mapTargetError)); + if (target.id === undefined) return yield* new StackCommandDestroyError({ reason: "flags", - message: `No managed stack${Option.isSome(flags.stack) ? ` named "${flags.stack.value}"` : ""} was found for this project.`, - suggestion: "Choose an existing --stack name or omit --stack for the current project.", + message: Option.isSome(flags.stack) + ? `No managed stack named "${flags.stack.value}" was found for this project.` + : "No managed stack was found for this project.", + suggestion: "Choose an existing --stack name or --stack-id.", }); - }); const yes = yield* resolveYes; const tty = yield* Tty; if (!yes && (!tty.stdinIsTty || !output.interactive || output.format !== "text")) @@ -114,7 +75,7 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( const confirmed = yield* promptYesNo( output, yes, - `Permanently destroy stack "${target.name}" at ${target.projectRoot} (${target.id}) and all of its data?`, + `Permanently destroy stack ${target.id} at ${target.projectRoot} and its owned data? Storage upload files will be preserved.`, false, ); if (!confirmed) @@ -123,12 +84,21 @@ export const stackDestroy = Effect.fn("experimental.stack.destroy")(function* ( message: "Stack destruction was not confirmed.", }); const stack = yield* api - .openStack(StackIdSchema.make(target.id)) + .open({ + id: target.id, + stateRoot: path.join(settings.supabaseHome, "stacks"), + cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), + }) .pipe(Effect.mapError(destroyError)); const destroying = yield* output.task(`Destroying stack ${target.id}...`); yield* stack.destroy.pipe( - Effect.tapError((error) => destroying.fail(error.message)), - Effect.tap(() => destroying.clear()), + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? destroying.clear() + : Cause.hasInterruptsOnly(exit.cause) + ? destroying.cancel() + : destroying.fail(Option.getOrUndefined(Exit.findErrorOption(exit))?.message), + ), Effect.mapError(destroyError), ); if (output.format === "text") yield* output.raw(`Stack ${target.id} destroyed.\n`); diff --git a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts index 4ba59a508c..70d10bc54c 100644 --- a/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/destroy/destroy.integration.test.ts @@ -1,221 +1,133 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer, Option, Stream } from "effect"; -import { ContainerEngineError, StackDestructionError, StackIdSchema } from "@supabase/stack/effect"; -import type { EffectStack } from "@supabase/stack/effect"; +import { Effect, FileSystem, Layer, Option, Path } from "effect"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; -import { OutputFlag, YesFlag } from "../../../../command-internal/global-flags.ts"; -import { - ErrorActionabilityId, - actionability, -} from "../../../../shared/telemetry/error-actionability.ts"; +import { YesFlag } from "../../../../command-internal/global-flags.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; -import { - mockOutput, - mockStdin, - mockTty, - processEnvLayer, -} from "../../../../../tests/helpers/mocks.ts"; -import { StackApi } from "../stack.shared.ts"; +import { mockOutput, mockStdin, mockTty } from "../../../../../tests/helpers/mocks.ts"; +import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackDestroy } from "./destroy.handler.ts"; -import { StackCommandDestroyError } from "./destroy.errors.ts"; - -const id = StackIdSchema.make("a".repeat(64)); -const descriptor = { - id, - projectRoot: "/project", - name: "feature-a", - branchContext: "ordinary-workspace" as const, - runtime: { kind: "native" as const }, - desiredLifecycle: "stopped" as const, -}; - -const flags = (stackId = Option.none(), stack = Option.none()) => ({ - stack, - stackId, -}); -function setup(options: { - yes: boolean; - destroyFailure?: boolean; - destroyContainerFailure?: boolean; - interactive?: boolean; - outputInteractive?: boolean; - promptConfirmResponses?: ReadonlyArray; - outputFormat?: "text" | "json"; - found?: boolean; -}) { - const output = mockOutput({ - format: options.outputFormat, - interactive: options.outputInteractive, - promptConfirmResponses: options.promptConfirmResponses, - }); +const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +const fixture = Effect.fn("StackDestroyTest.fixture")(function* (yes: boolean) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-destroy-" }); + const api = yield* StackApi; + const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; + const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "native" }); + const output = mockOutput({ interactive: false }); const telemetry = mockTelemetryStateTracked(); - const state = { destroyed: 0, opened: 0 }; - const stack: EffectStack = { - id, - status: Effect.die("unused"), - credentials: Effect.die("unused"), - prepare: () => Effect.die("unused"), - start: () => Effect.die("unused"), - stop: Effect.die("unused"), - destroy: options.destroyContainerFailure - ? Effect.fail(new ContainerEngineError({ message: "container engine unavailable" })) - : options.destroyFailure - ? Effect.fail(new StackDestructionError({ message: "destroy failed" })) - : Effect.sync(() => void state.destroyed++), - resetDatabase: Effect.die("unused"), - logs: () => Effect.die("unused"), - followLogs: () => Stream.empty, - }; + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); + const layer = Layer.mergeAll( + output.layer, + telemetry.layer, + settings, + stackTargetResolverLayer.pipe(Layer.provide(settings)), + mockTty({ stdinIsTty: false }), + mockStdin(false), + Layer.succeed(YesFlag, yes), + Layer.succeed(CliArgs, { args: yes ? ["--yes"] : ["--yes=false"] }), + ); return { + root, + fs, + path, + api, + locations, + stack, output, - state, telemetry, - layer: Layer.mergeAll( - output.layer, - telemetry.layer, - mockCommandSettings({ workdir: descriptor.projectRoot }), - mockTty({ - stdinIsTty: options.interactive ?? false, - stdoutIsTty: options.interactive ?? false, - }), - mockStdin(options.interactive ?? false), - processEnvLayer({}), - Layer.succeed(YesFlag, options.yes), - Layer.succeed(CliArgs, { args: options.yes ? ["--yes"] : [] }), - Layer.succeed(StackApi, { - findStack: () => - Effect.succeed(options.found === false ? Option.none() : Option.some(descriptor)), - createStack: () => Effect.die("unused"), - inspectStack: () => Effect.succeed({ descriptor, owner: "absent" as const }), - openStack: () => - Effect.sync(() => { - state.opened++; - return stack; - }), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - }), - BunServices.layer, - ), + layer, + flags: { stack: Option.none(), stackId: Option.some(stack.id) }, }; -} +}); describe("stack destroy", () => { - it.live("requires confirmation before mutating a noninteractive invocation", () => { - const fixture = setup({ yes: false }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags()).pipe(Effect.flip); - expect(failure).toBeInstanceOf(StackCommandDestroyError); - expect(failure.message).toContain("requires confirmation"); - expect(fixture.state.destroyed).toBe(0); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("destroys the selected stack after --yes", () => { - const fixture = setup({ yes: true }); - return Effect.gen(function* () { - yield* stackDestroy(flags(Option.some(id))); - expect(fixture.state.destroyed).toBe(1); - expect(fixture.output.stdoutText).toContain("destroyed"); - expect(fixture.telemetry.flushed).toBe(true); - }).pipe(Effect.provide(fixture.layer)); - }); + it.live("requires confirmation before starting an owner or changing saved state", () => + Effect.gen(function* () { + const f = yield* fixture(false); + const error = yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer), Effect.flip); + expect(error.reason).toBe("confirmation"); + const saved = yield* f.api.discover(f.locations); + expect(saved.map(({ definition }) => definition.id)).toEqual([f.stack.id]); + expect(saved[0]?.host).toBeUndefined(); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); - it.live("emits a machine-readable result after --yes", () => { - const fixture = setup({ yes: true, outputFormat: "json" }); - return Effect.gen(function* () { - yield* stackDestroy(flags(Option.some(id))); - expect(fixture.output.messages).toEqual( - expect.arrayContaining([ - expect.objectContaining({ type: "success", data: { destroyed: true, id } }), - ]), + it.live("destroys an offline namespace without affecting a different stack", () => + Effect.gen(function* () { + const f = yield* fixture(true); + const other = yield* f.api.create({ + ...f.locations, + projectRoot: f.root, + name: "other", + runtime: "native", + }); + yield* Effect.acquireUseRelease( + Effect.succeed(f.stack), + () => stackDestroy(f.flags).pipe(Effect.provide(f.layer)), + () => + f.api + .discover(f.locations) + .pipe( + Effect.flatMap((entries) => + entries.some( + ({ definition, host }) => definition.id === f.stack.id && host !== undefined, + ) + ? f.stack.stop + : Effect.void, + ), + ), ); - }).pipe(Effect.provide(fixture.layer)); - }); + expect((yield* f.api.discover(f.locations)).map(({ definition }) => definition.id)).toEqual([ + other.id, + ]); + expect(f.output.stdoutText).toContain(`Stack ${f.stack.id} destroyed.`); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); - it.live("does not destroy a stack when interactive confirmation is declined", () => { - const fixture = setup({ yes: false, interactive: true, promptConfirmResponses: [false] }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags()).pipe(Effect.flip); - expect(failure.message).toContain("not confirmed"); - expect(failure.reason).toBe("cancelled"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.cancelled); - expect(fixture.state.destroyed).toBe(0); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("destroys a stack after interactive confirmation is accepted", () => { - const fixture = setup({ yes: false, interactive: true, promptConfirmResponses: [true] }); - return Effect.gen(function* () { - yield* stackDestroy(flags()); - expect(fixture.state.destroyed).toBe(1); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("requires --yes when stdout is redirected", () => { - const fixture = setup({ - yes: false, - interactive: true, - outputInteractive: false, - promptConfirmResponses: [true], - }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags()).pipe(Effect.flip); - expect(failure.reason).toBe("confirmation"); - expect(fixture.state.destroyed).toBe(0); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("flushes telemetry when destruction fails", () => { - const fixture = setup({ yes: true, destroyFailure: true }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags(Option.some(id))).pipe(Effect.flip); - expect(failure.message).toContain("destroy failed"); - expect(fixture.telemetry.flushed).toBe(true); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("classifies container engine failures with runtime guidance", () => { - const fixture = setup({ yes: true, destroyContainerFailure: true }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags(Option.some(id))).pipe(Effect.flip); - expect(failure.reason).toBe("runtime"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.dockerNotRunning); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("rejects the legacy output flag before opening a stack", () => { - const fixture = setup({ yes: true }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags(Option.some(id))).pipe(Effect.flip); - expect(failure.reason).toBe("flags"); - expect(fixture.state.opened).toBe(0); - }).pipe( - Effect.provide(Layer.mergeAll(fixture.layer, Layer.succeed(OutputFlag, Option.some("json")))), - ); - }); - - it.live("rejects malformed and conflicting targets without opening a stack", () => { - const fixture = setup({ yes: true }); - return Effect.gen(function* () { - const malformed = yield* stackDestroy(flags(Option.some("invalid"))).pipe(Effect.flip); - expect(malformed.message).toContain("lowercase SHA-256"); - const conflicting = yield* stackDestroy( - flags(Option.some(id), Option.some("feature-a")), - ).pipe(Effect.flip); - expect(conflicting.message).toContain("cannot be used together"); - }).pipe(Effect.provide(fixture.layer)); - }); - - it.live("reports an absent named target", () => { - const fixture = setup({ yes: true, found: false }); - return Effect.gen(function* () { - const failure = yield* stackDestroy(flags()).pipe(Effect.flip); - expect(failure.message).toContain("No managed stack"); - }).pipe(Effect.provide(fixture.layer)); - }); + it.live( + "destroys a live namespace and standalone services while preserving caller-owned uploads", + () => + Effect.gen(function* () { + const f = yield* fixture(true); + const uploads = f.path.join(f.root, "uploads"); + yield* f.fs.makeDirectory(uploads); + yield* f.fs.writeFileString(f.path.join(uploads, "object.txt"), "keep this upload"); + yield* Effect.acquireUseRelease( + f.stack.services.create({ + service: "storage", + config: { + databaseUrl: "postgresql://unused", + jwtSecret: "test-secret", + filePath: uploads, + }, + endpoints: {}, + }), + () => + Effect.gen(function* () { + yield* f.stack.services.create({ service: "mail", config: {}, endpoints: {} }); + yield* stackDestroy(f.flags).pipe(Effect.provide(f.layer)); + expect(yield* f.api.discover(f.locations)).toEqual([]); + expect(yield* f.fs.readFileString(f.path.join(uploads, "object.txt"))).toBe( + "keep this upload", + ); + }), + () => + f.api + .discover(f.locations) + .pipe( + Effect.flatMap((entries) => + entries.some(({ host }) => host !== undefined) ? f.stack.stop : Effect.void, + ), + ), + ); + }).pipe(Effect.provide(live)), + ); }); diff --git a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md index 0f9a2a37e4..547285590d 100644 --- a/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/list/SIDE_EFFECTS.md @@ -1,102 +1,41 @@ # `supabase stack list` -Lists every persisted managed local stack. The command is registered only when the -`experimental.stack` family is enabled; that gate also controls help and completion. It reads -the registry without loading a project configuration, contacting an owner, or changing a -lifecycle. +Lists persisted local stacks when `experimental.stack` is enabled. Discovery probes +existing owners but never launches an owner or starts, stops, or prepares services. +The handler does not load project configuration. -## Files Read +## Files and network -| Path | Format | When | -| --------------------------------------------------------------------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/config.toml` or `config.json` | TOML or JSON | During the `experimental.stack` family gate when `SUPABASE_EXPERIMENTAL_STACK` is unset or empty; the nearest project is discovered from the default workdir, while explicit `--workdir`/`SUPABASE_WORKDIR` is used as given. | -| `/profile` | plain text | By shared `CommandSettings` when `--profile` and `SUPABASE_PROFILE` are unset; an empty or missing file selects `supabase`. | -| Value of `--profile`, `SUPABASE_PROFILE`, or persisted profile | YAML profile | By shared `CommandSettings` when the selected profile is not a built-in name; an unreadable, malformed, or invalid explicit profile fails the command. | -| `/supabase/config.toml` | existence probe | By shared `CommandSettings` while resolving a default workdir from its ancestor directories. The list handler does not perform a full project config load. | -| `/managed/stacks/` | directory | Registry enumeration. Entries that are not stack IDs are ignored. | -| `/managed/stacks//state.json` | JSON | Each stack ID entry; readable descriptors use `identity.projectRoot`, `identity.stackName`, `identity.branchContext`, `runtime`, and `desiredLifecycle`; missing state files are ignored as remnants; decode or validation failures remain unreadable entries. | -| `/telemetry.json` | JSON | Shared telemetry state load, regardless of telemetry delivery being enabled. | - -## Files Written - -| Path | Format | When | -| ----------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `/telemetry.json` | JSON | Once the handler is entered, always best-effort through its flush finalizer, on success and failure, including when telemetry is disabled. Startup or profile-resolution failures can occur before this finalizer. | - -## API Routes - -| Method | Path | Auth | Request body | Response (used fields) | -| ------ | ---- | ---- | ------------ | ----------------------------------------------------------------------------- | -| — | — | — | — | No Management API calls or owner RPCs; shared telemetry delivery is separate. | - -## Environment Variables - -| Variable | Purpose | Required? | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | -| `SUPABASE_EXPERIMENTAL_STACK` | Family registration (`1` enable, `0` disable; unset/empty uses project config; other values fail selection) | no | -| `SUPABASE_HOME` | Global registry, profile, and telemetry root | no (falls back to `~/.supabase`) | -| `SUPABASE_PROFILE` | Built-in profile name or YAML profile path | no (falls back to the persisted profile file, then `supabase`) | -| `SUPABASE_WORKDIR` | Explicit workdir for shared settings and feature discovery | no (used after `--workdir`, then ancestor probes from CWD) | -| `SUPABASE_TELEMETRY_DISABLED` | Suppresses telemetry delivery while retaining telemetry state persistence | no | -| `DO_NOT_TRACK` | Suppresses telemetry delivery while retaining telemetry state persistence | no | - -## Exit Codes - -| Code | Condition | -| ---- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `0` | Registry enumeration succeeds, including readable entries, unreadable entries, only unreadable entries, and a fully empty registry. | -| `1` | Registry root existence/read failure, mapped to `StackCommandListError` with `invalid-config`. | -| `1` | The legacy `-o/--output` flag is supplied; use `--output-format json`, `text`, or `stream-json`. | -| `1` | Shared profile loading or explicit feature selection fails. | - -## Telemetry Events Fired - -| Event | When | Notable properties / groups | -| ---------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| `cli_command_executed` | Post-run, success or failure, through the command telemetry wrapper | `exit_code`, `duration_ms`, command flags, run identity; capture is consent-gated. | - -No custom events are emitted. +Reads `/stacks//state.json` and probes existing +local owner control endpoints. It makes no hosted API calls. Shared command setup +may read project configuration for the feature gate and the selected profile for +CLI settings. Discovery creates the registry directory if absent and sets its mode to 0700; +no stack state files are written. Telemetry flushes to +`/telemetry.json` on success and failure. ## Output -Readable entries retain `id`, `project_root`, `name`, `branch_context`, `runtime`, and -`desired_lifecycle`, plus `readable: true`. Unreadable entries contain only `id`, `readable: false`, -and `error: {code, message}`; no metadata is inferred from a failed state document. Readable -entries sort by project root, name, and ID. Unreadable entries follow them, sorted by ID. - -### `--output-format text` - -Readable entries show a table with NAME, PROJECT, BRANCH, RUNTIME, DESIRED, and a compact ID. -Every unreadable entry is shown in a diagnostic section with its error code and the package-enriched -message carrying its full ID. Underlying filesystem diagnostics may also contain paths with that ID. -The `No managed stacks found.` message is emitted only when both readable and unreadable entry arrays -are empty. +Entries contain `id`, `project_root`, `name`, `branch_context`, `runtime` (native, +docker, or podman), and `owner` (reachable or unavailable). Owner availability is +not service lifecycle or health. Entries sort by project root, name, then ID. +Text shows NAME, PROJECT, BRANCH, RUNTIME, OWNER, and a compact ID. An empty +registry prints `No managed stacks found.` Use `--output-format json` to obtain +the full ID required by `--stack-id`; the text column shows only a prefix. -### `--output-format json` +JSON emits `{ "stacks": [...], "message": "" }`; stream-json wraps that data in +one result event. Invalid state documents are skipped with a warning on stderr identifying each stack. +Other registry read failures still fail discovery. State is never repaired or deleted; +opening stacks and allocating ports still reject invalid documents. Target resolution for +other stack commands also remains strict. -The success data contains a single `stacks` array with the discriminated entries described above. -The shared `Output.success` JSON envelope also carries its empty `message` field: +## Flags and exit codes -```json -{ - "stacks": [ - { - "id": "…", - "readable": true, - "project_root": "…", - "name": "…", - "branch_context": "…", - "runtime": { "kind": "native" }, - "desired_lifecycle": "stopped" - }, - { "id": "…", "readable": false, "error": { "code": "StackStateInvalidError", "message": "…" } } - ], - "message": "" -} -``` +Explicit legacy `-o/--output` is rejected; use `--output-format text`, `json`, or +`stream-json`. Exit status is 0 on successful discovery, including an empty +registry or unavailable owners; 1 on invalid flags, registry failure, or shared +configuration failure; 130 on interruption. -### `--output-format stream-json` +## Telemetry -One `result` NDJSON event contains the same complete `{stacks: [...]}` success data (plus the -shared empty `message` field). Per-entry issues are successful enumeration and therefore do not -emit an error event or change the exit code. +The existing command wrapper emits `cli_command_executed` subject to consent. +No custom events are emitted and stack identities are not telemetry properties. diff --git a/apps/cli/src/commands/experimental/stack/list/list.handler.ts b/apps/cli/src/commands/experimental/stack/list/list.handler.ts index 76b9448cb5..0d8b268265 100644 --- a/apps/cli/src/commands/experimental/stack/list/list.handler.ts +++ b/apps/cli/src/commands/experimental/stack/list/list.handler.ts @@ -1,10 +1,5 @@ -import { Effect, Match } from "effect"; -import { - type StackDescriptor, - type StackDiscoveryError, - type StackDiscoveryIssue, - type StackRuntime, -} from "@supabase/stack/effect"; +import { Effect, Path } from "effect"; +import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { renderGlamourTable } from "../../../../output/glamour-table.ts"; @@ -12,113 +7,74 @@ import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts import { StackApi, rejectStackOutput } from "../stack.shared.ts"; import { StackCommandListError } from "./list.errors.ts"; -const readableEntry = (descriptor: StackDescriptor) => ({ - id: descriptor.id, - readable: true as const, - project_root: descriptor.projectRoot, - name: descriptor.name, - branch_context: descriptor.branchContext, - runtime: descriptor.runtime, - desired_lifecycle: descriptor.desiredLifecycle, -}); - -const unreadableEntry = ({ id, error }: StackDiscoveryIssue) => ({ - id, - readable: false as const, - error: { - code: error._tag, - message: error.message, - }, -}); - -type ReadableEntry = ReturnType; -type UnreadableEntry = ReturnType; -type StackEntry = ReadableEntry | UnreadableEntry; - -const compareCodeunit = (left: string, right: string): number => - left === right ? 0 : left < right ? -1 : 1; - -const compareEntries = (left: ReadableEntry, right: ReadableEntry): number => { - const project = compareCodeunit(left.project_root, right.project_root); - if (project !== 0) return project; - const name = compareCodeunit(left.name, right.name); - return name !== 0 ? name : compareCodeunit(left.id, right.id); -}; - -const mapStackError = (error: StackDiscoveryError) => - new StackCommandListError({ - reason: "invalid-config", - message: error.message, - suggestion: - "Inspect the managed stack registry under $SUPABASE_HOME/managed/stacks or ~/.supabase/managed/stacks.", - cause: error, - }); - -const renderRuntime = (runtime: StackRuntime): string => - Match.value(runtime).pipe( - Match.when({ kind: "native" }, () => "native"), - Match.when({ kind: "container" }, ({ engine }) => `container (${engine})`), - Match.exhaustive, - ); - -const compactId = (id: string): string => id.slice(0, 8); - -const render = (stacks: ReadonlyArray): string => { - if (stacks.length === 0) return "No managed stacks found.\n"; - const readable = stacks.filter((stack): stack is ReadableEntry => stack.readable); - const unreadable = stacks.filter((stack): stack is UnreadableEntry => !stack.readable); - const lines: string[] = []; - if (readable.length > 0) { - lines.push( - renderGlamourTable( - ["NAME", "PROJECT", "BRANCH", "RUNTIME", "DESIRED", "ID"], - readable.map((stack) => [ - stack.name, - stack.project_root, - stack.branch_context, - renderRuntime(stack.runtime), - stack.desired_lifecycle, - compactId(stack.id), - ]), - ).trimEnd(), - ); - } - if (unreadable.length > 0) { - if (lines.length > 0) lines.push(""); - lines.push("Unreadable stacks:"); - for (const stack of unreadable) { - lines.push(` ${stack.error.code}: ${stack.error.message}`); - } - } - return `${lines.join("\n")}\n`; -}; - export const stackList = Effect.fn("experimental.stack.list")(function* () { - const telemetryState = yield* TelemetryState; - const body = Effect.gen(function* () { + const telemetry = yield* TelemetryState; + return yield* Effect.gen(function* () { const output = yield* Output; - const outputFlag = yield* Effect.serviceOption(OutputFlag); - yield* rejectStackOutput(outputFlag).pipe( + const settings = yield* CommandSettings; + const path = yield* Path.Path; + const api = yield* StackApi; + yield* rejectStackOutput(yield* Effect.serviceOption(OutputFlag)).pipe( Effect.mapError( - (error) => + (cause) => new StackCommandListError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, + reason: cause.reason, + message: cause.message, + ...(cause.suggestion === undefined ? {} : { suggestion: cause.suggestion }), + cause, }), ), ); - const api = yield* StackApi; - const result = yield* api.discoverStacks().pipe(Effect.mapError(mapStackError)); - const readable = result.stacks.map(readableEntry).sort(compareEntries); - const unreadable = result.errors - .map(unreadableEntry) - .sort((left, right) => compareCodeunit(left.id, right.id)); - const stacks: ReadonlyArray = [...readable, ...unreadable]; - if (output.format === "text") yield* output.raw(render(stacks)); - else yield* output.success("", { stacks }); + const discovered = yield* api + .discover({ + stateRoot: path.join(settings.supabaseHome, "stacks"), + onInvalidState: (id, error) => + output.raw(`Warning: skipping invalid stack ${id}: ${error.message}\n`, "stderr"), + }) + .pipe( + Effect.mapError( + (cause) => + new StackCommandListError({ + reason: "invalid-config", + message: cause.message, + suggestion: + "Inspect the stack registry under $SUPABASE_HOME/stacks or ~/.supabase/stacks.", + cause, + }), + ), + ); + const stacks = discovered + .map(({ definition, host }) => ({ + id: definition.id, + project_root: definition.identity.projectRoot, + name: definition.identity.stackName, + branch_context: definition.identity.branchContext, + runtime: definition.runtime, + owner: host === undefined ? "unavailable" : "reachable", + })) + .sort((left, right) => { + for (const field of ["project_root", "name", "id"] as const) { + if (left[field] !== right[field]) return left[field] < right[field] ? -1 : 1; + } + return 0; + }); + if (output.format === "text") { + yield* output.raw( + stacks.length === 0 + ? "No managed stacks found.\n" + : `${renderGlamourTable( + ["NAME", "PROJECT", "BRANCH", "RUNTIME", "OWNER", "ID"], + stacks.map((stack) => [ + stack.name, + stack.project_root, + stack.branch_context, + stack.runtime, + stack.owner, + stack.id.slice(0, 8), + ]), + ).trimEnd()}\n`, + ); + } else yield* output.success("", { stacks }); return stacks; - }); - return yield* body.pipe(Effect.ensuring(telemetryState.flush)); + }).pipe(Effect.ensuring(telemetry.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/list/list.integration.test.ts b/apps/cli/src/commands/experimental/stack/list/list.integration.test.ts index 3ae03db451..40d0f41840 100644 --- a/apps/cli/src/commands/experimental/stack/list/list.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/list/list.integration.test.ts @@ -1,304 +1,135 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { - ConfigProvider, - Effect, - FileSystem, - Layer, - Option, - Path, - Schema, - Sink, - Stdio, - Stream, -} from "effect"; -import { - createStack, - StackIdSchema, - StackStateInvalidError, - type StackDescriptor, - type StackDiscoveryIssue, -} from "@supabase/stack/effect"; +import { Effect, FileSystem, Layer, Option, Path } from "effect"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; -import { mockTelemetryStateTracked } from "../../../../../tests/helpers/command-mocks.ts"; +import { + mockCommandSettings, + mockTelemetryStateTracked, +} from "../../../../../tests/helpers/command-mocks.ts"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; -import { jsonOutputLayer, streamJsonOutputLayer } from "../../../../shared/output/output.layer.ts"; import { StackApi, stackApiLayer } from "../stack.shared.ts"; import { stackList } from "./list.handler.ts"; -import { StackCommandListError } from "./list.errors.ts"; -const descriptor = ( - id: string, - projectRoot: string, - name: string, - desiredLifecycle: StackDescriptor["desiredLifecycle"], -): StackDescriptor => ({ - id: StackIdSchema.make(id.repeat(64)), - projectRoot, - name, - branchContext: `${name}-branch`, - runtime: { kind: "native" }, - desiredLifecycle, +const registry = Effect.fn("StackListTest.registry")(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs + .makeTempDirectoryScoped({ prefix: "stack-list-" }) + .pipe(Effect.flatMap(fs.realPath)); + const api = yield* StackApi; + const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; + const telemetry = mockTelemetryStateTracked(); + const layer = Layer.mergeAll( + mockCommandSettings({ workdir: root, supabaseHome: root }), + telemetry.layer, + ); + return { root, api, fs, path, locations, telemetry, layer }; }); -const setup = (options: { - stacks?: ReadonlyArray; - errors?: ReadonlyArray; - discoveryError?: StackDiscoveryIssue["error"]; - outputFormat?: "text" | "json" | "stream-json"; - outputFlag?: boolean; -}) => { - const output = mockOutput({ format: options.outputFormat }); - const telemetry = mockTelemetryStateTracked(); - return { - output, - telemetry, - layer: Layer.mergeAll( - output.layer, - telemetry.layer, - Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - createStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - inspectStack: () => Effect.die("unused"), - discoverStacks: () => - options.discoveryError === undefined - ? Effect.succeed({ stacks: options.stacks ?? [], errors: options.errors ?? [] }) - : Effect.fail(options.discoveryError), - }), - ...(options.outputFlag ? [Layer.succeed(OutputFlag, Option.some("json"))] : []), - BunServices.layer, - ), - }; -}; +const live = Layer.provideMerge(stackApiLayer, BunServices.layer); describe("stack list", () => { - it.live("sorts persisted stacks and includes stopped and unconfigured lifecycles", () => { - const fixture = setup({ - stacks: [ - descriptor("e", "/work/z", "aaa", "stopped"), - descriptor("a", "/work/a", "beta", "running"), - descriptor("c", "/work/a", "alpha", "unconfigured"), - descriptor("b", "/work/a", "alpha", "stopped"), - ], - }); - return stackList().pipe( - Effect.tap((entries) => - Effect.sync(() => { - expect(entries.map(({ id }) => id)).toEqual( - ["b", "c", "a", "e"].map((id) => id.repeat(64)), - ); - expect(fixture.output.stdoutText.indexOf("beta")).toBeLessThan( - fixture.output.stdoutText.indexOf("aaa"), - ); - expect(fixture.output.stdoutText.indexOf("alpha")).toBeLessThan( - fixture.output.stdoutText.indexOf("beta"), - ); - expect(fixture.output.stdoutText.indexOf("bbbbbbbb")).toBeLessThan( - fixture.output.stdoutText.indexOf("cccccccc"), - ); - expect(fixture.output.stdoutText).toContain("DESIRED"); - expect(fixture.output.stdoutText).toContain("stopped"); - expect(fixture.output.stdoutText).toContain("unconfigured"); - expect(fixture.telemetry.flushed).toBe(true); - }), - ), - Effect.provide(fixture.layer), - ); - }); - - it.live("reports an empty registry", () => { - const fixture = setup({}); - return stackList().pipe( - Effect.tap(() => - Effect.sync(() => expect(fixture.output.stdoutText).toBe("No managed stacks found.\n")), - ), - Effect.provide(fixture.layer), - ); - }); - - it.live("lists a real registry exhaustively in text, JSON, and NDJSON", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped({ prefix: "stack-list-registry-" }); - const project = path.join(home, "project"); - yield* fs.makeDirectory(project); - const telemetry = mockTelemetryStateTracked(); - yield* Effect.gen(function* () { - const healthy = yield* createStack({ - projectRoot: project, - name: "healthy", - runtime: { kind: "native" }, - }); - const registry = path.join(home, "managed", "stacks"); - const corrupt = "b".repeat(64); - const unsupported = "c".repeat(64); - const remnant = "d".repeat(64); - for (const id of [corrupt, unsupported, remnant, "not-a-stack-id"]) { - yield* fs.makeDirectory(path.join(registry, id)); - } - yield* fs.writeFileString(path.join(registry, corrupt, "state.json"), "{corrupt"); - yield* fs.writeFileString( - path.join(registry, unsupported, "state.json"), - '{"format":"unsupported-example"}', - ); - const text = mockOutput(); - const entries = yield* stackList().pipe(Effect.provide(text.layer)); - expect(entries.map(({ id }) => id)).toEqual([healthy.id, corrupt, unsupported]); - expect(text.stdoutText).toContain("NAME"); - expect(text.stdoutText).toContain("healthy"); - expect(text.stdoutText).toContain("unconfigured"); - expect(text.stdoutText).toContain(healthy.id.slice(0, 8)); - expect(text.stdoutText).not.toContain(healthy.id); - expect(text.stdoutText).toContain("Unreadable stacks:"); - for (const id of [corrupt, unsupported]) { - expect(text.stdoutText.split(id)).toHaveLength(2); - } - expect(text.stdoutText).toContain("Unable to parse state document"); - expect(text.stdoutText).toContain("Unsupported stack state format"); - expect(text.stdoutText).not.toContain(remnant); - expect(entries).toEqual([ - expect.objectContaining({ - id: healthy.id, - readable: true, - name: "healthy", - runtime: { kind: "native" }, - desired_lifecycle: "unconfigured", - }), - { - id: corrupt, - readable: false, - error: { - code: "StackStateInvalidError", - message: expect.stringContaining(`Failed to read managed stack ${corrupt}:`), - }, - }, - { - id: unsupported, - readable: false, - error: { - code: "StackStateFormatUnsupportedError", - message: expect.stringContaining(`Failed to read managed stack ${unsupported}:`), - }, - }, - ]); - for (const format of ["json", "stream-json"] as const) { - const stdout: string[] = []; - const stderr: string[] = []; - const capture = (target: string[]) => - Sink.forEach((chunk: string | Uint8Array) => - Effect.sync(() => { - target.push(typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)); - }), - ); - const stdio = Layer.succeed( - Stdio.Stdio, - Stdio.make({ - args: Effect.succeed([]), - stdin: Stream.empty, - stdout: () => capture(stdout), - stderr: () => capture(stderr), - }), - ); - const output = (format === "json" ? jsonOutputLayer : streamJsonOutputLayer).pipe( - Layer.provide(stdio), - ); - yield* stackList().pipe(Effect.provide(output)); - const lines = stdout.join("").trim().split("\n"); - expect(lines).toHaveLength(1); - const result = yield* Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Json))( - lines[0], - ); - if (format === "json") expect(result).toEqual({ stacks: entries, message: "" }); - else - expect(result).toEqual({ - type: "result", - timestamp: expect.any(String), - data: { stacks: entries, message: "" }, - }); - expect(stderr).toEqual([]); - } - yield* fs.remove(path.join(registry, healthy.id), { recursive: true }); - const unreadableText = mockOutput(); - const unreadable = yield* stackList().pipe(Effect.provide(unreadableText.layer)); - expect(unreadable.map(({ id }) => id)).toEqual([corrupt, unsupported]); - expect(unreadableText.stdoutText).not.toContain("NAME"); - expect(unreadableText.stdoutText).not.toContain("No managed stacks found."); - expect(telemetry.flushed).toBe(true); - }).pipe( - Effect.provide( - Layer.mergeAll( - stackApiLayer, - telemetry.layer, - ConfigProvider.layer(ConfigProvider.fromUnknown({ SUPABASE_HOME: home })), - ), - ), + it.live("lists real saved identities without starting owners in text and machine output", () => + Effect.gen(function* () { + const fixture = yield* registry(); + for (const name of ["beta", "alpha"]) { + yield* fixture.api.create({ + ...fixture.locations, + projectRoot: fixture.root, + name, + runtime: "native", + }); + } + for (const format of ["text", "json", "stream-json"] as const) { + const output = mockOutput({ format }); + const entries = yield* stackList().pipe( + Effect.provide(Layer.mergeAll(fixture.layer, output.layer)), ); - }), - ).pipe(Effect.provide(BunServices.layer)), + expect(entries.map(({ name }) => name)).toEqual(["alpha", "beta"]); + expect(entries.every(({ owner }) => owner === "unavailable")).toBe(true); + expect(entries.every(({ project_root }) => project_root === fixture.root)).toBe(true); + if (format === "text") { + expect(output.stdoutText).toContain("OWNER"); + expect(output.stdoutText).toContain("unavailable"); + } else + expect(output.messages).toContainEqual( + expect.objectContaining({ data: { stacks: entries } }), + ); + } + const discovered = yield* fixture.api.discover(fixture.locations); + expect(discovered.every(({ host }) => host === undefined)).toBe(true); + expect(fixture.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), ); - it.live("orders unreadable IDs after readable stacks regardless of discovery order", () => { - const healthy = descriptor("f", "/work", "healthy", "stopped"); - const fixture = setup({ - stacks: [healthy], - errors: ["c", "b"].map((prefix) => { - const id = StackIdSchema.make(prefix.repeat(64)); - return { - id, - error: new StackStateInvalidError({ - message: `Failed to read managed stack ${id}: corrupt state`, - }), - }; - }), - }); - return stackList().pipe( - Effect.tap((entries) => - Effect.sync(() => { - expect(entries.map(({ id }) => id)).toEqual([healthy.id, "b".repeat(64), "c".repeat(64)]); - expect(fixture.output.stdoutText.indexOf("b".repeat(64))).toBeLessThan( - fixture.output.stdoutText.indexOf("c".repeat(64)), + it.live("lists healthy stacks and warns about invalid entries without altering state", () => + Effect.gen(function* () { + const fixture = yield* registry(); + const healthy = yield* fixture.api.create({ + ...fixture.locations, + projectRoot: fixture.root, + name: "healthy", + runtime: "native", + }); + const broken = yield* fixture.api.create({ + ...fixture.locations, + projectRoot: fixture.root, + name: "broken", + runtime: "native", + }); + const file = fixture.path.join(fixture.locations.stateRoot, broken.id, "state.json"); + yield* fixture.fs.writeFileString(file, "{broken"); + const mismatchRoot = fixture.path.join(fixture.locations.stateRoot, "mismatched"); + yield* fixture.fs.makeDirectory(mismatchRoot); + const mismatchFile = fixture.path.join(mismatchRoot, "state.json"); + const healthyState = yield* fixture.fs.readFileString( + fixture.path.join(fixture.locations.stateRoot, healthy.id, "state.json"), + ); + yield* fixture.fs.writeFileString(mismatchFile, healthyState); + for (const format of ["text", "json", "stream-json"] as const) { + const output = mockOutput({ format }); + const entries = yield* stackList().pipe( + Effect.provide(Layer.mergeAll(fixture.layer, output.layer)), + ); + expect(entries.map(({ id }) => id)).toEqual([healthy.id]); + expect(output.stderrText).toContain(`skipping invalid stack ${broken.id}`); + expect(output.stderrText).toContain("skipping invalid stack mismatched"); + expect(output.stdoutText).not.toContain("Warning"); + if (format !== "text") + expect(output.messages).toContainEqual( + expect.objectContaining({ data: { stacks: entries } }), ); - }), - ), - Effect.provide(fixture.layer), - ); - }); - - it.live("fails registry discovery without emitting a partial result", () => { - const fixture = setup({ - discoveryError: new StackStateInvalidError({ message: "registry unavailable" }), - }); - return stackList().pipe( - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error).toBeInstanceOf(StackCommandListError); - expect(error.reason).toBe("invalid-config"); - expect(error.message).toContain("registry unavailable"); - expect(fixture.output.stdoutText).toBe(""); - expect(fixture.output.messages).toHaveLength(0); - expect(fixture.telemetry.flushed).toBe(true); - }), - ), - Effect.provide(fixture.layer), - ); - }); + } + expect(yield* fixture.fs.readFileString(file)).toBe("{broken"); + expect(yield* fixture.fs.readFileString(mismatchFile)).toBe(healthyState); + const discoveryError = yield* fixture.api.discover(fixture.locations).pipe(Effect.flip); + expect(discoveryError.operation).toBe("discover"); + const openError = yield* fixture.api + .open({ ...fixture.locations, id: broken.id }) + .pipe(Effect.flip); + expect(openError.operation).toBe("open"); + expect(fixture.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); - it.live("rejects the legacy output flag", () => { - const fixture = setup({ outputFlag: true }); - return stackList().pipe( - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error).toBeInstanceOf(StackCommandListError); - expect(error.reason).toBe("flags"); - expect(fixture.telemetry.flushed).toBe(true); - }), - ), - Effect.provide(fixture.layer), - ); - }); + it.live("reports an empty registry and rejects the legacy output flag", () => + Effect.gen(function* () { + const fixture = yield* registry(); + const output = mockOutput(); + yield* stackList().pipe(Effect.provide(Layer.mergeAll(fixture.layer, output.layer))); + expect(output.stdoutText).toBe("No managed stacks found.\n"); + const error = yield* stackList().pipe( + Effect.provide( + Layer.mergeAll( + fixture.layer, + output.layer, + Layer.succeed(OutputFlag, Option.some("json")), + ), + ), + Effect.flip, + ); + expect(error.reason).toBe("flags"); + expect(fixture.telemetry.flushCount).toBe(2); + }).pipe(Effect.provide(live)), + ); }); diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index bbc408bfce..2a9a9e92d3 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -1,97 +1,42 @@ # `supabase stack logs` -This command is available only when the `experimental.stack` feature flag is enabled. It reads -retained logs from the managed stack identified by the current project, an optional `--stack` -name, or `--stack-id`. It calls the public `@supabase/stack` logs and followLogs APIs without -starting the stack, stopping it, or changing its owner lifecycle. +Streams live stdout/stderr from the selected saved stack. The experimental +feature flag controls command registration. It connects to an existing owner; +it never launches an owner or starts/stops a service. -## Files read and written +## Selection and files -The stack package reads its normal durable state under `` and -the selected stack's persisted log state. The CLI reads the current workdir and experimental -stack feature setting through its normal settings resolution. This command writes no project -files, stack state, credentials, or runtime resources. +Select the current project/branch/name, `--stack `, or `--stack-id `. +The selectors are mutually exclusive. By default, only composition members are +included. `--service ` can also select standalone instances. +An unavailable owner, missing stack, or unmatched service fails with status 1. -## Output +Reads saved definitions under `/stacks//`. +Discovery ensures the registry directory exists with mode 0700 and probes local +owners. Shared routing/settings may read project config and profiles. No project +files, service configuration, artifacts, or data are changed. -Text mode writes ` /: ` followed by a newline for each -retained or followed entry. Terminal escape sequences and C0/C1 controls are removed from text -messages while tabs and newlines are preserved. JSON and bounded stream-json modes each write -one success/result payload containing `message`, `found`, and `entries`; when a stack is found, -it also contains `id`, `cursor`, and `running`. Log entry `message` values are preserved. `--follow` is -rejected with `--output-format json`; follow stream-json emits one `log-entry` event per entry. -Each event has `type: "log-entry"`, `timestamp`, `service`, `stream`, `line`, and `source`; -`line` preserves the original message, `stream` is `stdout`, `stderr`, or `internal`, and -`source` is `history` or `live`. +## Output and cancellation -A found stack's bounded JSON result is shaped as follows (the entry message is raw): +This is a live-only stream with no retained history, cursor, or `--tail` option. +It continues until interrupted or the selected streams close. `--follow` is +unnecessary and is not accepted. Legacy `-o/--output` and finite JSON output are +rejected; use text or `--output-format stream-json`. -```json -{ - "found": true, - "id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "entries": [ - { - "cursor": { "opaque": "1" }, - "timestamp": "2026-09-08T00:00:00.000Z", - "source": "database", - "stream": "stdout", - "message": "database ready" - } - ], - "cursor": { "opaque": "1" }, - "running": false, - "message": "" -} -``` +Text writes ` //: `. Terminal +control sequences are stripped from text. Stream JSON emits `log-entry` events +with `timestamp`, `service`, `instance_id`, `stream`, `line`, and `source: "live"`. +Lines preserve their content in machine output. UTF-8 and line fragments are +assembled separately for each instance and stdout/stderr channel. Timestamps +reflect receipt by the CLI. Delivery is best effort: slow subscribers can lose entries. Ordering across +stdout/stderr channels and different services is not guaranteed. -With no default stack, JSON output is: - -```json -{ - "found": false, - "entries": [], - "message": "No managed stack found for this context." -} -``` - -Follow prints the retained history first and then resumes from its returned cursor. If the stack -is already stopped, it prints the retained history and exits successfully. With no `--stack` or -`--stack-id`, an absent default stack prints a successful empty result. In stream-json mode this -is the standard result envelope: - -```json -{ - "type": "result", - "data": { - "found": false, - "entries": [], - "message": "No managed stack found for this context." - }, - "timestamp": "..." -} -``` - -A finite stream-json read emits one result event for a found stack, including when its entries -are empty. Follow mode emits only `log-entry` events; a found stack with no retained entries -emits no follow events, and a stopped stack exits successfully. A missing named stack fails with -status `1`. The legacy `-o`/`--output` flag is rejected; use -`--output-format`. - -`--service` accepts one capability name and excludes supervisor and gateway entries, including -their startup diagnostics. Omit it to include all sources. Retained logs are bounded to the -newest 1000 entries or 1 MiB, whichever is reached first; `--tail` further limits the returned -entries. - -Successful reads, including an absent default stack and a stopped stack, exit with status `0`. -Invalid flags, missing named stacks, and stack read failures exit with status `1`. Interrupting -follow cancels the log reader, exits with status `130`, and leaves the managed stack owner -untouched. +Interrupting the command cancels its subscriptions, exits with status 130, and +leaves the owner and services running. Successful stream completion exits 0; +selection, connection, or stream failures exit 1. ## Telemetry -On normal command completion, the command wrapper fires the standard `cli_command_executed` event -with the command name, exit code, duration, and safe flag metadata. An interrupted follow may -terminate before that event is captured. Stack log contents and messages are not sent as custom -telemetry properties. Telemetry state is flushed to -`/telemetry.json` after both successful and failed command runs. +Standard command telemetry is retained; log contents are not custom telemetry +properties. Telemetry flushes on every exit to +`/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.command.ts b/apps/cli/src/commands/experimental/stack/logs/logs.command.ts index 3a5333ce24..f895e7151d 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.command.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.command.ts @@ -1,12 +1,9 @@ import { Command, Flag } from "effect/unstable/cli"; import type * as CliCommand from "effect/unstable/cli/Command"; -import { CAPABILITY_NAMES } from "@supabase/stack/effect"; import { withJsonErrorHandling } from "../../../../shared/output/json-error-handling.ts"; import { withCommandTelemetry } from "../../../../telemetry/command-telemetry.ts"; import { stackLogs } from "./logs.handler.ts"; -const MAX_TAIL = 1000; - const config = { stack: Flag.string("stack").pipe( Flag.withDescription( @@ -18,48 +15,30 @@ const config = { Flag.withDescription("Read logs from an existing stack by id."), Flag.optional, ), - service: Flag.choice("service", CAPABILITY_NAMES).pipe( + service: Flag.string("service").pipe( Flag.withDescription( - "Limit logs to one stack service; omit this flag to include supervisor and gateway diagnostics.", + "Stream one service kind or instance ID; defaults to composition members.", ), Flag.optional, ), - tail: Flag.integer("tail").pipe( - Flag.filter( - (value) => value >= 0 && value <= MAX_TAIL, - (value) => `Expected --tail between 0 and ${MAX_TAIL}, got ${value}`, - ), - Flag.withDescription( - "Number of retained log entries to print. Use 0 with --follow to skip retained history.", - ), - Flag.withDefault(100), - ), - follow: Flag.boolean("follow").pipe( - Flag.withAlias("f"), - Flag.withDescription("Continue printing new log entries until interrupted."), - Flag.withDefault(false), - ), } as const; export type StackLogsFlags = CliCommand.Command.Config.Infer; export const stackLogsCommand = Command.make("logs", config).pipe( - Command.withDescription("Read logs from a managed local Supabase stack."), - Command.withShortDescription("Read managed local stack logs"), + Command.withDescription("Stream live logs until interrupted; retained history is unavailable."), + Command.withShortDescription("Stream managed local stack logs"), Command.withExamples([ { - command: "supabase stack logs --service database --tail 50", - description: "Print the latest database logs", + command: "supabase stack logs --service database", + description: "Stream new database logs", }, { - command: "supabase stack logs --follow --output-format stream-json", + command: "supabase stack logs --output-format stream-json", description: "Stream new stack logs as structured events", }, ]), Command.withHandler((flags) => - stackLogs(flags).pipe( - withCommandTelemetry({ flags, config, aliases: { f: "follow" } }), - withJsonErrorHandling, - ), + stackLogs(flags).pipe(withCommandTelemetry({ flags, config }), withJsonErrorHandling), ), ); diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts index eb17e81c0b..262bb07549 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts @@ -1,10 +1,4 @@ -import { Effect, Match, Option, Stream } from "effect"; -import type { - OpenStackError, - StackDiscoveryError, - StackLogEntry, - StackLogsError as ApiStackLogsError, -} from "@supabase/stack/effect"; +import { DateTime, Effect, Option, Path, Stream } from "effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; @@ -13,155 +7,121 @@ import { stripControlSequences } from "../../../../shared/output/strip-control-s import { StackApi, StackTargetError, + StackTargetResolver, rejectStackOutput, - validateStackId, validateStackTarget, } from "../stack.shared.ts"; import { StackCommandLogsError } from "./logs.errors.ts"; import type { StackLogsFlags } from "./logs.command.ts"; -const mapTargetError = (error: StackTargetError) => +const targetError = (cause: StackTargetError) => new StackCommandLogsError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, + reason: cause.reason, + message: cause.message, + ...(cause.suggestion === undefined ? {} : { suggestion: cause.suggestion }), + cause, }); - -const logsError = ( - error: StackDiscoveryError | OpenStackError | ApiStackLogsError, -): StackCommandLogsError => { - const classification = Match.value(error).pipe( - Match.tag("StackNotFoundError", () => ({ - reason: "flags" as const, - suggestion: - "Choose an existing stack with --stack or --stack-id, or omit both to use the current project.", - })), - Match.tag("InvalidStackIdentityError", () => ({ reason: "flags" as const })), - Match.tag("InvalidLogCursorError", () => ({ reason: "impossible-state" as const })), - Match.tag("StackNotRunningError", () => ({ - reason: "lifecycle" as const, - suggestion: "Run supabase stack start before reading logs.", - })), - Match.tag("StackOwnershipConflictError", () => ({ - reason: "lifecycle" as const, - suggestion: - "Retry with --debug. If the owner is no longer running, stop the selected stack to reconcile its state, then retry.", - })), - Match.tag("StackLifecycleConflictError", () => ({ - reason: "lifecycle" as const, - suggestion: "The stack owner is shutting down; retry shortly.", - })), - Match.tag("StackUpgradeRequiredError", () => ({ - reason: "lifecycle" as const, - suggestion: - "Stop and restart the selected stack using this CLI so its owner matches this release.", - })), - Match.tag("StackRuntimeMismatchError", () => ({ reason: "unknown" as const })), - Match.tag("StackStateInvalidError", "StackStateFormatUnsupportedError", () => ({ - reason: "invalid-config" as const, - })), - Match.tag("InvalidProjectRootError", () => ({ reason: "invalid-config" as const })), - Match.exhaustive, - ); - return new StackCommandLogsError({ - ...classification, - message: error.message, - cause: error, - }); -}; - -const renderEntry = (entry: StackLogEntry) => - `${entry.timestamp} ${entry.source}/${entry.stream}: ${stripControlSequences(entry.message)}\n`; - -const eventForEntry = (entry: StackLogEntry, source: "history" | "live") => ({ - type: "log-entry" as const, - timestamp: entry.timestamp, - service: entry.source, - stream: entry.stream, - line: entry.message, - source, -}); +const logsError = (cause: { readonly message: string }) => + new StackCommandLogsError({ reason: "unknown", message: cause.message, cause }); export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: StackLogsFlags) { - const telemetryState = yield* TelemetryState; - const body = Effect.gen(function* () { + const telemetry = yield* TelemetryState; + return yield* Effect.gen(function* () { const output = yield* Output; const settings = yield* CommandSettings; - const stackApi = yield* StackApi; - const outputFlag = yield* Effect.serviceOption(OutputFlag); - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + const path = yield* Path.Path; + const api = yield* StackApi; + const resolver = yield* StackTargetResolver; + yield* rejectStackOutput(yield* Effect.serviceOption(OutputFlag)).pipe( + Effect.mapError(targetError), + ); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); - if (flags.follow && output.format === "json") + }).pipe(Effect.mapError(targetError)); + if (output.format === "json") return yield* new StackCommandLogsError({ reason: "flags", - message: "--follow cannot be combined with --output-format json.", - suggestion: "Use --output-format stream-json for follow mode, or omit --follow.", + message: "Live logs require text or stream-json output.", + suggestion: "Use --output-format stream-json.", }); - - const id = Option.isSome(flags.stackId) - ? yield* validateStackId(flags.stackId.value).pipe(Effect.mapError(mapTargetError)) - : undefined; - const targetOption = - id === undefined - ? yield* stackApi - .findStack({ - projectRoot: settings.workdir, - ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), - }) - .pipe(Effect.mapError(logsError)) - : Option.some({ id }); - if (Option.isNone(targetOption)) { - if (Option.isSome(flags.stack)) - return yield* new StackCommandLogsError({ - reason: "flags", - message: `No managed stack named "${flags.stack.value}" was found for this project.`, - suggestion: - "Choose an existing stack with --stack or --stack-id, or omit both to use the current project.", - }); - yield* output.success("No managed stack found for this context.", { - found: false, - entries: [], + const target = yield* resolver + .resolve({ + projectRoot: settings.workdir, + runtime: "auto", + ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), + ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), + }) + .pipe(Effect.mapError(targetError)); + if (target.id === undefined) + return yield* new StackCommandLogsError({ + reason: "flags", + message: "No managed stack exists for the selected project.", + suggestion: "Run supabase stack start first.", }); - return; - } - const stack = yield* stackApi.openStack(targetOption.value.id).pipe(Effect.mapError(logsError)); - const query = { - ...(Option.isSome(flags.service) ? { capabilities: [flags.service.value] } : {}), - tail: flags.tail, - }; - const batch = yield* stack.logs(query).pipe(Effect.mapError(logsError)); - const emitEntries = (source: "history" | "live", entries: ReadonlyArray) => - output.format === "stream-json" - ? Effect.forEach(entries, (entry) => output.event(eventForEntry(entry, source)), { - discard: true, - }) - : Effect.forEach(entries, (entry) => output.raw(renderEntry(entry)), { discard: true }); - if (!flags.follow) { - if (output.format === "json" || output.format === "stream-json") { - yield* output.success("", { found: true, id: stack.id, ...batch }); - } else { - yield* emitEntries("history", batch.entries); - } - return; - } - yield* emitEntries("history", batch.entries); - if (!batch.running) return; - const followQuery = { - ...(Option.isSome(flags.service) ? { capabilities: [flags.service.value] } : {}), - cursor: batch.cursor, + const locations = { + stateRoot: path.join(settings.supabaseHome, "stacks"), + cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), }; - yield* stack.followLogs(followQuery).pipe( + const discovery = yield* api.discover(locations).pipe(Effect.mapError(logsError)); + if ( + !discovery.some(({ definition, host }) => definition.id === target.id && host !== undefined) + ) + return yield* new StackCommandLogsError({ + reason: "lifecycle", + message: "No owner is reachable; live logs are unavailable.", + suggestion: "Run supabase stack start first.", + }); + const stack = yield* api.open({ ...locations, id: target.id }).pipe(Effect.mapError(logsError)); + const instances = yield* stack.services.list.pipe(Effect.mapError(logsError)); + const composition = yield* stack.composition.describe.pipe(Effect.mapError(logsError)); + const requested = Option.getOrUndefined(flags.service); + const selected = instances.filter((instance) => + requested === undefined + ? composition.members.some(({ id }) => id === instance.id) + : instance.service === requested || instance.id === requested, + ); + if (selected.length === 0) + return yield* new StackCommandLogsError({ + reason: "flags", + message: + requested === undefined + ? "The stack has no composition members to stream." + : `No service matches ${requested}.`, + }); + const streams = selected.map((instance) => + instance.logs.pipe( + Stream.groupByKey((entry) => entry.stream), + Stream.flatMap( + ([channel, chunks]) => + chunks.pipe( + Stream.map(({ bytes }) => bytes), + Stream.decodeText, + Stream.splitLines, + Stream.map((line) => ({ + service: instance.service, + instance_id: instance.id, + stream: channel, + line, + })), + ), + { concurrency: 2 }, + ), + ), + ); + yield* Stream.mergeAll(streams, { concurrency: "unbounded" }).pipe( Stream.mapError(logsError), Stream.runForEach((entry) => - output.format === "stream-json" - ? output.event(eventForEntry(entry, "live")) - : output.raw(renderEntry(entry)), + Effect.gen(function* () { + const timestamp = DateTime.formatIso(yield* DateTime.now); + if (output.format === "stream-json") + yield* output.event({ type: "log-entry", timestamp, source: "live", ...entry }); + else + yield* output.raw( + `${timestamp} ${entry.service}/${entry.instance_id}/${entry.stream}: ${stripControlSequences(entry.line)}\n`, + ); + }), ), ); - }); - return yield* body.pipe(Effect.ensuring(telemetryState.flush)); + }).pipe(Effect.ensuring(telemetry.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index a5a3b6d635..c15462dd43 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -1,623 +1,218 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { - Deferred, - Effect, - FileSystem, - Fiber, - Layer, - Option, - Schema, - Sink, - Stdio, - Stream, -} from "effect"; -import { CliError, CliOutput, Command } from "effect/unstable/cli"; -import { - InvalidProjectRootError, - StackIdSchema, - StackNotFoundError, - StackOwnershipConflictError, - StackUpgradeRequiredError, -} from "@supabase/stack/effect"; -import type { - EffectStack, - OpenStackError, - StackLogBatch, - StackLogEntry, - StackDiscoveryError, - StackStatus, -} from "@supabase/stack/effect"; +import { Deferred, Effect, Fiber, FileSystem, Layer, Option, Path, Ref, Stream } from "effect"; +import type { Stack } from "@supabase/stack/effect"; +import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, - mockTelemetryStateLayer, + mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; -import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; -import { OutputFlag } from "../../../../command-internal/global-flags.ts"; -import { - ErrorActionabilityId, - actionability, -} from "../../../../shared/telemetry/error-actionability.ts"; -import { StackApi } from "../stack.shared.ts"; +import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackLogs } from "./logs.handler.ts"; -import { StackCommandLogsError } from "./logs.errors.ts"; -import { stackLogsCommand } from "./logs.command.ts"; -import { textCliOutputFormatter } from "../../../../shared/output/text-formatter.ts"; -import { streamJsonOutputLayer } from "../../../../shared/output/output.layer.ts"; -const id = StackIdSchema.make("a".repeat(64)); -const streamResultSchema = Schema.Struct({ - type: Schema.Literal("result"), - data: Schema.Unknown, - timestamp: Schema.String, -}); -const entries: ReadonlyArray = [ - { - cursor: { opaque: "1" }, - timestamp: "2026-09-08T00:00:00.000Z", - source: "database", - stream: "stdout", - message: "database ready", - }, - { - cursor: { opaque: "2" }, - timestamp: "2026-09-08T00:00:01.000Z", - source: "functions", - stream: "stderr", - message: "function failed", - }, -]; -const internalEntry: StackLogEntry = { - cursor: { opaque: "internal-1" }, - timestamp: "2026-09-08T00:00:02.000Z", - source: "supervisor", - stream: "internal", - message: "stack supervisor ready", -}; - -const status: StackStatus = { - id, - lifecycle: "running", - desiredLifecycle: "running", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: [], - artifacts: [], -}; - -const flags = (overrides: Partial[0]> = {}) => ({ - stack: Option.none(), - stackId: Option.none(), - service: Option.none<"database" | "functions">(), - tail: 100, - follow: false, - ...overrides, +const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +const fixture = Effect.fn("StackLogsTest.fixture")(function* ( + format: "text" | "stream-json" | "json" = "text", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-logs-" }); + const api = yield* StackApi; + const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; + const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "native" }); + const output = mockOutput({ format }); + const telemetry = mockTelemetryStateTracked(); + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); + const layer = Layer.mergeAll( + output.layer, + telemetry.layer, + settings, + stackTargetResolverLayer.pipe(Layer.provide(settings)), + ); + return { + api, + locations, + stack, + output, + telemetry, + layer, + flags: { + stack: Option.none(), + stackId: Option.some(stack.id), + service: Option.none(), + }, + }; }); - -const parseErrorMessages = (error: CliError.CliError): ReadonlyArray => - Schema.is(CliError.ShowHelp)(error) ? error.errors.map(({ message }) => message) : []; - -function setup(opts: { - root?: string; - logs?: (query: unknown) => Effect.Effect; - followLogs?: (query: unknown) => Stream.Stream; - openFailure?: OpenStackError; - findFailure?: StackDiscoveryError; - noDefault?: boolean; -}) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = - opts.root ?? (yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-logs-" })); - const out = mockOutput(); - const calls: { - readonly queries: unknown[]; - opened: string[]; - stopCalls: number; - destroyCalls: number; - } = { queries: [], opened: [], stopCalls: 0, destroyCalls: 0 }; - const stack = { - id, - status: Effect.succeed(status), - credentials: Effect.die("unused"), - prepare: () => Effect.die("unused"), - start: () => Effect.die("must not start"), - stop: Effect.sync(() => void calls.stopCalls++), - destroy: Effect.sync(() => void calls.destroyCalls++), - resetDatabase: Effect.die("unused"), - logs: (query?: unknown) => { - calls.queries.push(query); - return ( - opts.logs?.(query) ?? Effect.succeed({ entries, cursor: { opaque: "2" }, running: false }) - ); - }, - followLogs: (query?: unknown) => { - calls.queries.push(query); - return opts.followLogs?.(query) ?? Stream.fromIterable(entries); - }, - } satisfies EffectStack; - const descriptor = { - id, - projectRoot: root, - name: "feature-a", - branchContext: "ordinary-workspace", - runtime: { kind: "native" as const }, - desiredLifecycle: "running" as const, - }; - const layer = Layer.mergeAll( - out.layer, - mockCommandSettings({ workdir: root }), - mockTelemetryStateLayer, - Layer.succeed(StackApi, { - createStack: () => Effect.die("must not create"), - findStack: (query) => - opts.findFailure === undefined - ? Effect.succeed( - query.name === "missing" || (query.name === undefined && opts.noDefault) - ? Option.none() - : Option.some(descriptor), - ) - : Effect.fail(opts.findFailure), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - openStack: (stackId) => - opts.openFailure === undefined - ? Effect.sync(() => { - calls.opened.push(stackId); - return stack; - }) - : Effect.fail(opts.openFailure), - inspectStack: () => Effect.die("must not inspect"), - }), - BunServices.layer, - ); - return { layer, out, calls, root }; - }).pipe(Effect.provide(BunServices.layer)); -} - -function capturedStdio() { - const stdout: string[] = []; - const layer = Layer.succeed( - Stdio.Stdio, - Stdio.make({ - args: Effect.succeed([]), - stdin: Stream.empty, - stdout: () => - Sink.forEach((item: string | Uint8Array) => - Effect.sync(() => - stdout.push(typeof item === "string" ? item : new TextDecoder().decode(item)), - ), - ), - stderr: () => Sink.forEach(() => Effect.void), +const loggingApi = ( + api: StackApi["Service"], + stack: Stack, + instances: Effect.Success, +) => + Layer.succeed( + StackApi, + StackApi.of({ + ...api, + open: () => + Effect.succeed({ + ...stack, + services: { ...stack.services, list: Effect.succeed([...instances]) }, + }), }), ); - return { layer, stdout }; -} -describe("experimental stack logs", () => { - it.live("parses --service as a value-consuming flag", () => { - let parsed: { service: Option.Option; tail: number } | undefined; - const command = stackLogsCommand.pipe( - Command.withHandler((flags) => - Effect.sync(() => { - parsed = { service: flags.service, tail: flags.tail }; - }), - ), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--service", - "database", - "--tail", - "4", - ]); - expect(parsed?.service).toEqual(Option.some("database")); - expect(parsed?.tail).toBe(4); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); +describe("stack logs", () => { + it.live("rejects finite JSON before starting an owner", () => + Effect.gen(function* () { + const f = yield* fixture("json"); + const error = yield* stackLogs(f.flags).pipe(Effect.provide(f.layer), Effect.flip); + expect(error.reason).toBe("flags"); + expect(error.suggestion).toBe("Use --output-format stream-json."); + expect((yield* f.api.discover(f.locations))[0]?.host).toBeUndefined(); + }).pipe(Effect.provide(live)), + ); - it.live( - "accepts zero and maximum tail with the follow alias and rejects out-of-range tails", - () => { - let parsed: { tail: number; follow: boolean } | undefined; - const command = stackLogsCommand.pipe( - Command.withHandler((flags) => - Effect.sync(() => { - parsed = { tail: flags.tail, follow: flags.follow }; + it.live("selects all instances of a kind and sanitizes terminal text", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.acquireUseRelease( + Effect.succeed(f.stack), + (stack) => + Effect.gen(function* () { + const first = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: {}, + }); + const second = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: {}, + }); + const api = loggingApi( + f.api, + stack, + [first, second].map((instance) => ({ + ...instance, + logs: Stream.make({ + stream: "stderr" as const, + bytes: new TextEncoder().encode("\u001b[31mwarning\u001b[0m\n"), + }), + })), + ); + yield* stackLogs({ ...f.flags, service: Option.some("mail") }).pipe( + Effect.provide(Layer.provideMerge(f.layer, api)), + ); + expect(f.output.stdoutText).toContain(`mail/${first.id}/stderr: warning\n`); + expect(f.output.stdoutText).toContain(`mail/${second.id}/stderr: warning\n`); + expect(f.output.stdoutText).not.toContain("\u001b"); + const error = yield* stackLogs({ ...f.flags, service: Option.some("missing") }).pipe( + Effect.provide(f.layer), + Effect.flip, + ); + expect(error.message).toBe("No service matches missing."); }), - ), + (stack) => stack.destroy, ); - const outputLayer = Layer.mergeAll( - BunServices.layer, - CliOutput.layer(textCliOutputFormatter()), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })(["--tail", "0", "-f"]); - expect(parsed).toEqual({ tail: 0, follow: true }); - yield* Command.runWith(command, { version: "0.0.0-test" })(["--tail", "1000"]); - expect(parsed).toEqual({ tail: 1000, follow: false }); - const belowMinimum = yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--tail=-1", - ]).pipe(Effect.flip); - const aboveMaximum = yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--tail=1001", - ]).pipe(Effect.flip); - expect(belowMinimum).toBeInstanceOf(CliError.ShowHelp); - expect(aboveMaximum).toBeInstanceOf(CliError.ShowHelp); - expect(parseErrorMessages(belowMinimum)).toContain( - 'Invalid value for flag --tail: "-1". Expected --tail between 0 and 1000, got -1', - ); - expect(parseErrorMessages(aboveMaximum)).toContain( - 'Invalid value for flag --tail: "1001". Expected --tail between 0 and 1000, got 1001', - ); - }).pipe(Effect.provide(outputLayer)); - }, + }).pipe(Effect.provide(live)), ); - it.live("reads a finite tail and passes the service filter without starting or stopping", () => { - return setup({}).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - yield* stackLogs(flags({ service: Option.some("database"), tail: 2 })); - expect(setupResult.calls.queries).toEqual([{ capabilities: ["database"], tail: 2 }]); - expect(setupResult.calls.opened).toEqual([id]); - expect(setupResult.out.stdoutText).toContain("database ready"); - expect(setupResult.out.stdoutText).toContain("function failed"); - }).pipe(Effect.provide(setupResult.layer)), - ), - ); - }); - - it.live("sanitizes text messages while preserving structured log content", () => { - const message = - "safe\u009d0;bel\u0007-red\u009d0;c1\u009c-blue\u009d0;esc\u001b\\\u009b38:2::255:0:0m-color\u009b0m\tcolumn\rnext\u0000\u000b\u0085end"; - const sanitized = "safe-red-blue-color\tcolumnnextend"; - return setup({ - logs: () => - Effect.succeed({ - entries: [{ ...entries[0]!, message }], - cursor: { opaque: "1" }, - running: false, - }), - }).pipe( - Effect.flatMap((setupResult) => { - const text = mockOutput(); - const json = mockOutput({ format: "json" }); - const structured = mockOutput({ format: "stream-json" }); - return Effect.gen(function* () { - yield* stackLogs(flags()).pipe( - Effect.provide(Layer.mergeAll(setupResult.layer, text.layer)), - ); - expect(text.stdoutText).toContain(sanitized); - expect(text.stdoutText).not.toContain("\u001b"); - - yield* stackLogs(flags()).pipe( - Effect.provide(Layer.mergeAll(setupResult.layer, json.layer)), - ); - const jsonResult = json.messages.find((entry) => entry.type === "success")?.data; - expect(jsonResult).toEqual( - expect.objectContaining({ entries: [expect.objectContaining({ message })] }), - ); - - yield* stackLogs(flags()).pipe( - Effect.provide(Layer.mergeAll(setupResult.layer, structured.layer)), - ); - const streamResult = structured.messages.find((entry) => entry.type === "success")?.data; - expect(streamResult).toEqual( - expect.objectContaining({ entries: [expect.objectContaining({ message })] }), - ); - }).pipe(Effect.provide(setupResult.layer)); - }), - ); - }); - - it.live("rejects conflicting targets before resolving a stack", () => { - return setup({}).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - const failure = yield* stackLogs( - flags({ stack: Option.some("feature-a"), stackId: Option.some(id) }), - ).pipe(Effect.flip); - expect(failure.reason).toBe("flags"); - expect(setupResult.calls.queries).toEqual([]); - expect(setupResult.calls.opened).toEqual([]); - }).pipe(Effect.provide(setupResult.layer)), - ), - ); - }); - - it.live("distinguishes an absent default stack from a missing named stack", () => { - return Effect.gen(function* () { - const absent = yield* setup({ noDefault: true }); - const named = yield* setup({}); - yield* stackLogs(flags()).pipe(Effect.provide(absent.layer)); - expect(absent.out.messages).toContainEqual( - expect.objectContaining({ - type: "success", - message: "No managed stack found for this context.", - }), - ); - const absentStream = capturedStdio(); - yield* stackLogs(flags()).pipe( - Effect.provide( - Layer.mergeAll( - absent.layer, - absentStream.layer, - streamJsonOutputLayer.pipe(Layer.provide(absentStream.layer)), - ), - ), - ); - const absentResult = yield* Schema.decodeEffect(Schema.fromJsonString(streamResultSchema))( - absentStream.stdout.join(""), - ); - expect(absentResult).toEqual({ - type: "result", - data: { - found: false, - entries: [], - message: "No managed stack found for this context.", - }, - timestamp: expect.any(String), - }); - const failure = yield* stackLogs(flags({ stack: Option.some("missing") })).pipe( - Effect.flip, - Effect.provide(named.layer), + it.live("frames split UTF-8 and stdout/stderr lines with instance identity", () => + Effect.gen(function* () { + const f = yield* fixture("stream-json"); + yield* Effect.acquireUseRelease( + Effect.succeed(f.stack), + (stack) => + Effect.gen(function* () { + const member = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: {}, + }); + const standalone = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: {}, + }); + yield* stack.composition.configure({ + members: [{ id: member.id, activation: "eager" }], + dependencies: [], + }); + const utf8 = new TextEncoder().encode("hello 🐘\nlast line"); + const memberLogs = Stream.make( + { stream: "stdout" as const, bytes: utf8.slice(0, 8) }, + { stream: "stderr" as const, bytes: new TextEncoder().encode("warning\n") }, + { stream: "stdout" as const, bytes: utf8.slice(8) }, + ); + const api = loggingApi(f.api, stack, [ + { ...member, logs: memberLogs }, + { ...standalone, logs: Stream.die("standalone must not be subscribed") }, + ]); + yield* stackLogs(f.flags).pipe(Effect.provide(Layer.provideMerge(f.layer, api))); + const entries = f.output.events.filter((entry) => entry.type === "log-entry"); + expect(entries).toHaveLength(3); + expect(entries.map(({ line }) => line)).toEqual( + expect.arrayContaining(["hello 🐘", "last line", "warning"]), + ); + expect( + entries.every( + (entry) => + entry.instance_id === member.id && + entry.service === "mail" && + entry.source === "live", + ), + ).toBe(true); + expect(entries.find(({ line }) => line === "warning")?.stream).toBe("stderr"); + expect(f.telemetry.flushed).toBe(true); + }), + (stack) => stack.destroy, ); - expect(failure.reason).toBe("flags"); - expect(failure.message).toContain("No managed stack named"); - }); - }); + }).pipe(Effect.provide(live)), + ); - it.live( - "streams finite follow output and supports interruption without owner lifecycle calls", - () => { - return setup({ - logs: () => - Effect.succeed({ entries: [entries[0]!], cursor: { opaque: "1" }, running: true }), - followLogs: (query) => { - expect(query).toEqual({ cursor: { opaque: "1" } }); - return Stream.fromIterable([ - { ...entries[1]!, message: "function\u001b[2K failed\rretry" }, - ]); - }, - }).pipe( - Effect.flatMap((setupResult) => + it.live("interrupts a selected log subscription without stopping or destroying its service", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.acquireUseRelease( + Effect.succeed(f.stack), + (stack) => Effect.gen(function* () { - yield* stackLogs(flags({ follow: true })); - expect(setupResult.out.stdoutText).toContain("function failedretry"); - const followStarted = yield* Deferred.make(); - let finalized = false; - const interrupted = yield* setup({ - root: setupResult.root, - logs: () => - Effect.succeed({ entries: [entries[0]!], cursor: { opaque: "1" }, running: true }), - followLogs: (): Stream.Stream => - Stream.fromEffect( - Effect.as(Deferred.succeed(followStarted, undefined), undefined), - ).pipe( - Stream.flatMap(() => Stream.empty), - Stream.concat(Stream.never), - Stream.ensuring(Effect.sync(() => void (finalized = true))), - ), + const service = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: {}, }); - const fiber = yield* Effect.forkChild( - stackLogs(flags({ follow: true })).pipe(Effect.provide(interrupted.layer)), + const subscribed = yield* Deferred.make(); + const released = yield* Ref.make(false); + const logs = Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( + Stream.drain, + Stream.concat(Stream.never), + Stream.ensuring(Ref.set(released, true)), ); - yield* Deferred.await(followStarted); - expect(interrupted.calls.opened).toEqual([id]); + const api = loggingApi(f.api, stack, [{ ...service, logs }]); + const fiber = yield* stackLogs({ ...f.flags, service: Option.some(service.id) }).pipe( + Effect.provide(Layer.provideMerge(f.layer, api)), + Effect.forkChild, + ); + yield* Deferred.await(subscribed); yield* Fiber.interrupt(fiber); - expect(finalized).toBe(true); - expect(interrupted.calls.stopCalls).toBe(0); - expect(interrupted.calls.destroyCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)), - ), + expect(yield* Ref.get(released)).toBe(true); + expect((yield* service.status).registered).toBe(true); + expect((yield* f.api.discover(f.locations))[0]?.host).toBeDefined(); + expect(f.telemetry.flushed).toBe(true); + }), + (stack) => stack.destroy, ); - }, + }).pipe(Effect.provide(live)), ); - it.live("emits a bounded JSON result", () => { - const output = mockOutput({ format: "json" }); - return setup({}).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - yield* stackLogs(flags({ tail: 2 })); - expect(output.messages.find((message) => message.type === "success")?.data).toEqual({ - found: true, - id, - entries, - cursor: { opaque: "2" }, - running: false, - }); - }).pipe(Effect.provide(Layer.mergeAll(setupResult.layer, output.layer))), - ), - ); - }); - - it.live("emits one bounded stream-json result", () => { - return setup({ - logs: () => - Effect.succeed({ - entries: [internalEntry], - cursor: { opaque: "internal-1" }, - running: false, - }), - }).pipe( - Effect.flatMap((setupResult) => { - const output = capturedStdio(); - return Effect.gen(function* () { - yield* stackLogs(flags({ tail: 2 })); - const result = yield* Schema.decodeEffect(Schema.fromJsonString(streamResultSchema))( - output.stdout.join(""), - ); - expect(result).toEqual({ - type: "result", - data: { - found: true, - id, - entries: [internalEntry], - cursor: { opaque: "internal-1" }, - running: false, - message: "", - }, - timestamp: expect.any(String), - }); - expect(output.stdout.join("")).not.toContain('"type":"log-entry"'); - }).pipe( - Effect.provide( - Layer.mergeAll( - setupResult.layer, - output.layer, - streamJsonOutputLayer.pipe(Layer.provide(output.layer)), - ), - ), - ); - }), - ); - }); - - it.live("emits history and live events while following stream-json", () => { - return setup({ - logs: () => - Effect.succeed({ entries: [entries[0]!], cursor: { opaque: "1" }, running: true }), - followLogs: () => Stream.fromIterable([entries[1]!]), - }).pipe( - Effect.flatMap((setupResult) => { - const output = mockOutput({ format: "stream-json" }); - return Effect.gen(function* () { - yield* stackLogs(flags({ follow: true })).pipe( - Effect.provide(Layer.mergeAll(setupResult.layer, output.layer)), - ); - expect(output.events).toEqual([ - expect.objectContaining({ - type: "log-entry", - source: "history", - line: entries[0]!.message, - }), - expect.objectContaining({ - type: "log-entry", - source: "live", - line: entries[1]!.message, - }), - ]); - }); - }), - ); - }); - - it.live("finishes follow after printing retained history when the stack is stopped", () => { - return setup({ - logs: () => - Effect.succeed({ entries: [entries[0]!], cursor: { opaque: "1" }, running: false }), - followLogs: () => Stream.die("follow must not be opened for a stopped stack"), - }).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - yield* stackLogs(flags({ follow: true })); - expect(setupResult.out.stdoutText).toContain("database ready"); - expect(setupResult.calls.queries).toEqual([{ tail: 100 }]); - }).pipe(Effect.provide(setupResult.layer)), - ), - ); - }); - - it.live("rejects legacy output before selecting a stack", () => { - return setup({}).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - const failure = yield* stackLogs(flags()).pipe(Effect.flip); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(setupResult.calls.opened).toEqual([]); - }).pipe( - Effect.provide( - Layer.mergeAll(setupResult.layer, Layer.succeed(OutputFlag, Option.some("json"))), - ), - ), - ), - ); - }); - - it.live("rejects JSON follow mode before selecting a stack", () => { - const out = mockOutput({ format: "json" }); - return setup({}).pipe( - Effect.flatMap((setupResult) => { - const layer = Layer.mergeAll(setupResult.layer, out.layer); - return Effect.gen(function* () { - const failure = yield* stackLogs(flags({ follow: true })).pipe(Effect.flip); - expect(failure).toBeInstanceOf(StackCommandLogsError); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(failure.suggestion).toContain("stream-json"); - expect(setupResult.calls.opened).toEqual([]); - }).pipe(Effect.provide(layer)); - }), - ); - }); - - it.live("rejects invalid targets without package calls", () => + it.live("reports unavailable logs without starting an offline owner", () => Effect.gen(function* () { - const result = yield* setup({}); - const failure = yield* stackLogs(flags({ stackId: Option.some("invalid") })).pipe( - Effect.flip, - Effect.provide(result.layer), - ); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(result.calls.queries).toEqual([]); - expect(result.calls.opened).toEqual([]); - }), + const f = yield* fixture(); + const error = yield* stackLogs(f.flags).pipe(Effect.provide(f.layer), Effect.flip); + expect(error.reason).toBe("lifecycle"); + expect((yield* f.api.discover(f.locations))[0]?.host).toBeUndefined(); + expect(f.output.stdoutText).toBe(""); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), ); - - it.live("classifies an invalid project root as invalid config", () => { - return setup({ - findFailure: new InvalidProjectRootError({ message: "Project root is invalid" }), - }).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - const failure = yield* stackLogs(flags()).pipe(Effect.flip); - expect(failure.reason).toBe("invalid-config"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.invalidConfig); - expect(setupResult.calls.opened).toEqual([]); - }).pipe(Effect.provide(setupResult.layer)), - ), - ); - }); - - it.live("gives retry guidance for busy owners and upgrade guidance for old owners", () => { - return Effect.gen(function* () { - const busy = yield* setup({ - openFailure: new StackOwnershipConflictError({ message: "Stack owner is busy" }), - }); - const upgrade = yield* setup({ - openFailure: new StackUpgradeRequiredError({ - expectedRelease: "next", - actualRelease: "current", - message: "Stack upgrade required", - }), - }); - const busyFailure = yield* stackLogs(flags()).pipe(Effect.flip, Effect.provide(busy.layer)); - const upgradeFailure = yield* stackLogs(flags({ stack: Option.some("feature-a") })).pipe( - Effect.flip, - Effect.provide(upgrade.layer), - ); - expect(busyFailure.suggestion).toContain("no longer running"); - expect(busyFailure.suggestion).toContain("reconcile"); - expect(upgradeFailure.suggestion).toContain("Stop and restart the selected stack"); - expect(upgradeFailure.suggestion).toContain("matches this release"); - }); - }); - - it.live("suggests an explicit target when an addressed stack is missing", () => { - return setup({ - openFailure: new StackNotFoundError({ message: "Stack state was not found" }), - }).pipe( - Effect.flatMap((setupResult) => - Effect.gen(function* () { - const failure = yield* stackLogs(flags({ stackId: Option.some(id) })).pipe(Effect.flip); - expect(failure.suggestion).toBe( - "Choose an existing stack with --stack or --stack-id, or omit both to use the current project.", - ); - }).pipe(Effect.provide(setupResult.layer)), - ), - ); - }); }); diff --git a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md index f7a3484ce6..1a41ef9324 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md @@ -1,70 +1,57 @@ # `supabase stack prepare` -The command is available only when the `experimental.stack` family is enabled. It resolves the -selected project and stack, loads the effective project configuration, and delegates immutable -artifact preparation to `@supabase/stack`. If the target does not exist, it creates and registers -the stack so it appears in `supabase stack list` and can later be removed with `supabase stack -destroy`. It downloads or pulls artifacts without starting, stopping, destroying, or activating -any service. +The command is available when the experimental stack family is enabled. It resolves the selected +project and stack, loads the effective project configuration, and asks the stack package to prepare +the selected service artifacts without starting or changing the saved composition. A new target gets +a managed descriptor so it can later be opened, started, listed, or destroyed. + +For each selected creation, the CLI creates a temporary standalone service instance in the target +namespace, runs that instance's `prepare` operation, and destroys that same instance in a scoped +finalizer. Existing composition members and unrelated standalone instances are never adopted or +modified. A preparation failure still runs cleanup for every acquired temporary instance. ## Files read -- `/supabase/config.toml`, plus project dotenv files and supported `SUPABASE_*` overrides - used by the shared stack config loader. The feature gate may inspect `config.json`; it does not - supply prepare settings. -- `/managed/stacks//state.json` for an existing stack. -- `/profile` and an explicitly selected profile when shared settings - resolve them. -- `/telemetry.json` for shared telemetry state. +- `/supabase/config.toml`, project dotenv files, and supported `SUPABASE_*` overrides + used by the shared stack config loader. +- Managed stack state under `/stacks//` when opening an + existing target. +- Shared profile and telemetry state under `/`. ## Files written -- A managed stack descriptor/state record when the current or named target does not exist. -- Native runtime artifacts in `/managed/stacks/artifacts`; container - images are stored by the selected container engine. -- `/telemetry.json` after success or failure once the handler starts. +- A managed stack descriptor/state record when the target does not exist. Existing records are + updated while temporary instances and their instance directories are created and removed. +- Runtime artifacts through the selected native or container runtime's package-owned cache. +- Telemetry state after success or failure once the handler starts. ## Network and subprocesses -Native preparation may download runtime artifacts. Container preparation may contact the configured -container registry and invoke the selected container engine. The package owns preparation, -dependency selection, cache, and cancellation behavior. - -## Environment variables - -- `SUPABASE_EXPERIMENTAL_STACK` enables (`1`) or disables (`0`) the command family; an unset value - uses project configuration. -- `SUPABASE_HOME` selects the managed state, artifact cache, profile, and telemetry root. -- `SUPABASE_WORKDIR` selects the project root when no `--workdir` is supplied. -- `SUPABASE_PROFILE` selects the CLI profile. -- `SUPABASE_TELEMETRY_DISABLED` and `DO_NOT_TRACK` suppress telemetry delivery while retaining - local telemetry state. +The resident owner process may be launched and remains available after preparation; `stack stop` +shuts it down. No composition service is started. Native preparation may download runtime artifacts. Container preparation may pull images from the +configured registry and invoke the selected container engine. The stack package owns artifact +caching, preparation, and cancellation; the CLI owns the temporary instance cleanup boundary. ## Flags and output `--stack` and `--stack-id` are mutually exclusive. `--runtime` defaults to `auto` for new stacks; -existing stacks reuse their persisted runtime, and an explicit runtime that differs from an existing -stack fails. When no project configuration exists, the stack package's default settings are used. -With no `--capability`, all enabled capabilities are -prepared. `--capability` is a bounded repeatable choice flag with at most ten occurrences; each -occurrence names one capability, and the package includes dependencies and deduplicates its closure. -Unknown capability names are rejected by the flag parser; disabled capabilities are rejected by the package. -The legacy `-o/--output` flag is rejected; use +existing stacks reuse their persisted runtime, and an explicit mismatch fails. With no +`--capability`, every enabled creation from the effective project configuration is prepared. +Repeated `--capability` includes each capability's configured companion services (Storage/Imgproxy, +Studio/Pgmeta, Analytics/Vector); requesting a disabled service fails before instance preparation. The legacy `-o/--output` flag is rejected; use `--output-format`. -Text output lists the stack ID and each prepared capability, version, and outcome (`cached`, -`downloaded`, or `pulled`). JSON output returns `{ id, capabilities, message: "" }`. Stream-JSON -output returns `{ type: "result", data: { id, capabilities, message: "" }, timestamp }`. Errors -use typed actionability and retain package diagnostics. +Text output lists the stack ID and each prepared service and version. JSON output returns the stack +ID and prepared capabilities. Errors retain typed actionability and package diagnostics. ## Exit codes -- `0` when preparation completes. +- `0` when all selected temporary instances prepare and are cleaned up. - `130` when preparation is interrupted. - `1` for invalid targets/configuration, unavailable runtime or registry, preparation failures, or rejected output flags. ## Telemetry -The command emits the standard `cli_command_executed` event through `withCommandTelemetry` and -flushes telemetry state after successful and failed handler execution. It emits no custom event. +The command emits the standard `cli_command_executed` event through command telemetry and flushes +telemetry state after successful and failed handler execution. It emits no custom event. diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts index ad8057907f..3c5ac534c5 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.command.ts @@ -1,9 +1,9 @@ import { Command, Flag } from "effect/unstable/cli"; import type * as CliCommand from "effect/unstable/cli/Command"; -import { CAPABILITY_NAMES } from "@supabase/stack/effect"; import { withJsonErrorHandling } from "../../../../shared/output/json-error-handling.ts"; import { withCommandTelemetry } from "../../../../telemetry/command-telemetry.ts"; import { stackPrepare } from "./prepare.handler.ts"; +import { STACK_PREPARABLE_CAPABILITIES } from "./prepare.options.ts"; const config = { stack: Flag.string("stack").pipe(Flag.withDescription("Name this stack."), Flag.optional), @@ -16,8 +16,8 @@ const config = { Flag.withDefault("auto" as const), ), capability: Flag.atMost( - Flag.choice("capability", CAPABILITY_NAMES), - CAPABILITY_NAMES.length, + Flag.choice("capability", STACK_PREPARABLE_CAPABILITIES), + STACK_PREPARABLE_CAPABILITIES.length, ).pipe(Flag.withDescription("Capability to prepare (repeatable).")), } as const; diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.errors.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.errors.ts index 60f0ff6dbd..7ceff0b000 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.errors.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.errors.ts @@ -1,5 +1,5 @@ -import { Data, Match } from "effect"; -import { isStackError } from "@supabase/stack/effect"; +import { StackError } from "@supabase/stack/effect"; +import { Data } from "effect"; import { actionability, type CliErrorActionabilityDeclaration, @@ -7,14 +7,7 @@ import { } from "../../../../shared/telemetry/error-actionability.ts"; export class StackCommandPrepareError extends Data.TaggedError("ExperimentalStackPrepareError")<{ - readonly reason: - | "flags" - | "invalid-config" - | "runtime" - | "registry" - | "artifact" - | "lifecycle" - | "unknown"; + readonly reason: "flags" | "invalid-config" | "artifact" | "lifecycle" | "unknown"; readonly message: string; readonly suggestion?: string; readonly cause?: unknown; @@ -24,75 +17,33 @@ export class StackCommandPrepareError extends Data.TaggedError("ExperimentalStac case "flags": return actionability.provideFlags; case "invalid-config": + case "lifecycle": return actionability.invalidConfig; - case "runtime": - return actionability.dockerNotRunning; - case "registry": case "artifact": return actionability.externalNetwork; - case "lifecycle": - return actionability.invalidConfig; case "unknown": return actionability.unknown; } } } +/** Maps an internal stack failure to the prepare command's stable error boundary. */ export const stackPrepareError = (error: unknown): StackCommandPrepareError => { - const stackError = isStackError(error) ? error : undefined; - const classification = - stackError === undefined - ? { reason: "unknown" as const } - : Match.value(stackError).pipe( - Match.tag("ContainerEngineError", () => ({ - reason: "runtime" as const, - suggestion: - "Check that the selected container engine is installed and its daemon is running, then retry the command.", - })), - Match.tag("ContainerPullError", () => ({ - reason: "registry" as const, - suggestion: - "Check registry connectivity and image availability, then retry the command.", - })), - Match.tag("StackPreparationError", "ArtifactIntegrityError", () => ({ - reason: "artifact" as const, - suggestion: - "Retry the stack preparation with --debug if the artifact cannot be prepared.", - })), - Match.tag( - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "InvalidStackIdentityError", - "InvalidProjectRootError", - () => ({ reason: "invalid-config" as const }), - ), - Match.tag("StackStateInvalidError", () => ({ - reason: "invalid-config" as const, - suggestion: - "Inspect the reported state error and restore a valid state record before retrying.", - })), - Match.tag("StackStateFormatUnsupportedError", () => ({ - reason: "invalid-config" as const, - suggestion: "Use a CLI version compatible with the persisted stack state.", - })), - Match.tag("StackNotFoundError", () => ({ - reason: "flags" as const, - })), - Match.tag("StackRuntimeMismatchError", () => ({ - reason: "flags" as const, - suggestion: - "Omit --runtime to reuse the existing runtime, or choose a different --stack name.", - })), - Match.tag("StackOwnershipConflictError", "StackUpgradeRequiredError", () => ({ - reason: "lifecycle" as const, - suggestion: "Resolve the existing stack state before preparing it again.", - })), - Match.orElse(() => ({ reason: "unknown" as const })), - ); + const message = + typeof error === "object" && + error !== null && + "message" in error && + typeof error.message === "string" + ? error.message + : String(error); return new StackCommandPrepareError({ - ...classification, - message: stackError?.message ?? String(error), - ...("suggestion" in classification ? { suggestion: classification.suggestion } : {}), + reason: + error instanceof StackError + ? error.operation === "prepareService" + ? "artifact" + : "unknown" + : "unknown", + message, cause: error, }); }; diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.handler.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.handler.ts index 46d1a63c72..19f437d2d7 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.handler.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.handler.ts @@ -1,32 +1,26 @@ -import { Cause, Effect, Exit, Option } from "effect"; -import type { PrepareStackResult, StackRuntimePreference } from "@supabase/stack/effect"; +import { Cause, Effect, Exit, Option, Path } from "effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; +import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; +import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import { StackApi, + stackCapabilityForService, StackTargetError, rejectStackOutput, StackTargetResolver, validateStackTarget, } from "../stack.shared.ts"; -import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import type { StackPrepareFlags } from "./prepare.command.ts"; import { StackCommandPrepareError, stackPrepareError } from "./prepare.errors.ts"; +import { defaultStackRuntime } from "../../../../command-internal/stack-runtime.ts"; -const payload = (id: string, result: PrepareStackResult) => ({ - id, - capabilities: result.capabilities, -}); -const render = (id: string, result: PrepareStackResult) => { - const lines = [`Stack ${id} prepared.`]; - if (result.capabilities.length > 0) { - lines.push("Capabilities:"); - for (const capability of result.capabilities) - lines.push(` ${capability.capability} ${capability.version} (${capability.outcome})`); - } - return `${lines.join("\n")}\n`; +type PreparedCapability = { + readonly capability: string; + readonly version?: string; + readonly outcome: "prepared"; }; const mapTargetError = (error: StackTargetError) => @@ -37,6 +31,19 @@ const mapTargetError = (error: StackTargetError) => cause: error, }); +const render = (id: string, capabilities: ReadonlyArray) => { + const lines = [`Stack ${id} prepared.`]; + if (capabilities.length > 0) { + lines.push("Capabilities:"); + for (const capability of capabilities) + lines.push( + ` ${capability.capability} ${capability.version ?? "catalog default"} (${capability.outcome})`, + ); + } + return `${lines.join("\n")}\n`; +}; + +/** Prepares selected stack artifacts without starting service processes. */ export const stackPrepare = Effect.fn("experimental.stack.prepare")(function* ( flags: StackPrepareFlags, ) { @@ -44,6 +51,8 @@ export const stackPrepare = Effect.fn("experimental.stack.prepare")(function* ( const body = Effect.gen(function* () { const output = yield* Output; const settings = yield* CommandSettings; + const path = yield* Path.Path; + const runtime = yield* RuntimeInfo; const resolver = yield* StackTargetResolver; const api = yield* StackApi; const outputFlag = yield* Effect.serviceOption(OutputFlag); @@ -70,37 +79,88 @@ export const stackPrepare = Effect.fn("experimental.stack.prepare")(function* ( }), ), ); - const runtime: StackRuntimePreference | undefined = target.runtime; + const stateRoot = path.join(settings.supabaseHome, "stacks"); + const cacheRoot = path.join(settings.supabaseHome, "cache", "stack"); const stack = - target.id !== undefined - ? yield* api.openStack(target.id).pipe(Effect.mapError(stackPrepareError)) - : yield* api - .createStack({ + target.id === undefined + ? yield* api + .create({ projectRoot: target.projectRoot, + stateRoot, + cacheRoot, + runtime: target.runtime ?? defaultStackRuntime(runtime), ...(target.name === undefined ? {} : { name: target.name }), - ...(runtime === undefined ? {} : { runtime }), }) + .pipe(Effect.mapError(stackPrepareError)) + : yield* api + .open({ id: target.id, stateRoot, cacheRoot }) .pipe(Effect.mapError(stackPrepareError)); + const creations = yield* config.creations(stack.id).pipe( + Effect.mapError( + (error) => + new StackCommandPrepareError({ + reason: "invalid-config", + message: error.message, + cause: error, + }), + ), + ); + const selectedServices = flags.capability.length === 0 ? undefined : new Set(flags.capability); + for (const capability of flags.capability) { + if (!creations.some(({ service }) => stackCapabilityForService(service) === capability)) + return yield* new StackCommandPrepareError({ + reason: "invalid-config", + message: `Service ${capability} is disabled in the project configuration`, + suggestion: "Enable the service in supabase/config.toml before preparing it.", + }); + } + const requested = + selectedServices === undefined + ? creations + : creations.filter(({ service }) => + selectedServices.has(stackCapabilityForService(service)), + ); const task = yield* output.task("Preparing local Supabase stack..."); - const result = yield* stack - .prepare({ - config, - ...(flags.capability.length === 0 ? {} : { capabilities: flags.capability }), - }) - .pipe( - Effect.onExit((exit) => - Exit.isSuccess(exit) - ? task.clear() - : Option.match(Cause.findErrorOption(exit.cause), { - onNone: () => (Cause.hasInterruptsOnly(exit.cause) ? task.cancel() : task.fail()), - onSome: (error) => task.fail(error.message), - }), - ), - Effect.mapError(stackPrepareError), - ); - if (output.format === "text") yield* output.raw(render(stack.id, result)); - else yield* output.success("", payload(stack.id, result)); - return result; + const capabilities = yield* Effect.forEach(requested, (creation) => + Effect.acquireUseRelease( + stack.services.create(creation).pipe(Effect.mapError(stackPrepareError)), + (instance) => + instance.prepare.pipe(Effect.mapError(stackPrepareError), Effect.as(creation)), + (instance) => + instance.destroy.pipe( + Effect.mapError( + (cause) => + new StackCommandPrepareError({ + reason: "lifecycle", + message: `Failed to remove temporary ${instance.service} instance ${instance.id}: ${cause.message}`, + suggestion: `Inspect stack ${stack.id}; run supabase stack destroy --stack-id ${stack.id} to remove it if no existing data must be retained.`, + cause, + }), + ), + ), + ), + ).pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? task.clear() + : Option.match(Cause.findErrorOption(exit.cause), { + onNone: () => (Cause.hasInterruptsOnly(exit.cause) ? task.cancel() : task.fail()), + onSome: (error) => task.fail(error.message), + }), + ), + ); + const prepared = capabilities.map((creation): PreparedCapability => ({ + capability: creation.service, + ...(creation.service === "database" + ? { version: creation.config.version } + : creation.version === undefined + ? {} + : { version: creation.version }), + outcome: "prepared", + })); + if (output.format === "text") yield* output.raw(render(stack.id, prepared)); + else yield* output.success("", { id: stack.id, capabilities: prepared }); + return prepared; }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index c8dfcd94ac..329bc9bbba 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -1,652 +1,275 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { CliOutput, Command } from "effect/unstable/cli"; +import { Effect, FileSystem, Layer, Option, Path, Redacted, Stream } from "effect"; import { - Cause, - Deferred, - Effect, - FileSystem, - Exit, - Fiber, - Layer, - Option, - Path, - Result, - Schema, - Sink, - Stdio, - Stream, -} from "effect"; -import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; -import { - InvalidStackConfigError, - StackIdSchema, - StackPreparationError, - StackRuntimeMismatchError, - type EffectStack, + StackError, + type Stack, + type ServiceCreation, + type ServiceInstances, } from "@supabase/stack/effect"; -import { OutputFlag } from "../../../../command-internal/global-flags.ts"; -import { mockOutput, processEnvLayer } from "../../../../../tests/helpers/mocks.ts"; +import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; -import { StackApi, StackTargetResolver, stackTargetResolverLayer } from "../stack.shared.ts"; +import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; +import { StackApi, StackTargetResolver } from "../stack.shared.ts"; import { stackPrepare } from "./prepare.handler.ts"; -import { stackPrepareCommand, type StackPrepareFlags } from "./prepare.command.ts"; +import type { StackPrepareFlags } from "./prepare.command.ts"; import { StackCommandPrepareError } from "./prepare.errors.ts"; -import { jsonOutputLayer, streamJsonOutputLayer } from "../../../../shared/output/output.layer.ts"; -import { textCliOutputFormatter } from "../../../../shared/output/text-formatter.ts"; -import { Output } from "../../../../shared/output/output.service.ts"; -import { - actionability, - ErrorActionabilityId, -} from "../../../../shared/telemetry/error-actionability.ts"; -const makeProject = (config = 'project_id = "prepare-test"\n') => +const id = "a".repeat(64); +const flags = (overrides: Partial = {}): StackPrepareFlags => ({ + stack: Option.none(), + stackId: Option.none(), + runtime: "native", + capability: ["database"], + ...overrides, +}); + +const makeProject = (config: string) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-prepare-" }); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-prepare-" }); yield* fs.makeDirectory(path.join(root, "supabase")); yield* fs.writeFileString(path.join(root, "supabase", "config.toml"), config); return root; - }).pipe(Effect.provide(BunServices.layer)); - -const makeStack = ( - id: string, - prepare: EffectStack["prepare"], - calls: { start: number; stop: number; destroy: number } = { start: 0, stop: 0, destroy: 0 }, -): EffectStack => ({ - id: StackIdSchema.make(id), - status: Effect.die("unused"), - credentials: Effect.die("unused"), - prepare, - start: () => - Effect.sync(() => { - calls.start += 1; - }).pipe(Effect.flatMap(() => Effect.die("start should not be called"))), - stop: Effect.sync(() => { - calls.stop += 1; - }), - destroy: Effect.sync(() => { - calls.destroy += 1; - }), - resetDatabase: Effect.die("unused"), - logs: () => Effect.die("unused"), - followLogs: () => Stream.empty, -}); - -const flags = (overrides: Partial = {}): StackPrepareFlags => ({ - stack: Option.none(), - stackId: Option.none(), - runtime: "auto", - capability: [], - ...overrides, -}); + }); -const handlerLayer = (opts: { - readonly root: string; - readonly stack: EffectStack; - readonly output?: ReturnType; - readonly telemetry?: ReturnType; - readonly resolver?: Layer.Layer; - readonly api?: Layer.Layer; -}) => { - const output = opts.output ?? mockOutput(); - const telemetry = opts.telemetry ?? mockTelemetryStateTracked(); - return { - output, - telemetry, - layer: Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - output.layer, - telemetry.layer, - mockCommandSettings({ workdir: opts.root }), - processEnvLayer({}), - opts.resolver ?? - Layer.succeed(StackTargetResolver, { - resolve: () => Effect.succeed({ projectRoot: opts.root }), - }), - opts.api ?? - Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - discoverStacks: () => Effect.die("unused"), - inspectStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - createStack: () => Effect.succeed(opts.stack), - }), +const makeFixture = (root: string, failPreparation = false) => { + let prepareCount = 0; + let startCount = 0; + let destroyCount = 0; + const database = { + id: "database-id", + service: "database" as const, + start: Effect.sync(() => { + startCount += 1; + }), + ready: Effect.void, + stop: Effect.void, + restart: () => Effect.void, + destroy: Effect.sync(() => { + destroyCount += 1; + }), + prepare: Effect.sync(() => { + prepareCount += 1; + }).pipe( + Effect.andThen(() => + failPreparation + ? Effect.fail(new StackError({ operation: "prepareService", message: "download failed" })) + : Effect.void, + ), ), + status: Effect.succeed({ + id: "database-id", + endpoints: [], + config: { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("password"), + jwtSecret: Redacted.make("jwt-secret"), + jwtExpiry: 3600, + }, + }, + lifecycle: "stopped" as const, + health: undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: false, + registered: true, + }), + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => Effect.succeed({}), + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: Effect.die("unused"), }; -}; + function create( + creation: Input, + ): Effect.Effect; + function create( + creation: ServiceCreation, + ): Effect.Effect { + switch (creation.service) { + case "database": + return Effect.succeed(database); + case "studio": + return Effect.succeed({ ...database, service: "studio" as const }); + case "pgmeta": + return Effect.succeed({ ...database, service: "pgmeta" as const }); + default: + return Effect.die(`fixture does not support ${creation.service}`); + } + } -const captureStdio = (stdout: string[], stderr: string[]) => { - const capture = (target: string[]) => - Sink.forEach((chunk: string | Uint8Array) => - Effect.sync(() => { - target.push(typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)); + const stack: Stack = { + id, + services: { + create, + get: () => Effect.succeed(database), + list: Effect.succeed([]), + }, + composition: { + supabase: () => Effect.die("prepare must not change the composition"), + configure: () => Effect.void, + describe: Effect.succeed({ + members: [], + dependencies: [], }), - ); - return Layer.succeed( - Stdio.Stdio, - Stdio.make({ - args: Effect.succeed([]), - stdin: Stream.empty, - stdout: () => capture(stdout), - stderr: () => capture(stderr), + start: Effect.succeed([]), + stop: Effect.succeed([]), + restart: Effect.succeed([]), + }, + stop: Effect.void, + destroy: Effect.void, + tools: { run: () => Effect.die("unused") }, + }; + const output = mockOutput(); + const telemetry = mockTelemetryStateTracked(); + const layer = Layer.mergeAll( + BunServices.layer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, + telemetry.layer, + Layer.succeed(StackTargetResolver, { + resolve: () => Effect.succeed({ projectRoot: root }), }), - ); -}; - -const statefulTaskOutput = () => { - const state = { active: false, settled: false, failed: false, canceled: false, cleared: false }; - const layer = Layer.succeed( - Output, - Output.of({ - format: "text" as const, - interactive: false, - intro: () => Effect.void, - outro: () => Effect.void, - info: () => Effect.void, - warn: () => Effect.void, - error: () => Effect.void, - event: () => Effect.void, - task: () => - Effect.sync(() => { - state.active = true; - return { - message: () => Effect.void, - succeed: () => - Effect.sync(() => { - state.settled = true; - }), - fail: () => - Effect.sync(() => { - state.failed = true; - state.settled = true; - }), - info: () => Effect.void, - cancel: () => - Effect.sync(() => { - state.canceled = true; - state.settled = true; - }), - clear: () => - Effect.sync(() => { - state.cleared = true; - state.settled = true; - }), - }; - }), - promptText: () => Effect.die("unused"), - promptPassword: () => Effect.die("unused"), - promptConfirm: () => Effect.die("unused"), - promptSelect: () => Effect.die("unused"), - promptMultiSelect: () => Effect.die("unused"), - progress: () => Effect.die("unused"), - result: () => Effect.void, - success: () => Effect.void, - fail: () => Effect.void, - raw: () => Effect.void, - rawBytes: () => Effect.void, + Layer.succeed(StackApi, { + create: () => Effect.succeed(stack), + open: () => Effect.succeed(stack), + discover: () => Effect.succeed([]), + resolveIdentity: () => Effect.die("unused"), }), ); - return { layer, state }; + return { + layer, + get prepareCount() { + return prepareCount; + }, + get startCount() { + return startCount; + }, + get destroyCount() { + return destroyCount; + }, + }; }; +const databaseOnlyConfig = `project_id = "prepare-test"\n\n[api]\nenabled = false\n`; + describe("stack prepare", () => { - it.live("uses all default capabilities and renders the text result", () => { - return makeProject().pipe( + it.live("prepares a temporary Database without changing the composition", () => + makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { - let options: Parameters>[0]; - const id = "a".repeat(64); - const stack = makeStack(id, (value) => - Effect.sync(() => { - options = value; - return { - capabilities: [ - { capability: "database", version: "1", outcome: "cached" as const }, - { capability: "rest", version: "1", outcome: "cached" as const }, - ], - }; - }), - ); - const fixture = handlerLayer({ root, stack }); + const fixture = makeFixture(root); return stackPrepare(flags()).pipe( Effect.provide(fixture.layer), - Effect.tap(() => + Effect.tap((prepared) => Effect.sync(() => { - expect(options).toMatchObject({ config: expect.anything() }); - expect(options).not.toHaveProperty("capabilities"); - expect(fixture.output.stdoutText).toContain(`Stack ${id} prepared.`); - expect(fixture.output.stdoutText).toContain("database 1 (cached)"); - expect(fixture.telemetry.flushed).toBe(true); + expect(prepared.length).toBeGreaterThan(0); + expect(fixture.prepareCount).toBe(prepared.length); + expect(fixture.startCount).toBe(0); + expect(fixture.destroyCount).toBe(prepared.length); }), ), ); }), - ); - }); + Effect.provide(BunServices.layer), + ), + ); - it.live("maps a package config error for a disabled capability", () => { - return makeProject("[studio]\nenabled = false\n").pipe( + it.live("prepares Studio together with its configured Pgmeta companion", () => + makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { - let created = false; - let prepared = false; - let receivedConfig: unknown; - const calls = { start: 0, stop: 0, destroy: 0 }; - const stack = makeStack( - "2".repeat(64), - (options) => { - prepared = true; - receivedConfig = options?.config; - return Effect.fail( - new InvalidStackConfigError({ - message: "Capability studio is disabled", - capability: "studio", - }), - ); - }, - calls, - ); - const telemetry = mockTelemetryStateTracked(); - const fixture = handlerLayer({ - root, - stack, - telemetry, - api: Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - discoverStacks: () => Effect.die("unused"), - inspectStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - createStack: () => { - created = true; - return Effect.succeed(stack); - }, - }), - }); + const fixture = makeFixture(root); return stackPrepare(flags({ capability: ["studio"] })).pipe( - Effect.flip, Effect.provide(fixture.layer), - Effect.tap((error) => + Effect.tap((prepared) => Effect.sync(() => { - expect(error).toBeInstanceOf(StackCommandPrepareError); - expect(error.reason).toBe("invalid-config"); - expect(error[ErrorActionabilityId]).toEqual(actionability.invalidConfig); - expect(error.message).toContain("Capability studio is disabled"); - expect(error.suggestion).toBeUndefined(); - expect(receivedConfig).toMatchObject({ - capabilities: { studio: { enabled: false } }, - }); - expect(created).toBe(true); - expect(prepared).toBe(true); - expect(calls).toEqual({ start: 0, stop: 0, destroy: 0 }); - expect(telemetry.flushed).toBe(true); + expect(prepared.map((result) => result.capability).sort()).toEqual([ + "pgmeta", + "studio", + ]); + expect(prepared.every((result) => result.version !== "catalog default")).toBe(true); + expect(fixture.prepareCount).toBe(2); + expect(fixture.destroyCount).toBe(2); + expect(fixture.startCount).toBe(0); }), ), ); }), - ); - }); - - it.live("parses repeated capabilities into the handler and serializes JSON results", () => { - const formats = ["json", "stream-json"] as const; - return Effect.forEach(formats, (format) => { - return makeProject().pipe( - Effect.flatMap((root) => { - const stdout: string[] = []; - let options: Parameters>[0]; - const id = "b".repeat(64); - const stack = makeStack(id, (value) => - Effect.sync(() => { - options = value; - return { - capabilities: [ - { capability: "rest", version: "1", outcome: "downloaded" as const }, - { capability: "auth", version: "2", outcome: "cached" as const }, - ], - }; - }), - ); - const telemetry = mockTelemetryStateTracked(); - const machineOutput = (format === "json" ? jsonOutputLayer : streamJsonOutputLayer).pipe( - Layer.provide(captureStdio(stdout, [])), - ); - const command = stackPrepareCommand.pipe( - Command.withHandler((value) => stackPrepare(value)), - ); - const layer = Layer.mergeAll( - BunServices.layer, - runtimeInfoLayer, - CliOutput.layer(textCliOutputFormatter()), - machineOutput, - telemetry.layer, - mockCommandSettings({ workdir: root }), - processEnvLayer({}), - Layer.succeed(StackTargetResolver, { - resolve: () => Effect.succeed({ projectRoot: root }), - }), - Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - discoverStacks: () => Effect.die("unused"), - inspectStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - createStack: () => Effect.succeed(stack), - }), - ); - return Command.runWith(command, { version: "0.0.0-test" })([ - "--capability", - "rest", - "--capability", - "auth", - ]).pipe( - Effect.provide(layer), - Effect.tap(() => - Effect.sync(() => { - expect(options).toMatchObject({ capabilities: ["rest", "auth"] }); - expect(telemetry.flushed).toBe(true); - }), - ), - Effect.flatMap(() => - Effect.sync(() => { - if (stdout.length === 0) throw new Error("no stdout"); - return stdout.at(-1)!.trim(); - }), - ), - Effect.flatMap((line) => - Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(line), - ), - Effect.tap((result) => - Effect.sync(() => { - const data = { - id, - capabilities: [ - { capability: "rest", version: "1", outcome: "downloaded" }, - { capability: "auth", version: "2", outcome: "cached" }, - ], - }; - expect(result).toEqual( - format === "json" - ? { ...data, message: "" } - : { - type: "result", - data: { ...data, message: "" }, - timestamp: expect.any(String), - }, - ); - }), - ), - ); - }), - ); - }); - }); - - it.live("opens an inspected stack and loads configuration from its project root", () => { - return Effect.gen(function* () { - const callerRoot = yield* makeProject(); - const inspectedRoot = yield* makeProject("[api]\nport = 55432\n"); - const id = "c".repeat(64); - let inspected = false; - let opened: string | undefined; - let preparedConfig: unknown; - const stack = makeStack(id, (value) => - Effect.sync(() => { - preparedConfig = value?.config; - return { capabilities: [] }; - }), - ); - const api = Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - discoverStacks: () => Effect.die("unused"), - inspectStack: () => - Effect.sync(() => { - inspected = true; - return { - descriptor: { - id: StackIdSchema.make(id), - projectRoot: inspectedRoot, - name: "existing", - branchContext: "existing-branch", - runtime: { kind: "native" as const }, - desiredLifecycle: "stopped" as const, - }, - owner: "absent" as const, - }; - }), - openStack: (stackId) => - Effect.sync(() => { - opened = stackId; - return stack; - }), - createStack: () => Effect.die("must not create an existing stack"), - }); - const fixture = handlerLayer({ - root: callerRoot, - stack, - resolver: stackTargetResolverLayer, - api, - }); - return yield* stackPrepare(flags({ stackId: Option.some(id) })).pipe( - Effect.provide(fixture.layer), - Effect.tap(() => - Effect.sync(() => { - expect(inspected).toBe(true); - expect(opened).toBe(id); - expect(preparedConfig).toMatchObject({ listeners: { api: { port: 55432 } } }); - }), - ), - ); - }); - }); + Effect.provide(BunServices.layer), + ), + ); - it.live("rejects invalid config before creating a stack", () => { - return makeProject("project_id = [\n").pipe( + it.live("removes its temporary instance after a preparation failure", () => + makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { - let created = false; - const stack = makeStack("d".repeat(64), () => Effect.die("must not prepare")); - const fixture = handlerLayer({ - root, - stack, - api: Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - discoverStacks: () => Effect.die("unused"), - inspectStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - createStack: () => { - created = true; - return Effect.succeed(stack); - }, - }), - }); + const fixture = makeFixture(root, true); return stackPrepare(flags()).pipe( - Effect.flip, Effect.provide(fixture.layer), + Effect.flip, Effect.tap((error) => Effect.sync(() => { - expect(error.reason).toBe("invalid-config"); - expect(created).toBe(false); - expect(fixture.telemetry.flushed).toBe(true); + expect(error.message).toContain("download failed"); + expect(error.reason).toBe("artifact"); + expect(fixture.destroyCount).toBe(1); + expect(fixture.startCount).toBe(0); }), ), ); }), - ); - }); - - it.live("rejects invalid targets and the legacy output flag before preparation", () => { - return makeProject().pipe( - Effect.flatMap((targetRoot) => { - let resolved = false; - const stack = makeStack("e".repeat(64), () => Effect.die("must not prepare")); - const telemetry = mockTelemetryStateTracked(); - const output = mockOutput(); - const fixture = handlerLayer({ - root: targetRoot, - stack, - telemetry, - output, - resolver: Layer.succeed(StackTargetResolver, { - resolve: () => { - resolved = true; - return Effect.die("must not resolve invalid flags"); - }, - }), - }); - const invalidTarget = stackPrepare( - flags({ stack: Option.some("feature"), stackId: Option.some("e".repeat(64)) }), - ).pipe(Effect.flip, Effect.provide(fixture.layer)); - const legacy = stackPrepare(flags()).pipe( - Effect.flip, - Effect.provide( - Layer.mergeAll(fixture.layer, Layer.succeed(OutputFlag, Option.some("json"))), - ), - ); - return Effect.gen(function* () { - const targetError = yield* invalidTarget; - expect(targetError.reason).toBe("flags"); - expect(resolved).toBe(false); - expect(telemetry.flushed).toBe(true); - const legacyError = yield* legacy; - expect(legacyError.reason).toBe("flags"); - expect(output.stdoutText).toBe(""); - }); - }), - ); - }); + Effect.provide(BunServices.layer), + ), + ); - it.live("reports typed preparation failures through the task and flushes telemetry", () => { - return makeProject().pipe( + it.live("rejects a disabled requested service before preparing an instance", () => + makeProject(databaseOnlyConfig).pipe( Effect.flatMap((root) => { - const telemetry = mockTelemetryStateTracked(); - const output = mockOutput(); - const stack = makeStack("f".repeat(64), () => - Effect.fail(new StackPreparationError({ message: "artifact failed" })), - ); - const fixture = handlerLayer({ root, stack, output, telemetry }); - return stackPrepare(flags()).pipe( - Effect.flip, + const fixture = makeFixture(root); + return stackPrepare(flags({ capability: ["rest"] })).pipe( Effect.provide(fixture.layer), + Effect.flip, Effect.tap((error) => Effect.sync(() => { - expect(error).toBeInstanceOf(StackCommandPrepareError); - expect(error.reason).toBe("artifact"); - expect(output.messages).toEqual( - expect.arrayContaining([ - expect.objectContaining({ type: "error", message: "artifact failed" }), - ]), - ); - expect(telemetry.flushed).toBe(true); + expect(error.message).toContain("rest is disabled"); + expect(fixture.prepareCount).toBe(0); + expect(fixture.destroyCount).toBe(0); }), ), ); }), - ); - }); + Effect.provide(BunServices.layer), + ), + ); - it.live("suggests omitting runtime when a named stack has a runtime mismatch", () => { - return makeProject().pipe( + it.live("rejects malformed config before creating a stack", () => + makeProject("project_id = [\n").pipe( Effect.flatMap((root) => { - const stack = makeStack("0".repeat(64), () => Effect.die("must not prepare")); - const telemetry = mockTelemetryStateTracked(); - const fixture = handlerLayer({ - root, - stack, - telemetry, - resolver: Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed({ projectRoot: root, name: "feature", runtime: { kind: "native" } }), - }), - api: Layer.succeed(StackApi, { - findStack: () => Effect.die("unused"), - discoverStacks: () => Effect.die("unused"), - inspectStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - createStack: () => - Effect.fail(new StackRuntimeMismatchError({ message: "runtime mismatch" })), - }), - }); - return stackPrepare(flags({ stack: Option.some("feature"), runtime: "native" })).pipe( - Effect.flip, + const fixture = makeFixture(root); + return stackPrepare(flags()).pipe( Effect.provide(fixture.layer), + Effect.flip, Effect.tap((error) => Effect.sync(() => { - expect(error.reason).toBe("flags"); - expect(error.suggestion).toBe( - "Omit --runtime to reuse the existing runtime, or choose a different --stack name.", - ); - expect(telemetry.flushed).toBe(true); + expect(error).toBeInstanceOf(StackCommandPrepareError); + expect(error.reason).toBe("invalid-config"); + expect(fixture.prepareCount).toBe(0); + expect(fixture.startCount).toBe(0); }), ), ); }), - ); - }); - - it.live( - "cancels preparation on interruption without lifecycle calls and flushes telemetry", - () => { - return makeProject().pipe( - Effect.flatMap((root) => { - const started = Deferred.makeUnsafe(); - const calls = { start: 0, stop: 0, destroy: 0 }; - let canceled = false; - const telemetry = mockTelemetryStateTracked(); - const stack = makeStack( - "1".repeat(64), - () => - Effect.gen(function* () { - yield* Deferred.succeed(started, undefined); - return yield* Effect.never.pipe(Effect.as({ capabilities: [] })); - }).pipe(Effect.ensuring(Effect.sync(() => (canceled = true)))), - calls, - ); - const fixture = handlerLayer({ root, stack, telemetry }); - const taskOutput = statefulTaskOutput(); - return Effect.gen(function* () { - const fiber = yield* Effect.forkChild( - Effect.provide( - stackPrepare(flags()), - Layer.mergeAll(fixture.layer, taskOutput.layer), - ), - ); - yield* Deferred.await(started); - yield* Fiber.interrupt(fiber); - expect(canceled).toBe(true); - expect(taskOutput.state.active).toBe(true); - expect(taskOutput.state.settled).toBe(true); - expect(taskOutput.state.failed).toBe(false); - expect(taskOutput.state.canceled || taskOutput.state.cleared).toBe(true); - expect(calls).toEqual({ start: 0, stop: 0, destroy: 0 }); - expect(telemetry.flushed).toBe(true); - }); - }), - ); - }, + Effect.provide(BunServices.layer), + ), ); - - it.live("settles the task on a preparation defect while preserving the defect", () => { - return makeProject().pipe( - Effect.flatMap((root) => { - const stack = makeStack("3".repeat(64), () => Effect.die("preparation defect")); - const fixture = handlerLayer({ root, stack }); - const taskOutput = statefulTaskOutput(); - return Effect.gen(function* () { - const exit = yield* Effect.exit( - Effect.provide(stackPrepare(flags()), Layer.mergeAll(fixture.layer, taskOutput.layer)), - ); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const defect = Cause.findDefect(exit.cause); - expect(Result.isSuccess(defect)).toBe(true); - if (Result.isSuccess(defect)) expect(defect.success).toBe("preparation defect"); - } - expect(taskOutput.state.settled).toBe(true); - expect(taskOutput.state.canceled).toBe(false); - expect(taskOutput.state.failed).toBe(true); - }); - }), - ); - }); }); diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.options.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.options.ts new file mode 100644 index 0000000000..2c64ed47f4 --- /dev/null +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.options.ts @@ -0,0 +1,6 @@ +import { STACK_START_EXCLUDABLE_CAPABILITIES } from "../start/start.options.ts"; + +export const STACK_PREPARABLE_CAPABILITIES = [ + "database", + ...STACK_START_EXCLUDABLE_CAPABILITIES, +] as const; diff --git a/apps/cli/src/commands/experimental/stack/restart/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/restart/SIDE_EFFECTS.md index 821c2b699e..33a48bb19e 100644 --- a/apps/cli/src/commands/experimental/stack/restart/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/restart/SIDE_EFFECTS.md @@ -1,76 +1,39 @@ # `supabase stack restart` -The command is available only when `experimental.stack` is enabled through project -configuration or `SUPABASE_EXPERIMENTAL_STACK=1`. It has no top-level alias. - -## Files Read - -Reads the selected stack's descriptor and `/managed/stacks//state.json`, -plus owner metadata in `control.json` when present. Restart does not load project -configuration; it uses the selected stack's saved effective configuration and preparation policy. -Feature routing and workdir discovery may still read `supabase/config.toml` or -`supabase/config.json`. Use `SUPABASE_EXPERIMENTAL_STACK=1` with `--stack-id` when -project configuration is invalid or unavailable. Runtime startup can read files -referenced by the saved definition, such as signing material. - -## Files Written - -The CLI does not rewrite project configuration. The stack package updates its -state record, owner metadata, runtime files, logs, and service data beneath the -selected stack directory. Restart preserves the stack ID and data; it never calls -create or destroy. There is no explicit `prepare()` call. Runtime startup reuses -cached artifacts and may fetch missing ones; the saved `background` preparation -policy can also prefetch enabled lazy services, while `on-demand` skips that prefetch. - -## API Routes - -No Management API routes. The command uses local stack control RPC and delegates -artifact downloads, container operations, and service startup to the package. -Artifact URLs and registry requests depend on the selected runtime and releases. - -## Environment Variables - -- `SUPABASE_HOME`: managed state location; defaults to the user's `.supabase` directory. -- `HOME`: participates in default home resolution. -- Project configuration and function dotenv overrides are not reloaded by the - restart handler. Saved secret values are not emitted. -- Standard CLI settings, output, and telemetry environment controls apply through - the existing CLI layers; restart adds no command-specific environment variables. - -## Exit Codes - -| Code | Condition | -| ----- | --------------------------------------------------------- | -| `0` | The selected stack restarted successfully. | -| `1` | Invalid flags, missing stack, or a stop or start failure. | -| `130` | The CLI waiter was interrupted. | - -## Telemetry Events Fired - -Telemetry state is flushed to `/telemetry.json` -after successful and failed runs. Standard command instrumentation emits `cli_command_executed` for success or -failure, with canonical command identity `stack restart`, duration, sanitized flags, and error classification. Restart adds -no custom telemetry event and does not emit configuration or credential values. - -## Output - -- `--output-format text`: stack ID, runtime, lifecycle, configured endpoints, and - dormant capabilities. Progress is cleared after success or failed before propagation. -- `--output-format json`: one status object containing `id`, `lifecycle`, - `desired_lifecycle`, `runtime`, `endpoints`, `versions`, `capabilities`, and `artifacts`. -- `--output-format stream-json`: standard progress events, followed by a `result` - event carrying the same status object, or an `error` event on failure. - -Legacy `-o/--output` is rejected with guidance to use `--output-format`. - -## Notes - -Targets one existing stack through `--stack`, `--stack-id`, or the current -project. The command stops and starts without an explicit configuration. Stop failure prevents -start; start failure leaves the same stack stopped and available for recovery. Interrupting -the CLI waiter follows the package's owner lifecycle contract and does not invoke -destroy from the command handler. -An unconfigured stack must be initialized with `supabase stack start` before it can be restarted. - -Restart reuses saved one-off `start` flags such as `--exclude`, `--eager`, and -`--preparation`; a normal `start` reloads project configuration and current flags. +Restarts the selected saved application composition. It preserves member IDs, +data, ports, dependency bindings, and activation settings. Standalone instances +and their jobs are not part of this restart. The experimental feature flag +controls command registration. + +## Selection and configuration + +Select the current project/branch/name, `--stack `, or `--stack-id `. +The selectors are mutually exclusive. Missing or unconfigured stacks fail with +guidance to run `stack start`. Explicit legacy `-o/--output` is rejected in favor +of `--output-format`. + +Restart uses saved configuration; it does not load, compare, or apply project +config changes. To change participating services, use `stack start --exclude` +with the desired selection. Configuration replacement is not implied by restart. + +## Files and network + +Reads the selected state under `/stacks//`. +Discovery ensures the registry directory exists with mode 0700 and probes local +owners. An owner may be launched by restart when none exists. Existing runtime +resources are stopped/recreated for composition members only. Artifacts are +resolved from `/cache/stack`; missing artifacts may +be downloaded. Shared routing/settings may read project config and profiles. +No hosted project API calls or caller-owned data deletion occur. + +## Output, exit codes and telemetry + +Text identifies that saved configuration was used and lists each member's +lifecycle and health separately. JSON/stream-json results contain stack `id` and +`services` entries with instance `id`, `service`, `lifecycle`, `health`, and +`wake_enabled`. Runtime secrets/configuration are not serialized into results. + +Exit 0 on success, 1 on selection/configuration/runtime failure, 130 on +interruption. Standard command telemetry is retained, with no custom events. +Telemetry flushes on every exit to +`/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.command.ts b/apps/cli/src/commands/experimental/stack/restart/restart.command.ts index e96199c7ec..c0ac0707bd 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.command.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.command.ts @@ -19,7 +19,7 @@ export type StackRestartFlags = CliCommand.Command.Config.Infer; export const stackRestartCommand = Command.make("restart", config).pipe( Command.withDescription( - "Restart an existing managed local Supabase stack using its saved configuration, including the previous start's service selection and preparation policy.", + "Restart an existing managed local Supabase stack using its saved configuration, including the previous start's service selection and activation policy. Project config changes are not applied.", ), Command.withShortDescription("Restart a managed local stack"), Command.withExamples([ diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.errors.ts b/apps/cli/src/commands/experimental/stack/restart/restart.errors.ts index f5a6560bcb..fd82d74ca4 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.errors.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.errors.ts @@ -17,6 +17,7 @@ export class StackCommandRestartError extends Data.TaggedError("ExperimentalStac | "artifact" | "unknown"; readonly message: string; + readonly detail?: string; readonly suggestion?: string; readonly cause?: unknown; }> { diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.handler.ts b/apps/cli/src/commands/experimental/stack/restart/restart.handler.ts index 8354213d9b..93a6d2099f 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.handler.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.handler.ts @@ -1,154 +1,114 @@ -import { Effect, Match, Option } from "effect"; -import type { StackDescriptor, StackError, StackId } from "@supabase/stack/effect"; +import { Cause, Effect, Exit, Option, Path } from "effect"; +import type { StackError } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; import { StackApi, - StackTargetError, + StackTargetResolver, rejectStackOutput, - renderStackStatus, - stackStatusPayload, - validateStackId, validateStackTarget, } from "../stack.shared.ts"; -import type { StackRestartFlags } from "./restart.command.ts"; import { StackCommandRestartError } from "./restart.errors.ts"; +import type { StackRestartFlags } from "./restart.command.ts"; -const mapTargetError = (error: StackTargetError) => +const runtimeError = (cause: StackError) => new StackCommandRestartError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - -const mapStackError = (error: StackError) => { - const classification = Match.value(error).pipe( - Match.tag("StackNotFoundError", () => ({ reason: "not-found" as const })), - Match.tag("InvalidStackIdentityError", () => ({ reason: "flags" as const })), - Match.tag("PortUnavailableError", "PortAllocationError", () => ({ - reason: "port" as const, - suggestion: - "Free the conflicting port and retry, or stop the stack and use supabase stack start to apply updated project port configuration.", - })), - Match.tag( - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "InvalidProjectRootError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - () => ({ reason: "invalid-config" as const }), - ), - Match.tag("StackRuntimeMismatchError", () => ({ - reason: "flags" as const, - suggestion: - "Restart preserves the existing runtime; choose a different existing stack if needed.", - })), - Match.tag( - "StackLifecycleConflictError", - "StackNotRunningError", - "StackMustBeStoppedError", - "StackOwnershipConflictError", - "StackUpgradeRequiredError", - () => ({ - reason: "lifecycle" as const, - suggestion: "Run supabase stack status to inspect the stack state.", - }), - ), - Match.tag("ContainerEngineError", () => ({ - reason: "runtime" as const, - suggestion: "Ensure the selected container engine is running and retry the command.", - })), - Match.tag("ContainerPullError", () => ({ - reason: "registry" as const, - suggestion: "Check registry connectivity and image availability, then retry the command.", - })), - Match.tag("ArtifactIntegrityError", "StackPreparationError", () => ({ - reason: "artifact" as const, - suggestion: "Retry the stack restart with --debug if the artifact cannot be prepared.", - })), - Match.tag("StackRuntimeError", () => ({ - reason: "unknown" as const, - suggestion: "Retry the stack restart with --debug and inspect the runtime diagnostics.", - })), - Match.tag("StackCleanupError", () => ({ - reason: "unknown" as const, - suggestion: "Retry the stack restart with --debug and inspect cleanup diagnostics.", - })), - Match.orElse(() => ({ reason: "unknown" as const })), - ); - return new StackCommandRestartError({ - ...classification, - message: error.message, - cause: error, + reason: "unknown", + message: cause.message, + ...(cause.outcomes === undefined + ? {} + : { + detail: cause.outcomes + .filter((outcome) => !outcome.succeeded) + .map((outcome) => `${outcome.id}: ${outcome.error ?? "failed"}`) + .join("\n"), + }), + cause, }); -}; export const stackRestart = Effect.fn("experimental.stack.restart")(function* ( flags: StackRestartFlags, ) { - const telemetryState = yield* TelemetryState; - const body = Effect.gen(function* () { + const telemetry = yield* TelemetryState; + return yield* Effect.gen(function* () { const output = yield* Output; const settings = yield* CommandSettings; + const path = yield* Path.Path; const api = yield* StackApi; - const outputFlag = yield* Effect.serviceOption(OutputFlag); - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + const resolver = yield* StackTargetResolver; + const mapTargetError = (cause: { + readonly reason: "flags" | "invalid-config"; + readonly message: string; + readonly suggestion?: string; + }) => + new StackCommandRestartError({ + reason: cause.reason, + message: cause.message, + ...(cause.suggestion === undefined ? {} : { suggestion: cause.suggestion }), + cause, + }); + yield* rejectStackOutput(yield* Effect.serviceOption(OutputFlag)).pipe( + Effect.mapError(mapTargetError), + ); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), }).pipe(Effect.mapError(mapTargetError)); - - let id: StackId; - let desiredLifecycle: StackDescriptor["desiredLifecycle"]; - if (Option.isSome(flags.stackId)) { - id = yield* validateStackId(flags.stackId.value).pipe(Effect.mapError(mapTargetError)); - const inspection = yield* api.inspectStack(id).pipe(Effect.mapError(mapStackError)); - desiredLifecycle = inspection.descriptor.desiredLifecycle; - } else { - const found = yield* api - .findStack({ - projectRoot: settings.workdir, - ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), - }) - .pipe(Effect.mapError(mapStackError)); - if (Option.isNone(found)) - return yield* new StackCommandRestartError({ - reason: "not-found", - message: Option.isSome(flags.stack) - ? `No managed stack named "${flags.stack.value}" was found for this project.` - : "No managed stack exists for the selected project.", - suggestion: Option.isSome(flags.stack) - ? "Choose an existing --stack name or omit --stack for the current project." - : "Run supabase stack start first.", - }); - id = found.value.id; - desiredLifecycle = found.value.desiredLifecycle; - } - if (desiredLifecycle === "unconfigured") + const target = yield* resolver + .resolve({ + projectRoot: settings.workdir, + runtime: "auto", + ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), + ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), + }) + .pipe(Effect.mapError(mapTargetError)); + if (target.id === undefined) + return yield* new StackCommandRestartError({ + reason: "not-found", + message: "No managed stack exists for the selected project.", + suggestion: "Run supabase stack start first.", + }); + const stack = yield* api + .open({ + id: target.id, + stateRoot: path.join(settings.supabaseHome, "stacks"), + cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), + }) + .pipe(Effect.mapError(runtimeError)); + const composition = yield* stack.composition.describe.pipe(Effect.mapError(runtimeError)); + if (composition.members.length === 0) return yield* new StackCommandRestartError({ reason: "lifecycle", message: "The selected stack has not been configured yet.", - suggestion: "Run supabase stack start to configure the stack first.", + suggestion: "Run supabase stack start first.", }); - const stack = yield* api.openStack(id).pipe(Effect.mapError(mapStackError)); - const task = yield* output.task("Restarting local Supabase stack..."); - yield* stack.stop.pipe( - Effect.mapError(mapStackError), - Effect.tapError((error) => task.fail(error.message)), + const task = yield* output.task("Restarting the saved stack composition..."); + const observations = yield* stack.composition.restart.pipe( + Effect.onExit((exit) => + Exit.isSuccess(exit) + ? task.clear() + : Cause.hasInterruptsOnly(exit.cause) + ? task.cancel() + : task.fail(Option.getOrUndefined(Exit.findErrorOption(exit))?.message), + ), + Effect.mapError(runtimeError), ); - const status = yield* stack.start().pipe( - Effect.mapError(mapStackError), - Effect.tapError((error) => task.fail(error.message)), - Effect.tap(() => task.clear()), - ); - if (output.format === "text") yield* output.raw(renderStackStatus(status)); - else yield* output.success("", stackStatusPayload(status)); - return status; - }); - return yield* body.pipe(Effect.ensuring(telemetryState.flush)); + const services = observations.map(({ id, config, lifecycle, health, wakeEnabled }) => ({ + id, + service: config.service, + lifecycle, + health, + wake_enabled: wakeEnabled, + })); + if (output.format === "text") { + yield* output.raw(`Stack ${stack.id} restarted using its saved configuration.\n`); + for (const service of services) + yield* output.raw( + ` ${service.service}: ${service.lifecycle}, health ${service.health ?? "unavailable"}\n`, + ); + } else yield* output.success("", { id: stack.id, services }); + return services; + }).pipe(Effect.ensuring(telemetry.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts index e71ae3e232..5f6033f159 100644 --- a/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/restart/restart.integration.test.ts @@ -1,553 +1,149 @@ +import { tmpdir } from "node:os"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, FileSystem, Layer, Option, Path, Redacted, Stream } from "effect"; -import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; -import { - StackIdSchema, - StackPreparationError, - StackRuntimeError, - StackStateInvalidError, - StackCleanupError, - type EffectStack, - type StackStatus, - type StackConfig, -} from "@supabase/stack/effect"; -import { mockOutput, mockTty } from "../../../../../tests/helpers/mocks.ts"; +import { StackError } from "@supabase/stack/effect"; +import { Effect, FileSystem, Layer, Option, Path } from "effect"; +import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; -import * as HttpClient from "effect/unstable/http/HttpClient"; -import { StackApi, StackTargetResolver } from "../stack.shared.ts"; +import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackRestart } from "./restart.handler.ts"; -import { stackStart } from "../start/start.handler.ts"; -import { stackStop } from "../stop/stop.handler.ts"; -import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; -import { - ExperimentalFlag, - OutputFlag, - YesFlag, -} from "../../../../command-internal/global-flags.ts"; -import { DbConnection } from "../../../../command-internal/db-connection.service.ts"; -import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; -import { CommandPlatformApiFactory } from "../../../../auth/command-platform-api-factory.service.ts"; -import { noopStackCatalogSetupLayer } from "../../../../command-internal/stack-catalog-setup.ts"; - -const id = StackIdSchema.make("a".repeat(64)); -const project = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-restart-" }); - yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - "[api]\nmax_rows = 1234\n", - ); - return root; - }); -const status = (): StackStatus => ({ - id, - lifecycle: "running", - desiredLifecycle: "running", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: [], - artifacts: [], -}); -const flags = (overrides: Partial[0]> = {}) => ({ - stack: Option.none(), - stackId: Option.none(), - ...overrides, +const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +const fixture = Effect.fn("StackRestartTest.fixture")(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-restart-" }); + const api = yield* StackApi; + const locations = { + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(tmpdir(), "supabase-stack-artifacts"), + }; + const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "native" }); + const output = mockOutput(); + const telemetry = mockTelemetryStateTracked(); + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); + const layer = Layer.mergeAll( + output.layer, + telemetry.layer, + settings, + stackTargetResolverLayer.pipe(Layer.provide(settings)), + ); + return { + api, + locations, + stack, + output, + telemetry, + layer, + flags: { stack: Option.none(), stackId: Option.some(stack.id) }, + }; }); -const fixture = (options: { - stop?: "ok" | "fail"; - start?: "ok" | "fail"; - format?: "text" | "json"; - config?: "valid" | "invalid"; - found?: boolean; - unconfigured?: boolean; - startRuntimeFailure?: boolean; - inspectFailure?: boolean; -}) => - Effect.gen(function* () { - const root = yield* project(); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - if (options.config === "invalid") - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - 'project_id = "unterminated\n', - ); - const calls: string[] = []; - let lifecycle: StackStatus["lifecycle"] = "running"; - let startedConfig: unknown; - let selectedName: string | undefined; - const output = mockOutput({ format: options.format }); - const telemetry = mockTelemetryStateTracked(); - const stack: EffectStack = { - id, - status: Effect.sync(() => ({ ...status(), lifecycle })), - credentials: Effect.die("unused"), - prepare: () => Effect.die("restart must not prepare explicitly"), - resetDatabase: Effect.die("unused"), - stop: Effect.gen(function* () { - calls.push("stop"); - if (options.stop === "fail") - return yield* new StackCleanupError({ message: "stop failed" }); - lifecycle = "stopped"; - }), - start: (input) => - Effect.sync(() => calls.push("start")).pipe( - Effect.tap(() => - Effect.sync(() => { - startedConfig = input?.config; - }), - ), - Effect.flatMap(() => - options.start === "fail" - ? Effect.fail( - options.startRuntimeFailure === true - ? new StackRuntimeError({ message: "runtime failed" }) - : new StackPreparationError({ message: "start failed" }), - ) - : Effect.sync(() => { - lifecycle = "running"; - return status(); - }), - ), - ), - destroy: Effect.die("unused"), - logs: () => Effect.die("unused"), - followLogs: () => Stream.empty, - }; - const layer = Layer.mergeAll( - output.layer, - telemetry.layer, - mockCommandSettings({ workdir: root }), - Layer.succeed(StackApi, { - findStack: ({ projectRoot, name }) => - Effect.sync(() => { - selectedName = name; - return options.found === false || projectRoot !== root - ? Option.none() - : Option.some({ - id, - projectRoot: root, - name: name ?? "restart-test", - branchContext: "default", - runtime: { kind: "native" as const }, - desiredLifecycle: options.unconfigured - ? ("unconfigured" as const) - : ("running" as const), - }); - }), - createStack: () => Effect.die("create must not run"), - openStack: () => Effect.succeed(stack), - inspectStack: () => - options.inspectFailure - ? Effect.fail(new StackStateInvalidError({ message: "inspect failed" })) - : Effect.succeed({ - descriptor: { - id, - projectRoot: root, - name: "restart-test", - branchContext: "default", - runtime: { kind: "native" as const }, - desiredLifecycle: options.unconfigured - ? ("unconfigured" as const) - : ("running" as const), - }, - owner: "running" as const, - }), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - }), - BunServices.layer, - runtimeInfoLayer, - ); - return { - root, - calls, - output, - telemetry, - layer, - get lifecycle() { - return lifecycle; - }, - get startedConfig() { - return startedConfig; - }, - get selectedName() { - return selectedName; - }, - }; - }).pipe(Effect.provide(BunServices.layer)); - describe("stack restart", () => { - it.live("stops and starts the saved stack configuration", () => { - return fixture({}).pipe( - Effect.flatMap((setup) => - stackRestart(flags()).pipe( - Effect.provide(setup.layer), - Effect.tap(() => - Effect.sync(() => { - expect(setup.calls).toEqual(["stop", "start"]); - expect(setup.lifecycle).toBe("running"); - expect(setup.startedConfig).toBeUndefined(); - expect(setup.telemetry.flushed).toBe(true); - expect(setup.output.stdoutText).toContain(`Stack ${id}`); - }), - ), - ), - ), - ); - }); - - it.live("does not start when stopping fails", () => { - return fixture({ stop: "fail" }).pipe( - Effect.flatMap((setup) => - stackRestart(flags()).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.message).toContain("stop failed"); - expect(error.reason).toBe("unknown"); - expect(error.suggestion).toContain("--debug"); - expect(error.suggestion).toContain("cleanup"); - expect(setup.calls).toEqual(["stop"]); - }), - ), - ), - ), - ); - }); - - it.live("leaves the stack stopped and recoverable when start fails", () => { - return fixture({ start: "fail" }).pipe( - Effect.flatMap((setup) => - stackRestart(flags()).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.message).toContain("start failed"); - expect(setup.calls).toEqual(["stop", "start"]); - expect(setup.lifecycle).toBe("stopped"); - expect(setup.telemetry.flushed).toBe(true); - }), - ), - ), - ), - ); - }); - - it.live("reuses saved configuration for an id target", () => { - return fixture({ format: "json", config: "invalid" }).pipe( - Effect.flatMap((setup) => - stackRestart(flags({ stackId: Option.some(id) })).pipe( - Effect.provide(setup.layer), - Effect.tap(() => - Effect.sync(() => { - expect(setup.startedConfig).toBeUndefined(); - expect( - setup.output.messages.find(({ type }) => type === "success")?.data, - ).toMatchObject({ - id, - lifecycle: "running", + it.live( + "restarts saved members while leaving a running standalone service untouched", + () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* Effect.acquireUseRelease( + Effect.succeed(f.stack), + (stack) => + Effect.gen(function* () { + const member = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, }); - }), - ), - ), - ), - ); - }); - - it.live("fails without lifecycle calls when no existing stack is found", () => { - return fixture({ found: false }).pipe( - Effect.flatMap((setup) => - stackRestart(flags()).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason).toBe("not-found"); - expect(setup.calls).toEqual([]); - }), - ), - ), - ), - ); - }); - - it.live("selects a named existing stack", () => { - return fixture({}).pipe( - Effect.flatMap((setup) => - stackRestart(flags({ stack: Option.some("feature-a") })).pipe( - Effect.provide(setup.layer), - Effect.tap(() => Effect.sync(() => expect(setup.selectedName).toBe("feature-a"))), - ), - ), - ); - }); - - it.live("reports an actionable error for a missing named stack", () => { - return fixture({ found: false }).pipe( - Effect.flatMap((setup) => - stackRestart(flags({ stack: Option.some("missing") })).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(setup.selectedName).toBe("missing"); - expect(error.message).toContain('No managed stack named "missing"'); - expect(error.suggestion).toContain("existing --stack name"); - }), - ), - ), - ), - ); - }); - - it.live("rejects an unconfigured current stack before stopping", () => { - return fixture({ unconfigured: true }).pipe( - Effect.flatMap((setup) => - stackRestart(flags()).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason).toBe("lifecycle"); - expect(error.suggestion).toContain("Run supabase stack start"); - expect(setup.calls).toEqual([]); - }), - ), - ), - ), - ); - }); - - it.live("rejects an unconfigured id target before stopping", () => { - return fixture({ unconfigured: true }).pipe( - Effect.flatMap((setup) => - stackRestart(flags({ stackId: Option.some(id) })).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason).toBe("lifecycle"); - expect(setup.calls).toEqual([]); - }), - ), - ), - ), - ); - }); - - it.live("provides retry guidance for runtime failures", () => { - return fixture({ start: "fail", startRuntimeFailure: true }).pipe( - Effect.flatMap((setup) => - stackRestart(flags()).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason).toBe("unknown"); - expect(error.suggestion).toContain("--debug"); - }), - ), - ), - ), - ); - }); - - it.live("short circuits invalid target and output flags before lifecycle", () => { - return fixture({}).pipe( - Effect.flatMap((setup) => { - const invalidTarget = stackRestart( - flags({ stack: Option.some("named"), stackId: Option.some(id) }), - ).pipe(Effect.provide(setup.layer), Effect.flip); - const outputRejected = stackRestart(flags()).pipe( - Effect.provide(Layer.merge(setup.layer, Layer.succeed(OutputFlag, Option.some("json")))), - Effect.flip, + const standalone = yield* stack.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* stack.composition.configure({ + members: [{ id: member.id, activation: "eager" }], + dependencies: [], + }); + yield* stack.composition.start; + yield* standalone.start; + yield* standalone.ready; + const before = yield* member.status; + const standaloneBefore = yield* standalone.status; + const result = yield* stackRestart(f.flags).pipe(Effect.provide(f.layer)); + const after = yield* member.status; + const standaloneAfter = yield* standalone.status; + expect(result.map(({ id }) => id)).toEqual([member.id]); + expect(after.lifecycle).toBe("running"); + expect(after.health).toBe("healthy"); + expect(after.launchId).not.toBe(before.launchId); + expect(after.endpoints).toEqual(before.endpoints); + expect(standaloneAfter.lifecycle).toBe("running"); + expect(standaloneAfter.launchId).toBe(standaloneBefore.launchId); + expect(f.output.stdoutText).toContain("using its saved configuration"); + expect(f.telemetry.flushed).toBe(true); + yield* stack.stop; + const reopened = yield* stackRestart(f.flags).pipe(Effect.provide(f.layer)); + expect(reopened.map(({ id }) => id)).toEqual([member.id]); + const resumed = yield* member.status; + expect(resumed.endpoints).toEqual(before.endpoints); + expect(resumed.lifecycle).toBe("running"); + expect(resumed.health).toBe("healthy"); + expect((yield* standalone.status).lifecycle).toBe("stopped"); + }), + (stack) => stack.destroy, ); - return Effect.gen(function* () { - expect((yield* invalidTarget).reason).toBe("flags"); - expect((yield* outputRejected).reason).toBe("flags"); - expect(setup.calls).toEqual([]); - }); - }), - ); - }); + }).pipe(Effect.provide(live)), + 60_000, + ); - it.live("short circuits an ID inspection failure before lifecycle", () => { - return fixture({ inspectFailure: true }).pipe( - Effect.flatMap((setup) => - stackRestart(flags({ stackId: Option.some(id) })).pipe( - Effect.provide(setup.layer), - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.reason).toBe("invalid-config"); - expect(setup.calls).toEqual([]); - }), - ), - ), - ), - ); - }); + it.live("rejects an unconfigured namespace without starting an owner", () => + Effect.gen(function* () { + const f = yield* fixture(); + const error = yield* stackRestart(f.flags).pipe(Effect.provide(f.layer), Effect.flip); + expect(error.reason).toBe("lifecycle"); + expect((yield* f.api.discover(f.locations))[0]?.host).toBeUndefined(); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); - it.live( - "preserves effective start options through restart and reloads them on plain start", - () => { - return project().pipe( - Effect.flatMap((root) => { - const output = mockOutput({ format: "json" }); - const telemetry = mockTelemetryStateTracked(); - const calls: string[] = []; - let lifecycle: StackStatus["lifecycle"] = "stopped"; - const persisted: { config?: StackConfig } = {}; - const state = () => ({ - id, - lifecycle, - desiredLifecycle: "running" as const, - runtime: { kind: "native" as const }, - endpoints: {}, - versions: {}, - capabilities: [], - artifacts: [], - }); - const stack: EffectStack = { - id, - status: Effect.sync(state), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), - password: Redacted.make("secret"), - }, - }), - prepare: () => Effect.die("restart must not prepare explicitly"), - resetDatabase: Effect.die("unused"), - stop: Effect.sync(() => { - calls.push("stop"); - lifecycle = "stopped"; - }), - start: (input) => - Effect.sync(() => { - calls.push("start"); - if (input?.config !== undefined) persisted.config = input.config; - lifecycle = "running"; - return state(); - }), - destroy: Effect.die("unused"), - logs: () => Effect.die("unused"), - followLogs: () => Stream.empty, - }; - const descriptor = { - id, - projectRoot: root, - name: "restart-flow", - branchContext: "default", - runtime: { kind: "native" as const }, - desiredLifecycle: "running" as const, - }; - const layer = Layer.mergeAll( - output.layer, - telemetry.layer, - mockCommandSettings({ workdir: root }), - // `stackStart`'s bucket-seeding path statically requires these even though this - // fixture always resumes an existing stack, so seeding never runs. - Layer.succeed( - HttpClient.HttpClient, - HttpClient.make(() => Effect.die("unused")), - ), - Layer.succeed(CommandPlatformApiFactory, { make: Effect.die("unused") }), - stdinLayer.pipe(Layer.provide(mockTty({ stdinIsTty: false, stdoutIsTty: false }))), - mockTty({ stdinIsTty: false, stdoutIsTty: false }), - Layer.succeed(YesFlag, false), - Layer.succeed(StackTargetResolver, { - resolve: ({ id: targetId }) => - Effect.succeed({ - projectRoot: root, - ...(targetId === undefined ? {} : { id: StackIdSchema.make(targetId) }), + it.live("identifies failed members when a composition restart fails", () => + Effect.gen(function* () { + const f = yield* fixture(); + const failure = new StackError({ + operation: "composition.restart", + message: "Some services failed to restart", + outcomes: [ + { id: "database-instance", succeeded: true }, + { id: "rest-instance", succeeded: false, error: "Port is occupied" }, + ], + }); + const api = Layer.succeed( + StackApi, + StackApi.of({ + ...f.api, + open: () => + Effect.succeed({ + ...f.stack, + composition: { + ...f.stack.composition, + describe: Effect.succeed({ + members: [{ id: "rest-instance", activation: "lazy" }], + dependencies: [], }), - }), - Layer.succeed(StackApi, { - findStack: () => Effect.succeed(Option.some(descriptor)), - createStack: () => Effect.succeed(stack), - openStack: () => Effect.succeed(stack), - inspectStack: () => Effect.die("inspect unused"), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - }), - BunServices.layer, - runtimeInfoLayer, - noopStackCatalogSetupLayer, - Layer.succeed(ExperimentalFlag, false), - Layer.succeed(CliArgs, { args: ["stack", "start"] }), - Layer.succeed(DbConnection, { - connect: () => - Effect.succeed({ - exec: () => Effect.void, - query: () => Effect.succeed([]), - execBatch: () => Effect.void, - extensionExists: () => Effect.succeed(false), - copyToCsv: () => Effect.succeed(new Uint8Array()), - queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), - }), - }), - ); - const initialStart = { - exclude: ["studio"], - stack: Option.none(), - stackId: Option.none(), - runtime: "auto" as const, - preparation: "on-demand" as const, - eager: true, - }; - const plainStart = { - exclude: [], - stack: Option.none(), - stackId: Option.none(), - runtime: "auto" as const, - preparation: "background" as const, - eager: false, - }; - const stopFlags = { - all: Option.none(), - stack: Option.none(), - stackId: Option.none(), - }; - return Effect.gen(function* () { - yield* stackStart(initialStart); - expect(persisted.config).toMatchObject({ - preparation: "on-demand", - capabilities: { - studio: { enabled: false }, - rest: { activation: "eager" }, - }, - }); - const saved = persisted.config; - yield* stackRestart(flags()); - expect(persisted.config).toBe(saved); - yield* stackStop(stopFlags); - yield* stackStart(plainStart); - expect(persisted.config).toMatchObject({ - preparation: "background", - capabilities: { - studio: { settings: {} }, - rest: { settings: { max_rows: 1234 } }, + restart: Effect.fail(failure), }, - }); - expect(persisted.config?.capabilities?.studio?.enabled).not.toBe(false); - expect(persisted.config?.capabilities?.rest).not.toHaveProperty("activation", "eager"); - expect(calls).toEqual(["start", "stop", "start", "stop", "start"]); - }).pipe(Effect.provide(layer)); + }), }), - Effect.provide(BunServices.layer), ); - }, + const error = yield* stackRestart(f.flags).pipe( + Effect.provide(Layer.provideMerge(f.layer, api)), + Effect.flip, + ); + expect(error.message).toBe("Some services failed to restart"); + expect(error.detail).toBe("rest-instance: Port is occupied"); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), ); }); diff --git a/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts index 8c3d1c1efa..7a659649c6 100644 --- a/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts @@ -309,6 +309,22 @@ stack = true ); }); + it.effect("routes only functions serve through the selected Stack backend", () => + withServices( + Effect.gen(function* () { + const root = yield* project("[experimental]\nstack = true\n"); + expect(yield* resolve({ args: ["functions", "serve"], cwd: root, env: {} })).toBe("stack"); + expect(yield* resolve({ args: ["help", "functions", "serve"], cwd: root, env: {} })).toBe( + "stack", + ); + expect( + yield* resolve({ args: ["__complete", "functions", "serve", "--"], cwd: root, env: {} }), + ).toBe("stack"); + expect(yield* resolve({ args: ["functions", "list"], cwd: root, env: {} })).toBe("legacy"); + }), + ), + ); + it("selects matching command trees for completion", () => { expect(completionFlags("stack", "start")).toEqual( expect.arrayContaining(["--stack", "--runtime", "--preparation", "--eager"]), diff --git a/apps/cli/src/commands/experimental/stack/stack-command-telemetry.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-command-telemetry.integration.test.ts index 2719b39d97..40c4ea8b7b 100644 --- a/apps/cli/src/commands/experimental/stack/stack-command-telemetry.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-command-telemetry.integration.test.ts @@ -166,16 +166,11 @@ describe("stack command telemetry", () => { Effect.flatMap((fixture) => { const command = stackCommand.pipe(Command.provide(fixture.layer)); return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })(["logs"]); + yield* Command.runWith(command, { version: "0.0.0-test" })(["logs"]).pipe(Effect.flip); const event = fixture.analytics.captured.find( (candidate) => candidate.event === EventCommandExecuted, ); expect(event?.properties[PropCommand]).toBe("stack logs"); - expect(fixture.output.messages).toEqual( - expect.arrayContaining([ - expect.objectContaining({ type: "success", data: { found: false, entries: [] } }), - ]), - ); }).pipe(Effect.provide(fixture.layer)); }), Effect.provide(BunServices.layer), diff --git a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts index 94e2ec35b1..e55fc5ff70 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts @@ -1,34 +1,16 @@ -import { Buffer } from "node:buffer"; - -import { encrypt, PrivateKey } from "eciesjs"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer, Option, Redacted } from "effect"; +import { Effect, Exit, FileSystem, Layer } from "effect"; +import type { ServiceCreation } from "@supabase/stack/effect"; import { runtimeInfoLayer } from "../../../shared/runtime/runtime-info.layer.ts"; -import { compileStack } from "../../../../../../packages/stack/src/model/Compiler.ts"; import { withEnvVar } from "../../../../tests/helpers/command-mocks.ts"; -import { StackConfigError, loadStackConfig } from "../../../command-internal/stack-config.ts"; +import { loadStackConfig } from "../../../command-internal/stack-config.ts"; import { createStackConfigProject } from "../../../../tests/helpers/stack-config.ts"; -function withEnvironment( - values: Readonly>, - body: Effect.Effect, -): Effect.Effect { - return Object.entries(values).reduce( - (effect, [name, value]) => withEnvVar(name, value, effect), - body, - ); -} - const project = ( config: string, - options: { - readonly rootEnv?: string; - readonly supabaseEnv?: string; - readonly sharedFunctionEnvironment?: string; - readonly functionEnvironments?: Readonly>; - } = {}, + options: { readonly rootEnv?: string; readonly supabaseEnv?: string } = {}, ) => createStackConfigProject(config, { prefix: "supabase-stack-config-env-", @@ -40,19 +22,31 @@ const load = (projectRoot: string) => Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer)), ); -const encrypted = (privateKey: string, plaintext: string): string => - `encrypted:${Buffer.from( - encrypt(PrivateKey.fromHex(privateKey).publicKey.toHex(false), Buffer.from(plaintext, "utf8")), - ).toString("base64")}`; +const withEnvironment = ( + values: Readonly>, + body: Effect.Effect, +): Effect.Effect => + Object.entries(values).reduce((effect, [name, value]) => withEnvVar(name, value, effect), body); -const privateKey = "7fd7210cef8f331ee8c55897996aaaafd853a2b20a4dc73d6d75759f65d2a7eb"; -const wrongPrivateKey = "11".repeat(32); +const service = (services: ReadonlyArray, name: string) => + services.find((entry) => entry.service === name); describe("loadStackConfig environment overrides", () => { - it.effect( - "uses shell > supabase dotenv > project-root dotenv, with empty and indirect values handled", - () => { - return project( + it.live("keeps malformed Functions env out of database and config preparation", () => + Effect.gen(function* () { + const root = yield* project('project_id = "functions-env-malformed"\n'); + const fs = yield* FileSystem.FileSystem; + yield* fs.makeDirectory(`${root}/supabase/functions`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/functions/.env`, 'BROKEN="unterminated\n'); + const config = yield* load(root); + const creations = yield* config.creations("functions-env-malformed"); + expect(service(creations, "database")?.service).toBe("database"); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("uses shell > supabase dotenv > project-root dotenv precedence", () => + Effect.gen(function* () { + const root = yield* project( `project_id = "stack-config-env-precedence" [auth] site_url = "from-config" @@ -61,955 +55,96 @@ site_url = "from-config" rootEnv: "SUPABASE_AUTH_SITE_URL=root\nAUTH_SITE_URL=root-indirect\n", supabaseEnv: "SUPABASE_AUTH_SITE_URL=supabase\nAUTH_SITE_URL=supabase-indirect\n", }, - ).pipe( - Effect.flatMap((root) => - withEnvVar( - "SUPABASE_AUTH_SITE_URL", - "shell", - Effect.gen(function* () { - const shell = yield* load(root); - if ( - shell.capabilities?.auth === undefined || - !("settings" in shell.capabilities.auth) - ) - throw new Error("auth settings missing"); - expect(shell.capabilities.auth.settings?.site_url).toBe("shell"); - - const indirect = yield* withEnvVar( - "SUPABASE_AUTH_SITE_URL", - "env(AUTH_SITE_URL)", - load(root), - ); - if ( - indirect.capabilities?.auth === undefined || - !("settings" in indirect.capabilities.auth) - ) - throw new Error("auth settings missing"); - expect(indirect.capabilities.auth.settings?.site_url).toBe("supabase-indirect"); - - const empty = yield* withEnvVar("SUPABASE_AUTH_SITE_URL", "", load(root)); - if ( - empty.capabilities?.auth === undefined || - !("settings" in empty.capabilities.auth) - ) - throw new Error("auth settings missing"); - expect(empty.capabilities.auth.settings?.site_url).toBe("from-config"); - }), - ), - ), - ); - }, - ); - - it.effect( - "applies defaults-backed auth.email overrides but does not materialize absent SMTP, providers, or hooks", - () => { - return project( - `project_id = "stack-config-auth-presence" -`, - { - supabaseEnv: [ - "SUPABASE_AUTH_EMAIL_ENABLE_SIGNUP=false", - "SUPABASE_AUTH_EMAIL_SMTP_ENABLED=true", - "SUPABASE_AUTH_EXTERNAL_GITHUB_ENABLED=true", - "SUPABASE_AUTH_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED=true", - "", - ].join("\n"), - }, - ).pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const config = yield* load(root); - if ( - config.capabilities?.auth === undefined || - !("settings" in config.capabilities.auth) - ) - throw new Error("auth settings missing"); - const auth = config.capabilities.auth.settings; - expect(auth?.email?.enable_signup).toBe(false); - expect(auth?.email?.smtp).toBeUndefined(); - expect(auth?.external?.github).toBeUndefined(); - expect(auth?.hook?.custom_access_token).toBeUndefined(); - }), - ), ); - }, + const shell = yield* withEnvVar("SUPABASE_AUTH_SITE_URL", "shell", load(root)); + expect(shell.source.auth.site_url).toBe("shell"); + const indirect = yield* withEnvVar( + "SUPABASE_AUTH_SITE_URL", + "env(AUTH_SITE_URL)", + load(root), + ); + expect(indirect.source.auth.site_url).toBe("supabase-indirect"); + const empty = yield* withEnvVar("SUPABASE_AUTH_SITE_URL", "", load(root)); + expect(empty.source.auth.site_url).toBe("from-config"); + const dotenv = yield* withEnvVar("SUPABASE_AUTH_SITE_URL", undefined, load(root)); + expect(dotenv.source.auth.site_url).toBe("supabase"); + }), ); - it.effect("disables and re-enables a service through the effective environment layer", () => { - return Effect.gen(function* () { - const disabledRoot = yield* project('project_id = "stack-config-env-disable-file"\n', { - supabaseEnv: "SUPABASE_AUTH_ENABLED=false\n", - }); - const enabledRoot = yield* project( - 'project_id = "stack-config-env-enable-shell"\n[auth]\nenabled = false\n', - ); - return yield* withEnvVar( - "SUPABASE_AUTH_ENABLED", - undefined, - Effect.gen(function* () { - const disabled = yield* load(disabledRoot); - expect(disabled.capabilities?.auth).toEqual({ enabled: false }); - - const enabled = yield* withEnvVar("SUPABASE_AUTH_ENABLED", "true", load(enabledRoot)); - if ( - enabled.capabilities?.auth === undefined || - !("settings" in enabled.capabilities.auth) - ) - throw new Error("auth settings missing"); - expect(enabled.capabilities.auth.settings).toBeDefined(); - }), - ); - }); - }); - - it.effect("applies JWT environment overrides while auth is disabled", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-disabled-auth-jwt" -`, + it.live("applies env-only listener ports and leaves omitted ports automatic", () => + Effect.gen(function* () { + const root = yield* project('project_id = "stack-config-env-only-ports"\n'); + const config = yield* withEnvironment( { - supabaseEnv: [ - "SUPABASE_AUTH_ENABLED=false", - "SUPABASE_AUTH_JWT_ISSUER=https://issuer.example.test", - "SUPABASE_AUTH_JWT_SECRET=01234567890123456789012345678901", - "", - ].join("\n"), + SUPABASE_API_PORT: "54321", + SUPABASE_DB_PORT: "54322", + SUPABASE_ANALYTICS_PORT: "55555", + SUPABASE_ANALYTICS_VECTOR_PORT: "55556", }, + load(root), ); - const signingPathRoot = yield* project( - `project_id = "stack-config-disabled-auth-signing-path" -[auth] -enabled = false -`, - { supabaseEnv: "SUPABASE_AUTH_SIGNING_KEYS_PATH=keys.json\n" }, - ); - return yield* withEnvVar( - "SUPABASE_AUTH_ENABLED", - undefined, - Effect.gen(function* () { - const config = yield* load(root); - expect(config.capabilities?.auth).toEqual(expect.objectContaining({ enabled: false })); - expect(config.security?.jwt?.issuer).toBe("https://issuer.example.test"); - const signing = config.security?.jwt?.signing; - expect(signing?.kind).toBe("symmetric"); - if (signing?.kind !== "symmetric") throw new Error("symmetric signing missing"); - expect(Redacted.value(signing.secret)).toBe("01234567890123456789012345678901"); - const signingPath = yield* load(signingPathRoot); - expect(signingPath.capabilities?.auth).toEqual( - expect.objectContaining({ enabled: false }), - ); - expect(signingPath.security?.jwt?.signing).toEqual({ - kind: "jwks-file", - path: "supabase/keys.json", - }); - }), - ); - }); - }); + const recipes = yield* config.creations("env-only-ports"); + expect(service(recipes, "database")?.endpoints).toEqual({ sql: { port: 54322 } }); + expect(service(recipes, "rest")?.endpoints).toEqual({ http: { port: 54321 } }); + expect(service(recipes, "analytics")?.endpoints).toEqual({ http: { port: 55555 } }); - it.effect("rejects an env-enabled SMTP section without a port", () => { - return project( - `project_id = "stack-config-smtp-env-enable-missing-port" -[auth.email.smtp] -enabled = false -host = "smtp.example.test" -user = "smtp-user" -pass = "smtp-pass" -admin_email = "admin@example.test" -`, - { supabaseEnv: "SUPABASE_AUTH_EMAIL_SMTP_ENABLED=true\n" }, - ).pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("auth.email.smtp.port"); - }), - ), - ); - }); - - it.effect("does not decrypt an unused provider secret when auth is disabled", () => { - return project( - `project_id = "stack-config-disabled-auth-provider-secret" -[auth] -enabled = false -[auth.external.github] -enabled = true -client_id = "github-client" -secret = "encrypted:not-a-real-ciphertext" -`, - ).pipe( - Effect.flatMap((root) => - withEnvVar( - "DOTENV_PRIVATE_KEY", - undefined, - Effect.gen(function* () { - const config = yield* load(root); - expect(config.capabilities?.auth).toEqual(expect.objectContaining({ enabled: false })); - }), - ), - ), - ); - }); - - it.effect("re-enables a file-disabled Studio service with its env-provided listener port", () => { - return project( - `project_id = "stack-config-studio-env-reenable" -[studio] -enabled = false -port = 55440 -`, - ).pipe( - Effect.flatMap((root) => - withEnvironment( - { SUPABASE_STUDIO_ENABLED: "true", SUPABASE_STUDIO_PORT: "55441" }, - Effect.gen(function* () { - const config = yield* load(root); - if ( - config.capabilities?.studio === undefined || - !("settings" in config.capabilities.studio) - ) - throw new Error("Studio settings missing"); - expect(config.listeners?.studio).toEqual({ port: 55441 }); - }), - ), - ), - ); - }); - - it.effect("validates unsupported Figma from effective provider presence", () => { - return Effect.gen(function* () { - const disabledInFile = yield* project( - `project_id = "stack-config-figma-env-enable" -[auth.external.figma] -enabled = false -`, - ); - const enabledInFile = yield* project( - `project_id = "stack-config-figma-env-disable" -[auth.external.figma] -enabled = true -client_id = "figma-client" -secret = "figma-secret" -`, - ); - return yield* withEnvVar( - "SUPABASE_AUTH_EXTERNAL_FIGMA_ENABLED", - "true", - Effect.gen(function* () { - const enabled = yield* load(disabledInFile).pipe(Effect.exit); - expect(Exit.isFailure(enabled)).toBe(true); - if (Exit.isFailure(enabled)) - expect(String(enabled.cause)).toContain("auth.external.figma"); - - const disabled = yield* withEnvVar( - "SUPABASE_AUTH_EXTERNAL_FIGMA_ENABLED", - "false", - load(enabledInFile), - ); - if ( - disabled.capabilities?.auth === undefined || - !("settings" in disabled.capabilities.auth) - ) - throw new Error("auth settings missing"); - expect(Object.hasOwn(disabled.capabilities.auth.settings?.external ?? {}, "figma")).toBe( - false, - ); - }), - ); - }); - }); - - it.effect("overrides present SMTP, provider, and hook fields from SUPABASE_* values", () => { - return project( - `project_id = "stack-config-auth-nested-overrides" -[auth.email.smtp] -enabled = true -host = "config-smtp" -port = 2525 -user = "config-user" -pass = "config-pass" -admin_email = "config-admin@example.test" -sender_name = "Config Sender" -[auth.external.github] -enabled = true -client_id = "config-client" -secret = "config-secret" -[auth.hook.custom_access_token] -enabled = true -uri = "config-hook" -`, - { - supabaseEnv: [ - "SUPABASE_AUTH_EMAIL_SMTP_HOST=env-smtp", - "SUPABASE_AUTH_EMAIL_SMTP_PORT=2526", - "SUPABASE_AUTH_EMAIL_SMTP_USER=env-user", - "SUPABASE_AUTH_EMAIL_SMTP_PASS=env-pass", - "SUPABASE_AUTH_EMAIL_SMTP_ADMIN_EMAIL=env-admin@example.test", - "SUPABASE_AUTH_EMAIL_SMTP_SENDER_NAME=Env Sender", - "SUPABASE_AUTH_EXTERNAL_GITHUB_ENABLED=false", - "SUPABASE_AUTH_EXTERNAL_GITHUB_CLIENT_ID=env-client", - "SUPABASE_AUTH_EXTERNAL_GITHUB_SECRET=env-secret", - "SUPABASE_AUTH_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED=false", - "SUPABASE_AUTH_HOOK_CUSTOM_ACCESS_TOKEN_URI=env-hook", - "", - ].join("\n"), - }, - ).pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const config = yield* load(root); - if (config.capabilities?.auth === undefined || !("settings" in config.capabilities.auth)) - throw new Error("auth settings missing"); - const auth = config.capabilities.auth.settings; - expect(auth?.email?.smtp).toMatchObject({ - enabled: true, - host: "env-smtp", - port: 2526, - user: "env-user", - admin_email: "env-admin@example.test", - sender_name: "Env Sender", - }); - expect(auth?.email?.smtp?.pass).toBeDefined(); - if (auth?.email?.smtp?.pass === undefined) throw new Error("SMTP password missing"); - expect(Redacted.value(auth.email.smtp.pass)).toBe("env-pass"); - expect(auth?.external?.github).toMatchObject({ - enabled: false, - client_id: "env-client", - }); - expect(auth?.hook?.custom_access_token).toMatchObject({ - enabled: false, - uri: "env-hook", - }); - }), - ), - ); - }); - - it.effect("rejects incomplete auth sections after effective environment overrides", () => { - const cases = [ - { - name: "external provider", - config: `[auth.external.github]\nenabled = false\n`, - env: "SUPABASE_AUTH_EXTERNAL_GITHUB_ENABLED=true\n", - message: "auth.external.github.client_id", - }, - { - name: "SMS provider", - config: `[auth.sms.vonage]\nenabled = false\n`, - env: "SUPABASE_AUTH_SMS_VONAGE_ENABLED=true\n", - message: "auth.sms.vonage.from", - }, - ]; - return Effect.gen(function* () { - for (const testCase of cases) { - const root = yield* project( - `project_id = "stack-config-auth-invalid-${testCase.name}"\n${testCase.config}`, - { - supabaseEnv: testCase.env, - }, - ); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit), testCase.name).toBe(true); - if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain(testCase.message); - } - }); - }); - - it.effect("gates auth validation on the effective auth capability", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-auth-effective-gate" -[auth] -enabled = false -[auth.external.github] -enabled = false -`, - { - supabaseEnv: [ - "SUPABASE_AUTH_ENABLED=true", - "SUPABASE_AUTH_EXTERNAL_GITHUB_ENABLED=true", - "", - ].join("\n"), - }, - ); - const enabled = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(enabled)).toBe(true); - if (Exit.isFailure(enabled)) expect(String(enabled.cause)).toContain("auth.external.github"); - - const disabled = yield* withEnvVar("SUPABASE_AUTH_ENABLED", "false", load(root)); - expect(disabled.capabilities?.auth).toEqual(expect.objectContaining({ enabled: false })); - }); - }); - - it.effect("accepts a valid encrypted HTTPS auth hook", () => { - const hookSecret = `v1,whsec_${"A".repeat(32)}`; - return project( - `project_id = "stack-config-auth-encrypted-hook" -[auth.hook.custom_access_token] -enabled = true -uri = "https://hooks.example.test" -secrets = "${encrypted(privateKey, hookSecret)}" -`, - { supabaseEnv: `DOTENV_PRIVATE_KEY=${privateKey}\n` }, - ).pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const config = yield* load(root); - if (config.capabilities?.auth === undefined || !("settings" in config.capabilities.auth)) - throw new Error("auth settings missing"); - const hook = config.capabilities.auth.settings?.hook?.custom_access_token; - expect(hook?.uri).toBe("https://hooks.example.test"); - expect(hook?.secrets).toBeDefined(); - if (hook?.secrets === undefined) throw new Error("hook secret missing"); - expect(Redacted.value(hook.secrets)).toBe(hookSecret); - }), - ), - ); - }); - - it.effect( - "redacts plaintext and encrypted Vonage API keys and preserves disabled SMTP shape", - () => { - const encryptedApiKey = encrypted(privateKey, "encrypted-vonage-api-key"); - return Effect.gen(function* () { - const plaintextRoot = yield* project(`project_id = "stack-config-vonage-plaintext" -[auth.email.smtp] -enabled = false -[auth.sms.vonage] -enabled = true -from = "from" -api_key = "plaintext-vonage-api-key" -api_secret = "vonage-api-secret" -`); - const encryptedRoot = yield* project( - `project_id = "stack-config-vonage-encrypted" -[auth.sms.vonage] -enabled = true -from = "from" -api_key = "${encryptedApiKey}" -api_secret = "vonage-api-secret" -`, - { supabaseEnv: `DOTENV_PRIVATE_KEY=${privateKey}\n` }, - ); - const plaintext = yield* load(plaintextRoot); - const plaintextAuth = plaintext.capabilities?.auth; - if (plaintextAuth === undefined || !("settings" in plaintextAuth)) - throw new Error("plaintext auth settings missing"); - expect(plaintextAuth.settings?.sms?.vonage?.api_key).toBeDefined(); - expect(Redacted.value(plaintextAuth.settings!.sms!.vonage!.api_key!)).toBe( - "plaintext-vonage-api-key", - ); - expect(plaintextAuth.settings?.email?.smtp).not.toHaveProperty("port"); - - const encryptedConfig = yield* load(encryptedRoot); - const encryptedAuth = encryptedConfig.capabilities?.auth; - if (encryptedAuth === undefined || !("settings" in encryptedAuth)) - throw new Error("encrypted auth settings missing"); - expect(encryptedAuth.settings?.sms?.vonage?.api_key).toBeDefined(); - expect(Redacted.value(encryptedAuth.settings!.sms!.vonage!.api_key!)).toBe( - "encrypted-vonage-api-key", - ); - }); - }, - ); - - it.effect("applies stack environment overrides while preserving optional settings", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-stack-overrides" -[api] -enabled = true -auto_expose_new_tables = false -[storage] -[storage.image_transformation] -enabled = false -[db] -health_timeout = "2m" -[auth] -signing_keys_path = "keys.json" -`, - { - supabaseEnv: [ - "SUPABASE_API_AUTO_EXPOSE_NEW_TABLES=true", - "SUPABASE_STORAGE_IMAGE_TRANSFORMATION_ENABLED=true", - "SUPABASE_DB_HEALTH_TIMEOUT=45s", - "SUPABASE_AUTH_SIGNING_KEYS_PATH=overridden-keys.json", - "SUPABASE_AUTH_EXTERNAL_APPLE_ENABLED=true", - "SUPABASE_AUTH_EXTERNAL_APPLE_CLIENT_ID=apple-client", - "SUPABASE_AUTH_EXTERNAL_APPLE_SECRET=apple-secret", - "", - ].join("\n"), - }, - ); - const absentImageRoot = yield* project('project_id = "stack-config-absent-image"\n', { - supabaseEnv: "SUPABASE_STORAGE_IMAGE_TRANSFORMATION_ENABLED=true\n", - }); - const absentApiRoot = yield* project('project_id = "stack-config-absent-api-field"\n'); - const explicitFalseApiRoot = yield* project( - `project_id = "stack-config-explicit-false-api-field" -[api] -auto_expose_new_tables = true -`, - { supabaseEnv: "SUPABASE_API_AUTO_EXPOSE_NEW_TABLES=false\n" }, - ); - const config = yield* load(root); - if (config.capabilities?.rest === undefined || !("settings" in config.capabilities.rest)) - throw new Error("REST settings missing"); - expect(config.capabilities.rest.settings).not.toHaveProperty("auto_expose_new_tables"); - if (config.capabilities.storage === undefined || !("settings" in config.capabilities.storage)) - throw new Error("storage settings missing"); - expect(config.capabilities.storage.settings?.image_transformation).toEqual({ enabled: true }); - expect(config.capabilities.database?.settings?.health_timeout).toBe("45s"); - expect(config.security?.jwt?.signing).toEqual({ - kind: "jwks-file", - path: "supabase/overridden-keys.json", - }); - if (config.capabilities.auth === undefined || !("settings" in config.capabilities.auth)) - throw new Error("auth settings missing"); - expect(config.capabilities.auth.settings?.signing_keys_path).toBe("overridden-keys.json"); - expect(config.capabilities.auth.settings?.external?.apple).toMatchObject({ - enabled: true, - client_id: "apple-client", - }); + expect(service(recipes, "vector")?.endpoints).toEqual({ http: { port: 55556 } }); - const absentImage = yield* load(absentImageRoot); - if ( - absentImage.capabilities?.storage === undefined || - !("settings" in absentImage.capabilities.storage) - ) - throw new Error("absent-image storage settings missing"); - expect(absentImage.capabilities.storage.settings?.image_transformation).toEqual({ - enabled: true, + const defaults = yield* load(root); + const defaultRecipes = yield* defaults.creations("automatic-ports"); + expect(service(defaultRecipes, "analytics")?.endpoints).toEqual({ + http: { port: "auto" }, }); - - const absentApi = yield* load(absentApiRoot); - if ( - absentApi.capabilities?.rest === undefined || - !("settings" in absentApi.capabilities.rest) - ) - throw new Error("absent-api REST settings missing"); - expect(absentApi.capabilities.rest.settings).not.toHaveProperty("auto_expose_new_tables"); - - const explicitFalseApi = yield* load(explicitFalseApiRoot); - if ( - explicitFalseApi.capabilities?.rest === undefined || - !("settings" in explicitFalseApi.capabilities.rest) - ) - throw new Error("explicit-false REST settings missing"); - expect(explicitFalseApi.capabilities.rest.settings).not.toHaveProperty( - "auto_expose_new_tables", - ); - }); - }); - - it.effect("rejects an invalid analytics backend environment override", () => { - return project('project_id = "stack-config-invalid-analytics-backend"\n').pipe( - Effect.flatMap((root) => - withEnvVar( - "SUPABASE_ANALYTICS_BACKEND", - "invalid-backend", - Effect.gen(function* () { - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(String(exit.cause)).toContain("analytics.backend"); - } - }), - ), - ), - ); - }); - - it.effect("honors representative API, analytics, database, and Studio overrides", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-representative-overrides" -[api] -schemas = ["config-schema"] -[db] -major_version = 15 -[studio] -openai_api_key = "config-studio-key" -`, - { - supabaseEnv: [ - "SUPABASE_API_SCHEMAS=public,storage", - "SUPABASE_ANALYTICS_BACKEND=bigquery", - "SUPABASE_ANALYTICS_GCP_PROJECT_ID=env-project", - "SUPABASE_DB_MAJOR_VERSION=17", - "SUPABASE_STUDIO_OPENAI_API_KEY=env-studio-key", - "", - ].join("\n"), - }, - ); - const config = yield* load(root); - if (config.capabilities?.rest === undefined || !("settings" in config.capabilities.rest)) - throw new Error("REST settings missing"); - if (config.capabilities?.studio === undefined || !("settings" in config.capabilities.studio)) - throw new Error("Studio settings missing"); - expect(config.capabilities.rest.settings?.schemas).toEqual(["public", "storage"]); - expect(config.capabilities.database?.version).toBe("17"); - if ( - config.capabilities.analytics === undefined || - !("settings" in config.capabilities.analytics) - ) - throw new Error("analytics settings missing"); - expect(config.capabilities.analytics.settings?.backend).toBe("bigquery"); - expect(config.capabilities.analytics.settings?.gcp_project_id).toBe("env-project"); - expect(config.capabilities.studio.settings?.openai_api_key).toBeDefined(); - if (config.capabilities.studio.settings?.openai_api_key === undefined) - throw new Error("Studio secret missing"); - expect(Redacted.value(config.capabilities.studio.settings.openai_api_key)).toBe( - "env-studio-key", - ); - }); - }); - - it.effect("creates listeners from env-only ports and leaves omitted defaults dynamic", () => { - return project('project_id = "stack-config-env-only-ports"\n').pipe( - Effect.flatMap((root) => - withEnvironment( - { SUPABASE_API_PORT: "0xD431", SUPABASE_DB_PORT: "010" }, - Effect.gen(function* () { - const overridden = yield* load(root); - expect(overridden.listeners?.api).toEqual({ port: 54321 }); - expect(overridden.listeners?.database).toEqual({ port: 8 }); - - const defaults = yield* withEnvironment( - { SUPABASE_API_PORT: undefined, SUPABASE_DB_PORT: undefined }, - load(root), - ); - expect(defaults.listeners).toEqual({}); - }), - ), - ), - ); - }); - - it.effect( - "creates an env-only pooler capability and listener without a raw db.pooler table", - () => { - return project('project_id = "stack-config-env-only-pooler"\n').pipe( - Effect.flatMap((root) => - withEnvironment( - { SUPABASE_DB_POOLER_ENABLED: "true", SUPABASE_DB_POOLER_PORT: "55450" }, - Effect.gen(function* () { - const config = yield* load(root); - if ( - config.capabilities?.pooler === undefined || - !("settings" in config.capabilities.pooler) - ) - throw new Error("pooler settings missing"); - expect(config.listeners?.pooler).toEqual({ port: 55450 }); - }), - ), - ), - ); - }, + }), ); - it.effect("defers omitted and empty pooler settings to the stack package", () => { - return Effect.gen(function* () { - const omitted = yield* project('project_id = "stack-config-omitted-pooler"\n'); - const empty = yield* project('project_id = "stack-config-empty-pooler"\n[db.pooler]\n'); - expect((yield* load(omitted)).capabilities?.pooler).toEqual({ - settings: { pool_mode: "transaction", default_pool_size: 20, max_client_conn: 100 }, - }); - expect((yield* load(empty)).capabilities?.pooler).toEqual({ - settings: { pool_mode: "transaction", default_pool_size: 20, max_client_conn: 100 }, - }); - }); - }); - - it.effect("compiles package defaults from the CLI adapter", () => { - return Effect.gen(function* () { - const root = yield* project('project_id = "stack-config-compiler-defaults"\n'); - const config = yield* load(root); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config, - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))); - expect(compiled.definition.capabilities.pooler.enabled).toBe(true); - expect(compiled.definition.capabilities.pooler.activation).toBe("lazy"); - expect(compiled.definition.capabilities.storage.settings.image_transformation?.enabled).toBe( - true, - ); - expect(compiled.definition.capabilities.storage.activation).toBe("lazy"); - expect(compiled.executionPlan.workloads.some(({ id }) => id === "storage:imgproxy")).toBe( - true, - ); - expect(compiled.executionPlan.workloads.some(({ id }) => id === "analytics:vector")).toBe( - true, - ); - }); - }); - - it.effect("keeps pooler tuning while inheriting its enabled default", () => { - return Effect.gen(function* () { - const root = yield* project( - 'project_id = "stack-config-compiler-pooler-tuning"\n[db.pooler]\ndefault_pool_size = 37\n', - ); - const config = yield* load(root); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config, - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))); - expect(compiled.definition.capabilities.pooler.enabled).toBe(true); - expect(compiled.definition.capabilities.pooler.settings.default_pool_size).toBe(37); - expect(compiled.definition.listeners.pooler.enabled).toBe(true); - }); - }); - - it.effect("honors explicit capability overrides through compilation", () => { - return Effect.gen(function* () { - const enabled = yield* project( - `project_id = "stack-config-compiler-overrides" -[db.pooler] -enabled = false -[storage.image_transformation] -enabled = false -`, - { - supabaseEnv: - "SUPABASE_DB_POOLER_ENABLED=true\nSUPABASE_STORAGE_IMAGE_TRANSFORMATION_ENABLED=true\n", - }, - ); - const disabled = yield* project('project_id = "stack-config-compiler-disabled"\n', { - supabaseEnv: - "SUPABASE_DB_POOLER_ENABLED=false\nSUPABASE_STORAGE_IMAGE_TRANSFORMATION_ENABLED=false\n", + it.live("disables and re-enables Auth through the effective environment layer", () => + Effect.gen(function* () { + const disabledRoot = yield* project('project_id = "stack-config-env-disable"\n', { + supabaseEnv: "SUPABASE_AUTH_ENABLED=false\n", }); - const enabledConfig = yield* load(enabled); - const enabledCompiled = yield* compileStack({ - projectRoot: enabled, - runtime: { kind: "native" }, - config: enabledConfig, - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))); - expect(enabledCompiled.definition.capabilities.pooler.enabled).toBe(true); + const disabled = yield* load(disabledRoot); + expect(disabled.source.auth.enabled).toBe(false); expect( - enabledCompiled.definition.capabilities.storage.settings.image_transformation?.enabled, - ).toBe(true); - expect( - enabledCompiled.executionPlan.workloads.some(({ id }) => id === "storage:imgproxy"), - ).toBe(true); - - const disabledConfig = yield* load(disabled); - const disabledCompiled = yield* compileStack({ - projectRoot: disabled, - runtime: { kind: "native" }, - config: disabledConfig, - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))); - expect(disabledCompiled.definition.capabilities.pooler.enabled).toBe(false); - expect( - disabledCompiled.definition.capabilities.storage.settings.image_transformation?.enabled, - ).toBe(false); - expect( - disabledCompiled.executionPlan.workloads.some(({ id }) => id === "storage:imgproxy"), + (yield* disabled.creations("disabled")).some(({ service }) => service === "auth"), ).toBe(false); - }); - }); - it.effect("keeps an absent raw Studio section disabled through env overrides", () => { - return project('project_id = "stack-config-absent-studio-disabled"\n').pipe( - Effect.flatMap((root) => - withEnvironment( - { SUPABASE_STUDIO_ENABLED: "false", SUPABASE_STUDIO_PORT: "55451" }, - Effect.gen(function* () { - const config = yield* load(root); - expect(config.capabilities?.studio).toEqual( - expect.objectContaining({ enabled: false }), - ); - expect(config.listeners?.studio).toEqual({ enabled: false }); - }), - ), - ), - ); - }); - - it.effect("fails a malformed env-only port override as a config error", () => { - return project('project_id = "stack-config-malformed-env-port"\n').pipe( - Effect.flatMap((root) => - withEnvVar( - "SUPABASE_DB_PORT", - "not-a-port", - Effect.gen(function* () { - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(String(exit.cause)).toContain("db.port"); - expect(String(exit.cause)).not.toContain("StackConfigError: StackConfigError"); - } - }), - ), - ), - ); - }); - - it.effect( - "decrypts config, shared function, per-function, and config env secrets into Redacted values", - () => { - const authSecret = "auth-secret-that-is-long-enough"; - const studioSecret = "studio-api-key"; - const edgeSecret = "edge-config-secret"; - const functionConfigSecret = "function-config-secret"; - const sharedSecret = "function-shared-secret"; - const localOverrideSecret = "function-local-override-secret"; - const localSecret = "function-local-secret"; - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-encrypted-values" -[auth] -jwt_secret = "${encrypted(privateKey, authSecret)}" -[studio] -openai_api_key = "${encrypted(privateKey, studioSecret)}" -[edge_runtime] -enabled = true -secrets = { EDGE_SECRET = "${encrypted(privateKey, edgeSecret)}" } -[functions.hello] -env = { CONFIG_SECRET = "env(CONFIG_FN_SECRET)" } -`, - { - supabaseEnv: `CONFIG_FN_SECRET=${encrypted(privateKey, functionConfigSecret)}\n`, - sharedFunctionEnvironment: `SHARED_SECRET=${encrypted(privateKey, sharedSecret)}\n`, - functionEnvironments: { - hello: `SHARED_SECRET=${encrypted(privateKey, localOverrideSecret)}\nLOCAL_SECRET=${encrypted(privateKey, localSecret)}\n`, - world: "", - }, - }, - ); - return yield* withEnvironment( - { - DOTENV_PRIVATE_KEY: `${wrongPrivateKey},,${privateKey}`, - DOTENV_PRIVATE_KEY_TEST: wrongPrivateKey, - }, - Effect.gen(function* () { - const config = yield* load(root); - if ( - config.capabilities?.auth === undefined || - !("settings" in config.capabilities.auth) - ) - throw new Error("auth settings missing"); - if ( - config.capabilities?.studio === undefined || - !("settings" in config.capabilities.studio) - ) - throw new Error("studio settings missing"); - if ( - config.capabilities?.functions === undefined || - !("settings" in config.capabilities.functions) - ) - throw new Error("functions settings missing"); - const authSecretValue = config.capabilities.auth.settings?.jwt_secret; - const studioSecretValue = config.capabilities.studio.settings?.openai_api_key; - const functions = config.capabilities.functions.settings?.functions; - const edgeSecrets = config.capabilities.functions.settings?.edge_runtime?.secrets; - expect(authSecretValue).toBeDefined(); - expect(studioSecretValue).toBeDefined(); - expect(functions?.hello?.env?.CONFIG_SECRET).toBeDefined(); - expect(functions?.hello?.env?.SHARED_SECRET).toBeDefined(); - expect(functions?.hello?.env?.LOCAL_SECRET).toBeDefined(); - expect(functions?.world?.env?.SHARED_SECRET).toBeDefined(); - expect(edgeSecrets?.EDGE_SECRET).toBeDefined(); - if ( - authSecretValue === undefined || - studioSecretValue === undefined || - functions?.hello?.env?.CONFIG_SECRET === undefined || - functions.hello.env.SHARED_SECRET === undefined || - functions.hello.env.LOCAL_SECRET === undefined || - functions.world?.env?.SHARED_SECRET === undefined || - edgeSecrets?.EDGE_SECRET === undefined - ) - throw new Error("encrypted settings missing"); - expect(Redacted.value(authSecretValue)).toBe(authSecret); - expect(Redacted.value(studioSecretValue)).toBe(studioSecret); - expect(Redacted.value(functions.hello.env.CONFIG_SECRET)).toBe(functionConfigSecret); - expect(Redacted.value(functions.hello.env.SHARED_SECRET)).toBe(localOverrideSecret); - expect(Redacted.value(functions.hello.env.LOCAL_SECRET)).toBe(localSecret); - expect(Redacted.value(functions.world.env.SHARED_SECRET)).toBe(sharedSecret); - expect(Redacted.value(edgeSecrets.EDGE_SECRET)).toBe(edgeSecret); - }), - ); - }); - }, - ); - - it.effect("lets a valid encrypted auth env override replace invalid file ciphertext", () => { - const plaintext = "valid-env-auth-secret-that-is-long-enough"; - const ciphertext = encrypted(privateKey, plaintext); - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-encrypted-env-wins" + const enabledRoot = yield* project( + `project_id = "stack-config-env-enable" [auth] -jwt_secret = "encrypted:invalid-file-ciphertext" +enabled = false `, ); - return yield* withEnvironment( - { DOTENV_PRIVATE_KEY: privateKey, SUPABASE_AUTH_JWT_SECRET: ciphertext }, - Effect.gen(function* () { - const config = yield* load(root); - if (config.capabilities?.auth === undefined || !("settings" in config.capabilities.auth)) - throw new Error("auth settings missing"); - const signing = config.security?.jwt?.signing; - expect(signing?.kind).toBe("symmetric"); - if (signing?.kind !== "symmetric") throw new Error("symmetric signing missing"); - expect(Redacted.value(signing.secret)).toBe(plaintext); - }), + const enabled = yield* withEnvVar("SUPABASE_AUTH_ENABLED", "true", load(enabledRoot)); + expect(enabled.source.auth.enabled).toBe(true); + expect((yield* enabled.creations("enabled")).some(({ service }) => service === "auth")).toBe( + true, ); - }); - }); - - it.effect( - "does not disclose encrypted secret, plaintext, or key on missing or wrong key errors", - () => { - const plaintext = "secret-plaintext-that-must-not-appear"; - const ciphertext = encrypted(privateKey, plaintext); - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-encrypted-error" -[auth] -jwt_secret = "${ciphertext}" -`, - ); - for (const key of [undefined, wrongPrivateKey]) { - const exit = yield* withEnvVar("DOTENV_PRIVATE_KEY", key, load(root).pipe(Effect.exit)); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const message = String(exit.cause); - expect(message).toContain("could not be decrypted"); - expect(message).not.toContain(plaintext); - expect(message).not.toContain(ciphertext); - expect(message).not.toContain(privateKey); - expect(message).not.toContain(wrongPrivateKey); - const error = Cause.findErrorOption(exit.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(StackConfigError); - } - } - }); - }, + }), ); - it.effect("does not read a disabled function's dotenv file or unresolved env reference", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-disabled-function-env" -[functions.disabled] -enabled = false -env = { TOKEN = "env(MISSING_DISABLED_FUNCTION_ENV)" } -`, - { functionEnvironments: { disabled: "lowercase=value\n!=secret-value\n" } }, + it.live("reports malformed environment ports as configuration errors", () => + Effect.gen(function* () { + const root = yield* project('project_id = "stack-config-env-invalid-port"\n'); + const exit = yield* withEnvVar("SUPABASE_API_PORT", "not-a-port", load(root)).pipe( + Effect.exit, ); - const config = yield* load(root); - if ( - config.capabilities?.functions === undefined || - !("settings" in config.capabilities.functions) - ) - throw new Error("functions settings missing"); - expect(config.capabilities.functions.settings?.functions?.disabled?.enabled).toBe(false); - }); - }); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("api.port"); + }), + ); + + it.live("rejects the existing OrioleDB environment override", () => + Effect.gen(function* () { + const root = yield* project('project_id = "stack-config-env-orioledb"\n'); + const exit = yield* withEnvVar( + "SUPABASE_EXPERIMENTAL_ORIOLEDB_VERSION", + "15.1.1.14", + load(root), + ).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) + expect(String(exit.cause)).toContain("experimental.orioledb_version"); + }), + ); }); diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index 8d16637e65..0975e12e9b 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -1,552 +1,188 @@ -import { BunPath, BunServices } from "@effect/platform-bun"; +import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, FileSystem, Exit, Layer, Option, Path, Redacted } from "effect"; +import { Effect, Exit, Layer, Schema } from "effect"; +import { ServiceCreation } from "../../../../../../packages/stack/src/services/Catalog.ts"; import { runtimeInfoLayer } from "../../../shared/runtime/runtime-info.layer.ts"; import { renderCliConfigTemplate } from "../../../shared/init/project-init.templates.ts"; -import { StackConfigError, loadStackConfig } from "../../../command-internal/stack-config.ts"; +import { loadStackConfig } from "../../../command-internal/stack-config.ts"; import { createStackConfigProject } from "../../../../tests/helpers/stack-config.ts"; const load = (projectRoot: string) => loadStackConfig(projectRoot).pipe( Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer)), ); + const project = (contents: string) => - createStackConfigProject(contents, { - sharedFunctionEnvironment: 'SHARED=shared\nOVERRIDE=shared\nQUOTED="hello # world" # comment\n', - supabaseEnv: "CONFIG_FN=config-value\n", - functionEnvironments: { - hello: "LOCAL=local\nOVERRIDE=local\n", - world: "WORLD=yes\n", - }, - functionNames: ["hello", "world", "plain", "old.backup", "_shared"], - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))); + createStackConfigProject(contents).pipe( + Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer)), + ); + +const byService = (services: ReadonlyArray>) => + new Map(services.map((service) => [service.service, service])); describe("loadStackConfig", () => { - it.effect("maps service settings, secrets, function files, and explicit ports", () => { - return Effect.gen(function* () { - const root = yield* project(` -project_id = "stack-config-test" + it.live("decodes the default recipe and leaves listeners automatic", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-defaults" +[edge_runtime] +enabled = true +`); + const config = yield* load(root); + const services = yield* config.creations("stack-defaults"); + for (const service of services) yield* Schema.decodeEffect(ServiceCreation)(service); + + const recipes = byService(services); + expect(recipes.get("database")?.endpoints).toEqual({ sql: { port: "auto" } }); + expect(recipes.get("rest")?.endpoints).toEqual({ http: { port: "auto" } }); + expect(recipes.get("analytics")?.endpoints).toEqual({ http: { port: "auto" } }); + const storage = recipes.get("storage"); + expect(storage?.service === "storage" && storage.config.fileSizeLimit).toBe("52428800"); + expect(recipes.has("vector")).toBe(true); + expect(recipes.has("functions")).toBe(true); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("preserves explicit listener ports while translating every enabled service", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-ports" [api] port = 55421 -schemas = ["public", "private"] [db] port = 55422 [db.pooler] enabled = true -pool_mode = "session" -default_pool_size = 33 -max_client_conn = 222 -[auth] -jwt_secret = "01234567890123456789012345678901" -[auth.email.smtp] -enabled = true -host = "smtp.example.test" -port = 2525 -user = "smtp-user" -pass = "smtp-secret" -admin_email = "admin@example.test" -sender_name = "Test" -[auth.external.github] +port = 55423 +[analytics] +port = 55424 +[studio] enabled = true -client_id = "client" -secret = "secret" -[auth.hook.custom_access_token] +port = 55425 +[local_smtp] enabled = true -uri = "pg-functions://custom" -[edge_runtime] +port = 55426 +smtp_port = 55427 +pop3_port = 55428 +[storage.image_transformation] enabled = true -inspector_port = 58083 -[functions.hello] -verify_jwt = false -import_map = "./functions/import_map.json" -entrypoint = "./functions/hello/index.ts" -env = { API_KEY = "env(CONFIG_FN)" } `); const config = yield* load(root); - expect(config.listeners).toMatchObject({ - api: { port: 55421 }, - database: { port: 55422 }, - functionsInspector: { port: 58083 }, - }); - expect(config.listeners?.studio).toBeUndefined(); - if (config.capabilities?.rest === undefined || !("settings" in config.capabilities.rest)) - throw new Error("REST settings missing"); - expect(config.capabilities.rest.settings?.schemas).toEqual(["public", "private"]); - if ( - config.capabilities?.functions === undefined || - !("settings" in config.capabilities.functions) - ) - throw new Error("Functions settings missing"); - expect(config.capabilities.functions.settings?.functions?.hello).toMatchObject({ - verify_jwt: false, - import_map: "../import_map.json", - entrypoint: "index.ts", - env: { - API_KEY: expect.anything(), - SHARED: expect.anything(), - LOCAL: expect.anything(), - OVERRIDE: expect.anything(), - }, + const services = yield* config.creations("stack-ports"); + const recipes = byService(services); + expect(recipes.get("database")?.endpoints).toEqual({ sql: { port: 55422 } }); + expect(recipes.get("rest")?.endpoints).toEqual({ http: { port: 55421 } }); + expect(recipes.get("analytics")?.endpoints).toEqual({ http: { port: 55424 } }); + expect(recipes.get("pooler")?.endpoints).toEqual({ + http: { port: "auto" }, + sql: { port: 55423 }, }); - expect(config.capabilities.functions.settings?.functions?.world?.env).toMatchObject({ - WORLD: expect.anything(), + expect(recipes.get("studio")?.endpoints).toEqual({ http: { port: 55425 } }); + expect(recipes.get("mail")?.endpoints).toEqual({ + http: { port: 55426 }, + smtp: { port: 55427 }, + pop3: { port: 55428 }, }); - const helloEnv = config.capabilities.functions.settings?.functions?.hello?.env; - const plainEnv = config.capabilities.functions.settings?.functions?.plain?.env; - expect(config.capabilities.functions.settings?.functions?.["old.backup"]).toBeUndefined(); - expect(config.capabilities.functions.settings?.functions?._shared).toBeUndefined(); - expect(helloEnv).toBeDefined(); - expect(plainEnv).toBeDefined(); - if (helloEnv === undefined || plainEnv === undefined) throw new Error("function env missing"); - expect(Redacted.value(helloEnv.API_KEY!)).toBe("config-value"); - expect(Redacted.value(helloEnv.OVERRIDE!)).toBe("local"); - expect(Redacted.value(plainEnv.SHARED!)).toBe("shared"); - expect(Redacted.value(plainEnv.QUOTED!)).toBe("hello # world"); - if (config.capabilities.auth === undefined || !("settings" in config.capabilities.auth)) - throw new Error("auth settings missing"); - expect(config.capabilities.auth.settings?.email?.smtp).toMatchObject({ - enabled: true, - host: "smtp.example.test", - }); - expect(config.capabilities.auth.settings?.external?.github).toMatchObject({ - enabled: true, - client_id: "client", - }); - expect(config.capabilities.auth.settings?.hook?.custom_access_token).toMatchObject({ - enabled: true, - uri: "pg-functions://custom", - }); - if (config.capabilities.pooler === undefined || !("settings" in config.capabilities.pooler)) - throw new Error("pooler settings missing"); - expect(config.capabilities.pooler.settings).toMatchObject({ - pool_mode: "session", - default_pool_size: 33, - max_client_conn: 222, - }); - expect(config.security?.jwt?.signing?.kind).toBe("symmetric"); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("rejects an enabled provider the stack cannot represent", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-figma" -[auth.external.figma] -enabled = true -client_id = "figma-client" -secret = "figma-secret" -`); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("auth.external.figma"); - }).pipe(Effect.provide(BunServices.layer)); - }); + expect(recipes.has("pgmeta")).toBe(true); + expect(recipes.has("imgproxy")).toBe(true); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.effect("accepts hosted-only settings without forwarding them to the local stack", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-hosted-only" -[api] -auto_expose_new_tables = true -[api.tls] -enabled = true -cert_path = "./server.crt" -key_path = "./server.key" -[storage.analytics] -enabled = true -max_namespaces = 9 -max_tables = 12 -max_catalogs = 4 -`); + it.live("leaves stack-opt-in init listeners automatic except disabled pooler", () => + Effect.gen(function* () { + const root = yield* project(renderCliConfigTemplate("stack-config-init", false, true)); const config = yield* load(root); - if (config.capabilities?.rest === undefined || !("settings" in config.capabilities.rest)) - throw new Error("REST settings missing"); - expect(config.capabilities.rest.settings).toMatchObject({ - schemas: ["public", "graphql_public"], - extra_search_path: ["public", "extensions"], - max_rows: 1000, - }); - expect(config.capabilities.rest.settings).not.toHaveProperty("auto_expose_new_tables"); - expect(config.capabilities.rest.settings).not.toHaveProperty("tls"); - if ( - config.capabilities?.storage === undefined || - !("settings" in config.capabilities.storage) - ) - throw new Error("Storage settings missing"); - expect(config.capabilities.storage.settings).not.toHaveProperty("analytics"); - }).pipe(Effect.provide(BunServices.layer)); - }); + const services = byService(yield* config.creations("stack-config-init")); + expect(services.get("database")?.endpoints).toEqual({ sql: { port: "auto" } }); + expect(services.get("rest")?.endpoints).toEqual({ http: { port: "auto" } }); + expect(services.get("pooler")).toBeUndefined(); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.effect("keeps disabled unsupported providers harmless", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-disabled-figma" -[auth.external.figma] + it.live("keeps the shared API port when REST is disabled", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-auth-port" +[api] enabled = false +port = 55431 `); const config = yield* load(root); - if (config.capabilities?.auth === undefined || !("settings" in config.capabilities.auth)) - throw new Error("auth settings missing"); - expect(Object.hasOwn(config.capabilities.auth.settings?.external ?? {}, "figma")).toBe(false); - }).pipe(Effect.provide(BunServices.layer)); - }); + const recipes = byService(yield* config.creations("stack-auth-port")); + expect(recipes.has("rest")).toBe(false); + expect(recipes.get("auth")?.endpoints).toEqual({ http: { port: 55431 } }); + expect(recipes.get("storage")?.endpoints).toEqual({ http: { port: 55431 } }); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.effect("rejects an unset function env reference without dropping it", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-missing-env" -[functions.hello] -env = { TOKEN = "env(SUPABASE_STACK_TEST_MISSING_ENV)" } + it.live("rejects Auth email template content paths deferred by the stack", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-auth-template" +[auth.email.template.invite] +content_path = "./templates/invite.html" `); const exit = yield* load(root).pipe(Effect.exit); expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("functions.hello.env"); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("reports unsupported dotenv keys with the file and key only", () => { - return Effect.gen(function* () { - const root = yield* project('project_id = "stack-config-invalid-env-key"\n'); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "functions", ".env"), - "lowercase=value\nSECRET=value\n", - ); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const failure = Cause.findErrorOption(exit.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) expect(failure.value).toBeInstanceOf(StackConfigError); - const message = String(exit.cause); - expect(message).toContain("functions/.env"); - expect(message).toContain("lowercase"); - expect(message).not.toContain("value"); - } - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("sanitizes malformed dotenv parser errors", () => { - return Effect.gen(function* () { - const root = yield* project('project_id = "stack-config-malformed-env"\n'); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "functions", ".env"), - "BROKEN=value\n!=secret-value\n", - ); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(String(exit.cause)).toContain("Failed to parse environment file"); - expect(String(exit.cause)).not.toContain("secret-value"); - expect(String(exit.cause)).not.toContain("StackConfigError: StackConfigError"); - } - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("preserves platform-specific absolute signing paths", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-windows-signing-path" -[auth] -signing_keys_path = 'C:\\keys\\signing.json' -`, - ); - const nativePath = yield* Path.Path; - const winPath = yield* Path.Path.pipe(Effect.provide(BunPath.layerWin32)); - const config = yield* loadStackConfig(root).pipe( - Effect.provideService(Path.Path, { ...nativePath, isAbsolute: winPath.isAbsolute }), - Effect.provide(runtimeInfoLayer), - ); - expect(config.security?.jwt?.signing).toEqual({ - kind: "jwks-file", - path: "C:\\keys\\signing.json", - }); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("rejects encrypted secrets with a targeted diagnostic", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-encrypted-secret" -[auth] -jwt_secret = "encrypted:not-a-real-ciphertext" -`, - ); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const message = String(exit.cause); - expect(message).toContain("capabilities.auth.settings.jwt_secret uses an encrypted secret"); - expect(message).not.toContain("not-a-real-ciphertext"); - } - }); - }); - - it.effect("rejects encrypted function dotenv values", () => { - return Effect.gen(function* () { - const root = yield* project('project_id = "stack-config-encrypted-function-secret"\n'); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "functions", ".env"), - "DOTENV=encrypted:dotenv\n", - ); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const message = String(exit.cause); - expect(message).toContain("uses an encrypted secret"); - expect(message).not.toContain("encrypted:dotenv"); - } - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("rejects function paths outside the function root", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-outside-function" + if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("auth.email"); + }).pipe(Effect.provide(BunServices.layer)), + ); -[functions.hello] -import_map = "./import_map.json" + it.live("rejects Auth third-party providers deferred by the stack", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-auth-third-party" +[auth.third_party.firebase] +enabled = true +project_id = "firebase-project" `); const exit = yield* load(root).pipe(Effect.exit); expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("functions.hello.import_map"); - }); - }); - - it.effect( - "rejects supabase-prefixed function paths that resolve outside the project root", - () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-nested-supabase" - -[functions.hello] -entrypoint = "supabase/functions/hello/index.ts" -`); - const exit = yield* load(root).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) - expect(String(exit.cause)).toContain("functions.hello.entrypoint"); - }); - }, + if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("auth.third_party"); + }).pipe(Effect.provide(BunServices.layer)), ); - it.effect("resolves supabase-prefixed signing paths beneath the config directory", () => { - return Effect.gen(function* () { - const root = yield* project( - `project_id = "stack-config-signing-path" -[auth] -signing_keys_path = "supabase/signing-keys.json" -`, - ); - const config = yield* load(root); - expect(config.security?.jwt?.signing).toEqual({ - kind: "jwks-file", - path: "supabase/supabase/signing-keys.json", - }); - }); - }); - - it.effect("lets the stack runtime resolve the API port for Studio's default URL", () => { - return Effect.gen(function* () { - const root = yield* project('project_id = "stack-config-studio-default"\n'); - const config = yield* load(root); - if (config.capabilities?.studio === undefined || !("settings" in config.capabilities.studio)) - throw new Error("Studio settings missing"); - expect(config.capabilities.studio.settings).toEqual({ - api_url: undefined, - openai_api_key: undefined, - }); - }); - }); - - it.effect("parses legacy dotenv expansion and colon assignments", () => { - return Effect.gen(function* () { - const root = yield* project('project_id = "stack-config-dotenv-compat"\n'); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "functions", ".env"), - "BASE=shared\nEXPANDED=$BASE\nCOLON: colon-value\n", - ); - const config = yield* load(root); - if ( - config.capabilities?.functions === undefined || - !("settings" in config.capabilities.functions) - ) - throw new Error("Functions settings missing"); - const env = config.capabilities.functions.settings?.functions?.hello?.env; - expect(env).toBeDefined(); - if (env === undefined) throw new Error("Function env missing"); - expect(Redacted.value(env.EXPANDED!)).toBe("shared"); - expect(Redacted.value(env.COLON!)).toBe("colon-value"); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("keeps the gateway listener when API service is disabled for auth", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-gateway" -[api] -enabled = false -port = 55430 -[auth] -enabled = true -`); - const config = yield* load(root); - expect(config.listeners?.api).toEqual({ port: 55430 }); - }); - }); - - it.effect("keeps the gateway listener when analytics is enabled", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-analytics-gateway" -[api] -enabled = false -port = 55431 -[auth] -enabled = false -[realtime] -enabled = false -[storage] -enabled = false -[edge_runtime] -enabled = false + it.live("rejects an analytics backend the stack cannot represent", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-analytics" [analytics] enabled = true +backend = "bigquery" `); - const config = yield* load(root); - expect(config.listeners?.api).toEqual({ port: 55431 }); - }); - }); - - it.effect("keeps nested settings on a disabled functions capability", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-disabled-functions" -[edge_runtime] -enabled = false -`); - const config = yield* load(root); - expect(config.capabilities?.functions).toEqual({ enabled: false }); - }); - }); - - it.effect("leaves listeners absent when ports are omitted", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-defaults" -[edge_runtime] -enabled = true -`); - const config = yield* load(root); - expect(config.listeners).toEqual({}); - expect(config.listeners?.functionsInspector).toBeUndefined(); - if (config.capabilities?.database !== undefined && "settings" in config.capabilities.database) - expect(config.capabilities.database.settings?.health_timeout).toBe("2m"); - }); - }); - - it.effect("loads the actual initialized stack config template", () => { - return Effect.gen(function* () { - const root = yield* project(renderCliConfigTemplate("stack-config-init", false)); - const config = yield* load(root); - expect(config.listeners?.api).toEqual({ port: 54321 }); - expect(config.listeners?.database).toEqual({ port: 54322 }); - expect(config.listeners?.pooler).toEqual({ enabled: false }); - expect(config.listeners?.smtp).toBeUndefined(); - expect(config.listeners?.pop3).toBeUndefined(); - expect(config.listeners?.functionsInspector).toEqual({ port: 8083 }); - }); - }); - - it.effect("leaves stack-opt-in init listeners automatic except disabled pooler", () => { - return Effect.gen(function* () { - const root = yield* project(renderCliConfigTemplate("stack-config-init", false, true)); - const config = yield* load(root); - expect(config.listeners).toEqual({ pooler: { enabled: false } }); - }); - }); + const exit = yield* load(root).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) + expect(String(exit.cause)).toContain("analytics.backend must be postgres"); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.effect("ignores unresolved function env references when edge runtime is disabled", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-disabled-functions-env" -[edge_runtime] -enabled = false -[functions.hello] -env = { TOKEN = "env(SUPABASE_STACK_TEST_DISABLED_MISSING_ENV)" } + it.live("rejects unsupported database versions before creating recipes", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-db-version" +[db] +major_version = 14 `); - const config = yield* load(root); - expect(config.capabilities?.functions).toEqual({ enabled: false }); - }); - }); + const exit = yield* load(root).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) + expect(String(exit.cause)).toContain("db.major_version must be 15 or 17"); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.effect("does not read disabled edge runtime dotenv files", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-disabled-dotenv" -[edge_runtime] -enabled = false + it.live("rejects unsupported OrioleDB and experimental S3 settings", () => + Effect.gen(function* () { + const orioledb = yield* project(`project_id = "stack-config-orioledb" +[experimental] +orioledb_version = "15.1.1.14" `); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "functions", ".env"), - "lowercase=value\n", - ); - const config = yield* load(root); - expect(config.capabilities?.functions).toEqual({ enabled: false }); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("accepts the initialized disabled service listeners with explicit ports", () => { - return Effect.gen(function* () { - const root = yield* project(`project_id = "stack-config-disabled-listeners" -[api] -enabled = false -port = 55431 -[auth] -enabled = false -[realtime] -enabled = false -[storage] -enabled = false -[db] -port = 55432 -[db.pooler] -enabled = false -port = 55433 -[studio] -enabled = false -port = 55434 -[local_smtp] -enabled = false -port = 55435 -smtp_port = 55436 -pop3_port = 55437 -[edge_runtime] -enabled = false -inspector_port = 55438 -[analytics] -enabled = false + const orioledbExit = yield* load(orioledb).pipe(Effect.exit); + expect(Exit.isFailure(orioledbExit)).toBe(true); + if (Exit.isFailure(orioledbExit)) + expect(String(orioledbExit.cause)).toContain("experimental.orioledb_version"); + + const s3 = yield* project(`project_id = "stack-config-experimental-s3" +[experimental] +s3_host = "s3.example.test" `); - const config = yield* load(root); - expect(config.listeners).toEqual({ - api: { enabled: false }, - database: { port: 55432 }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }); - }); - }); - - it.effect("uses default stack settings when no project config exists", () => { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-config-" }); - const config = yield* loadStackConfig(root); - expect(config.listeners).toEqual({}); - expect(config.capabilities?.database).toMatchObject({ settings: { health_timeout: "2m" } }); - }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer))); - }); + const s3Exit = yield* load(s3).pipe(Effect.exit); + expect(Exit.isFailure(s3Exit)).toBe(true); + if (Exit.isFailure(s3Exit)) expect(String(s3Exit.cause)).toContain("experimental.s3_host"); + }).pipe(Effect.provide(BunServices.layer)), + ); }); diff --git a/apps/cli/src/commands/experimental/stack/stack-endpoints.format.ts b/apps/cli/src/commands/experimental/stack/stack-endpoints.format.ts new file mode 100644 index 0000000000..869d6b8ee1 --- /dev/null +++ b/apps/cli/src/commands/experimental/stack/stack-endpoints.format.ts @@ -0,0 +1,17 @@ +import type { Observation } from "@supabase/stack/effect"; + +const endpointUrl = (endpoint: Observation["endpoints"][number]) => + `${endpoint.protocol}://${endpoint.host}:${endpoint.port}`; + +export const endpointReports = (observation: Observation | undefined) => + Object.fromEntries( + (observation?.endpoints ?? []).map((endpoint) => [ + endpoint.name, + { + protocol: endpoint.protocol, + address: endpoint.host, + port: endpoint.port, + url: endpointUrl(endpoint), + }, + ]), + ); diff --git a/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts new file mode 100644 index 0000000000..caa05f621a --- /dev/null +++ b/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts @@ -0,0 +1,347 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Redacted, Schema } from "effect"; +import { ServiceCreation } from "../../../../../../packages/stack/src/services/Catalog.ts"; +import { makeSpec as authSpec } from "../../../../../../packages/stack/src/services/Auth.ts"; +import { makeSpec as restSpec } from "../../../../../../packages/stack/src/services/Rest.ts"; +import { makeSpec as poolerSpec } from "../../../../../../packages/stack/src/services/Pooler.ts"; +import { makeSpec as realtimeSpec } from "../../../../../../packages/stack/src/services/Realtime.ts"; +import { makeSpec as storageSpec } from "../../../../../../packages/stack/src/services/Storage.ts"; +import { makeSpec as studioSpec } from "../../../../../../packages/stack/src/services/Studio.ts"; +import { loadStackConfig } from "../../../command-internal/stack-config.ts"; +import { runtimeInfoLayer } from "../../../shared/runtime/runtime-info.layer.ts"; +import { createStackConfigProject } from "../../../../tests/helpers/stack-config.ts"; + +const layer = Layer.merge(BunServices.layer, runtimeInfoLayer); + +describe("stack service configuration", () => { + it.live("carries editable TOML settings into native and container service environments", () => + Effect.gen(function* () { + const root = yield* createStackConfigProject(`project_id = "forwarding" +[api] +extra_search_path = ["public", "extensions", "custom"] +[db] +health_timeout = "12.5s" +root_key = "${"a".repeat(64)}" +[db.pooler] +enabled = true +default_pool_size = 7 +max_client_conn = 42 +[realtime] +ip_version = "IPv6" +max_header_length = 8192 +[studio] +api_url = "https://public.example.test" +openai_api_key = "test-openai-key" +[storage.s3_protocol] +enabled = false +[storage.vector] +enabled = false +max_buckets = 3 +max_indexes = 2 +[analytics] +vector_port = 59001 +`); + const loaded = yield* loadStackConfig(root); + const services = yield* loaded.creations("forwarding"); + const database = services.find((s) => s.service === "database"); + expect(database?.config.healthTimeoutMs).toBe(12500); + expect(database?.config.rootKey && Redacted.value(database.config.rootKey)).toBe( + "a".repeat(64), + ); + expect(services.find((s) => s.service === "vector")?.endpoints?.http?.port).toBe(59001); + for (const container of [false, true]) { + for (const service of services) { + switch (service.service) { + case "rest": + expect(yield* restSpec().env(service, new Map(), container)).toMatchObject({ + PGRST_DB_EXTRA_SEARCH_PATH: "public,extensions,custom", + }); + break; + case "pooler": + expect(yield* poolerSpec().env(service, new Map(), container)).toMatchObject({ + TENANT_ID: "pooler-dev", + DEFAULT_POOL_SIZE: "7", + MAX_CLIENT_CONN: "42", + }); + break; + case "realtime": + expect(yield* realtimeSpec().env(service, new Map(), container)).toMatchObject({ + MAX_HEADER_LENGTH: "8192", + ERL_AFLAGS: "-proto_dist inet6_tcp", + }); + break; + case "storage": + expect(yield* storageSpec().env(service, new Map(), container)).toMatchObject({ + S3_PROTOCOL_ENABLED: "false", + VECTOR_ENABLED: "false", + VECTOR_MAX_BUCKETS: "3", + VECTOR_MAX_INDEXES: "2", + }); + break; + case "studio": + expect(yield* studioSpec().env(service, new Map(), container)).toMatchObject({ + SUPABASE_PUBLIC_URL: "https://public.example.test", + OPENAI_API_KEY: "test-openai-key", + }); + break; + } + } + } + }).pipe(Effect.provide(layer)), + ); + + it.live( + "resolves per-function configuration and shared secrets without enabling the inspector", + () => + Effect.gen(function* () { + const root = yield* createStackConfigProject( + `project_id = "functions-forwarding" +[edge_runtime] +inspector_port = 59229 +[edge_runtime.secrets] +SHARED = "shared-secret" +[functions.hello] +verify_jwt = false +entrypoint = "functions/hello/main.ts" +import_map = "import_map.json" +static_files = ["functions/hello/*.txt"] +[functions.hello.env] +LOCAL = "env(FUNCTION_VALUE)" +[functions.disabled] +enabled = false +`, + { rootEnv: "FUNCTION_VALUE=function-value\n" }, + ); + const fs = yield* FileSystem.FileSystem; + yield* fs.writeFileString( + `${root}/supabase/functions/hello/main.ts`, + 'Deno.serve(() => new Response("hello"));', + ); + yield* fs.writeFileString(`${root}/supabase/import_map.json`, "{}"); + const config = yield* loadStackConfig(root); + const services = yield* config.creations("functions-forwarding"); + const functions = services.find((service) => service.service === "functions"); + expect(functions?.config).toMatchObject({ + filesRoot: root, + env: { SHARED: "shared-secret" }, + functions: { + hello: { + verifyJWT: false, + entrypoint: `${root}/supabase/functions/hello/main.ts`, + import_map: `${root}/supabase/import_map.json`, + static_files: [`${root}/supabase/functions/hello/*.txt`], + env: { LOCAL: "function-value" }, + }, + disabled: { enabled: false }, + }, + }); + expect(functions?.config.inspector).not.toBe(true); + expect(functions?.endpoints?.inspector?.port).toBe(59229); + }).pipe(Effect.provide(layer)), + ); + it.live( + "decrypts a configured signing secret before deriving service credentials and reopening", + () => + Effect.gen(function* () { + const plaintext = "test-jwt-secret-with-more-than-32-characters"; + const root = yield* createStackConfigProject( + `project_id = "encrypted-jwt" +[auth] +jwt_secret = "encrypted:BOsrXIZY2BNTW43BeRhMbfvlOIUjwI7GCyFHxJD/Ik+UQ4mqkgVl2+61WWhEf3+8SEDngaEMZnSWajCMCInbHJbRnH+C1xgcAZlWKR0qLcHanvkM+zDKWxcQgMbN5AmOqwn3olCjpHbqkSzoyPri015szpcZMp5JKGmUsw6KEwTFE7LyQwRlTbqlVn7u" +`, + { + rootEnv: + "DOTENV_PRIVATE_KEY=7fd7210cef8f331ee8c55897996aaaafd853a2b20a4dc73d6d75759f65d2a7eb\n", + }, + ); + const config = yield* loadStackConfig(root); + expect(Redacted.value(config.jwtSecret)).toBe(plaintext); + const services = yield* config.creations("encrypted-jwt", { + jwtSecret: Redacted.make(plaintext), + }); + expect(services.find((service) => service.service === "auth")?.config.jwtSecret).toBe( + plaintext, + ); + }).pipe(Effect.provide(layer)), + ); + + it.live("reopens with an empty configured JWT secret and omits an empty database root key", () => + Effect.gen(function* () { + const root = yield* createStackConfigProject(`project_id = "empty-secrets" +[auth] +jwt_secret = "" +[db] +root_key = "" +`); + const config = yield* loadStackConfig(root); + const creations = yield* config.creations("empty-secrets", { jwtSecret: config.jwtSecret }); + const database = creations.find((creation) => creation.service === "database"); + expect(database).toBeDefined(); + expect(database?.config.rootKey).toBeUndefined(); + }).pipe(Effect.provide(layer)), + ); + + it.live("forwards Auth policies, providers and custom SMTP ahead of local mail", () => + Effect.gen(function* () { + const root = yield* createStackConfigProject( + `project_id = "auth-forwarding" +[auth] +additional_redirect_urls = ["https://app.example.test/callback"] +enable_refresh_token_rotation = false +refresh_token_reuse_interval = 15 +enable_manual_linking = true +enable_anonymous_sign_ins = true +minimum_password_length = 12 +password_requirements = "letters_digits" +jwt_issuer = "https://issuer.example.test" +[auth.rate_limit] +email_sent = 55 +sms_sent = 44 +sign_in_sign_ups = 33 +[auth.email] +enable_confirmations = true +double_confirm_changes = false +secure_password_change = true +otp_length = 8 +otp_expiry = 120 +max_frequency = "10s" +[auth.email.smtp] +enabled = true +host = "smtp.example.test" +port = 2525 +user = "test-user" +pass = "env(SMTP_SECRET)" +admin_email = "admin@example.test" +sender_name = "Custom sender" +[auth.email.template.confirmation] +subject = "Confirm your account" +[auth.email.notification.password_changed] +enabled = true +subject = "Password changed" +[auth.sms] +enable_signup = true +otp_length = 7 +otp_expiry = 180 +[auth.sms.test_otp] +"12345678901" = "1234567" +[auth.sms.twilio] +enabled = true +account_sid = "AC123" +auth_token = "env(SMS_SECRET)" +message_service_sid = "MG123" +content_sid = "HX123" +[auth.captcha] +enabled = true +provider = "hcaptcha" +secret = "env(CAPTCHA_SECRET)" +[auth.hook.custom_access_token] +enabled = true +uri = "pg-functions://postgres/public/custom_access_token" +[auth.mfa.totp] +enroll_enabled = true +verify_enabled = true +[auth.mfa.phone] +enroll_enabled = true +verify_enabled = true +otp_length = 7 +max_frequency = "10s" +[auth.sessions] +timebox = "24h" +inactivity_timeout = "1h" +[auth.external.github] +enabled = true +client_id = "github-client" +secret = "env(GITHUB_SECRET)" +[auth.web3.solana] +enabled = true +[auth.oauth_server] +enabled = true +authorization_url_path = "/oauth/consent" +allow_dynamic_registration = true +`, + { + rootEnv: + "SMTP_SECRET=smtp-secret\nSMS_SECRET=sms-secret\nCAPTCHA_SECRET=captcha-secret\nGITHUB_SECRET=github-secret\n", + }, + ); + const config = yield* loadStackConfig(root); + const services = yield* config.creations("auth-forwarding"); + const auth = services.find((service) => service.service === "auth"); + expect(auth).toBeDefined(); + if (auth === undefined) return; + const decoded = yield* Schema.decodeEffect(ServiceCreation)(auth); + expect(decoded).toEqual(auth); + if (decoded.service !== "auth") return; + expect(decoded.config.settings?.email?.template.confirmation?.subject).toBe( + "Confirm your account", + ); + expect(decoded.config.settings?.email?.notification.password_changed?.subject).toBe( + "Password changed", + ); + expect(decoded.config.smtp).toMatchObject({ + host: "smtp.example.test", + port: 2525, + adminEmail: "admin@example.test", + }); + const env = yield* authSpec().env( + { + ...auth, + config: { + ...auth.config, + smtpUrl: "smtp://127.0.0.1:1025", + externalApiUrl: "http://localhost:54321/auth/v1", + }, + }, + new Map(), + false, + ); + expect(env).toMatchObject({ + GOTRUE_URI_ALLOW_LIST: "https://app.example.test/callback", + GOTRUE_SECURITY_REFRESH_TOKEN_ROTATION_ENABLED: "false", + GOTRUE_SECURITY_REFRESH_TOKEN_REUSE_INTERVAL: "15", + GOTRUE_SECURITY_MANUAL_LINKING_ENABLED: "true", + GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: "true", + GOTRUE_PASSWORD_MIN_LENGTH: "12", + GOTRUE_PASSWORD_REQUIRED_CHARACTERS: + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ:0123456789", + GOTRUE_JWT_ISSUER: "https://issuer.example.test", + GOTRUE_RATE_LIMIT_EMAIL_SENT: "55", + GOTRUE_RATE_LIMIT_SMS_SENT: "44", + GOTRUE_RATE_LIMIT_OTP: "33", + GOTRUE_MAILER_AUTOCONFIRM: "false", + GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: "false", + GOTRUE_SECURITY_UPDATE_PASSWORD_REQUIRE_REAUTHENTICATION: "true", + GOTRUE_MAILER_OTP_LENGTH: "8", + GOTRUE_MAILER_OTP_EXP: "120", + GOTRUE_SMTP_MAX_FREQUENCY: "10s", + GOTRUE_SMTP_HOST: "smtp.example.test", + GOTRUE_SMTP_PORT: "2525", + GOTRUE_SMTP_PASS: "smtp-secret", + GOTRUE_SMTP_SENDER_NAME: "Custom sender", + GOTRUE_MAILER_SUBJECTS_CONFIRMATION: "Confirm your account", + GOTRUE_MAILER_NOTIFICATIONS_PASSWORD_CHANGED_ENABLED: "true", + GOTRUE_MAILER_SUBJECTS_PASSWORD_CHANGED_NOTIFICATION: "Password changed", + GOTRUE_SMS_OTP_LENGTH: "7", + GOTRUE_SMS_OTP_EXP: "180", + GOTRUE_SMS_TEST_OTP: "12345678901:1234567", + GOTRUE_SMS_PROVIDER: "twilio", + GOTRUE_SMS_TWILIO_AUTH_TOKEN: "sms-secret", + GOTRUE_SMS_TWILIO_CONTENT_SID: "HX123", + GOTRUE_SECURITY_CAPTCHA_SECRET: "captcha-secret", + GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED: "true", + GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_URI: "pg-functions://postgres/public/custom_access_token", + GOTRUE_MFA_TOTP_ENROLL_ENABLED: "true", + GOTRUE_MFA_PHONE_OTP_LENGTH: "7", + GOTRUE_MFA_PHONE_MAX_FREQUENCY: "10s", + GOTRUE_SESSIONS_TIMEBOX: "24h", + GOTRUE_SESSIONS_INACTIVITY_TIMEOUT: "1h", + GOTRUE_EXTERNAL_GITHUB_CLIENT_ID: "github-client", + GOTRUE_EXTERNAL_GITHUB_SECRET: "github-secret", + GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI: "http://localhost:54321/auth/v1/callback", + GOTRUE_EXTERNAL_WEB3_SOLANA_ENABLED: "true", + GOTRUE_OAUTH_SERVER_AUTHORIZATION_PATH: "/oauth/consent", + }); + }).pipe(Effect.provide(layer)), + ); +}); diff --git a/apps/cli/src/commands/experimental/stack/stack.command.ts b/apps/cli/src/commands/experimental/stack/stack.command.ts index 267cd01b0b..28f3cbf6eb 100644 --- a/apps/cli/src/commands/experimental/stack/stack.command.ts +++ b/apps/cli/src/commands/experimental/stack/stack.command.ts @@ -1,4 +1,4 @@ -import { Layer } from "effect"; +import { Effect, Layer } from "effect"; import { Command } from "effect/unstable/cli"; import { commandRuntimeLayer } from "../../../shared/runtime/command-runtime.layer.ts"; import { commandSettingsLayer } from "../../../config/command-settings.layer.ts"; @@ -19,13 +19,22 @@ import { stackListCommand as stackListCommandBase } from "./list/list.command.ts import { stackRestartCommand as stackRestartCommandBase } from "./restart/restart.command.ts"; import { stackPrepareCommand as stackPrepareCommandBase } from "./prepare/prepare.command.ts"; import { stackApiLayer, stackTargetResolverLayer } from "./stack.shared.ts"; +import { configureLoopbackProxyBypass } from "../../../command-internal/hostname.ts"; + +const stackCommandSettings = commandSettingsLayer.pipe(Layer.provide(debugLoggerLayer)); +const stackTarget = stackTargetResolverLayer.pipe( + Layer.provideMerge(stackApiLayer), + Layer.provideMerge(stackCommandSettings), +); +const stackLoopbackProxyLayer = Layer.effectDiscard( + Effect.sync(() => configureLoopbackProxyBypass()), +); export const stackRuntimeLayer = Layer.mergeAll( - stackTargetResolverLayer, - stackApiLayer, + stackLoopbackProxyLayer, + stackTarget, dbConnectionLayer, stackCatalogSetupLayer, - commandSettingsLayer.pipe(Layer.provide(debugLoggerLayer)), telemetryStateLayer, ); diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts new file mode 100644 index 0000000000..90eca5507a --- /dev/null +++ b/apps/cli/src/commands/experimental/stack/stack.shared.integration.test.ts @@ -0,0 +1,165 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Layer, Option, Path } from "effect"; +import { resolveStackIdentity } from "@supabase/stack/internal/identity"; +import { + StackApi, + StackTargetError, + StackTargetResolver, + stackApiLayer, + stackTargetResolverLayer, +} from "./stack.shared.ts"; +import { mockCommandSettings } from "../../../../tests/helpers/command-mocks.ts"; + +type TargetInput = { + readonly projectRoot: string; + readonly name?: string; + readonly id?: string; + readonly runtime: "auto" | "docker" | "native"; +}; + +type DiscoveredStack = Effect.Success< + ReturnType +>[number]["definition"]; + +const stack = ( + id: string, + identity: DiscoveredStack["identity"], + runtime: DiscoveredStack["runtime"] = "native", +): DiscoveredStack => ({ + id, + identity, + runtime, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], +}); + +const resolverLayer = ( + discovered: ReadonlyArray, + workdir: string, + supabaseHome: string, +) => { + const api = Layer.succeed(StackApi, { + create: () => Effect.die("unused"), + open: () => Effect.die("unused"), + discover: () => + Effect.succeed(discovered.map((definition) => ({ definition, host: undefined }))), + resolveIdentity: (options) => + resolveStackIdentity(options).pipe(Effect.provide(BunServices.layer)), + }); + return stackTargetResolverLayer.pipe( + Layer.provide(mockCommandSettings({ workdir, supabaseHome })), + Layer.provide(api), + Layer.provide(BunServices.layer), + ); +}; + +const resolveTarget = (layer: Layer.Layer, input: TargetInput) => + Effect.gen(function* () { + const resolver = yield* StackTargetResolver; + return yield* resolver.resolve(input); + }).pipe(Effect.provide(layer)); + +describe("stack target resolver", () => { + it.live("resolves an explicit id without reading the current project identity", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-target-id-" }); + const storedRoot = path.join(root, "stored-project"); + const id = "a".repeat(64); + const target = yield* resolveTarget( + resolverLayer( + [stack(id, { projectRoot: storedRoot, branchContext: "main", stackName: "default" })], + root, + path.join(root, ".supabase"), + ), + { projectRoot: path.join(root, "missing-current-project"), id, runtime: "auto" }, + ); + expect(target).toEqual({ projectRoot: storedRoot, id, runtime: "native" }); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("rejects an explicit id when its saved runtime differs", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-target-runtime-" }); + const id = "b".repeat(64); + const exit = yield* resolveTarget( + resolverLayer( + [stack(id, { projectRoot: root, branchContext: "main", stackName: "default" }, "docker")], + root, + path.join(root, ".supabase"), + ), + { projectRoot: root, id, runtime: "native" }, + ).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + const failure = Exit.findErrorOption(exit); + expect(Option.isSome(failure)).toBe(true); + if (Option.isSome(failure)) { + expect(failure.value).toBeInstanceOf(StackTargetError); + if (failure.value instanceof StackTargetError) { + expect(failure.value.reason).toBe("flags"); + expect(failure.value.message).toContain("Requested runtime native"); + expect(failure.value.message).toContain("existing stack runtime docker"); + expect(failure.value.suggestion).toContain("--runtime auto"); + expect(failure.value.suggestion).toContain("different --stack name"); + } + } + } + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("selects a saved stack by the package identity tuple", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-target-identity-" }); + const canonicalRoot = yield* fs.realPath(root); + const identity = yield* resolveStackIdentity({ projectRoot: root, name: "feature" }).pipe( + Effect.provide(BunServices.layer), + ); + const id = "c".repeat(64); + const target = yield* resolveTarget( + resolverLayer( + [stack("d".repeat(64), { ...identity, branchContext: "other" }), stack(id, identity)], + root, + path.join(root, ".supabase"), + ), + { projectRoot: root, name: "feature", runtime: "auto" }, + ); + expect(target).toMatchObject({ projectRoot: canonicalRoot, id, name: "feature" }); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); + +describe("stack API layer", () => { + it.live("creates, discovers, and opens state without starting an owner", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-api-layer-" }); + const api = yield* StackApi.pipe(Effect.provide(stackApiLayer)); + const locations = { + projectRoot: root, + stateRoot: path.join(root, ".supabase", "stacks"), + cacheRoot: path.join(root, ".supabase", "cache"), + runtime: "native" as const, + }; + const created = yield* api.create(locations); + const discovered = yield* api.discover({ stateRoot: locations.stateRoot }); + const opened = yield* api.open({ + stateRoot: locations.stateRoot, + cacheRoot: locations.cacheRoot, + id: created.id, + }); + expect(discovered).toHaveLength(1); + expect(discovered[0]?.definition.id).toBe(created.id); + expect(discovered[0]?.host).toBeUndefined(); + expect(opened.id).toBe(created.id); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); diff --git a/apps/cli/src/commands/experimental/stack/stack.shared.ts b/apps/cli/src/commands/experimental/stack/stack.shared.ts index 1d1fe94f0d..75582d3809 100644 --- a/apps/cli/src/commands/experimental/stack/stack.shared.ts +++ b/apps/cli/src/commands/experimental/stack/stack.shared.ts @@ -1,11 +1,6 @@ -import { Context, Data, Effect, Layer, Option } from "effect"; -import { - isStackId, - StackNotFoundError, - type StackRuntimePreference, - type StackStatus, -} from "@supabase/stack/effect"; -import type { StackId } from "@supabase/stack"; +import type { ServiceCreation } from "@supabase/stack/effect"; +import { Context, Data, Effect, Layer, Option, Path } from "effect"; +import { CommandSettings } from "../../../config/command-settings.service.ts"; import { actionability, type CliErrorActionabilityDeclaration, @@ -15,12 +10,15 @@ import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts" export { StackApi, stackApiLayer }; +type StackId = string; +const isStackId = (id: string): boolean => /^[0-9a-f]{64}$/u.test(id); + /** The target selected by the CLI adapter for one stack command. */ -interface StackTarget { +export interface StackTarget { readonly projectRoot: string; readonly id?: StackId; readonly name?: string; - readonly runtime?: StackRuntimePreference; + readonly runtime?: "native" | "docker" | "podman"; } export class StackTargetError extends Data.TaggedError("ExperimentalStackTargetError")<{ @@ -34,17 +32,13 @@ export class StackTargetError extends Data.TaggedError("ExperimentalStackTargetE } } -/** - * Configuration and targeting are supplied by the CLI adapter so later stack - * commands can reuse the same project, name, id, and environment rules. - */ interface StackTargetResolverShape { readonly resolve: (input: { readonly projectRoot: string; readonly name?: string; readonly id?: string; readonly runtime: "auto" | "docker" | "native"; - }) => Effect.Effect; + }) => Effect.Effect; } export class StackTargetResolver extends Context.Service< @@ -56,16 +50,16 @@ export const validateStackTarget = (input: { readonly stack?: string; readonly stackId?: string; }): Effect.Effect => - Effect.gen(function* () { - if (input.stack !== undefined && input.stackId !== undefined) { - return yield* new StackTargetError({ - message: "--stack and --stack-id cannot be used together", - reason: "flags", - }); - } - }); + input.stack !== undefined && input.stackId !== undefined + ? Effect.fail( + new StackTargetError({ + message: "--stack and --stack-id cannot be used together", + reason: "flags", + }), + ) + : Effect.void; -export const validateStackId = (id: string): Effect.Effect => +const validateStackId = (id: string): Effect.Effect => isStackId(id) ? Effect.succeed(id) : Effect.fail( @@ -89,107 +83,109 @@ export const rejectStackOutput = ( ) : Effect.void; -export const stackStatusPayload = (status: StackStatus) => ({ - id: status.id, - lifecycle: status.lifecycle, - desired_lifecycle: status.desiredLifecycle, - runtime: status.runtime, - endpoints: status.endpoints, - versions: status.versions, - capabilities: status.capabilities, - artifacts: status.artifacts, - ...(status.recovery === undefined ? {} : { recovery: status.recovery }), -}); - -export const stackStatusIssueLines = (status: StackStatus): ReadonlyArray => { - const lines: Array = []; - const diagnostics = status.capabilities.filter(({ error }) => error !== undefined); - if (diagnostics.length > 0) { - lines.push("Capability diagnostics:"); - for (const capability of diagnostics) { - const detail = capability.error?.split(/\r?\n/u)[0] ?? "No diagnostic was recorded."; - lines.push(` ${capability.name}: ${capability.state} — ${detail}`); - } - } - if (status.recovery !== undefined) { - lines.push(`Recovery: ${status.recovery.message}`); - if (status.recovery.operation === "stop") { - lines.push( - `Recovery command: supabase stack stop --stack-id ${status.id} && supabase stack start --stack-id ${status.id}`, - ); - } else { - lines.push(`Recovery command: supabase stack destroy --stack-id ${status.id}`); - lines.push("Warning: destroy is destructive and removes the stack data."); - } - } - return lines; -}; +const runtimeForFlag = ( + runtime: "auto" | "docker" | "native", +): StackTarget["runtime"] | undefined => + runtime === "auto" ? undefined : runtime === "docker" ? "docker" : "native"; -export const renderStackStatus = (status: StackStatus): string => { - const lines = [ - `Stack ${status.id}`, - `Runtime: ${status.runtime.kind}`, - `Lifecycle: ${status.lifecycle}`, - ]; - const endpoints = Object.entries(status.endpoints); - if (endpoints.length > 0) { - lines.push("Endpoints:"); - for (const [name, endpoint] of endpoints) - if (endpoint !== undefined) lines.push(` ${name}: ${endpoint.url}`); - } - const dormant = status.capabilities.filter(({ state }) => state === "dormant"); - if (dormant.length > 0) - lines.push(`Dormant capabilities: ${dormant.map(({ name }) => name).join(", ")}`); - lines.push(...stackStatusIssueLines(status)); - return `${lines.join("\n")}\n`; -}; +const runtimeMatches = ( + saved: StackTarget["runtime"], + requested: StackTarget["runtime"], +): boolean => requested === undefined || saved === requested; -/** Runtime configuration for the first stack command. Later commands reuse this layer. */ -export const stackTargetResolverLayer = Layer.succeed(StackTargetResolver, { - resolve: (input) => - Effect.gen(function* () { +/** Resolves an existing stack by its persisted canonical package identity. */ +export const stackTargetResolverLayer = Layer.effect( + StackTargetResolver, + Effect.gen(function* () { + const settings = yield* CommandSettings; + const stackApi = yield* StackApi; + const path = yield* Path.Path; + const resolve = Effect.fn("StackTargetResolver.resolve")(function* (input: { + readonly projectRoot: string; + readonly name?: string; + readonly id?: string; + readonly runtime: "auto" | "docker" | "native"; + }) { const id = input.id === undefined ? undefined : yield* validateStackId(input.id); - const stackApi = yield* StackApi; - const inspection = + const requestedRuntime = runtimeForFlag(input.runtime); + const identity = + id === undefined + ? yield* stackApi + .resolveIdentity({ + projectRoot: input.projectRoot, + ...(input.name === undefined ? {} : { name: input.name }), + }) + .pipe( + Effect.mapError( + (cause) => + new StackTargetError({ + message: cause.message, + reason: "invalid-config", + cause, + }), + ), + ) + : undefined; + const discovered = yield* stackApi + .discover({ stateRoot: path.join(settings.supabaseHome, "stacks") }) + .pipe( + Effect.mapError( + (cause) => + new StackTargetError({ + message: cause.message, + reason: "invalid-config", + cause, + }), + ), + ); + const found = id === undefined - ? undefined - : yield* stackApi.inspectStack(id).pipe( - Effect.mapError( - (error) => - new StackTargetError({ - message: `Unable to inspect stack ${id}: ${error.message}`, - reason: error instanceof StackNotFoundError ? "flags" : "invalid-config", - cause: error, - }), - ), - ); - const projectRoot = inspection?.descriptor.projectRoot ?? input.projectRoot; - const requestedRuntime = - input.runtime === "auto" - ? undefined - : input.runtime === "native" - ? { kind: "native" as const } - : { kind: "container" as const, engine: "docker" as const }; - if ( - inspection !== undefined && - requestedRuntime !== undefined && - (inspection.descriptor.runtime.kind !== requestedRuntime.kind || - (requestedRuntime.kind === "container" && - inspection.descriptor.runtime.kind === "container" && - inspection.descriptor.runtime.engine !== requestedRuntime.engine)) - ) { + ? discovered.find( + ({ definition }) => + identity !== undefined && + definition.identity.projectRoot === identity.projectRoot && + definition.identity.branchContext === identity.branchContext && + definition.identity.stackName === identity.stackName, + ) + : discovered.find(({ definition }) => definition.id === id); + if (id !== undefined && found === undefined) return yield* new StackTargetError({ - message: "The requested runtime does not match the existing stack", + message: `Stack ${id} was not found`, + reason: "flags", + }); + if (found !== undefined && !runtimeMatches(found.definition.runtime, requestedRuntime)) + return yield* new StackTargetError({ + message: `Requested runtime ${requestedRuntime} does not match existing stack runtime ${found.definition.runtime}`, + suggestion: + "Use --runtime auto to reuse the saved runtime, or omit --stack-id and choose a different --stack name.", reason: "flags", }); - } return { - projectRoot, - ...(id === undefined ? {} : { id }), + projectRoot: + found?.definition.identity.projectRoot ?? identity?.projectRoot ?? input.projectRoot, + ...(found === undefined ? {} : { id: found.definition.id }), ...(input.name === undefined ? {} : { name: input.name }), - ...(id === undefined && requestedRuntime !== undefined - ? { runtime: requestedRuntime } - : {}), + ...(found === undefined && requestedRuntime === undefined + ? {} + : found === undefined + ? { runtime: requestedRuntime } + : { runtime: found.definition.runtime }), }; - }), -}); + }); + return StackTargetResolver.of({ resolve }); + }), +); + +/** Groups companion services under their user-facing stack capability. */ +export const stackCapabilityForService = (service: ServiceCreation["service"]) => { + switch (service) { + case "imgproxy": + return "storage"; + case "vector": + return "analytics"; + case "pgmeta": + return "studio"; + default: + return service; + } +}; diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 28c6ab4826..0dacf507db 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -1,116 +1,88 @@ # `supabase stack start` -The command is available only when the `experimental.stack` feature flag is enabled. The -top-level `supabase start` command uses this handler when the same flag is enabled. - -This command creates or resumes the managed stack identified by the current -project and optional `--stack`, or opens an existing stack with `--stack-id`. -It loads `supabase/config.toml` for the target project when present and uses -default settings when it is absent; starting the stack does not create a config -file. It resolves explicit -`env(NAME)` references plus supported automatic `SUPABASE_*` overrides. Shell -values take precedence over values from `supabase/` dotenv files, which take -precedence over project-root dotenv files. Empty automatic overrides are -ignored. Explicit `env(NAME)` references remain available for values that are -not directly overridden. -The `@supabase/stack` Effect API owns persistent state, the detached -Supervisor, runtime resources, readiness, and cleanup. The CLI only resolves -the project configuration and renders the resulting status. - -Durable stack state lives under `$SUPABASE_HOME/managed/stacks//` -(`~/.supabase/managed/stacks//` by default). Ephemeral shadows use -`$SUPABASE_HOME/managed/ephemeral-postgres//`. - -`SUPABASE_HOME` controls the package's durable stack state through its normal -runtime composition boundary. The stack owner is deliberately detached from -the command waiter, so returning from a successful start leaves the stack -running for later commands. An interrupted start is handled by the package's -owner cleanup contract. - -The config loader reads the project environment files used for config -resolution, including the shared `supabase/.env` and `supabase/.env.local` -files when applicable. Function environment settings additionally read the -shared `supabase/functions/.env` and each enabled function's -`supabase/functions//.env`; per-function values override shared values. -Encrypted values use dotenvx decryption with keys from `DOTENV_PRIVATE_KEY` -and `DOTENV_PRIVATE_KEY_*`; each variable may contain comma-separated keys. -Failed decryption returns a typed configuration error without logging the -plaintext, ciphertext, or private key. Listener port numbers remain dynamically -allocated unless explicitly configured or supplied through a supported -`SUPABASE_*_PORT` override. -The CLI resolves supported environment overrides into a complete plain config -document and validates that effective document with `@supabase/config` before -projecting it into the stack runtime shape. Package validation errors are -reported with field paths and generic values so secrets are not exposed. The -stack projection then wraps consumed secrets, decrypts values at capability -boundaries, and preserves `env(NAME)` function references until runtime settings -are assembled. Provider secrets are resolved when the Auth capability is enabled; -disabled capabilities skip their unconsumed secrets. JWT issuer and signing -overrides are always applied because stack security consumes them even when Auth -itself is disabled. The CLI preserves optional-section presence metadata while -projecting the validated document. - -Text output includes the stack id, lifecycle, endpoints, and dormant -capabilities. Structured output includes the same status fields. The command reads configured -credentials and function/provider secrets to pass them to the stack runtime, but -never emits those values. - -`--stack` and `--stack-id` are mutually exclusive. For a new stack, `--runtime auto` -selects Docker when the daemon is reachable and native otherwise; a present Docker -client with a dead daemon persists native and prints a notice that destroy-and-recreate -(or a new `--stack` name) is required to use Docker later. Existing stacks reuse their -persisted runtime. `docker` and `native` select the requested runtime without fallback. -Native start is refused as uid 0 because `initdb` refuses root. -`--preparation` controls background versus -on-demand artifact preparation, and `--eager` requests enabled capabilities be -activated before the command returns. Eager capabilities do not receive automatic -idle stops. Per-capability `idleTimeoutSeconds` values are available through the -package's Effect API only; the CLI does not expose them as command or project -configuration settings. -`--exclude` accepts repeated or comma-separated capability names (`rest`, `auth`, `realtime`, -`storage`, `functions`, `studio`, `mail`, `analytics`, and `pooler`) and disables those services -in the effective start configuration. The database cannot be excluded. Exclusions are applied in -memory and persisted with the stack state; the project configuration file is unchanged. A capability -and its dependents are disabled together, so excluding `rest` also disables `studio`. Excluding -`analytics` does not. Analytics and pooler catalog downloads follow the excluded start config; -the platform trio still fail-closes against the full enabled config. -Listeners are derived by the runtime from enabled capability routes; route-less listeners are therefore omitted. -Eager activation never re-enables an excluded capability. - -The command owns only the start request. Once the package reports readiness, -the detached stack owner remains alive after the CLI process exits. If the CLI -caller is interrupted while waiting, the package's owner lifecycle decides -whether the start can complete or must clean up; the CLI does not call stop or -destroy as a cancellation handler. - -Database `network_restrictions`, `ssl_enforcement`, and `vault` settings are -accepted by the project config model but are not implemented by the local -runtime and are not forwarded. They do not enforce database security for this -command. - -## Bucket seeding on stack creation - -When this invocation runs the stack's first configured start (`desiredLifecycle` was -`unconfigured`, including after `stack prepare`) and Storage is not `disabled`, the -command seeds `[storage.buckets]` — creating or updating buckets and uploading their -`objects_path` files, non-interactively with auto-confirm — against the stack's gateway -using its service-role JWT, before printing the resulting status. Auto-confirm is safe -here because a first-start stack has no pre-existing buckets to overwrite or prune. This -reuses the same seeding core as `supabase seed buckets` and `db reset --local`. Files -read: `supabase/config.toml` `[storage.buckets]`, the configured `objects_path` files, -and the project dotenv files used for config resolution. Network calls: -`POST`/`GET /storage/v1/bucket` and `POST /storage/v1/object/...` against the stack's -API URL. - -A project with no `[storage.buckets]` or `[storage.vector.buckets]` configured resolves -no credentials and prints nothing — there is nothing to seed. - -A resumed stack is never re-seeded by `start`. Storage `disabled` skips seeding -silently; any other unusable capability state, a missing capability/credentials, or a -stack-gateway activation failure prints a stderr warning and skips seeding without -failing the command. Any other seeding failure (e.g. an invalid bucket entry) fails the -command with exit code `1`, but the stack itself is left running — a seeding failure -never stops or destroys it. - -Telemetry state is flushed to `/telemetry.json` -after both successful and failed command runs. +The experimental stack family and the enabled top-level `supabase start` alias use this handler. +It creates or resumes the stack for the project and optional `--stack` name, or opens the selected +`--stack-id`. Those selectors are mutually exclusive. Starting does not create a project config file. + +## Configuration and state + +The CLI loads the target project's `supabase/config.toml`, supported environment overrides, and +project dotenv files. It validates the supported configuration before creating service definitions. +Auth policies, OAuth providers, hooks, MFA, SMTP, email subjects and notification controls are +forwarded to Auth. REST search paths, pooler limits, Realtime settings, Studio settings, Storage +S3 protocol/vector controls, and configured Vector ports are forwarded to their services. +Encrypted JWT secrets are decrypted before shared credentials are derived. `db.health_timeout` +controls database readiness; an explicit `db.root_key` is supplied through a stack-owned key file. +Studio receives the Functions management directory/URL and Analytics credentials when present. +Email template `content_path` values and third-party identity providers remain unsupported: they +require template serving and shared external JWKS verification respectively. + +Secrets needed by enabled services are passed to the runtime. State and service data live under +`$SUPABASE_HOME/stacks//` (`~/.supabase/stacks//` by default); native artifacts +use `$SUPABASE_HOME/cache/stack`. Storage files use the caller-owned project directory +`supabase/.temp/stack-uploads//`. Functions preparation may build the project's source. + +For a new stack, `--runtime auto` selects native on Linux x64/arm64 and macOS arm64, and Docker +elsewhere. An existing stack keeps its saved runtime. Explicit runtime selection has no fallback. +Native startup refuses root because PostgreSQL `initdb` cannot run as root. + +Database is eager by default. Other services are lazy; traffic wakes them through their listeners. +Lazy services with idle policies stop after 60 seconds without traffic; Functions has no automatic +idle stop. `--eager` makes all selected services eager. Changing activation policy can stop and +restart the composition, including when a later invocation omits an earlier `--eager` flag. +`--preparation` selects on-demand or background artifact preparation. + +When Functions is selected, the CLI reads and validates `supabase/functions/.env`, ignoring reserved +`SUPABASE_*` entries. `edge_runtime.secrets` overrides that file, while `functions..env` +provides per-function values from project environment references. Per-function enabled/JWT policies, +entrypoints, import maps and static files are forwarded to the worker bootstrap. Configured paths +are relative to `supabase/` and must remain within the project; Docker mounts that project read-only. +The inspector port is retained as an endpoint intent and does not enable debugging by itself. +If custom env values, per-function settings or default JWT verification differ from the saved member, +start restarts Functions in place before ordinary composition start. A stopped member briefly launches +and stops again so normal lazy activation is retained; its identity and endpoints stay unchanged. + +## Service selection + +`--exclude` accepts repeated or comma-separated capability names: `rest`, `auth`, `realtime`, +`storage`, `functions`, `studio`, `mail`, `analytics`, and `pooler`. Database cannot be excluded. +Storage includes its Imgproxy companion, Studio includes Pgmeta, and Analytics includes Vector. +Studio requires REST; excluding REST while keeping Studio fails before stopping the composition. +The native Vector artifact currently ships a demo configuration with its health API disabled. +CLI log collection configuration remains unimplemented; exclude Analytics to avoid Vector readiness +timeouts until that configuration is supplied. + +Changing exclusions stops the composition and reuses the existing service identities, data, and +ports. Removed services remain saved and stopped so including them again can reuse them. The +project configuration file is unchanged. Incompatible version, endpoint, or supported configuration +changes fail before stopping existing services; they are not silently applied to saved instances. + +## First startup and retries + +The first configured startup prepares the database catalog, temporarily runs configured schema-owning +services, applies the database overlay, and runs project migrations and seeds. Membership changes +apply needed catalog and webhook setup without replaying project migrations or seeds. An unchanged +composition reapplies the webhook setting before activation. + +When configured, initial Storage bucket seeding creates buckets and uploads their `objects_path` +files using the service-role JWT. Storage is started and made ready before those requests. A resumed +stack is not re-seeded. Projects without configured buckets make no bucket-seeding requests. + +A failure or interruption during the first startup stops and unconfigures that initial composition, +then destroys only the service instances created by this invocation. Cleanup diagnostics name any +instance that could not be removed. Existing instances and their data are retained. After successful +cleanup, fixing the cause and retrying starts from an empty composition. Failed resumes do not +destroy existing data. Abrupt process termination that bypasses finalizers requires manual inspection +and, for an incomplete first bootstrap, destruction before retrying; there is no recovery journal. + +## Processes, network, and output + +The package's detached owner manages service processes, listeners, readiness, and runtime resources. +It remains available after the CLI exits. Preparation downloads native artifacts or pulls container +images. Catalog setup and project SQL connect to the primary database. Bucket seeding uses the local +Storage HTTP endpoint. The CLI does not remove caller-owned Storage files during cleanup. + +Text output reports progress and `Stack is ready.`. JSON output returns the stack `id`, assigned `endpoints` keyed by service and endpoint name +(for example `database.sql`), and an empty message. Endpoints contain protocol, address, port, +and URL, matching `stack status`; use status for service observations. Failures retain typed command +errors and package diagnostics. Telemetry state is flushed after success or failure. diff --git a/apps/cli/src/commands/experimental/stack/start/start.command.ts b/apps/cli/src/commands/experimental/stack/start/start.command.ts index a36f062137..3b81180aed 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.command.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.command.ts @@ -42,7 +42,7 @@ export const stackStartCommand = Command.make("start", config).pipe( Command.withDescription( "Create or resume a managed local Supabase stack using supabase/config.toml when present. " + "Without a config file, default settings are used and no file is created. " + - "For a new stack, auto selects Docker when its daemon is reachable and native otherwise; the choice is persisted for that stack. " + + "For a new stack, auto selects native on supported platforms and Docker elsewhere; the choice is persisted for that stack. " + "Explicit runtime choices are honored. Values support explicit env(NAME) references and automatic SUPABASE_* overrides.", ), Command.withShortDescription("Start a managed local stack"), diff --git a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts index 2c07194e38..1dc0a26fcc 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.e2e.test.ts @@ -1,6 +1,5 @@ // Starts a native stack through the compiled CLI binary, checks its status and connection-variable -// export, stops it, checks status again, then uses the package's public Promise API to inspect and -// destroy that stack. +// export, then stops and destroys that stack through the CLI. import { BunServices } from "@effect/platform-bun"; import { Data, @@ -12,9 +11,8 @@ import { Path, Predicate, Schema, - Stream, } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { parse as parseDotenv } from "dotenv"; import { afterAll, afterEach, describe, expect, test } from "vitest"; import { @@ -73,64 +71,17 @@ const makeTempDirectory = (prefix: string) => return yield* fs.makeTempDirectory({ directory: dirname(prefix), prefix: basename(prefix) }); }); -const execFileEffect = ( - command: string, - args: ReadonlyArray, - options: { - readonly cwd?: string; - readonly env?: Record; - readonly timeout?: number; - } = {}, -) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn( - ChildProcess.make(command, args, { - cwd: options.cwd, - env: options.env, - extendEnv: options.env !== undefined, - stdin: "ignore", - detached: false, - }), - ); - const [stdout, stderr, exitCode] = yield* Effect.all( - [ - child.stdout.pipe( - Stream.decodeText(), - Stream.runCollect, - Effect.map((chunks) => chunks.join("")), - ), - child.stderr.pipe( - Stream.decodeText(), - Stream.runCollect, - Effect.map((chunks) => chunks.join("")), - ), - child.exitCode, - ], - { concurrency: 3 }, - ); - if (exitCode !== 0) { - return yield* new StartE2eProcessError({ - message: `${command} exited ${exitCode}: ${stderr}`, - }); - } - return { stdout, stderr }; - }).pipe(Effect.scoped, Effect.timeout(options.timeout ?? CLEANUP_TIMEOUT_MS)); - const StackInspectionSchema = Schema.Struct({ - owner: Schema.String, - projectRoot: Schema.String, - runtime: Schema.Struct({ kind: Schema.String }), - lifecycle: Schema.String, - database: Schema.optionalKey(Schema.String), - databaseUrl: Schema.optionalKey(Schema.String), - hasApi: Schema.Boolean, -}); - -const LogDataSchema = Schema.Struct({ - found: Schema.Boolean, - id: Schema.String, - entries: Schema.Array(Schema.Struct({ source: Schema.String, message: Schema.String })), + identity: Schema.Struct({ project_root: Schema.String }), + runtime: Schema.String, + owner: Schema.Literals(["reachable", "unavailable"]), + lifecycle: Schema.NullOr(Schema.String), + composition: Schema.Struct({ + members: Schema.Array( + Schema.Struct({ id: Schema.String, service: Schema.String, state: Schema.String }), + ), + }), + endpoints: Schema.Record(Schema.String, Schema.Struct({ url: Schema.String })), }); const FollowEventSchema = Schema.Struct({ @@ -140,13 +91,7 @@ const FollowEventSchema = Schema.Struct({ }); const VariablesSchema = Schema.Record(Schema.String, Schema.String); - -const RetainedLogDataSchema = Schema.Struct({ - found: Schema.Boolean, - id: Schema.String, - running: Schema.Boolean, - entries: Schema.Array(Schema.Struct({ source: Schema.String })), -}); +const StartResultSchema = Schema.Struct({ id: Schema.String }); const minimalConfig = `project_id = "compiled-stack-start-e2e" @@ -154,7 +99,7 @@ const minimalConfig = `project_id = "compiled-stack-start-e2e" stack = true [api] -enabled = false +enabled = true [auth] enabled = false @@ -182,58 +127,40 @@ enabled = false `; const inspectStackState = (home: string, stackId: string) => { - const script = ` - import { inspectStack, openStack, StackIdSchema } from "@supabase/stack"; - const id = StackIdSchema.make(process.argv.at(-1)); - const inspection = await inspectStack(id); - const stack = await openStack(id); - const status = await stack.status(); - const credentials = - status.lifecycle === "running" ? await stack.credentials() : undefined; - console.log(JSON.stringify({ - owner: inspection.owner, - projectRoot: inspection.descriptor.projectRoot, - runtime: status.runtime, - lifecycle: status.lifecycle, - database: status.capabilities.find(({ name }) => name === "database")?.state, - databaseUrl: credentials?.database.url, - hasApi: credentials?.api !== undefined, - })); - `; return Effect.gen(function* () { - const result = yield* execFileEffect("bun", ["--bun", "-e", script, stackId], { - env: { - SUPABASE_HOME: home, - SUPABASE_NO_KEYRING: "1", - SUPABASE_TELEMETRY_DISABLED: "1", + const result = yield* runSupabaseEffect( + ["stack", "status", "--stack-id", stackId, "--output-format", "json"], + { + home, + env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, + exitTimeoutMs: CLEANUP_TIMEOUT_MS, }, - timeout: CLEANUP_TIMEOUT_MS, - }); - const line = result.stdout.trim().split("\n").at(-1); - if (line === undefined) { + ); + if (result.exitCode !== 0) { return yield* new StartE2eProcessError({ - message: `Stack probe returned no result:\n${result.stderr}`, + message: `Stack status returned ${result.exitCode}:\n${result.stderr}`, }); } - return yield* Schema.decodeEffect(Schema.fromJsonString(StackInspectionSchema))(line); + return yield* Schema.decodeEffect(Schema.fromJsonString(StackInspectionSchema))( + result.stdout.trim(), + ); }); }; -const destroyStack = (home: string, stackId: string) => { - const script = ` - import { openStack, StackIdSchema } from "@supabase/stack"; - const stack = await openStack(StackIdSchema.make(process.argv.at(-1))); - await stack.destroy(); - `; - return execFileEffect("bun", ["--bun", "-e", script, stackId], { - env: { - SUPABASE_HOME: home, - SUPABASE_NO_KEYRING: "1", - SUPABASE_TELEMETRY_DISABLED: "1", - }, - timeout: CLEANUP_TIMEOUT_MS, - }); -}; +const destroyStack = (home: string, stackId: string) => + Effect.flatMap( + runSupabaseEffect(["stack", "destroy", "--stack-id", stackId, "--yes"], { + home, + env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, + exitTimeoutMs: CLEANUP_TIMEOUT_MS, + }), + (result) => + result.exitCode === 0 + ? Effect.void + : new StartE2eProcessError({ + message: `stack destroy exited ${result.exitCode}: ${result.stderr}`, + }), + ); const waitForOutput = ( spawned: ReturnType, @@ -243,7 +170,10 @@ const waitForOutput = ( Effect.tryPromise({ try: () => spawned.waitForOutput(pattern, timeoutMs), catch: (cause) => - new StartE2eProcessError({ message: "stack logs follower did not produce history", cause }), + new StartE2eProcessError({ + message: "stack logs follower did not produce live output", + cause, + }), }); describe("stack start (compiled e2e)", () => { @@ -255,7 +185,7 @@ describe("stack start (compiled e2e)", () => { const cleanup = Effect.gen(function* () { let cleanupComplete = stackDestroyed; if (!cleanupComplete && home !== undefined) { - const candidates = yield* readDirectory(join(home.dir, "managed", "stacks")).pipe( + const candidates = yield* readDirectory(join(home.dir, "stacks")).pipe( Effect.catchIf( (cause) => Predicate.isTagged(cause, "PlatformError") && @@ -304,14 +234,24 @@ describe("stack start (compiled e2e)", () => { yield* makeDirectory(join(projectDir, "supabase"), { recursive: true }); yield* writeText(join(projectDir, "supabase", "config.toml"), minimalConfig); + const excluded = yield* runSupabaseEffect( + ["stack", "start", "--runtime", "native", "--exclude", "rest", "--eager"], + { cwd: projectDir, home: home.dir, exitTimeoutMs: START_TIMEOUT_MS }, + ); + expect( + excluded.exitCode, + `stdout:\n${excluded.stdout}\nstderr:\n${excluded.stderr}`, + ).toBe(0); + const result = yield* runSupabaseEffect( - ["stack", "start", "--runtime", "native", "--eager"], + ["stack", "start", "--runtime", "native", "--eager", "--output-format", "json"], { cwd: projectDir, home: home.dir, exitTimeoutMs: START_TIMEOUT_MS }, ); expect(result.exitCode, `stdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); - const idMatch = result.stdout.match(/Stack ([0-9a-f]{64})/u); - expect(idMatch, `stdout:\n${result.stdout}`).not.toBeNull(); - stackId = idMatch?.[1]; + const startResult = yield* Schema.decodeEffect(Schema.fromJsonString(StartResultSchema))( + result.stdout.trim(), + ); + stackId = startResult.id; const idText = stackId; const homeDir = home; const projectRoot = projectDir; @@ -319,46 +259,21 @@ describe("stack start (compiled e2e)", () => { throw new Error("compiled start did not return a stack id"); const running = yield* inspectStackState(homeDir.dir, idText); - expect(running.owner).toBe("running"); - expect(running.projectRoot).toBe(yield* realPath(projectRoot)); - expect(running.runtime).toEqual({ kind: "native" }); + expect(running.owner).toBe("reachable"); + expect(running.identity.project_root).toBe(yield* realPath(projectRoot)); + expect(running.runtime).toBe("native"); expect(running.lifecycle).toBe("running"); - expect(running.database).toBe("ready"); - expect(running.hasApi).toBe(false); - expect(running.databaseUrl).toMatch( - /^postgresql:\/\/postgres:.+@127\.0\.0\.1:\d+\/postgres$/, - ); - const databasePath = join(homeDir.dir, "managed", "stacks", idText, "data", "database"); - yield* access(join(databasePath, "PG_VERSION")); - - const logs = yield* runSupabaseEffect( - [ - "stack", - "logs", - "--stack-id", - idText, - "--service", - "database", - "--tail", - "100", - "--output-format", - "json", - ], - { - cwd: projectRoot, - home: homeDir.dir, - env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, - exitTimeoutMs: CLEANUP_TIMEOUT_MS, - }, - ); - expect(logs.exitCode, `stdout:\n${logs.stdout}\nstderr:\n${logs.stderr}`).toBe(0); - const logData = yield* Schema.decodeEffect(Schema.fromJsonString(LogDataSchema))( - logs.stdout, - ); - expect(logData.found).toBe(true); - expect(logData.id).toBe(idText); - expect(logData.entries.length).toBeGreaterThan(0); - expect(logData.entries.every((entry) => entry.source === "database")).toBe(true); + expect( + running.composition.members.find(({ service }) => service === "database")?.state, + ).toBe("running"); + expect(running.composition.members.some(({ service }) => service === "rest")).toBe(true); + expect(running.endpoints["database.sql"]?.url).toMatch(/^tcp:\/\/127\.0\.0\.1:\d+$/); + const databaseId = running.composition.members.find( + ({ service }) => service === "database", + )?.id; + if (databaseId === undefined) return yield* Effect.die("database member id is missing"); + const databasePath = join(homeDir.dir, "stacks", idText, "data", databaseId); + yield* access(join(databasePath, "data", "PG_VERSION")); const followResult = yield* Effect.scoped( Effect.gen(function* () { @@ -373,9 +288,6 @@ describe("stack start (compiled e2e)", () => { idText, "--service", "database", - "--tail", - "1", - "--follow", "--output-format", "stream-json", ], @@ -403,9 +315,22 @@ describe("stack start (compiled e2e)", () => { ), ), ); + const restarted = yield* runSupabaseEffect( + ["stack", "restart", "--stack-id", idText], + { + cwd: projectRoot, + home: homeDir.dir, + env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, + exitTimeoutMs: START_TIMEOUT_MS, + }, + ); + expect( + restarted.exitCode, + `stdout:\n${restarted.stdout}\nstderr:\n${restarted.stderr}`, + ).toBe(0); yield* waitForOutput( followed, - /"type":"log-entry".*"service":"database".*"source":"history"/u, + /"type":"log-entry".*"source":"live"/u, START_TIMEOUT_MS, ); yield* Effect.sync(() => followed.kill("SIGINT")); @@ -423,16 +348,18 @@ describe("stack start (compiled e2e)", () => { .filter((line) => line.length > 0), (line) => Schema.decodeEffect(Schema.fromJsonString(FollowEventSchema))(line), ); - const historyEntries = followEvents.filter( - (event) => event.type === "log-entry" && event.source === "history", + const liveEntries = followEvents.filter( + (event) => event.type === "log-entry" && event.source === "live", ); - expect(historyEntries).toHaveLength(1); - expect(historyEntries[0]).toEqual(expect.objectContaining({ service: "database" })); + expect(liveEntries.length).toBeGreaterThan(0); + expect(liveEntries.every((event) => event.service === "database")).toBe(true); const afterFollow = yield* inspectStackState(homeDir.dir, idText); - expect(afterFollow.owner).toBe("running"); + expect(afterFollow.owner).toBe("reachable"); expect(afterFollow.lifecycle).toBe("running"); - expect(afterFollow.database).toBe("ready"); + expect( + afterFollow.composition.members.find(({ service }) => service === "database")?.state, + ).toBe("running"); const status = yield* runSupabaseEffect(["stack", "status", "--stack-id", idText], { cwd: projectRoot, home: homeDir.dir, @@ -440,7 +367,7 @@ describe("stack start (compiled e2e)", () => { }); expect(status.exitCode, `stdout:\n${status.stdout}\nstderr:\n${status.stderr}`).toBe(0); expect(status.stdout).toContain(`(${idText})`); - expect(status.stdout).toContain("Owner: running"); + expect(status.stdout).toContain("Owner: reachable"); expect(status.stdout).toContain("Lifecycle: running"); expect(status.stdout).toContain("Readiness: ready"); expect(status.stdout).toMatch(/Config drift: (changed|unchanged)/u); @@ -456,7 +383,7 @@ describe("stack start (compiled e2e)", () => { `stdout:\n${topLevelStatus.stdout}\nstderr:\n${topLevelStatus.stderr}`, ).toBe(0); expect(topLevelStatus.stdout).toContain(`(${idText})`); - expect(topLevelStatus.stdout).toContain("Owner: running"); + expect(topLevelStatus.stdout).toContain("Owner: reachable"); expect(topLevelStatus.stdout).toContain("Lifecycle: running"); const env = yield* runSupabaseEffect( @@ -467,8 +394,15 @@ describe("stack start (compiled e2e)", () => { const variables = yield* Schema.decodeEffect(Schema.fromJsonString(VariablesSchema))( env.stdout, ); - expect(Object.keys(variables)).toEqual(["DB_URL"]); - expect(variables.DB_URL).toMatch(/^postgresql:\/\/postgres:.+@.+:\d+\/postgres$/u); + expect(Object.keys(variables)).toEqual([ + "DB_URL", + "ANON_KEY", + "SERVICE_ROLE_KEY", + "API_URL", + ]); + expect(variables.DB_URL).toMatch( + /^postgresql:\/\/supabase_admin:.+@.+:\d+\/postgres(?:\?.*)?$/u, + ); const dotenv = yield* runSupabaseEffect( ["stack", "status", "--env", "--stack-id", idText, "--output-format", "text"], @@ -490,11 +424,13 @@ describe("stack start (compiled e2e)", () => { expect(stop.exitCode, `stdout:\n${stop.stdout}\nstderr:\n${stop.stderr}`).toBe(0); const observed = yield* inspectStackState(homeDir.dir, idText); - expect(observed.owner).toBe("absent"); - expect(observed.projectRoot).toBe(yield* realPath(projectRoot)); - expect(observed.runtime).toEqual({ kind: "native" }); - expect(observed.lifecycle).toBe("stopped"); - expect(observed.database).toBe("stopped"); + expect(observed.owner).toBe("unavailable"); + expect(observed.identity.project_root).toBe(yield* realPath(projectRoot)); + expect(observed.runtime).toBe("native"); + expect(observed.lifecycle).toBeNull(); + expect( + observed.composition.members.find(({ service }) => service === "database")?.state, + ).toBe("unavailable"); const stoppedStatus = yield* runSupabaseEffect( ["stack", "status", "--stack-id", idText], @@ -509,9 +445,9 @@ describe("stack start (compiled e2e)", () => { stoppedStatus.exitCode, `stdout:\n${stoppedStatus.stdout}\nstderr:\n${stoppedStatus.stderr}`, ).toBe(0); - expect(stoppedStatus.stdout).toContain("Owner: absent"); + expect(stoppedStatus.stdout).toContain("Owner: unavailable"); expect(stoppedStatus.stdout).toContain("Lifecycle: unavailable"); - expect(stoppedStatus.stdout).toContain("Readiness: unknown"); + expect(stoppedStatus.stdout).toContain("Readiness: unavailable"); const stoppedEnv = yield* runSupabaseEffect( ["stack", "status", "--env", "--stack-id", idText], @@ -524,11 +460,13 @@ describe("stack start (compiled e2e)", () => { ); expect(stoppedEnv.exitCode).not.toBe(0); expect(stoppedEnv.stdout).not.toContain("DB_URL"); - expect(stoppedEnv.stderr).toContain("must be running"); + expect(stoppedEnv.stderr).toContain( + "The stack owner or primary database is unavailable for environment export.", + ); - yield* access(join(databasePath, "PG_VERSION")); + yield* access(join(databasePath, "data", "PG_VERSION")); - const retainedLogs = yield* runSupabaseEffect( + const stoppedLogs = yield* runSupabaseEffect( [ "stack", "logs", @@ -536,10 +474,8 @@ describe("stack start (compiled e2e)", () => { idText, "--service", "database", - "--tail", - "100", "--output-format", - "json", + "stream-json", ], { cwd: projectRoot, @@ -548,25 +484,15 @@ describe("stack start (compiled e2e)", () => { exitTimeoutMs: CLEANUP_TIMEOUT_MS, }, ); - expect( - retainedLogs.exitCode, - `stdout:\n${retainedLogs.stdout}\nstderr:\n${retainedLogs.stderr}`, - ).toBe(0); - const retainedData = yield* Schema.decodeEffect( - Schema.fromJsonString(RetainedLogDataSchema), - )(retainedLogs.stdout); - expect(retainedData.found).toBe(true); - expect(retainedData.id).toBe(idText); - expect(retainedData.running).toBe(false); - expect(retainedData.entries.length).toBeGreaterThan(0); - expect(retainedData.entries.every((entry) => entry.source === "database")).toBe(true); + expect(stoppedLogs.exitCode).not.toBe(0); + expect(`${stoppedLogs.stdout}\n${stoppedLogs.stderr}`).toMatch( + /unavailable|not running|must be running/iu, + ); yield* destroyStack(homeDir.dir, idText); stackDestroyed = true; - const destroyed = yield* Effect.exit( - access(join(homeDir.dir, "managed", "stacks", idText)), - ); + const destroyed = yield* Effect.exit(access(join(homeDir.dir, "stacks", idText))); expect(Exit.isFailure(destroyed)).toBe(true); }), ), diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 6f73b01c94..33c2e3164b 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -1,11 +1,8 @@ -import { Effect, FileSystem, Match, Option, Path } from "effect"; -import { - excludeStackCapabilities, - isStackError, - type StackConfig, - type StackRuntimePreference, - type StackStatus, -} from "@supabase/stack/effect"; +import { endpointReports } from "../stack-endpoints.format.ts"; +import { readStackFunctionsEnv } from "../../../../command-internal/stack-functions-env.ts"; +import { defaultStackRuntime } from "../../../../command-internal/stack-runtime.ts"; +import { Effect, Equal, FileSystem, Fiber, Option, Path, Redacted, Ref } from "effect"; +import type { ServiceCreation, Stack } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag, @@ -13,63 +10,36 @@ import { } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; +import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; import { readDbToml } from "../../../../command-internal/db-config.toml-read.ts"; import { StackCatalogSetup } from "../../../../command-internal/stack-catalog-setup.ts"; import { applyStackMigrateAndSeed, applyStackWebhooksOnly, -} from "../../../../command-internal/stack-local-database.ts"; -import { - classifyStorageCapability, - stackStorageEndpointFor, -} from "../../../../command-internal/stack-storage.ts"; +} from "../../../../command-internal/stack-bootstrap.ts"; +import { stackStorageCredentialsFor } from "../../../../command-internal/stack-storage.ts"; import { hasConfiguredBuckets, SeedConfigLoadError, seedBucketsRun, } from "../../../../command-internal/seed-buckets.ts"; import { loadLocalProjectContext } from "../../../../command-internal/local-project-context.ts"; -import { yellow } from "../../../../command-internal/colors.ts"; +import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import { StackApi, + stackCapabilityForService, StackTargetError, StackTargetResolver, rejectStackOutput, - renderStackStatus, - stackStatusPayload, validateStackTarget, } from "../stack.shared.ts"; -import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import type { StackStartFlags } from "./start.command.ts"; import { StackCommandStartError } from "./start.errors.ts"; import { STACK_START_EXCLUDABLE_CAPABILITIES } from "./start.options.ts"; -const eagerlyActivate = < - T extends { readonly enabled?: boolean; readonly activation?: "eager" | "lazy" }, ->( - value: T, -): T => (value.enabled === false ? value : Object.assign({}, value, { activation: "eager" })); - -const isPostgresOnlyStatus = (status: StackStatus): boolean => { - const database = status.capabilities.find((capability) => capability.name === "database"); - if (status.lifecycle !== "running" || database?.state !== "ready") return false; - return STACK_START_EXCLUDABLE_CAPABILITIES.every( - (name) => - status.capabilities.find((capability) => capability.name === name)?.state === "disabled", - ); -}; - -const isPostgresOnlyConfig = (config: StackConfig): boolean => - STACK_START_EXCLUDABLE_CAPABILITIES.every( - (name) => config.capabilities?.[name]?.enabled === false, - ); - const validateExclusions = (exclusions: ReadonlyArray) => { - const unknown = exclusions.filter( - (name) => - name !== "database" && - !STACK_START_EXCLUDABLE_CAPABILITIES.some((capability) => capability === name), - ); + const supported = new Set(STACK_START_EXCLUDABLE_CAPABILITIES); + const unknown = exclusions.filter((name) => name !== "database" && !supported.has(name)); if (unknown.length > 0) return Effect.fail( new StackCommandStartError({ @@ -99,21 +69,116 @@ const mapTargetError = (error: StackTargetError) => cause: error, }); +const stackError = (cause: { readonly message: string }) => + new StackCommandStartError({ + reason: "unknown", + message: cause.message, + cause, + }); + +const sameKinds = ( + left: ReadonlyArray<{ readonly service: string }>, + right: ReadonlyArray<{ readonly service: string }>, +): boolean => { + const leftKinds = new Set(left.map(({ service }) => service)); + const rightKinds = new Set(right.map(({ service }) => service)); + return leftKinds.size === rightKinds.size && [...leftKinds].every((kind) => rightKinds.has(kind)); +}; + +const sameBinding = ( + observed: { + readonly service: string; + readonly version?: string; + readonly config: unknown; + readonly endpoints?: unknown; + }, + requested: { + readonly service: string; + readonly version?: string; + readonly config: unknown; + readonly endpoints?: unknown; + }, +): boolean => + observed.service === requested.service && + observed.version === requested.version && + Equal.equals(observed.endpoints, requested.endpoints) && + Equal.equals( + comparableConfig(observed.service, observed.config), + comparableConfig(requested.service, requested.config), + ); + +const reconciledConfigKeys: Readonly>> = { + database: new Set(["databasePassword", "jwtSecret"]), + rest: new Set(["databaseUrl", "jwtSecret"]), + auth: new Set(["databaseUrl", "jwtSecret", "externalApiUrl", "smtpUrl"]), + realtime: new Set(["databaseUrl", "jwtSecret"]), + storage: new Set(["databaseUrl", "filePath", "jwtSecret", "imgproxyUrl", "vectorDatabaseUrl"]), + functions: new Set([ + "functionsRoot", + "filesRoot", + "functions", + "bootstrap", + "apiUrl", + "databaseUrl", + "jwtSecret", + "env", + "verifyJwt", + ]), + studio: new Set([ + "functionsRoot", + "pgmetaUrl", + "analyticsUrl", + "analyticsApiKey", + "functionsUrl", + "apiUrl", + "publicApiUrl", + "jwtSecret", + ]), + analytics: new Set(["databaseUrl"]), + vector: new Set(["analyticsUrl"]), + pgmeta: new Set(["databaseUrl"]), + pooler: new Set(["databaseUrl", "jwtSecret"]), +}; + +const comparableConfig = (service: string, value: unknown): unknown => { + if (typeof value !== "object" || value === null || Array.isArray(value)) return value; + const ignored = reconciledConfigKeys[service] ?? new Set(); + return Object.fromEntries(Object.entries(value).filter(([key]) => !ignored.has(key))); +}; + +const selectedCreations = ( + creations: ReadonlyArray, + exclusions: ReadonlyArray, +) => + creations.filter((creation) => { + const capability = stackCapabilityForService(creation.service); + return !exclusions.includes(capability); + }); + +const compose = ( + stack: Stack, + creations: ReadonlyArray, + reuseIds: ReadonlyArray, +) => stack.composition.supabase(creations, reuseIds.length === 0 ? undefined : { reuseIds }); + +/** Starts the selected managed stack and applies the local database overlays. */ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags: StackStartFlags) { const telemetryState = yield* TelemetryState; const body = Effect.gen(function* () { const output = yield* Output; const settings = yield* CommandSettings; + const runtime = yield* RuntimeInfo; const resolver = yield* StackTargetResolver; const stackApi = yield* StackApi; const outputFlag = yield* Effect.serviceOption(OutputFlag); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); const exclusions = yield* validateExclusions(flags.exclude); yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), }).pipe(Effect.mapError(mapTargetError)); - const target = yield* resolver .resolve({ projectRoot: settings.workdir, @@ -132,76 +197,6 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }), ), ); - const configuredStart = excludeStackCapabilities(config, exclusions); - const startConfig = flags.eager - ? { - ...configuredStart, - capabilities: { - ...configuredStart.capabilities, - ...(configuredStart.capabilities?.rest === undefined - ? {} - : { rest: eagerlyActivate(configuredStart.capabilities.rest) }), - ...(configuredStart.capabilities?.auth === undefined - ? {} - : { auth: eagerlyActivate(configuredStart.capabilities.auth) }), - ...(configuredStart.capabilities?.realtime === undefined - ? {} - : { realtime: eagerlyActivate(configuredStart.capabilities.realtime) }), - ...(configuredStart.capabilities?.storage === undefined - ? {} - : { storage: eagerlyActivate(configuredStart.capabilities.storage) }), - ...(configuredStart.capabilities?.functions === undefined - ? {} - : { functions: eagerlyActivate(configuredStart.capabilities.functions) }), - ...(configuredStart.capabilities?.studio === undefined - ? {} - : { studio: eagerlyActivate(configuredStart.capabilities.studio) }), - ...(configuredStart.capabilities?.mail === undefined - ? {} - : { mail: eagerlyActivate(configuredStart.capabilities.mail) }), - ...(configuredStart.capabilities?.analytics === undefined - ? {} - : { analytics: eagerlyActivate(configuredStart.capabilities.analytics) }), - ...(configuredStart.capabilities?.pooler === undefined - ? {} - : { pooler: eagerlyActivate(configuredStart.capabilities.pooler) }), - }, - preparation: flags.preparation, - } - : { ...configuredStart, preparation: flags.preparation }; - const runtime: StackRuntimePreference | undefined = target.runtime; - // The package's public Effect API reads SUPABASE_HOME only at its runtime - // composition boundary and launches the detached owner through the compiled - // dispatch sentinel. The CLI adapter resolves the target and config; it does - // not recreate package lifecycle or runtime ownership here. - const stack = - target.id !== undefined - ? yield* stackApi.openStack(target.id).pipe(Effect.mapError(stackStartError)) - : yield* stackApi - .createStack({ - projectRoot: target.projectRoot, - ...(target.name === undefined ? {} : { name: target.name }), - ...(runtime === undefined ? {} : { runtime }), - }) - .pipe(Effect.mapError(stackStartError)); - if (stack.dockerFallbackNotice !== undefined) - yield* output.raw(`${stack.dockerFallbackNotice}\n`, "stderr"); - // `--stack-id` addresses this identity, not findStack(projectRoot, name). - const addressed = yield* stack.status.pipe(Effect.mapError(stackStartError)); - const firstCreate = addressed.desiredLifecycle === "unconfigured"; - const starting = yield* output.task("Starting local Supabase stack..."); - if (isPostgresOnlyStatus(addressed) && !isPostgresOnlyConfig(startConfig)) { - yield* stack.stop.pipe( - Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(stackStartError), - ); - } - const status = yield* stack.start({ config: startConfig }).pipe( - Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(stackStartError), - ); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const toml = yield* readDbToml(fs, path, target.projectRoot).pipe( Effect.mapError( (error) => @@ -212,40 +207,329 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }), ), ); - const setupFailed = (error: { readonly message: string; readonly cause?: unknown }) => - isStackError(error.cause) - ? stackStartError(error.cause) - : new StackCommandStartError({ - reason: "unknown", - message: error.message, - suggestion: "The stack is running. Recover with db reset.", - cause: error, - }); - // Bucket seeding failures never stop or destroy the stack; report them as a distinct - // reason so telemetry doesn't fold them into the runtime-lifecycle "unknown" bucket. - const seedFailed = (error: { readonly message: string }) => - new StackCommandStartError({ - reason: "seed", - message: error.message, - suggestion: "The stack is running. Recover with supabase seed buckets or db reset.", - cause: error, + const stateRoot = path.join(settings.supabaseHome, "stacks"); + const cacheRoot = path.join(settings.supabaseHome, "cache", "stack"); + const stack = + target.id === undefined + ? yield* stackApi + .create({ + projectRoot: target.projectRoot, + stateRoot, + cacheRoot, + runtime: target.runtime ?? defaultStackRuntime(runtime), + ...(target.name === undefined ? {} : { name: target.name }), + }) + .pipe(Effect.mapError(stackError)) + : yield* stackApi + .open({ id: target.id, stateRoot, cacheRoot }) + .pipe(Effect.mapError(stackError)); + const existingServices = yield* stack.services.list.pipe(Effect.mapError(stackError)); + const composition = yield* stack.composition.describe.pipe(Effect.mapError(stackError)); + const currentInstances = yield* Effect.forEach(composition.members, ({ id }) => + stack.services.get(id).pipe(Effect.mapError(stackError)), + ); + const primaryDatabase = currentInstances.find((instance) => instance.service === "database"); + const shadowDatabase = + composition.members.length === 0 + ? existingServices.find((instance) => instance.service === "database") + : undefined; + if (shadowDatabase !== undefined) + return yield* new StackCommandStartError({ + reason: "lifecycle", + message: "A standalone database exists outside the saved stack composition", + suggestion: "Destroy the standalone database before starting this stack.", + }); + const persistedJwt = + primaryDatabase === undefined + ? undefined + : yield* primaryDatabase.status.pipe( + Effect.catchTag("StackError", () => + primaryDatabase + .credentials({ from: "host" }) + .pipe(Effect.andThen(primaryDatabase.status)), + ), + Effect.map((status) => + status.config.service === "database" + ? Redacted.isRedacted(status.config.config.jwtSecret) + ? status.config.config.jwtSecret + : Redacted.make(status.config.config.jwtSecret) + : undefined, + ), + Effect.mapError(stackError), + ); + if (primaryDatabase !== undefined && persistedJwt === undefined) + return yield* new StackCommandStartError({ + reason: "invalid-config", + message: "The existing database has no persisted JWT secret", }); - if (firstCreate) { - const catalog = yield* Effect.serviceOption(StackCatalogSetup); - if (Option.isNone(catalog)) + const creations = yield* config + .creations(stack.id, persistedJwt === undefined ? undefined : { jwtSecret: persistedJwt }) + .pipe( + Effect.mapError( + (error) => + new StackCommandStartError({ + reason: "invalid-config", + message: error.message, + cause: error, + }), + ), + ); + const requested = yield* Effect.forEach(selectedCreations(creations, exclusions), (creation) => + creation.service === "functions" + ? readStackFunctionsEnv(`${creation.config.functionsRoot}/.env`, true).pipe( + Effect.map((env): ServiceCreation => ({ + ...creation, + config: { ...creation.config, env: { ...env, ...creation.config.env } }, + })), + Effect.mapError( + (cause) => + new StackCommandStartError({ + reason: "invalid-config", + message: cause.message, + cause, + }), + ), + ) + : Effect.succeed(creation), + ); + if ( + requested.some(({ service }) => service === "studio") && + !requested.some(({ service }) => service === "rest") + ) + return yield* new StackCommandStartError({ + reason: "flags", + message: "Studio cannot be started without the REST API capability", + suggestion: "Remove --exclude rest or also exclude studio.", + }); + if (requested.some(({ service }) => service === "storage")) + yield* fs + .makeDirectory( + path.join(target.projectRoot, "supabase", ".temp", "stack-uploads", stack.id), + { + recursive: true, + }, + ) + .pipe( + Effect.mapError( + (cause) => + new StackCommandStartError({ + reason: "invalid-config", + message: `Unable to create the Storage uploads directory: ${cause.message}`, + cause, + }), + ), + ); + if (requested.some(({ service }) => service === "functions")) + yield* fs + .makeDirectory(path.join(target.projectRoot, "supabase", "functions"), { recursive: true }) + .pipe( + Effect.mapError( + (cause) => + new StackCommandStartError({ + reason: "invalid-config", + message: `Unable to create the Functions directory: ${cause.message}`, + cause, + }), + ), + ); + const initialComposition = composition.members.length === 0; + let compositionChanged = !sameKinds(currentInstances, requested); + for (const instance of currentInstances) { + const creation = requested.find(({ service }) => service === instance.service); + if (creation === undefined) continue; + const status = yield* instance.status.pipe(Effect.mapError(stackError)); + if (!sameBinding(status.config, creation)) return yield* new StackCommandStartError({ - reason: "unknown", - message: "stack catalog setup is unavailable", + reason: "invalid-config", + message: `The requested ${creation.service} configuration does not match the saved stack binding`, + suggestion: + "Keep the saved endpoint and version settings, or destroy the stack before changing them.", }); - yield* catalog.value + } + const reuseIds: Array = compositionChanged + ? currentInstances + .filter((instance) => requested.some((creation) => creation.service === instance.service)) + .map(({ id }) => id) + : []; + if (compositionChanged) { + const currentKinds = new Set(currentInstances.map(({ service }) => service)); + for (const creation of requested) { + if (currentKinds.has(creation.service)) continue; + const candidates = []; + for (const candidate of existingServices) { + if (candidate.service !== creation.service || reuseIds.includes(candidate.id)) continue; + const status = yield* candidate.status.pipe( + Effect.map(Option.some), + Effect.catchTag("StackError", () => Effect.succeed(Option.none())), + ); + if ( + Option.isSome(status) && + status.value.lifecycle === "stopped" && + !status.value.wakeEnabled && + sameBinding(status.value.config, creation) + ) + candidates.push(candidate); + } + if (candidates.length > 1) + return yield* new StackCommandStartError({ + reason: "lifecycle", + message: `Multiple stopped ${creation.service} instances match this stack configuration`, + suggestion: "Remove the unused stack service, then retry.", + }); + const candidate = candidates[0]; + if (candidate !== undefined) reuseIds.push(candidate.id); + } + } + const starting = yield* output.task("Starting local Supabase stack..."); + if (compositionChanged && composition.members.length > 0) { + const stopping = yield* output.task( + "Stopping local Supabase stack before applying changes...", + ); + yield* stack.composition.stop.pipe( + Effect.tapError((error) => stopping.fail(error.message)), + Effect.mapError(stackError), + ); + yield* stopping.succeed(); + } else if (compositionChanged && primaryDatabase !== undefined) { + const stopping = yield* output.task( + "Stopping local Supabase stack before applying changes...", + ); + yield* primaryDatabase.stop.pipe( + Effect.tapError((error) => stopping.fail(error.message)), + Effect.mapError(stackError), + ); + yield* stopping.succeed(); + } + const members = compositionChanged + ? yield* compose(stack, requested, reuseIds).pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + ) + : currentInstances; + const initialCleanupComplete = yield* Ref.make(!initialComposition); + if (initialComposition) { + const existingIds = new Set(existingServices.map(({ id }) => id)); + const owned = members.filter(({ id }) => !existingIds.has(id)); + const cleanupInitialSafe = Effect.gen(function* () { + yield* stack.composition.stop; + yield* stack.composition.configure({ members: [], dependencies: [] }); + yield* Effect.forEach( + owned, + (instance) => + instance.destroy.pipe( + Effect.catch((error) => + output.raw( + `Failed to remove initial ${instance.service} instance ${instance.id}: ${error.message}. Run supabase stack destroy --stack-id ${stack.id} before retrying.\n`, + "stderr", + ), + ), + ), + { discard: true }, + ); + }).pipe( + Effect.catch((error) => + output.raw( + `Failed to clean up initial stack ${stack.id}: ${error.message}. Run supabase stack destroy --stack-id ${stack.id} before retrying.\n`, + "stderr", + ), + ), + ); + yield* Effect.addFinalizer(() => + Ref.get(initialCleanupComplete).pipe( + Effect.flatMap((complete) => (complete ? Effect.void : cleanupInitialSafe)), + ), + ); + } + const functions = members.find((member) => member.service === "functions"); + const requestedFunctions = requested.find((creation) => creation.service === "functions"); + if (functions?.service === "functions" && requestedFunctions?.service === "functions") { + const before = yield* functions.status.pipe(Effect.mapError(stackError)); + if (before.config.service === "functions") { + const next = { + ...before.config.config, + env: requestedFunctions.config.env, + functions: requestedFunctions.config.functions, + filesRoot: requestedFunctions.config.filesRoot, + verifyJwt: requestedFunctions.config.verifyJwt, + }; + if (!Equal.equals(before.config.config, next)) { + yield* functions.restart({ config: next }).pipe(Effect.mapError(stackError)); + if (before.lifecycle === "stopped") + yield* functions.stop.pipe(Effect.mapError(stackError)); + } + } + } + const configured = yield* stack.composition.describe.pipe(Effect.mapError(stackError)); + const desiredMembers = configured.members.map(({ id }) => { + const member = members.find((entry) => entry.id === id); + const eager = flags.eager || member?.service === "database"; + return { + id, + activation: eager ? ("eager" as const) : ("lazy" as const), + ...(eager || member?.service === "functions" ? {} : { idleMillis: 60_000 }), + }; + }); + const activationChanged = desiredMembers.some((desired) => { + const current = configured.members.find(({ id }) => id === desired.id); + return ( + current?.activation !== desired.activation || current?.idleMillis !== desired.idleMillis + ); + }); + if (activationChanged) { + if (!compositionChanged && composition.members.length > 0) { + const stopping = yield* output.task( + "Stopping local Supabase stack before applying changes...", + ); + yield* stack.composition.stop.pipe( + Effect.tapError((error) => stopping.fail(error.message)), + Effect.mapError(stackError), + ); + yield* stopping.succeed(); + } + yield* stack.composition + .configure({ ...configured, members: desiredMembers }) + .pipe(Effect.mapError(stackError)); + } + const database = members.find((instance) => instance.service === "database"); + if (database === undefined) + return yield* new StackCommandStartError({ + reason: "invalid-config", + message: "The stack composition has no database service", + }); + const preparation = + flags.preparation === "background" + ? yield* Effect.forEach( + members, + (member) => member.prepare.pipe(Effect.mapError(stackError), Effect.forkScoped), + { concurrency: "unbounded" }, + ) + : []; + yield* database.start.pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + ); + yield* database.ready.pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + ); + const databaseServices = requested + .filter( + (creation) => + creation.service === "auth" || + creation.service === "storage" || + creation.service === "realtime", + ) + .map((creation) => creation.service); + const catalog = yield* Effect.service(StackCatalogSetup); + if (initialComposition) { + yield* catalog .apply({ target: { - kind: "live", stack, - projectRoot: target.projectRoot, - config, + database, + databaseServices, + jwtSecret: Redacted.value(persistedJwt ?? config.jwtSecret), }, - optionalConfig: startConfig, overlay: { webhooks: "config", webhooksEnabled: toml.webhooksEnabled, @@ -256,38 +540,64 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }) .pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(setupFailed), + Effect.mapError(stackError), ); const experimental = yield* resolveExperimentalWithProjectEnv({ ...toml.projectEnv }); - yield* applyStackMigrateAndSeed(stack, target.projectRoot, toml, experimental).pipe( + yield* applyStackMigrateAndSeed(database, target.projectRoot, toml, experimental).pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(setupFailed), + Effect.mapError(stackError), + ); + } else if (compositionChanged) { + yield* catalog + .apply({ + target: { + stack, + database, + databaseServices, + jwtSecret: Redacted.value(persistedJwt ?? config.jwtSecret), + }, + overlay: { + webhooks: "config", + webhooksEnabled: toml.webhooksEnabled, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + workdir: target.projectRoot, + }, + }) + .pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + ); + yield* applyStackWebhooksOnly(database, toml.webhooksEnabled).pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), ); } else { - yield* applyStackWebhooksOnly(stack, toml.webhooksEnabled).pipe( + yield* applyStackWebhooksOnly(database, toml.webhooksEnabled).pipe( Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(setupFailed), + Effect.mapError(stackError), ); } - const skipSeeding = (error: { readonly message: string; readonly suggestion?: string }) => - output.raw( - `${yellow("WARNING:")} skipped seeding storage buckets: ${error.message}${ - error.suggestion === undefined ? "" : ` ${error.suggestion}` - }\n`, - "stderr", - ); - if (firstCreate) { - const capability = status.capabilities.find((entry) => entry.name === "storage"); - if (classifyStorageCapability(capability) === "disabled") { - // Skip silently: the stack was started with Storage excluded. - } else { - yield* Effect.gen(function* () { - const context = yield* loadLocalProjectContext( - target.projectRoot, - (message) => new SeedConfigLoadError({ message }), + if (initialComposition) { + const storage = members.find((instance) => instance.service === "storage"); + if (storage !== undefined) { + const context = yield* loadLocalProjectContext( + target.projectRoot, + (message) => new SeedConfigLoadError({ message }), + ); + if (hasConfiguredBuckets(context.config)) { + yield* storage.start.pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), ); - if (!hasConfiguredBuckets(context.config)) return; - const credentials = yield* stackStorageEndpointFor(stack, status); + yield* storage.ready.pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + ); + const credentials = yield* stackStorageCredentialsFor( + storage, + Redacted.value(persistedJwt ?? config.jwtSecret), + ).pipe(Effect.mapError(stackError)); yield* seedBucketsRun({ projectRef: "", emitSummary: false, @@ -297,106 +607,32 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags resolvedConfig: { config: context.config, document: context.loaded?.document }, projectEnvValues: toml.projectEnv, workdir: target.projectRoot, - }); - }).pipe( - // Missing capability/credentials and gateway-activation failures never abort a - // successful start; report and continue, same as an underlying seed-config - // failure below. - Effect.catchTags({ - StackStorageCapabilityError: skipSeeding, - StackStorageUnavailableError: skipSeeding, - }), - Effect.tapError((error) => starting.fail(error.message)), - Effect.mapError(seedFailed), - ); + }).pipe(Effect.mapError(stackError)); + } } } + yield* stack.composition.start.pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + ); + yield* Effect.forEach(preparation, (fiber) => Fiber.join(fiber)); + yield* Ref.set(initialCleanupComplete, true); + const endpoints = Object.fromEntries( + (yield* Effect.forEach(members, (member) => + member.status.pipe( + Effect.tapError((error) => starting.fail(error.message)), + Effect.mapError(stackError), + Effect.map((observation) => + Object.entries(endpointReports(observation)).map( + ([name, endpoint]) => [`${member.service}.${name}`, endpoint] as const, + ), + ), + ), + )).flat(), + ); yield* starting.succeed("Stack is ready."); - if (output.format === "text") { - yield* output.raw(renderStackStatus(status)); - } else { - yield* output.success("", stackStatusPayload(status)); - } - return status; + yield* output.success("", { id: stack.id, endpoints }); + return stack.id; }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); }); - -const stackStartError = (error: unknown) => { - const stackError = isStackError(error) ? error : undefined; - const message = stackError === undefined ? String(error) : stackError.message; - const classification = - stackError === undefined - ? { reason: "unknown" as const } - : Match.value(stackError).pipe( - Match.tag("ContainerEngineError", () => ({ - reason: "runtime" as const, - suggestion: - "Check that the selected container engine is installed and its daemon is running, then retry the command.", - })), - Match.tag("ContainerPullError", () => ({ - reason: "registry" as const, - suggestion: - "Check registry connectivity and image availability, then retry the command.", - })), - Match.tag("PortUnavailableError", "PortAllocationError", () => ({ - reason: "port" as const, - suggestion: - "Free the conflicting port or update the local stack port configuration, then retry.", - })), - Match.tag("StackPreparationError", "ArtifactIntegrityError", () => ({ - reason: "artifact" as const, - suggestion: "Retry the stack start with --debug if the artifact cannot be prepared.", - })), - Match.tag("StackRuntimeMismatchError", () => ({ - reason: "flags" as const, - suggestion: - "Omit --runtime to reuse the existing runtime, or choose a different --stack name.", - })), - Match.tag( - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "InvalidStackIdentityError", - "InvalidProjectRootError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - () => ({ reason: "invalid-config" as const }), - ), - Match.tag("StackStateInvalidError", () => ({ - reason: "invalid-config" as const, - suggestion: - "Inspect the reported state error and restore a valid state record before retrying.", - })), - Match.tag("StackStateFormatUnsupportedError", () => ({ - reason: "invalid-config" as const, - suggestion: "Use a CLI version compatible with the persisted stack state.", - })), - Match.tag("StackNotFoundError", () => ({ reason: "flags" as const })), - Match.tag( - "StackOwnershipConflictError", - "StackNotRunningError", - "StackMustBeStoppedError", - "StackLifecycleConflictError", - "StackUpgradeRequiredError", - () => ({ - reason: "lifecycle" as const, - suggestion: "Stop the stack before starting it again.", - }), - ), - Match.tag("StackRuntimeError", () => ({ - reason: "unknown" as const, - suggestion: "Retry the stack start with --debug and inspect the runtime diagnostics.", - })), - Match.tag("StackCleanupError", () => ({ - reason: "unknown" as const, - suggestion: "Retry the stack start with --debug and inspect cleanup diagnostics.", - })), - Match.orElse(() => ({ reason: "unknown" as const })), - ); - return new StackCommandStartError({ - ...classification, - message, - ...("suggestion" in classification ? { suggestion: classification.suggestion } : {}), - cause: error, - }); -}; diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 6365b73145..1b8272acd2 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -1,1608 +1,396 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { - Deferred, - Effect, - FileSystem, - Fiber, - Layer, - Option, - Path, - Redacted, - Schema, - Stream, -} from "effect"; -import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; -import { CliOutput, Command } from "effect/unstable/cli"; -import { - ContainerEngineError, - ContainerPullError, - StackConfigSchema, - StackIdSchema, - StackRuntimeError, - StackStateInvalidError, -} from "@supabase/stack/effect"; +import { Effect, FileSystem, Layer, Option, Stream } from "effect"; import type { - CapabilityState, - EffectStack, - StackStartError as ApiStackStartError, - StackStatus, + ServiceCreation, + ServiceInstance, + ServiceInstances, + Stack, } from "@supabase/stack/effect"; -import * as HttpClient from "effect/unstable/http/HttpClient"; -import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; -import { mockOutput, mockTty } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, mockTelemetryStateTracked, - withEnvVar, } from "../../../../../tests/helpers/command-mocks.ts"; -import { mockContextualAnalytics, mockProcessControl } from "../../../../../tests/helpers/mocks.ts"; +import { mockOutput, mockTty } from "../../../../../tests/helpers/mocks.ts"; import { - stackApiLayer, - StackTargetError, - StackTargetResolver, - stackTargetResolverLayer, - StackApi, -} from "../stack.shared.ts"; + DbConnection, + type DbSession, +} from "../../../../command-internal/db-connection.service.ts"; +import { StackCatalogSetup } from "../../../../command-internal/stack-catalog-setup.ts"; +import { ExperimentalFlag, YesFlag } from "../../../../command-internal/global-flags.ts"; +import { CommandPlatformApiFactory } from "../../../../auth/command-platform-api-factory.service.ts"; +import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; +import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; +import { StackApi, StackTargetResolver } from "../stack.shared.ts"; import { stackStart } from "./start.handler.ts"; -import { compileStack } from "../../../../../../../packages/stack/src/model/Compiler.ts"; import { StackCommandStartError } from "./start.errors.ts"; -import { stackStartCommand } from "./start.command.ts"; -import { textCliOutputFormatter } from "../../../../shared/output/text-formatter.ts"; -import { commandRuntimeLayer } from "../../../../shared/runtime/command-runtime.layer.ts"; -import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; -import { - ExperimentalFlag, - OutputFlag, - YesFlag, -} from "../../../../command-internal/global-flags.ts"; -import { DbConnection } from "../../../../command-internal/db-connection.service.ts"; -import { stackBackendLayer } from "../../../../command-internal/stack-backend.ts"; -import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; -import { CommandPlatformApiFactory } from "../../../../auth/command-platform-api-factory.service.ts"; -import { - actionability, - ErrorActionabilityId, -} from "../../../../shared/telemetry/error-actionability.ts"; -import { - noopStackCatalogSetupLayer, - recordingStackCatalogSetup, -} from "../../../../command-internal/stack-catalog-setup.ts"; - -const project = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-experimental-stack-start-", - }); - yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - 'project_id = "start-test"\n', - ); - return root; - }).pipe(Effect.provide(BunServices.layer)); - -const emptyProject = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - return yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-experimental-stack-start-empty-", - }); - }).pipe(Effect.provide(BunServices.layer)); - -const writeStartMigration = (root: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.makeDirectory(path.join(root, "supabase", "migrations"), { recursive: true }); - yield* fs.writeFileString( - path.join(root, "supabase", "migrations", "20240101000000_dogfood.sql"), - "create table public.dogfood ();\n", - ); - }); - -/** Overwrites the project's `config.toml` with a single configured bucket and one seed file. */ -const writeBucketsConfig = (root: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - 'project_id = "start-test"\n[storage.buckets.assets]\nobjects_path = "buckets/assets"\n', - ); - yield* fs.makeDirectory(path.join(root, "supabase", "buckets", "assets"), { - recursive: true, - }); - yield* fs.writeFileString( - path.join(root, "supabase", "buckets", "assets", "logo.png"), - "fake-bytes", - ); - }); - -const resolverLayer = stackTargetResolverLayer.pipe( - Layer.provideMerge(stackApiLayer), - Layer.provide(BunServices.layer), -); - -const status = (id: string, runtime: "native" | "container" = "native") => - ({ - id: StackIdSchema.make(id), - lifecycle: "running", - desiredLifecycle: "running", - runtime: runtime === "native" ? { kind: "native" } : { kind: "container", engine: "docker" }, - endpoints: { - api: { - protocol: "http" as const, - address: "127.0.0.1", - port: 55420, - url: "http://127.0.0.1:55420", - }, - }, - versions: {}, - capabilities: ( - [ - "database", - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", - ] as const - ).map((name) => ({ - name, - activation: name === "database" ? ("eager" as const) : ("lazy" as const), - state: "ready" as const, - })), - artifacts: [], - }) satisfies StackStatus; - -/** `status()` with the `storage` capability's state overridden. */ -const statusWithStorageState = (id: string, storageState: CapabilityState): StackStatus => ({ - ...status(id), - capabilities: status(id).capabilities.map((capability) => - capability.name === "storage" ? { ...capability, state: storageState } : capability, - ), -}); -/** `status()` with the `storage` capability entry removed entirely. */ -const statusWithoutStorageCapability = (id: string): StackStatus => ({ - ...status(id), - capabilities: status(id).capabilities.filter((capability) => capability.name !== "storage"), -}); - -/** `status()` with no API gateway endpoint exposed. */ -const statusWithoutApiEndpoint = (id: string): StackStatus => ({ - ...status(id), - endpoints: {}, -}); - -/** Records every request the seed-buckets gateway client issues, responding 200 to all. */ -function recordingStackStorageHttpClient(opts: { readonly bucketCreateStatus?: number } = {}) { - const requests: Array<{ - readonly method: string; - readonly url: string; - readonly authorization: string; - }> = []; - const layer = Layer.succeed( - HttpClient.HttpClient, - HttpClient.make((request) => { - requests.push({ - method: request.method, - url: request.url, - authorization: request.headers["authorization"] ?? "", - }); - if (request.method !== "GET" && opts.bucketCreateStatus !== undefined) { - return Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response("server error", { status: opts.bucketCreateStatus }), - ), - ); - } - const body = request.method === "GET" ? "[]" : JSON.stringify({ name: "bucket" }); - return Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response(body, { status: 200, headers: { "content-type": "application/json" } }), - ), - ); - }), - ); - return { layer, requests }; -} - -/** Records every request the seed-buckets gateway client issues, responding 503 to every GET. */ -function recordingStackStorageHttpClientGet503() { - const requests: Array<{ readonly method: string; readonly url: string }> = []; - const layer = Layer.succeed( - HttpClient.HttpClient, - HttpClient.make((request) => { - requests.push({ method: request.method, url: request.url }); - if (request.method === "GET") { - return Effect.succeed( - HttpClientResponse.fromWeb(request, new Response("unavailable", { status: 503 })), - ); - } - return Effect.succeed( - HttpClientResponse.fromWeb( - request, - new Response(JSON.stringify({ name: "bucket" }), { - status: 200, - headers: { "content-type": "application/json" }, - }), - ), - ); - }), - ); - return { layer, requests }; -} - -function fakeStack( - id: string, - start: (config?: { readonly config?: unknown }) => Effect.Effect, - desiredLifecycle: "unconfigured" | "stopped" | "running" = "unconfigured", -) { - // Mirrors a real stack: `.status` reflects the pre-start lifecycle (read by `firstCreate`) - // until `start` succeeds, then reflects `start`'s own return value (read by the bucket-seeding - // gate that follows it). - let currentStatus: StackStatus = { - ...status(id), - lifecycle: desiredLifecycle === "unconfigured" ? "unconfigured" : desiredLifecycle, - desiredLifecycle, - }; - return { - id: StackIdSchema.make(id), - status: Effect.suspend(() => Effect.succeed(currentStatus)), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), - password: Redacted.make("secret"), - }, - api: { - publishableKey: "anon", - secretKey: Redacted.make("service"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service"), - }, - }), - prepare: () => Effect.die("prepare not used in start test"), - start: (config?: { readonly config?: unknown }) => - start(config).pipe( - Effect.tap((result) => - Effect.sync(() => { - currentStatus = result; - }), - ), - ), - stop: Effect.void, - destroy: Effect.die("destroy not used in start test"), - resetDatabase: Effect.die("resetDatabase not used in start test"), - logs: () => Effect.die("logs not used in start test"), - followLogs: () => Stream.empty, - } satisfies EffectStack; -} - -const flags = ( - overrides: Partial[0]> = {}, -): Parameters[0] => ({ - exclude: [], +const flags = (exclude: ReadonlyArray = []) => ({ + exclude, stack: Option.none(), stackId: Option.none(), - runtime: "auto" as const, + runtime: "native" as const, preparation: "background" as const, eager: false, - ...overrides, }); -function handlerLayer(opts: { - root: string; - target: { - projectRoot: string; - name?: string; - id?: string; - runtime?: { kind: "native" } | { kind: "container"; engine: "docker" }; - }; - stack: EffectStack; - onCreate?: (options: unknown) => void; - onOpen?: () => void; - /** Overrides the default dying `HttpClient` stub, for tests exercising bucket seeding. */ - httpClient?: Layer.Layer; -}) { - const out = mockOutput(); - const telemetry = mockTelemetryStateTracked(); - const { id, ...targetWithoutId } = opts.target; - const targetLayer = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.succeed( - id === undefined ? targetWithoutId : { ...targetWithoutId, id: StackIdSchema.make(id) }, - ), - }); - const apiLayer = Layer.succeed(StackApi, { - findStack: () => Effect.succeed(Option.none()), - createStack: (options) => { - opts.onCreate?.(options); - return Effect.succeed(opts.stack); +const session: DbSession = { + exec: () => Effect.void, + execBatch: () => Effect.void, + query: () => Effect.succeed([]), + extensionExists: () => Effect.succeed(false), + copyToCsv: () => Effect.succeed(new Uint8Array()), + queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), +}; + +const instance = ( + creation: ServiceCreation, + id: string, +): ServiceInstances[ServiceCreation["service"]] => { + const status = (config: ServiceCreation) => ({ + id, + endpoints: + config.service === "database" + ? [{ name: "sql", protocol: "tcp" as const, host: "127.0.0.1", port: 23456 }] + : [], + config, + lifecycle: "stopped" as const, + health: undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: false, + registered: true, + }); + const base = { + id, + start: Effect.void, + ready: Effect.void, + stop: Effect.void, + restart: () => Effect.void, + destroy: Effect.void, + prepare: Effect.void, + status: Effect.succeed(status(creation)), + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => Effect.succeed({}), + } satisfies Omit; + switch (creation.service) { + case "database": + return { + ...base, + service: "database", + credentials: () => + Effect.succeed({ databaseUrl: "postgresql://postgres:postgres@127.0.0.1:5432/postgres" }), + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: Effect.die("unused"), + }; + case "rest": + return { ...base, service: "rest" }; + case "auth": + return { ...base, service: "auth" }; + case "realtime": + return { ...base, service: "realtime" }; + case "storage": + return { ...base, service: "storage" }; + case "imgproxy": + return { ...base, service: "imgproxy" }; + case "functions": { + let current = creation; + return { + ...base, + service: "functions", + restart: (input?: Parameters[0]) => + Effect.sync(() => { + if (input !== undefined) current = { ...current, config: input.config }; + }), + status: Effect.sync(() => status(current)), + }; + } + case "studio": + return { ...base, service: "studio" }; + case "pgmeta": + return { ...base, service: "pgmeta" }; + case "mail": + return { ...base, service: "mail" }; + case "analytics": + return { ...base, service: "analytics" }; + case "vector": + return { ...base, service: "vector" }; + case "pooler": + return { ...base, service: "pooler" }; + } +}; + +const fakeStack = () => { + let members: Array = []; + let stopped = 0; + let composed = 0; + const stack: Stack = { + id: "a".repeat(64), + services: { + create: (_creation: Input) => Effect.die("unused"), + get: (id: string) => { + const found = members.find((entry) => entry.id === id); + return found === undefined ? Effect.die(`missing instance ${id}`) : Effect.succeed(found); + }, + get list() { + return Effect.succeed(members); + }, }, - openStack: () => { - opts.onOpen?.(); - return Effect.succeed(opts.stack); + composition: { + describe: Effect.sync(() => ({ + members: members.map(({ id }) => ({ id, activation: "eager" as const })), + dependencies: [], + })), + supabase: (creations: ReadonlyArray) => + Effect.sync(() => { + composed += 1; + members = creations.map((creation) => instance(creation, `${creation.service}-member`)); + return members; + }), + configure: () => Effect.void, + start: Effect.succeed([]), + stop: Effect.sync(() => { + stopped += 1; + return []; + }), + restart: Effect.succeed([]), }, - inspectStack: () => Effect.die("inspect not used in handler test"), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - }); + stop: Effect.void, + destroy: Effect.void, + tools: { run: () => Effect.die("tool not used") }, + }; return { - out, - telemetry, - layer: Layer.mergeAll( - out.layer, - telemetry.layer, - mockCommandSettings({ workdir: opts.root }), - targetLayer, - apiLayer, - // Root provides the stack backend for every `stack` command; seeding guidance reads it. - stackBackendLayer("stack"), - BunServices.layer, - runtimeInfoLayer, - noopStackCatalogSetupLayer, - Layer.succeed(ExperimentalFlag, false), - Layer.succeed(CliArgs, { args: ["stack", "start"] }), - // The bucket-seeding path statically requires these even on the default fixture's - // no-buckets `config.toml`, where the short-circuit never reaches them at runtime. - opts.httpClient ?? - Layer.succeed( - HttpClient.HttpClient, - HttpClient.make(() => Effect.die("unused")), - ), - Layer.succeed(CommandPlatformApiFactory, { make: Effect.die("unused") }), - stdinLayer.pipe(Layer.provide(mockTty({ stdinIsTty: false, stdoutIsTty: false }))), - mockTty({ stdinIsTty: false, stdoutIsTty: false }), - Layer.succeed(YesFlag, false), - Layer.succeed(DbConnection, { - connect: () => - Effect.succeed({ - exec: () => Effect.void, - query: () => Effect.succeed([]), - execBatch: () => Effect.void, - extensionExists: () => Effect.succeed(false), - copyToCsv: () => Effect.succeed(new Uint8Array()), - queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), - }), - }), - ), + stack, + get members() { + return members; + }, + get stopped() { + return stopped; + }, + get composed() { + return composed; + }, }; -} - -describe("stack start targeting", () => { - for (const { exclusion, studioEnabled } of [ - { exclusion: "rest" as const, studioEnabled: false }, - { exclusion: "analytics" as const, studioEnabled: true }, - ]) { - it.live(`compiles ${exclusion} exclusion with Studio ${studioEnabled ? "on" : "off"}`, () => { - return project().pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const configBefore = yield* fs.readFileString( - path.join(root, "supabase", "config.toml"), - ); - const stack = fakeStack("c".repeat(64), (input) => - Effect.gen(function* () { - const stackConfig = yield* Schema.decodeUnknownEffect(StackConfigSchema)( - input?.config, - ).pipe( - Effect.mapError( - (error) => new StackStateInvalidError({ message: error.message }), - ), - ); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: stackConfig, - }).pipe( - Effect.mapError( - (error) => new StackStateInvalidError({ message: error.message }), - ), - Effect.provide(Layer.mergeAll(BunServices.layer, runtimeInfoLayer)), - ); - expect(compiled.definition.capabilities[exclusion].enabled).toBe(false); - expect(compiled.definition.capabilities.studio.enabled).toBe(studioEnabled); - expect(compiled.definition.capabilities.auth.enabled).toBe(true); - return status("c".repeat(64)); - }), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - for (const eager of [false, true]) - yield* stackStart(flags({ exclude: [exclusion], eager })).pipe( - Effect.provide(setup.layer), - ); - expect(yield* fs.readFileString(path.join(root, "supabase", "config.toml"))).toBe( - configBefore, - ); - }), - ), - Effect.provide(BunServices.layer), - ); - }); - } - - it.live("applies exclusions to the effective config without changing the project file", () => { - return project().pipe( - Effect.flatMap((root) => { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const configBefore = yield* fs.readFileString(path.join(root, "supabase", "config.toml")); - let startedConfig: unknown; - const stack = fakeStack("e".repeat(64), (config) => - Effect.gen(function* () { - startedConfig = config; - yield* Schema.decodeUnknownEffect(StackConfigSchema)(config?.config, { - onExcessProperty: "error", - }).pipe( - Effect.mapError((error) => new StackStateInvalidError({ message: error.message })), - ); - return status("e".repeat(64)); - }), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags({ exclude: ["studio", "analytics"] })).pipe( - Effect.provide(setup.layer), - ); - expect(startedConfig).toEqual( - expect.objectContaining({ - config: expect.objectContaining({ - capabilities: expect.objectContaining({ - studio: expect.objectContaining({ enabled: false }), - analytics: expect.objectContaining({ enabled: false }), - }), - }), - }), - ); - expect(yield* fs.readFileString(path.join(root, "supabase", "config.toml"))).toBe( - configBefore, - ); - }); - }), - Effect.provide(BunServices.layer), - ); - }); - - it.live("applies catalog setup from pre-exclude config after start returns", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeStartMigration(root); - const catalog = recordingStackCatalogSetup((input) => ({ - kind: input.target.kind, - authEnabled: input.target.config.capabilities?.auth?.enabled, - })); - const stack = fakeStack("f".repeat(64), () => Effect.succeed(status("f".repeat(64)))); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags({ exclude: ["auth"] })).pipe( - Effect.provide(Layer.mergeAll(setup.layer, catalog.layer)), - ); - expect(catalog.applied).toEqual([{ kind: "live", authEnabled: undefined }]); - expect(setup.out.stderrText).toContain("Applying migration 20240101000000_dogfood.sql"); - }).pipe(Effect.provide(BunServices.layer)); - }); +}; - it.live("threads excluded analytics into optional catalog downloads", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeStartMigration(root); - const catalog = recordingStackCatalogSetup((input) => ({ - analytics: input.optionalConfig?.capabilities?.analytics?.enabled, - })); - const stack = fakeStack("a".repeat(64), () => Effect.succeed(status("a".repeat(64)))); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags({ exclude: ["analytics"] })).pipe( - Effect.provide(Layer.mergeAll(setup.layer, catalog.layer)), - ); - expect(catalog.applied).toEqual([{ analytics: false }]); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("honors SUPABASE_EXPERIMENTAL from project .env on first-create migrate", () => { - return Effect.gen(function* () { - const root = yield* project(); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* writeStartMigration(root); - yield* fs.writeFileString( - path.join(root, "supabase", ".env"), - "SUPABASE_EXPERIMENTAL=true\n", - ); - const catalog = recordingStackCatalogSetup((input) => input.target.kind); - const stack = fakeStack("e".repeat(64), () => Effect.succeed(status("e".repeat(64)))); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* withEnvVar( - "SUPABASE_EXPERIMENTAL", - undefined, - stackStart(flags()).pipe(Effect.provide(Layer.mergeAll(setup.layer, catalog.layer))), - ); - expect(setup.out.stderrText).not.toContain("Applying migration 20240101000000_dogfood.sql"); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("skips catalog setup when an existing cluster is already present", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeStartMigration(root); - const catalog = recordingStackCatalogSetup((input) => input.target.kind); - const stack = fakeStack( - "d".repeat(64), - () => Effect.succeed(status("d".repeat(64))), - "stopped", - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - }); - yield* stackStart(flags()).pipe(Effect.provide(Layer.mergeAll(setup.layer, catalog.layer))); - expect(catalog.applied).toEqual([]); - expect(setup.out.stderrText).not.toContain("Applying migration"); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("applies catalog and migrations when starting an unconfigured stack by id", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeStartMigration(root); - const catalog = recordingStackCatalogSetup((input) => input.target.kind); - const stack = fakeStack("c".repeat(64), () => Effect.succeed(status("c".repeat(64)))); - const setup = handlerLayer({ - root, - target: { projectRoot: root, id: "c".repeat(64) }, - stack, - }); - yield* stackStart(flags({ stackId: Option.some("c".repeat(64)) })).pipe( - Effect.provide(Layer.mergeAll(setup.layer, catalog.layer)), - ); - expect(catalog.applied).toEqual(["live"]); - expect(setup.out.stderrText).toContain("Applying migration 20240101000000_dogfood.sql"); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live( - "leaves listener configuration to the compiled runtime when capabilities are excluded", - () => { - return project().pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - 'project_id = "start-test"\n[api]\nport = 55421\n', - ); - let startedConfig: unknown; - const stack = fakeStack("7".repeat(64), (config) => - Effect.sync(() => { - startedConfig = config; - return status("7".repeat(64)); - }), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart( - flags({ exclude: ["rest", "auth", "realtime", "storage", "functions", "analytics"] }), - ).pipe(Effect.provide(setup.layer)); - expect(startedConfig).toEqual( - expect.objectContaining({ - config: expect.objectContaining({ - listeners: expect.objectContaining({ api: { port: 55421 } }), - }), - }), - ); - }), - ), - Effect.provide(BunServices.layer), - ); - }, +const layers = (root: string, fixture: ReturnType, output = mockOutput()) => { + const telemetry = mockTelemetryStateTracked(); + const target = Layer.succeed(StackTargetResolver, { + resolve: () => Effect.succeed({ projectRoot: root, runtime: "native" as const }), + }); + const api = Layer.succeed(StackApi, { + create: () => Effect.succeed(fixture.stack), + open: () => Effect.succeed(fixture.stack), + discover: () => Effect.succeed([]), + resolveIdentity: () => Effect.die("identity not used"), + }); + return Layer.mergeAll( + BunServices.layer, + runtimeInfoLayer, + output.layer, + telemetry.layer, + mockCommandSettings({ workdir: root }), + target, + api, + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(CliArgs, { args: ["stack", "start"] }), + Layer.succeed(StackCatalogSetup, { apply: () => Effect.void }), + Layer.succeed(DbConnection, { connect: () => Effect.scoped(Effect.succeed(session)) }), + Layer.succeed(YesFlag, false), + Layer.succeed(CommandPlatformApiFactory, { make: Effect.die("unused") }), + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make(() => Effect.die("unused")), + ), + stdinLayer.pipe(Layer.provide(mockTty({ stdinIsTty: false, stdoutIsTty: false }))), + mockTty({ stdinIsTty: false, stdoutIsTty: false }), ); +}; - it.live("preserves the configured API listener on partial gateway exclusion", () => { - return project().pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - 'project_id = "start-test"\n[api]\nport = 55421\n', - ); - let startedConfig: unknown; - const stack = fakeStack("8".repeat(64), (config) => - Effect.sync(() => { - startedConfig = config; - return status("8".repeat(64)); - }), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags({ exclude: ["rest"] })).pipe(Effect.provide(setup.layer)); - expect(startedConfig).toEqual( - expect.objectContaining({ - config: expect.objectContaining({ - listeners: expect.objectContaining({ api: { port: 55421 } }), - }), - }), - ); - }), - ), - Effect.provide(BunServices.layer), - ); - }); - - it.live("does not rewrite listeners for configured-disabled capabilities", () => { - return project().pipe( - Effect.flatMap((root) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - yield* fs.writeFileString( - path.join(root, "supabase", "config.toml"), - `project_id = "start-test" -[api] -port = 55421 -[auth] -enabled = false -[realtime] -enabled = false -[storage] -enabled = false -[edge_runtime] -enabled = false -[analytics] -enabled = false -`, - ); - let startedConfig: unknown; - const stack = fakeStack("9".repeat(64), (config) => - Effect.sync(() => { - startedConfig = config; - return status("9".repeat(64)); - }), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags({ exclude: ["rest"] })).pipe(Effect.provide(setup.layer)); - expect(startedConfig).toEqual( - expect.objectContaining({ - config: expect.objectContaining({ - listeners: expect.objectContaining({ api: { port: 55421 } }), - }), - }), - ); - }), - ), - Effect.provide(BunServices.layer), - ); - }); - - it.live("leaves auto runtime selection to the package for a new stack", () => { - return Effect.gen(function* () { - const root = yield* project(); - let createOptions: unknown; - const stack = fakeStack("f".repeat(64), () => Effect.succeed(status("f".repeat(64)))); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - onCreate: (options) => { - createOptions = options; - }, - }); - yield* stackStart(flags({ runtime: "auto" })).pipe(Effect.provide(setup.layer)); - expect(createOptions).toEqual({ projectRoot: root }); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("forwards an explicit Docker runtime to the package", () => { - return Effect.gen(function* () { - const root = yield* project(); - let createOptions: unknown; - const stack = fakeStack("d".repeat(64), () => Effect.succeed(status("d".repeat(64)))); - const setup = handlerLayer({ - root, - target: { - projectRoot: root, - name: "feature-docker", - runtime: { kind: "container", engine: "docker" }, - }, - stack, - onCreate: (options) => { - createOptions = options; - }, - }); - yield* stackStart(flags({ stack: Option.some("feature-docker"), runtime: "docker" })).pipe( - Effect.provide(setup.layer), - ); - expect(createOptions).toEqual({ - projectRoot: root, - name: "feature-docker", - runtime: { kind: "container", engine: "docker" }, - }); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.effect("resolves the current project target", () => { - return Effect.gen(function* () { - const root = yield* project(); - const resolver = yield* StackTargetResolver; - const target = yield* resolver.resolve({ - projectRoot: root, - runtime: "auto", - }); - expect(target.projectRoot).toBe(root); - expect(target.id).toBeUndefined(); - expect(target.name).toBeUndefined(); - expect(target.runtime).toBeUndefined(); - }).pipe(Effect.provide(resolverLayer)); - }); - - it.effect("keeps a named native stack target distinct", () => { - return Effect.gen(function* () { - const root = yield* project(); - const resolver = yield* StackTargetResolver; - const target = yield* resolver.resolve({ - projectRoot: root, - name: "feature-a", - runtime: "native", - }); - expect(target.name).toBe("feature-a"); - expect(target.runtime).toEqual({ kind: "native" }); - }).pipe(Effect.provide(resolverLayer)); - }); - - it.effect("rejects a malformed stack id before loading project configuration", () => +describe("experimental stack start", () => { + it.live("rejects incompatible Functions env before changing composition", () => Effect.gen(function* () { - const resolver = yield* StackTargetResolver; - const failure = yield* resolver - .resolve({ projectRoot: "/does/not/exist", id: "invalid", runtime: "auto" }) - .pipe(Effect.flip); - expect(failure).toBeInstanceOf(StackTargetError); - expect(failure.message).toContain("lowercase SHA-256"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - }).pipe(Effect.provide(resolverLayer)), + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-functions-env-" }); + yield* fs.makeDirectory(`${root}/supabase/functions`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "functions-env"\n'); + const fixture = fakeStack(); + for (const [contents, message] of [ + ["INVALID.KEY=value\n", "Environment names"], + ['VALUE="first\nsecond"\n', "Multiline"], + ] as const) { + yield* fs.writeFileString(`${root}/supabase/functions/.env`, contents); + const error = yield* stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + expect(error).toMatchObject({ reason: "invalid-config" }); + expect(error.message).toContain(message); + expect(fixture.composed).toBe(0); + } + }).pipe(Effect.provide(BunServices.layer)), ); - it.effect("classifies an existing stack runtime mismatch as provided flags", () => { - const api = Layer.succeed(StackApi, { - findStack: () => Effect.succeed(Option.none()), - createStack: () => Effect.die("unused"), - openStack: () => Effect.die("unused"), - inspectStack: () => - Effect.succeed({ - descriptor: { - id: StackIdSchema.make("b".repeat(64)), - projectRoot: "/tmp/existing-stack", - name: "existing", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - owner: "running", - }), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - }); - return Effect.gen(function* () { - const resolver = yield* StackTargetResolver; - const failure = yield* resolver - .resolve({ projectRoot: "/tmp/project", id: "b".repeat(64), runtime: "docker" }) - .pipe(Effect.flip); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - }).pipe( - Effect.provide( - stackTargetResolverLayer.pipe(Layer.provideMerge(api), Layer.provide(BunServices.layer)), - ), - ); - }); - - it.live("creates a named native stack with eager on-demand configuration", () => { - return Effect.gen(function* () { - const root = yield* project(); - let createOptions: unknown; - let startConfig: unknown; - const stack = fakeStack("a".repeat(64), (config) => { - startConfig = config; - return Effect.succeed(status("a".repeat(64))); - }); - const setup = handlerLayer({ - root, - target: { projectRoot: root, name: "feature-a", runtime: { kind: "native" } }, - stack, - onCreate: (options) => { - createOptions = options; - }, - }); - yield* stackStart( - flags({ - stack: Option.some("feature-a"), - runtime: "native", - preparation: "on-demand", - eager: true, - exclude: ["studio"], - }), - ).pipe(Effect.provide(setup.layer)); - expect(createOptions).toEqual({ - projectRoot: root, - name: "feature-a", - runtime: { kind: "native" }, - }); - expect(startConfig).toMatchObject({ config: { preparation: "on-demand" } }); - expect(startConfig).toEqual( - expect.objectContaining({ - config: expect.objectContaining({ - capabilities: expect.objectContaining({ - rest: expect.objectContaining({ activation: "eager" }), - studio: expect.objectContaining({ enabled: false }), - }), + it.live("rejects malformed configuration before creating a stack", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-invalid-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, "[db]\nmajor_version = 14\n"); + let created = false; + const fixture = fakeStack(); + const base = layers(root, fixture); + const api = Layer.succeed(StackApi, { + create: () => + Effect.sync(() => { + created = true; + return fixture.stack; }), - }), - ); - expect(setup.out.stdoutText).toContain("Stack"); - expect(setup.telemetry.flushed).toBe(true); - }); - }); - - it.live("renders stack recovery guidance returned by the Effect API", () => { - return Effect.gen(function* () { - const root = yield* project(); - const stackId = "d".repeat(64); - const stack = fakeStack(stackId, () => - Effect.succeed({ - ...status(stackId), - recovery: { - operation: "destroy" as const, - message: "Destroy is required before this stack can be started again.", - }, - }), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(setup.out.stdoutText).toContain( - "Recovery: Destroy is required before this stack can be started again.", + open: () => Effect.succeed(fixture.stack), + discover: () => Effect.succeed([]), + resolveIdentity: () => Effect.die("identity not used"), + }); + const result = yield* stackStart(flags()).pipe( + Effect.flip, + Effect.provide(Layer.merge(base, api)), ); - expect(setup.out.stdoutText).toContain(`supabase stack destroy --stack-id ${stackId}`); - expect(setup.out.stdoutText).toContain("Warning: destroy is destructive"); - }); - }); + expect(result).toMatchObject({ reason: "invalid-config" }); + expect(created).toBe(false); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.live("opens an addressed existing stack using its own project root", () => { - return Effect.gen(function* () { - const settingsRoot = yield* project(); - const targetRoot = yield* project(); + it.live("applies capability selection across repeated starts", () => + Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-db-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); yield* fs.writeFileString( - path.join(targetRoot, "supabase", "config.toml"), - 'project_id = "target"\n[api]\nport = 55421\n', - ); - let opened = false; - let startConfig: unknown; - const stack = fakeStack( - "b".repeat(64), - (config) => { - startConfig = config; - return Effect.succeed(status("b".repeat(64))); - }, - "running", + `${root}/supabase/config.toml`, + 'project_id = "start-test"\n[edge_runtime]\nenabled = false\n', ); - const setup = handlerLayer({ - root: settingsRoot, - target: { projectRoot: targetRoot, id: "b".repeat(64) }, - stack, - onOpen: () => { - opened = true; - }, - }); - yield* stackStart(flags({ stackId: Option.some("b".repeat(64)) })).pipe( - Effect.provide(setup.layer), - ); - expect(opened).toBe(true); - expect(startConfig).toMatchObject({ config: { listeners: { api: { port: 55421 } } } }); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("reports a typed runtime failure without success output or cleanup calls", () => { - return Effect.gen(function* () { - const root = yield* project(); - let stopped = false; - let destroyed = false; - const stack = { - ...fakeStack("c".repeat(64), () => - Effect.fail(new ContainerEngineError({ message: "Docker is unavailable" })), - ), - stop: Effect.sync(() => { - stopped = true; - }), - destroy: Effect.sync(() => { - destroyed = true; - }), - } satisfies EffectStack; - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - const failure = yield* stackStart(flags()).pipe(Effect.flip, Effect.provide(setup.layer)); - expect(failure).toBeInstanceOf(StackCommandStartError); - if (failure instanceof StackCommandStartError) { - expect(failure.reason).toBe("runtime"); - expect(failure.suggestion).toContain("container engine is installed"); - expect(failure.suggestion).toContain("daemon is running"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.dockerNotRunning); - } - expect(stopped).toBe(false); - expect(destroyed).toBe(false); - expect(setup.out.messages.filter((message) => message.type === "success")).toHaveLength(0); - expect(setup.telemetry.flushed).toBe(true); - }); - }); - - it.live("does not clean up a stack after a successful start", () => { - return Effect.gen(function* () { - const root = yield* project(); - let stopped = false; - let destroyed = false; - const stack = { - ...fakeStack("f".repeat(64), () => Effect.succeed(status("f".repeat(64)))), - stop: Effect.sync(() => { - stopped = true; - }), - destroy: Effect.sync(() => { - destroyed = true; - }), - } satisfies EffectStack; - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(stopped).toBe(false); - expect(destroyed).toBe(false); - }); - }); - - it.live("stops a running postgres-only stack before starting the full config", () => { - return Effect.gen(function* () { - const root = yield* project(); - const events: Array = []; - const id = "a".repeat(64); - const stack = { - ...fakeStack( - id, - () => { - events.push("start"); - return Effect.succeed(status(id)); + const fixture = fakeStack(); + const excluded = [ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "mail", + "analytics", + "pooler", + ]; + const output = mockOutput({ format: "json" }); + yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture, output))); + expect(output.messages).toContainEqual( + expect.objectContaining({ + data: { + id: fixture.stack.id, + endpoints: { + "database.sql": { + protocol: "tcp", + address: "127.0.0.1", + port: 23456, + url: "tcp://127.0.0.1:23456", + }, + }, }, - "running", - ), - status: Effect.succeed({ - ...status(id), - lifecycle: "running" as const, - desiredLifecycle: "running" as const, - capabilities: status(id).capabilities.map((capability) => ({ - ...capability, - state: capability.name === "database" ? ("ready" as const) : ("disabled" as const), - })), - }), - stop: Effect.sync(() => { - events.push("stop"); }), - } satisfies EffectStack; - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(events).toEqual(["stop", "start"]); - }); - }); - - it.live("classifies registry pull failures separately from engine failures", () => { - return Effect.gen(function* () { - const root = yield* project(); - const stack = fakeStack("8".repeat(64), () => - Effect.fail( - new ContainerPullError({ - message: "registry refused the workload image", - image: "example.test/workload:dev", - }), - ), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - const failure = yield* stackStart(flags()).pipe(Effect.flip, Effect.provide(setup.layer)); - expect(failure).toBeInstanceOf(StackCommandStartError); - if (failure instanceof StackCommandStartError) { - expect(failure.reason).toBe("registry"); - expect(failure.suggestion).toContain("registry connectivity"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.externalNetwork); - } - }); - }); - - it.live("reports runtime start failures with operational guidance", () => { - return Effect.gen(function* () { - const root = yield* project(); - const stack = fakeStack("b".repeat(64), () => - Effect.fail(new StackRuntimeError({ message: "runtime crashed" })), ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - const failure = yield* stackStart(flags()).pipe(Effect.flip, Effect.provide(setup.layer)); - expect(failure).toBeInstanceOf(StackCommandStartError); - if (failure instanceof StackCommandStartError) { - expect(failure.reason).toBe("unknown"); - expect(failure.suggestion).toContain("runtime diagnostics"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.unknown); - } - }); - }); - - it.live("classifies persisted state failures with recovery guidance", () => { - return Effect.gen(function* () { - const root = yield* project(); - const stack = fakeStack("c".repeat(64), () => - Effect.fail(new StackStateInvalidError({ message: "persisted state is invalid" })), - ); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - const failure = yield* stackStart(flags()).pipe(Effect.flip, Effect.provide(setup.layer)); - expect(failure).toBeInstanceOf(StackCommandStartError); - if (failure instanceof StackCommandStartError) { - expect(failure.reason).toBe("invalid-config"); - expect(failure.suggestion).toContain("restore a valid state record"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.invalidConfig); - } - }); - }); + expect(fixture.members.map(({ service }) => service)).toEqual(["database"]); + expect(fixture.composed).toBe(1); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.some(({ service }) => service === "rest")).toBe(true); + expect(fixture.composed).toBe(2); + yield* stackStart(flags(["studio"])).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.some(({ service }) => service === "studio")).toBe(false); + expect(fixture.stopped).toBe(2); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.some(({ service }) => service === "studio")).toBe(true); + expect(fixture.stopped).toBe(3); + yield* stackStart(flags(excluded)).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.composed).toBe(5); + }).pipe(Effect.provide(BunServices.layer)), + ); - it.live("starts with default configuration when no project config exists", () => { - return Effect.gen(function* () { - const root = yield* emptyProject(); - let started = false; - const stack = fakeStack("9".repeat(64), (config) => { - started = true; - expect(config).toMatchObject({ - config: { capabilities: { database: { settings: { health_timeout: "2m" } } } }, - }); - return Effect.succeed(status("9".repeat(64))); - }); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(started).toBe(true); + it.live("loads Functions env only when Functions are selected", () => + Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - expect(yield* fs.exists(path.join(root, "supabase", "config.toml"))).toBe(false); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live("keeps ownership with the package when the CLI caller is interrupted", () => { - return Effect.gen(function* () { - const root = yield* project(); - const started = yield* Deferred.make(); - let stopped = false; - let destroyed = false; - const stack = { - ...fakeStack("7".repeat(64), () => - Effect.gen(function* () { - yield* Deferred.succeed(started, undefined); - return yield* Effect.never.pipe(Effect.as(status("7".repeat(64)))); - }), - ), - stop: Effect.sync(() => { - stopped = true; - }), - destroy: Effect.sync(() => { - destroyed = true; - }), - } satisfies EffectStack; - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - const fiber = yield* Effect.forkChild(Effect.provide(stackStart(flags()), setup.layer)); - yield* Deferred.await(started); - yield* Fiber.interrupt(fiber); - expect(stopped).toBe(false); - expect(destroyed).toBe(false); - }); - }); - - it.live("rejects invalid target flags before resolving or mutating a stack", () => { - return Effect.gen(function* () { - const root = yield* project(); - let resolved = false; - let created = false; - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack: fakeStack("d".repeat(64), () => Effect.succeed(status("d".repeat(64)))), - }); - const layer = Layer.mergeAll( - setup.out.layer, - setup.telemetry.layer, - mockCommandSettings({ workdir: root }), - Layer.succeed(StackTargetResolver, { - resolve: () => { - resolved = true; - return Effect.die("resolver should not run"); - }, - }), - Layer.succeed(StackApi, { - findStack: () => Effect.succeed(Option.none()), - createStack: () => { - created = true; - return Effect.die("create should not run"); - }, - openStack: () => Effect.die("open should not run"), - inspectStack: () => Effect.die("inspect should not run"), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - }), - BunServices.layer, - runtimeInfoLayer, - Layer.succeed(ExperimentalFlag, false), - Layer.succeed(CliArgs, { args: ["stack", "start"] }), - Layer.succeed(DbConnection, { - connect: () => - Effect.succeed({ - exec: () => Effect.void, - query: () => Effect.succeed([]), - execBatch: () => Effect.void, - extensionExists: () => Effect.succeed(false), - copyToCsv: () => Effect.succeed(new Uint8Array()), - queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), - }), - }), - Layer.succeed( - HttpClient.HttpClient, - HttpClient.make(() => Effect.die("unused")), - ), - Layer.succeed(CommandPlatformApiFactory, { make: Effect.die("unused") }), - stdinLayer.pipe(Layer.provide(mockTty({ stdinIsTty: false, stdoutIsTty: false }))), - mockTty({ stdinIsTty: false, stdoutIsTty: false }), - Layer.succeed(YesFlag, false), - ); - const failure = yield* stackStart( - flags({ stack: Option.some("feature"), stackId: Option.some("e".repeat(64)) }), - ).pipe(Effect.flip, Effect.provide(layer)); - expect(failure.message).toContain("cannot be used together"); - expect(resolved).toBe(false); - expect(created).toBe(false); - expect(setup.telemetry.flushed).toBe(true); - }); - }); -}); - -describe("stack start bucket seeding", () => { - it.live("seeds configured buckets through the stack gateway when storage is dormant", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = { - ...fakeStack("1".repeat(64), () => - Effect.succeed(statusWithStorageState("1".repeat(64), "dormant")), - ), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), - password: Redacted.make("secret"), - }, - api: { - publishableKey: "anon", - secretKey: Redacted.make("service"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("created-stack-jwt"), - }, - }), - } satisfies EffectStack; - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - const result = yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(result.capabilities.find((c) => c.name === "storage")?.state).toBe("dormant"); - expect( - client.requests.some( - (r) => r.method === "POST" && r.url === "http://127.0.0.1:55420/storage/v1/bucket", - ), - ).toBe(true); - expect( - client.requests.some( - (r) => r.url === "http://127.0.0.1:55420/storage/v1/object/assets/logo.png", - ), - ).toBe(true); - expect(client.requests.every((r) => r.authorization.includes("created-stack-jwt"))).toBe( - true, + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-functions-env-" }); + yield* fs.makeDirectory(`${root}/supabase/functions`, { recursive: true }); + yield* fs.writeFileString( + `${root}/supabase/config.toml`, + 'project_id = "start-functions-env"\n[edge_runtime]\nenabled = true\n', ); - expect(setup.out.stdoutText).toContain("Stack"); - }).pipe(Effect.provide(BunServices.layer)); - }); + yield* fs.writeFileString(`${root}/supabase/functions/.env`, 'BROKEN="unterminated\n'); + const fixture = fakeStack(); - it.live("skips seeding without a warning when storage is disabled", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = fakeStack("2".repeat(64), () => - Effect.succeed(statusWithStorageState("2".repeat(64), "disabled")), - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags({ exclude: ["storage"] })).pipe(Effect.provide(setup.layer)); - expect(client.requests).toHaveLength(0); - expect(setup.out.stderrText).not.toContain("skipped seeding storage buckets"); - }).pipe(Effect.provide(BunServices.layer)); - }); + yield* stackStart(flags(["functions"])).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.some(({ service }) => service === "functions")).toBe(false); - it.live("warns and issues no requests when storage failed to start", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = fakeStack("3".repeat(64), () => - Effect.succeed(statusWithStorageState("3".repeat(64), "failed")), - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(setup.out.stderrText).toContain( - "WARNING: skipped seeding storage buckets: Storage failed to start for this stack.", + yield* fs.writeFileString( + `${root}/supabase/functions/.env`, + "CUSTOM_VALUE=hello\nSUPABASE_SERVICE_ROLE_KEY=ignored\n", ); - expect(client.requests).toHaveLength(0); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live( - "warns without failing when first-start seeding cannot resolve the stack's API credentials (Auth disabled)", - () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = { - ...fakeStack("a".repeat(64), () => - Effect.succeed(statusWithStorageState("a".repeat(64), "dormant")), - ), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:secret@127.0.0.1:54329/postgres"), - password: Redacted.make("secret"), - }, - }), - } satisfies EffectStack; - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(setup.out.stderrText).toContain("WARNING: skipped seeding storage buckets:"); - expect(setup.out.stderrText).toContain("API credentials"); - expect(setup.out.stderrText).toContain("Auth"); - expect(client.requests).toHaveLength(0); - }).pipe(Effect.provide(BunServices.layer)); - }, - ); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + const functions = fixture.members.find(({ service }) => service === "functions"); + if (functions?.service !== "functions") return yield* Effect.die("Functions missing"); + const database = fixture.members.find(({ service }) => service === "database"); + if (database === undefined) return yield* Effect.die("Database missing"); + const databaseId = database.id; + const functionsId = functions.id; + const status = yield* functions.status; + expect(status.config.service).toBe("functions"); + if (status.config.service === "functions") + expect(status.config.config.env).toEqual({ CUSTOM_VALUE: "hello" }); - it.live( - "does not warn or resolve credentials when no buckets are configured and the stack exposes no API endpoint", - () => { - return Effect.gen(function* () { - const root = yield* project(); - const client = recordingStackStorageHttpClient(); - const stack = fakeStack("0".repeat(64), () => - Effect.succeed(statusWithoutApiEndpoint("0".repeat(64))), - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(setup.out.stderrText).not.toContain("skipped seeding storage buckets"); - expect(client.requests).toHaveLength(0); - }).pipe(Effect.provide(BunServices.layer)); - }, - ); - - it.live("never seeds buckets when opening an existing stack", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = fakeStack( - "4".repeat(64), - () => Effect.succeed(statusWithStorageState("4".repeat(64), "dormant")), - "running", + yield* fs.writeFileString( + `${root}/supabase/functions/.env`, + "CUSTOM_VALUE=changed\nSUPABASE_SERVICE_ROLE_KEY=ignored-again\n", ); - const setup = handlerLayer({ - root, - target: { projectRoot: root, id: "4".repeat(64) }, - stack, - httpClient: client.layer, + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + expect(fixture.members.find(({ service }) => service === "database")?.id).toBe(databaseId); + const refreshed = fixture.members.find(({ service }) => service === "functions"); + if (refreshed?.service !== "functions") return yield* Effect.die("Functions missing"); + expect(refreshed.id).toBe(functionsId); + const refreshedStatus = yield* refreshed.status; + if (refreshedStatus.config.service === "functions") + expect(refreshedStatus.config.config.env).toEqual({ CUSTOM_VALUE: "changed" }); + + if (refreshedStatus.config.service !== "functions") + return yield* Effect.die("Functions configuration missing"); + yield* refreshed.restart({ + config: { ...refreshedStatus.config.config, verifyJwt: false }, }); - yield* stackStart(flags({ stackId: Option.some("4".repeat(64)) })).pipe( - Effect.provide(setup.layer), - ); - expect(client.requests).toHaveLength(0); - }).pipe(Effect.provide(BunServices.layer)); - }); - - it.live( - "never seeds a plain start of an already-configured stack (no --stack-id, dormant storage)", - () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = fakeStack( - "6".repeat(64), - () => Effect.succeed(statusWithStorageState("6".repeat(64), "dormant")), - "running", - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(client.requests).toHaveLength(0); - }).pipe(Effect.provide(BunServices.layer)); - }, - ); - - it.live( - "seeds an addressed stack prepared but never started, even though --stack-id names an existing id", - () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const id = "d".repeat(64); - const stack = fakeStack(id, () => Effect.succeed(statusWithStorageState(id, "dormant"))); - const setup = handlerLayer({ - root, - target: { projectRoot: root, id }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags({ stackId: Option.some(id) })).pipe(Effect.provide(setup.layer)); - expect( - client.requests.some( - (r) => r.method === "POST" && r.url === "http://127.0.0.1:55420/storage/v1/bucket", - ), - ).toBe(true); - }).pipe(Effect.provide(BunServices.layer)); - }, - ); - - it.live( - "warns and issues no requests when the started status reports no storage capability at all", - () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient(); - const stack = fakeStack("2".repeat(64), () => - Effect.succeed(statusWithoutStorageCapability("2".repeat(64))), - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(setup.out.stderrText).toContain("WARNING: skipped seeding storage buckets"); - expect(client.requests).toHaveLength(0); - }).pipe(Effect.provide(BunServices.layer)); - }, - ); - - it.live( - "warns and succeeds when the storage gateway returns 503 while activating during seeding", - () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClientGet503(); - const stack = fakeStack("3".repeat(64), () => - Effect.succeed(statusWithStorageState("3".repeat(64), "dormant")), - ); - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - yield* stackStart(flags()).pipe(Effect.provide(setup.layer)); - expect(setup.out.stderrText).toContain("WARNING:"); - expect(setup.out.stderrText).toContain("HTTP 503 for Storage"); - expect(setup.out.stderrText).not.toContain("activate"); - }).pipe(Effect.provide(BunServices.layer)); - }, + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + const restored = yield* refreshed.status; + expect(restored.config).toMatchObject({ service: "functions", config: { verifyJwt: true } }); + }).pipe(Effect.provide(BunServices.layer)), ); - it.live("fails with reason 'seed' and never stops/destroys the stack on a gateway error", () => { - return Effect.gen(function* () { - const root = yield* project(); - yield* writeBucketsConfig(root); - const client = recordingStackStorageHttpClient({ bucketCreateStatus: 500 }); - let stopped = false; - let destroyed = false; - const stack = { - ...fakeStack("5".repeat(64), () => - Effect.succeed(statusWithStorageState("5".repeat(64), "dormant")), - ), - stop: Effect.sync(() => { - stopped = true; - }), - destroy: Effect.sync(() => { - destroyed = true; - }), - } satisfies EffectStack; - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack, - httpClient: client.layer, - }); - const failure = yield* stackStart(flags()).pipe(Effect.flip, Effect.provide(setup.layer)); - expect(failure).toBeInstanceOf(StackCommandStartError); - if (failure instanceof StackCommandStartError) { - expect(failure.reason).toBe("seed"); - } - expect(stopped).toBe(false); - expect(destroyed).toBe(false); - }).pipe(Effect.provide(BunServices.layer)); - }); -}); - -describe("stack start parser", () => { - it.live("records the wired command identity with a fresh run id per invocation", () => { - return Effect.gen(function* () { - const root = yield* project(); - const analytics = mockContextualAnalytics(); - const processControl = mockProcessControl(); - const output = mockOutput(); - const stack = fakeStack("e".repeat(64), () => Effect.succeed(status("e".repeat(64)))); - const setup = handlerLayer({ root, target: { projectRoot: root }, stack }); - const command = stackStartCommand.pipe( - Command.provide(commandRuntimeLayer(["stack", "start"])), - Command.provide( - Layer.mergeAll(setup.layer, output.layer, analytics.layer, processControl.layer), - ), - ); - const run = Command.runWith(command, { version: "0.0.0-test" })([]); - const runtime = Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter())); - - yield* run.pipe(Effect.provide(runtime)); - yield* run.pipe(Effect.provide(runtime)); - const events = analytics.captured.filter((event) => event.event === "cli_command_executed"); - expect(events).toHaveLength(2); - expect(events[0]?.properties.command).toBe("stack start"); - expect(events[1]?.properties.command).toBe("stack start"); - expect(events[0]?.properties.command_run_id).toBeDefined(); - expect(events[1]?.properties.command_run_id).toBeDefined(); - expect(events[0]?.properties.command_run_id).not.toBe(events[1]?.properties.command_run_id); - }); - }); - - it.live("parses --stack and --runtime through the command", () => { - let parsed: { stack: Option.Option; runtime: string } | undefined; - const command = stackStartCommand.pipe( - Command.withHandler((flags) => - Effect.sync(() => { - parsed = { stack: flags.stack, runtime: flags.runtime }; - }), - ), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--stack", - "feature-a", - "--runtime", - "native", - ]); - expect(parsed?.stack).toEqual(Option.some("feature-a")); - expect(parsed?.runtime).toBe("native"); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); - - it.live("parses repeated, comma-separated, and short exclusion flags", () => { - let parsed: ReadonlyArray | undefined; - const command = stackStartCommand.pipe( - Command.withHandler((flags) => Effect.sync(() => void (parsed = flags.exclude))), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--exclude", - "studio,analytics", - "-x", - "mail", - ]); - expect(parsed).toEqual(["studio", "analytics", "mail"]); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); - - it.live("rejects a root legacy output value before resolving the stack target", () => { - return Effect.gen(function* () { - const root = yield* project(); - let resolved = false; - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack: fakeStack("a".repeat(64), () => Effect.succeed(status("a".repeat(64)))), - }); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.sync(() => { - resolved = true; - return { projectRoot: root }; - }), - }); - const failure = yield* stackStart(flags({ exclude: ["bogus"] })).pipe( + it.live("rejects Studio without REST before changing the composition", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-studio-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "studio-test"\n'); + const fixture = fakeStack(); + const result = yield* stackStart(flags(["rest"])).pipe( Effect.flip, - Effect.provide( - Layer.mergeAll(setup.layer, target, Layer.succeed(OutputFlag, Option.some("json"))), - ), + Effect.provide(layers(root, fixture)), ); - expect(failure.message).toContain("--output"); - expect(failure.message).not.toContain("Unknown stack capabilities"); - expect(failure).toBeInstanceOf(StackCommandStartError); - if (failure instanceof StackCommandStartError) { - expect(failure.reason).toBe("flags"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - } - expect(resolved).toBe(false); - expect(setup.telemetry.flushed).toBe(true); - }); - }); - - it.live("rejects unknown and database exclusions before resolving the target", () => { - return Effect.gen(function* () { - const root = yield* project(); - let resolved = false; - const setup = handlerLayer({ - root, - target: { projectRoot: root }, - stack: fakeStack("b".repeat(64), () => Effect.succeed(status("b".repeat(64)))), - }); - const target = Layer.succeed(StackTargetResolver, { - resolve: () => - Effect.sync(() => { - resolved = true; - return { projectRoot: root }; - }), - }); - for (const { exclusion, message } of [ - { exclusion: "", message: 'Unknown stack capabilities in --exclude: ""' }, - { - exclusion: " analytics", - message: 'Unknown stack capabilities in --exclude: " analytics"', - }, - { exclusion: "unknown", message: 'Unknown stack capabilities in --exclude: "unknown"' }, - { exclusion: "database", message: "database capability cannot be excluded" }, - ]) { - const failure = yield* stackStart(flags({ exclude: [exclusion] })).pipe( - Effect.flip, - Effect.provide(Layer.mergeAll(setup.layer, target)), - ); - expect(failure).toBeInstanceOf(StackCommandStartError); - expect(failure.reason).toBe("flags"); - expect(failure.message).toContain(message); - } - expect(resolved).toBe(false); - }); - }); + expect(result).toBeInstanceOf(StackCommandStartError); + expect(result).toMatchObject({ reason: "flags" }); + expect(fixture.stopped).toBe(0); + expect(fixture.composed).toBe(0); + }).pipe(Effect.provide(BunServices.layer)), + ); }); diff --git a/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts new file mode 100644 index 0000000000..2a79cfcac8 --- /dev/null +++ b/apps/cli/src/commands/experimental/stack/start/start.native.integration.test.ts @@ -0,0 +1,230 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Layer, Option, Path } from "effect"; +import { FetchHttpClient } from "effect/unstable/http"; +import { + mockCommandSettings, + mockTelemetryStateTracked, +} from "../../../../../tests/helpers/command-mocks.ts"; +import { mockOutput, mockTty } from "../../../../../tests/helpers/mocks.ts"; +import { CommandPlatformApiFactory } from "../../../../auth/command-platform-api-factory.service.ts"; +import { dbConnectionLayer } from "../../../../command-internal/db-connection.layer.ts"; +import { DbConnection } from "../../../../command-internal/db-connection.service.ts"; +import { parseConnectionString } from "../../../../command-internal/db-config.parse.ts"; +import { stackCatalogSetupLayer } from "../../../../command-internal/stack-catalog-setup.ts"; +import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; +import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; +import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; +import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; +import { ExperimentalFlag, YesFlag } from "../../../../command-internal/global-flags.ts"; +import { stackStart } from "./start.handler.ts"; +import { stackPrepare } from "../prepare/prepare.handler.ts"; + +const excluded = [ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "mail", + "analytics", + "pooler", +]; +const flags = (exclude: ReadonlyArray) => ({ + exclude, + stack: Option.none(), + stackId: Option.none(), + runtime: "native" as const, + preparation: "on-demand" as const, + eager: false, +}); + +const prepareFlags = () => ({ + stack: Option.none(), + stackId: Option.none(), + runtime: "native" as const, + capability: [], +}); + +const liveStackApi = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + +const projectConfig = ` +project_id = "stack-start-native-integration" + +[api] +enabled = true + +[auth] +enabled = false + +[realtime] +enabled = false + +[storage] +enabled = false + +[edge_runtime] +enabled = false + +[studio] +enabled = false + +[analytics] +enabled = false + +[db.pooler] +enabled = false + +[local_smtp] +enabled = false +`; + +const makeLayers = (root: string) => { + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); + const resolver = stackTargetResolverLayer.pipe( + Layer.provide(Layer.mergeAll(BunServices.layer, settings, liveStackApi)), + ); + const output = mockOutput(); + const telemetry = mockTelemetryStateTracked(); + const tty = mockTty({ stdinIsTty: false, stdoutIsTty: false }); + return { + output, + telemetry, + layer: Layer.mergeAll( + BunServices.layer, + FetchHttpClient.layer, + runtimeInfoLayer, + settings, + liveStackApi, + resolver, + output.layer, + telemetry.layer, + stackCatalogSetupLayer, + dbConnectionLayer, + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(YesFlag, false), + Layer.succeed(CliArgs, { args: ["stack", "start"] }), + Layer.succeed(CommandPlatformApiFactory, { make: Effect.die("unused") }), + stdinLayer.pipe(Layer.provide(tty)), + tty, + ), + }; +}; + +describe("experimental stack start native lifecycle", () => { + it.live( + "preserves database identity, data, and port while excluding then including REST", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-native-" }); + yield* fs.makeDirectory(path.join(root, "supabase", "migrations"), { recursive: true }); + yield* fs.writeFileString( + path.join(root, "supabase", "migrations", "20260919000000_native_start.sql"), + "THIS IS NOT SQL;\n", + ); + yield* fs.writeFileString(path.join(root, "supabase", "config.toml"), projectConfig); + const fixture = makeLayers(root); + let stackId: string | undefined; + yield* Effect.ensuring( + Effect.gen(function* () { + yield* Effect.scoped(stackPrepare(prepareFlags())); + const api = yield* StackApi; + const failedStart = yield* Effect.scoped(Effect.exit(stackStart(flags([])))); + expect(Exit.isFailure(failedStart)).toBe(true); + const failedDefinitions = yield* api.discover({ stateRoot: path.join(root, "stacks") }); + expect(failedDefinitions).toHaveLength(1); + const failedDefinition = failedDefinitions[0]; + if (failedDefinition === undefined) + return yield* Effect.die("registered stack missing"); + { + const failedStack = yield* api.open({ + id: failedDefinition.definition.id, + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(root, "cache"), + }); + expect(yield* failedStack.services.list).toHaveLength(0); + expect((yield* failedStack.composition.describe).members).toHaveLength(0); + } + yield* fs.writeFileString( + path.join(root, "supabase", "migrations", "20260919000000_native_start.sql"), + "CREATE TABLE native_migration_probe(value text NOT NULL); INSERT INTO native_migration_probe(value) VALUES ('once');\n", + ); + stackId = yield* stackStart(flags(excluded)); + const stack = yield* api.open({ + id: stackId, + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(root, "cache"), + }); + const firstDatabase = (yield* stack.services.list).find( + (instance) => instance.service === "database", + ); + if (firstDatabase === undefined) return yield* Effect.die("database missing"); + const firstCredentials = yield* firstDatabase.credentials({ from: "host" }); + const firstDatabaseUrl = firstCredentials.databaseUrl; + if (firstDatabaseUrl === undefined) return yield* Effect.die("database URL missing"); + const firstConnection = parseConnectionString(firstDatabaseUrl); + if (firstConnection === undefined) return yield* Effect.die("database URL invalid"); + const db = yield* DbConnection; + yield* Effect.scoped( + Effect.gen(function* () { + const session = yield* db.connect(firstConnection, { + isLocal: true, + dnsResolver: "native", + }); + yield* session.exec("CREATE TABLE native_start_probe(value text NOT NULL)"); + yield* session.exec("INSERT INTO native_start_probe(value) VALUES ('preserved')"); + const migrationRows = yield* session.query( + "SELECT value FROM native_migration_probe", + ); + expect(migrationRows).toEqual([{ value: "once" }]); + }), + ); + const secondId = yield* stackStart(flags([])); + expect(secondId).toBe(stackId); + const thirdId = yield* stackStart(flags([])); + expect(thirdId).toBe(stackId); + const reopened = yield* api.open({ + id: secondId, + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(root, "cache"), + }); + const services = yield* reopened.services.list; + const secondDatabase = services.find((instance) => instance.service === "database"); + if (secondDatabase === undefined) return yield* Effect.die("reopened database missing"); + expect(secondDatabase.id).toBe(firstDatabase.id); + const secondCredentials = yield* secondDatabase.credentials({ from: "host" }); + expect(secondCredentials.databaseUrl).toBe(firstDatabaseUrl); + const secondConnection = parseConnectionString(secondCredentials.databaseUrl ?? ""); + if (secondConnection === undefined) return yield* Effect.die("reopened URL invalid"); + const rows = yield* Effect.scoped( + Effect.gen(function* () { + const session = yield* db.connect(secondConnection, { + isLocal: true, + dnsResolver: "native", + }); + return yield* session.query("SELECT value FROM native_start_probe"); + }), + ); + expect(rows).toEqual([{ value: "preserved" }]); + expect(services.some((instance) => instance.service === "rest")).toBe(true); + }), + Effect.exit( + Effect.gen(function* () { + const api = yield* StackApi; + if (stackId === undefined) return; + const stack = yield* api.open({ + id: stackId, + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(root, "cache"), + }); + yield* Effect.exit(stack.destroy).pipe(Effect.asVoid); + }), + ).pipe(Effect.asVoid), + ).pipe(Effect.provide(fixture.layer)); + }).pipe(Effect.provide(BunServices.layer)), + { timeout: 180_000 }, + ); +}); diff --git a/apps/cli/src/commands/experimental/stack/start/start.options.ts b/apps/cli/src/commands/experimental/stack/start/start.options.ts index 55147662bb..13b810c47b 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.options.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.options.ts @@ -1 +1,11 @@ -export { STACK_START_EXCLUDABLE_CAPABILITIES } from "../../../../command-internal/stack-local-database.ts"; +export const STACK_START_EXCLUDABLE_CAPABILITIES = [ + "rest", + "auth", + "realtime", + "storage", + "functions", + "studio", + "mail", + "analytics", + "pooler", +] as const; diff --git a/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md index adf67e5036..d22a0e8997 100644 --- a/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/status/SIDE_EFFECTS.md @@ -1,109 +1,73 @@ # `supabase stack status` -Reports the persisted identity and current owner state of a managed local stack. -The command is read-only: it never creates, starts, prepares, stops, or destroys -a stack, and opens a stack handle only when `--env` is used. -The command is available only when the `experimental.stack` feature flag is -enabled. The top-level `supabase status` command uses this handler when the -same flag is enabled. Command routing may read `supabase/config.toml` or -`supabase/config.json` to select the experimental backend; set -`SUPABASE_EXPERIMENTAL_STACK=1` to inspect a stack addressed with `--stack-id` -when project configuration is missing or invalid. +Reports the saved identity and the observed state of a managed local stack. +The command is available when the `experimental.stack` feature flag is enabled. +The top-level `supabase status` command uses this handler when the same flag is +enabled. Routing may read `supabase/config.toml` or `supabase/config.json` to +select the experimental backend; set `SUPABASE_EXPERIMENTAL_STACK=1` to inspect +a stack addressed with `--stack-id` when project configuration is missing or +invalid. Target selection accepts the current project, `--stack `, or -`--stack-id `. `--stack` and `--stack-id` are mutually exclusive. Any -explicit legacy `-o/--output` value is rejected; use `--output-format` instead, -or `--env` in place of the `env` value. - -When the project configuration loads and the comparison accepts it, status -includes redacted config drift paths. An absent `supabase/config.toml` is -compared using default settings, matching `supabase stack start`. Drift output -contains statuses and paths only; secret values are never emitted. - -When the configuration cannot be loaded at all, the warning is `Project -configuration could not be loaded; fix it before checking drift.` When it -loads but the comparison rejects it, such as an invalid stack config or an -unsupported version, the warning is `Project configuration could not be -compared: `. Either warning leaves the persisted stack -inspection available, appears as `Config warning:` in text output, and as -`config_drift.message` with `status: "unavailable"` in JSON. - -Text output includes identity, runtime, owner, lifecycle, readiness, -endpoints, and config drift. JSON output nests only the identity fields under -`identity`; runtime, lifecycle, readiness, endpoints, and config drift remain -top-level fields. - -Each capability includes its current state and may include a diagnostic when -cleanup or runtime readiness failed. When recovery is required, JSON adds a -`recovery` object with the `operation` (`stop` or `destroy`) and its message. -Text output includes that message and the matching recovery command; stop -recovery prints `stop --stack-id && start --stack-id `, while destroy -recovery prints the destructive destroy command and warns that stack data is -removed. - -When a running stack has a capability transitioning through `stopping`, readiness is reported as -`stopping`. A failed capability takes precedence and reports `degraded`; otherwise readiness -reports `starting`, `stopped`, `dormant`, or `ready` according to the live capability states. +`--stack-id `. `--stack` and `--stack-id` are mutually exclusive. Explicit +legacy `-o/--output` values are rejected; use `--output-format` instead, or +`--env` in place of the `env` value. + +Status reads the persisted stack identity and composition membership, then +observes each service through the saved owner when that owner is reachable. It +never creates, starts, prepares, stops, destroys, or launches an owner for the +ordinary status view. A missing owner is reported as unavailable and its saved +members remain visible without live lifecycle or health values. Lifecycle, +health, endpoint, and aggregate readiness values are reported separately so a +stopped or unhealthy member is distinguishable from an unavailable owner. + +The command compares the database version and explicit numeric endpoint ports with the saved +configuration of existing composition members. It does not compare other service settings or +changes to membership; an excluded service is not considered drift. Live listener availability +does not affect this comparison. It reports `config_drift.status` as `unchanged` or +`changed`, with changed paths, when the comparison is possible. A configuration +loading failure or unavailable database observation keeps the saved stack report +available and is shown as `config_drift.status: "unavailable"` with a message in +JSON. Status does not apply current configuration. The `services` list may include +saved standalone instances; composition members identify the services used for +primary database, environment export, and drift comparisons. +The source checkout may bundle the default Functions bootstrap while translating +project configuration for drift; this is in-memory and writes no project files. + +Text output includes identity, runtime, owner, lifecycle, readiness, services, +endpoints, and config drift. JSON nests only identity fields under `identity`; +runtime, lifecycle, readiness, composition, services, endpoints, and config +drift remain top-level fields. Stack identity, endpoints, and credentials are +never telemetry properties. ## Exporting environment variables (`--env`) -`--env` opens the target stack, requires it to be running, and exports its -connection URLs and credentials instead of the ordinary identity/drift report. -It does not load or compare project configuration. Text output emits dotenv -assignments, quoting each value with single quotes, or with double quotes when -the value contains a single quote but none of `"`, `\`, `$`, backtick, or `!`, -so sourcing the file in a shell performs no expansion. Any other value, or one -containing a carriage return, fails the command with a pointer to -`--output-format json`. JSON and stream-JSON output, including automatic agent -detection, emit a plain variable map under a successful result. As described -above, the legacy `-o env` value is rejected with guidance to use `--env`. - -The exported variables are `DB_URL`, `API_URL`, `ANON_KEY`, `SERVICE_ROLE_KEY`, -`PUBLISHABLE_KEY`, `SECRET_KEY`, `STUDIO_URL`, `INBUCKET_URL`, -`S3_PROTOCOL_ACCESS_KEY_ID`, `S3_PROTOCOL_ACCESS_KEY_SECRET`, -`S3_PROTOCOL_REGION`, and `S3_PROTOCOL_URL`. `ANON_KEY`, `SERVICE_ROLE_KEY`, -`PUBLISHABLE_KEY`, and `SECRET_KEY` are omitted when the stack's Auth capability -is disabled. `API_URL`, `STUDIO_URL`, `INBUCKET_URL`, and the `S3_PROTOCOL_*` -variables are omitted when the corresponding endpoint or storage credentials are -unavailable. Values always come from the running stack; none are invented. +`--env` is the explicit environment-export operation. It requires a reachable +owner and a running primary database, then derives the database URL from the +observed SQL endpoint and saved database credentials, using the `supabase_admin` role. It does not request live +credentials or launch an owner, and it does not load or compare project +configuration. Text output emits dotenv assignments; +JSON and stream-JSON output emit a plain variable map under a successful result. +Unavailable optional credentials and endpoints are omitted. The derived +`ANON_KEY` and `SERVICE_ROLE_KEY` values are emitted from the required saved database JWT secret. `--override-name` renames an exported variable, accepting repeated flags or a comma-separated list of `EXPORTED_VARIABLE=VALID_ENV_NAME` entries. It requires -`--env` and rejects an unknown source variable, a source variable listed more -than once, an invalid target name, a missing or malformed entry, and a rename -that collides with another exported variable's name. - -Ordinary status (without `--env`) never opens a stack handle and never emits -credentials, regardless of the stack's lifecycle. A stopped stack or a -credentials failure with `--env` fails the command without emitting output. +`--env` and rejects an unknown source variable, duplicate source, invalid target, +malformed entry, or destination collision. Dotenv values that cannot be quoted +without shell expansion fail with a pointer to `--output-format json`. ## Files read and written -Without `--env`, the command reads the selected `supabase/config.toml` or -`supabase/config.json` and the project dotenv files and environment overrides -the shared config loader consults to resolve the target stack's configuration; -with `--env`, it skips config loading entirely. Either way, it -reads the target stack's persisted state under -`/managed/stacks//`, and when a live owner -exists, it reads the owner's local RPC endpoint for status and credentials. -The command calls no API routes and writes no files besides `telemetry.json`. -It reads the CLI's usual home, workdir, and experimental routing variables, -plus the environment variables used as project configuration overrides. - -## Output and telemetry +The command reads the selected project configuration for ordinary drift +messaging, the CLI's persisted stack state under +`/stacks//`, and the owner's local RPC +endpoint when it exists. `--env` skips project configuration loading. It calls +no management API routes and writes no files besides `telemetry.json`. Exit status is `0` for a successful report, including a stopped stack or an -absent or unreachable owner, and `130` if the command is interrupted. It is -`1` for a flag validation failure (`--stack` with `--stack-id`, any legacy -`-o/--output` value, `--override-name` without `--env` or with an unknown -source, an invalid target name, a duplicate source, or a colliding -destination), for no stack in the current context or an unknown `--stack-id`, -for a typed stack failure, for `--env` against a stack that is not running, or -for a value that dotenv cannot represent losslessly. - -Standard command instrumentation (`withCommandTelemetry`) records command -metadata and flag presence; stack identity, endpoints, and credentials are -never telemetry properties. - -Telemetry state is flushed to `/telemetry.json` -after both successful and failed command runs. +absent or unreachable owner, and `130` if interrupted. It is `1` for flag +validation, an unknown target, a typed stack failure, an invalid configuration, +a lifecycle failure during `--env`, or a value that dotenv cannot represent +losslessly. Standard command telemetry is flushed after successful and failed +runs. diff --git a/apps/cli/src/commands/experimental/stack/status/status.env.ts b/apps/cli/src/commands/experimental/stack/status/status.env.ts index 216a0d1012..117875a37d 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.env.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.env.ts @@ -1,5 +1,5 @@ -import type { EffectStackCredentials, StackStatus } from "@supabase/stack/effect"; -import { Effect, Redacted } from "effect"; +import { Effect } from "effect"; +import { generateGoJwt } from "../../../../command-internal/go-jwt.ts"; import { StackCommandStatusError } from "./status.errors.ts"; const variableNames = [ @@ -52,32 +52,26 @@ export const stackEnvOverrides = (entries: ReadonlyArray) => }); export const stackEnvValues = ( - status: StackStatus, - credentials: EffectStackCredentials, + status: { + readonly endpoints: Readonly<{ + readonly api?: { readonly url: string }; + readonly studio?: { readonly url: string }; + readonly mailUi?: { readonly url: string }; + }>; + readonly jwtSecret?: string; + }, + credentials: Readonly>, names: ReadonlyMap, ): Readonly> => { - const values: Record = { - DB_URL: Redacted.value(credentials.database.url), - ...(credentials.api === undefined - ? {} - : { - ANON_KEY: credentials.api.anonJwt, - SERVICE_ROLE_KEY: Redacted.value(credentials.api.serviceRoleJwt), - PUBLISHABLE_KEY: credentials.api.publishableKey, - SECRET_KEY: Redacted.value(credentials.api.secretKey), - }), - ...(status.endpoints.api === undefined ? {} : { API_URL: status.endpoints.api.url }), - ...(status.endpoints.studio === undefined ? {} : { STUDIO_URL: status.endpoints.studio.url }), - ...(status.endpoints.mailUi === undefined ? {} : { INBUCKET_URL: status.endpoints.mailUi.url }), - ...(credentials.storage === undefined - ? {} - : { - S3_PROTOCOL_ACCESS_KEY_ID: credentials.storage.accessKeyId, - S3_PROTOCOL_ACCESS_KEY_SECRET: Redacted.value(credentials.storage.secretAccessKey), - S3_PROTOCOL_REGION: credentials.storage.region, - S3_PROTOCOL_URL: credentials.storage.endpoint, - }), - }; + const values: Record = {}; + if (credentials.databaseUrl !== undefined) values.DB_URL = credentials.databaseUrl; + if (status.jwtSecret !== undefined) { + values.ANON_KEY = generateGoJwt(status.jwtSecret, "anon"); + values.SERVICE_ROLE_KEY = generateGoJwt(status.jwtSecret, "service_role"); + } + if (status.endpoints.api !== undefined) values.API_URL = status.endpoints.api.url; + if (status.endpoints.studio !== undefined) values.STUDIO_URL = status.endpoints.studio.url; + if (status.endpoints.mailUi !== undefined) values.INBUCKET_URL = status.endpoints.mailUi.url; return Object.fromEntries( Object.entries(values).map(([key, value]) => [names.get(key) ?? key, value]), ); diff --git a/apps/cli/src/commands/experimental/stack/status/status.env.unit.test.ts b/apps/cli/src/commands/experimental/stack/status/status.env.unit.test.ts index 5dba18aa48..0992cd5ec5 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.env.unit.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.env.unit.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import { parse } from "dotenv"; import { Effect } from "effect"; -import { encodeStackEnv } from "./status.env.ts"; +import { encodeStackEnv, stackEnvOverrides, stackEnvValues } from "./status.env.ts"; describe("stack dotenv encoding", () => { it.effect("round-trips literal credentials without expanding or changing characters", () => @@ -40,3 +40,26 @@ describe("stack dotenv encoding", () => { }), ); }); + +describe("stack environment overrides", () => { + it.effect("accepts renames and rejects malformed or colliding destinations", () => + Effect.gen(function* () { + const names = yield* stackEnvOverrides(["API_URL=NEXT_PUBLIC_API_URL"]); + expect(names.get("API_URL")).toBe("NEXT_PUBLIC_API_URL"); + expect( + stackEnvValues({ endpoints: { api: { url: "http://127.0.0.1:54321" } } }, {}, names), + ).toEqual({ NEXT_PUBLIC_API_URL: "http://127.0.0.1:54321" }); + + for (const entries of [ + ["UNKNOWN=value"], + ["API_URL="], + ["API_URL=ONE=two"], + ["API_URL=NAME", "API_URL=OTHER"], + ["API_URL=DB_URL"], + ]) { + const error = yield* stackEnvOverrides(entries).pipe(Effect.flip); + expect(error.reason).toBe("flags"); + } + }), + ); +}); diff --git a/apps/cli/src/commands/experimental/stack/status/status.handler.ts b/apps/cli/src/commands/experimental/stack/status/status.handler.ts index f9136f83d7..50ae474453 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.handler.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.handler.ts @@ -1,27 +1,83 @@ -import { Effect, Match, Option } from "effect"; -import { - isStackError, - type StackError, - type StackInspection, - type StackStatus, -} from "@supabase/stack/effect"; +import { endpointReports } from "../stack-endpoints.format.ts"; +import { Effect, Option, Path, Redacted } from "effect"; +import type { Observation, ServiceCreation, Stack } from "@supabase/stack/effect"; +import type { StackError } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; +import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; +import { toPostgresURL } from "../../../../command-internal/postgres-url.ts"; import { StackApi, StackTargetError, + StackTargetResolver, rejectStackOutput, - stackStatusIssueLines, - validateStackId, validateStackTarget, } from "../stack.shared.ts"; -import { loadStackConfig } from "../../../../command-internal/stack-config.ts"; import type { StackStatusFlags } from "./status.command.ts"; import { StackCommandStatusError } from "./status.errors.ts"; import { encodeStackEnv, stackEnvOverrides, stackEnvValues } from "./status.env.ts"; +type StackService = Effect.Success>; +type EndpointReport = { + readonly protocol: "tcp" | "http"; + readonly address: string; + readonly port: number; + readonly url: string; +}; +type ServiceReport = { + readonly id: string; + readonly service: ServiceCreation["service"]; + readonly state: + | "unavailable" + | "sleeping" + | "starting" + | "running" + | "stopping" + | "stopped" + | "unhealthy" + | "exited"; + readonly lifecycle: Observation["lifecycle"] | null; + readonly health: Observation["health"] | null; + readonly endpoints: Readonly>; + readonly error?: string; +}; +type ObservedService = { + readonly instance: StackService; + readonly observation: Observation | undefined; + readonly error?: StackError; +}; +type StackReport = { + readonly identity: { + readonly id: string; + readonly name: string; + readonly project_root: string; + readonly branch_context: string; + }; + readonly runtime: "native" | "docker" | "podman"; + readonly owner: "reachable" | "unavailable"; + readonly lifecycle: Observation["lifecycle"] | null; + readonly readiness: "unavailable" | "starting" | "sleeping" | "stopped" | "ready" | "unhealthy"; + readonly composition: { + readonly members: ReadonlyArray<{ + readonly id: string; + readonly service: ServiceCreation["service"] | "unknown"; + readonly activation: string; + readonly state: ServiceReport["state"]; + readonly lifecycle: ServiceReport["lifecycle"]; + readonly health: ServiceReport["health"]; + }>; + }; + readonly services: ReadonlyArray; + readonly endpoints: Readonly>; + readonly config_drift: { + readonly status: "unchanged" | "changed" | "unavailable"; + readonly message: string; + readonly paths?: ReadonlyArray; + }; +}; + const mapTargetError = (error: StackTargetError) => new StackCommandStatusError({ reason: error.reason, @@ -30,146 +86,319 @@ const mapTargetError = (error: StackTargetError) => cause: error, }); -const classifyStackError = (error: StackError) => - Match.value(error).pipe( - Match.tag("StackNotFoundError", () => ({ - reason: "not-found" as const, - suggestion: - "Choose an existing --stack-id, or run supabase stack start without --stack-id to create one.", - })), - Match.tag( - "InvalidStackIdentityError", - "InvalidProjectRootError", - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", - "StackUpgradeRequiredError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - () => ({ reason: "invalid-config" as const }), - ), - Match.tag("StackNotRunningError", "StackLifecycleConflictError", () => ({ - reason: "lifecycle" as const, - suggestion: "Run supabase stack start first.", - })), - Match.orElse(() => ({ - reason: "runtime" as const, - suggestion: "Retry the command and use --debug if the stack state remains unavailable.", - })), - ); - -const mapStackError = (error: StackError) => { - const classification = classifyStackError(error); - return new StackCommandStatusError({ - ...classification, +const mapStackError = (error: StackError) => + new StackCommandStatusError({ + reason: + error.operation === "open" || + error.operation === "discover" || + error.operation === "definition" || + error.operation === "getComposition" + ? "invalid-config" + : "runtime", message: error.message, + suggestion: + error.operation === "open" || + error.operation === "discover" || + error.operation === "definition" || + error.operation === "getComposition" + ? "Inspect the saved stack state under $SUPABASE_HOME/stacks." + : "Retry the command and use --debug if the stack remains unavailable.", cause: error, }); + +const serviceState = (observation: Observation | undefined): ServiceReport["state"] => { + if (observation === undefined) return "unavailable"; + if (observation.health === "unhealthy") return "unhealthy"; + if (observation.lifecycle === "stopped") { + if (observation.error?.operation === "exit") return "exited"; + return observation.wakeEnabled ? "sleeping" : "stopped"; + } + return observation.lifecycle; }; -const catchStackError = (effect: Effect.Effect) => - effect.pipe(Effect.catchIf(isStackError, (error) => Effect.fail(mapStackError(error)))); +const serviceReport = ({ instance, observation, error }: ObservedService): ServiceReport => ({ + id: instance.id, + service: instance.service, + state: serviceState(observation), + lifecycle: observation?.lifecycle ?? null, + health: observation?.health ?? null, + endpoints: endpointReports(observation), + ...(error === undefined && observation?.error === undefined + ? {} + : { error: error?.message ?? observation?.error?.message }), +}); -const readiness = (status: StackStatus | undefined): string => { - if (status === undefined) return "unknown"; - if (status.lifecycle !== "running") return status.lifecycle; - if (status.capabilities.some(({ state }) => state === "failed")) return "degraded"; - if (status.capabilities.some(({ state }) => state === "stopping")) return "stopping"; - if (status.capabilities.some(({ state }) => state === "starting")) return "starting"; - if (status.capabilities.some(({ state }) => state === "stopped")) return "stopped"; - if (status.capabilities.some(({ state }) => state === "dormant")) return "dormant"; - return "ready"; +const aggregateLifecycle = ( + reports: ReadonlyArray, + owner: StackReport["owner"], +): StackReport["lifecycle"] => { + if (owner === "unavailable" || reports.length === 0) return null; + if (reports.some(({ state }) => state === "unavailable")) return null; + if (reports.some(({ lifecycle }) => lifecycle === "starting")) return "starting"; + if (reports.some(({ lifecycle }) => lifecycle === "stopping")) return "stopping"; + if (reports.some(({ lifecycle }) => lifecycle === "running")) return "running"; + return "stopped"; }; -const configUnavailableWarning = - "Project configuration could not be loaded; fix it before checking drift."; +const aggregateReadiness = ( + reports: ReadonlyArray, + owner: StackReport["owner"], +): StackReport["readiness"] => { + if (owner === "unavailable" || reports.length === 0) return "unavailable"; + if (reports.some(({ state }) => state === "unavailable")) return "unavailable"; + if (reports.some(({ state }) => state === "unhealthy" || state === "exited")) return "unhealthy"; + if (reports.some(({ state }) => state === "starting")) return "starting"; + if (reports.some(({ state }) => state === "sleeping")) return "sleeping"; + if (reports.every(({ state }) => state === "stopped")) return "stopped"; + if (reports.some(({ state }) => state === "stopped")) return "stopped"; + return reports.every(({ state, health }) => state === "running" && health === "healthy") + ? "ready" + : "starting"; +}; -const configComparisonWarning = (detail: string) => - `Project configuration could not be compared: ${detail}`; +const endpointFor = ( + services: ReadonlyArray, + members: ReadonlyArray<{ readonly id: string }>, + service: ServiceCreation["service"], + endpoint: string, +) => { + const memberIds = new Set(members.map(({ id }) => id)); + return services.find((entry) => memberIds.has(entry.id) && entry.service === service)?.endpoints[ + endpoint + ]; +}; -const payload = (inspection: StackInspection, configWarning?: string) => ({ - identity: { - id: inspection.descriptor.id, - name: inspection.descriptor.name, - project_root: inspection.descriptor.projectRoot, - branch_context: inspection.descriptor.branchContext, +const reportFor = ( + definition: { + readonly id: string; + readonly identity: { + readonly projectRoot: string; + readonly branchContext: string; + readonly stackName: string; + }; + readonly runtime: "native" | "docker" | "podman"; }, - runtime: inspection.descriptor.runtime, - owner: inspection.owner, - lifecycle: inspection.status?.lifecycle ?? null, - desired_lifecycle: inspection.status?.desiredLifecycle ?? inspection.descriptor.desiredLifecycle, - readiness: readiness(inspection.status), - ...(inspection.status === undefined ? {} : { endpoints: inspection.status.endpoints }), - ...(inspection.status === undefined ? {} : { capabilities: inspection.status.capabilities }), - ...(inspection.status?.recovery === undefined ? {} : { recovery: inspection.status.recovery }), - config_drift: - inspection.configDrift ?? - ({ - status: "unavailable", - message: configWarning ?? "Configuration was not compared.", - } as const), -}); - -const comparedInspection = ( - inspection: StackInspection, -): { - readonly inspection: StackInspection; - readonly warning?: string; -} => ({ inspection }); + owner: StackReport["owner"], + observed: ReadonlyArray, + members: ReadonlyArray<{ readonly id: string; readonly activation: string }>, + configDrift: StackReport["config_drift"], +): StackReport => { + const services = observed.map(serviceReport); + const endpoints = Object.fromEntries( + services.flatMap((service) => + Object.entries(service.endpoints).map(([name, endpoint]) => [ + `${service.service}.${name}`, + endpoint, + ]), + ), + ); + return { + identity: { + id: definition.id, + name: definition.identity.stackName, + project_root: definition.identity.projectRoot, + branch_context: definition.identity.branchContext, + }, + runtime: definition.runtime, + owner, + lifecycle: aggregateLifecycle(services, owner), + readiness: aggregateReadiness(services, owner), + composition: { + members: members.map((member) => { + const service = services.find(({ id }) => id === member.id); + return { + id: member.id, + service: service?.service ?? "unknown", + activation: member.activation, + state: service?.state ?? "unavailable", + lifecycle: service?.lifecycle ?? null, + health: service?.health ?? null, + }; + }), + }, + services, + endpoints, + config_drift: configDrift, + }; +}; -const render = (inspection: StackInspection, configWarning?: string): string => { - const descriptor = inspection.descriptor; +const render = (report: StackReport): string => { const lines = [ - `Stack ${descriptor.name} (${descriptor.id})`, - `Project: ${descriptor.projectRoot}`, - `Branch: ${descriptor.branchContext}`, - `Runtime: ${descriptor.runtime.kind}`, - `Owner: ${inspection.owner}`, - `Lifecycle: ${inspection.status?.lifecycle ?? "unavailable"}`, - `Desired lifecycle: ${inspection.status?.desiredLifecycle ?? descriptor.desiredLifecycle}`, - `Readiness: ${readiness(inspection.status)}`, + `Stack ${report.identity.name} (${report.identity.id})`, + `Project: ${report.identity.project_root}`, + `Branch: ${report.identity.branch_context}`, + `Runtime: ${report.runtime}`, + `Owner: ${report.owner}`, + `Lifecycle: ${report.lifecycle ?? "unavailable"}`, + `Readiness: ${report.readiness}`, + "Services:", ]; - if (inspection.status !== undefined) { - const endpoints = Object.entries(inspection.status.endpoints); - if (endpoints.length > 0) { - lines.push("Endpoints:"); - for (const [name, endpoint] of endpoints) - if (endpoint !== undefined) lines.push(` ${name}: ${endpoint.url}`); - } - lines.push(...stackStatusIssueLines(inspection.status)); + const members = new Map(report.composition.members.map((member) => [member.id, member])); + for (const service of report.services) { + const details = [service.state, `lifecycle=${service.lifecycle ?? "unavailable"}`]; + const member = members.get(service.id); + details.push(member === undefined ? "standalone" : `activation=${member.activation}`); + if (service.health !== null) details.push(`health=${service.health}`); + lines.push(` ${service.service} (${service.id}): ${details.join(", ")}`); + for (const [name, endpoint] of Object.entries(service.endpoints)) + lines.push(` ${name}: ${endpoint.url}`); + if (service.error !== undefined) lines.push(` error: ${service.error}`); } - const drift = inspection.configDrift; - lines.push(`Config drift: ${drift?.status ?? "unavailable"}`); - if (drift !== undefined) for (const path of drift.paths) lines.push(` ${path}`); - if (configWarning !== undefined) lines.push(`Config warning: ${configWarning}`); + lines.push(`Config drift: ${report.config_drift.status}`); + lines.push(` ${report.config_drift.message}`); + for (const path of report.config_drift.paths ?? []) lines.push(` ${path}`); return `${lines.join("\n")}\n`; }; -const findDescriptor = (projectRoot: string, name: string | undefined, id: string | undefined) => +const findTarget = Effect.fn("experimental.stack.status.findTarget")(function* ( + projectRoot: string, + name: string | undefined, + id: string | undefined, +) { + const settings = yield* CommandSettings; + const path = yield* Path.Path; + const resolver = yield* StackTargetResolver; + const api = yield* StackApi; + const target = yield* resolver + .resolve({ + projectRoot, + ...(name === undefined ? {} : { name }), + ...(id === undefined ? {} : { id }), + runtime: "auto", + }) + .pipe(Effect.mapError(mapTargetError)); + if (target.id === undefined) + return yield* new StackCommandStatusError({ + reason: "not-found", + message: "No managed stack exists for the selected project.", + suggestion: "Run supabase stack start first.", + }); + const discovered = yield* api + .discover({ stateRoot: path.join(settings.supabaseHome, "stacks") }) + .pipe(Effect.mapError(mapStackError)); + const found = discovered.find(({ definition }) => definition.id === target.id); + if (found === undefined) + return yield* new StackCommandStatusError({ + reason: "not-found", + message: `Stack ${target.id} was not found.`, + suggestion: "Choose an existing --stack-id, or run supabase stack start first.", + }); + return { + ...target, + id: target.id, + definition: found.definition, + owner: found.host === undefined ? ("unavailable" as const) : ("reachable" as const), + }; +}); + +const observe = (stack: Stack, owner: StackReport["owner"]) => + Effect.gen(function* () { + const instances = yield* stack.services.list.pipe(Effect.mapError(mapStackError)); + const composition = yield* stack.composition.describe.pipe(Effect.mapError(mapStackError)); + const observed = yield* Effect.forEach(instances, (instance): Effect.Effect => + owner === "unavailable" + ? Effect.succeed({ instance, observation: undefined }) + : instance.status.pipe( + Effect.map((observation) => ({ instance, observation })), + Effect.catchTag("StackError", (error) => + Effect.succeed({ instance, observation: undefined, error }), + ), + ), + ); + return { observed, members: composition.members }; + }); + +const compareConfig = ( + creations: ReadonlyArray, + observed: ReadonlyArray, + members: ReadonlyArray<{ readonly id: string }>, +): StackReport["config_drift"] => { + const paths: string[] = []; + const memberIds = new Set(members.map(({ id }) => id)); + for (const creation of creations) { + const service = observed.find( + ({ instance }) => memberIds.has(instance.id) && instance.service === creation.service, + ); + if (service === undefined) continue; + if (service.observation === undefined) { + if (service?.error !== undefined) + return { + status: "unavailable", + message: `Configuration could not be compared because ${creation.service} status failed: ${service.error.message}`, + }; + paths.push(`services.${creation.service}`); + continue; + } + if ( + creation.service === "database" && + service.observation.config.service === "database" && + creation.config.version !== service.observation.config.config.version + ) + paths.push(`services.database.config.version`); + if (creation.endpoints === undefined) continue; + for (const name of Object.keys(creation.endpoints)) { + const endpoint = Reflect.get(creation.endpoints, name); + if ( + typeof endpoint !== "object" || + endpoint === null || + !("port" in endpoint) || + typeof endpoint.port !== "number" + ) + continue; + const savedEndpoints = service.observation.config.endpoints; + const actual = savedEndpoints === undefined ? undefined : Reflect.get(savedEndpoints, name); + if ( + typeof actual !== "object" || + actual === null || + !("port" in actual) || + actual.port !== endpoint.port + ) + paths.push(`services.${creation.service}.endpoints.${name}`); + } + } + return paths.length === 0 + ? { + status: "unchanged", + message: + "Configured database version and explicit endpoint ports match existing composition members.", + } + : { + status: "changed", + message: `${paths.length} configured service value${paths.length === 1 ? "" : "s"} differ from the saved stack.`, + paths, + }; +}; + +const configDrift = ( + projectRoot: string, + stackId: string, + observed: ReadonlyArray, + members: ReadonlyArray<{ readonly id: string }>, +) => Effect.gen(function* () { - const api = yield* StackApi; - if (id !== undefined) { - const validId = yield* validateStackId(id).pipe(Effect.mapError(mapTargetError)); - const inspection = yield* catchStackError(api.inspectStack(validId)); + const memberIds = new Set(members.map(({ id }) => id)); + const database = observed.find( + ({ instance }) => memberIds.has(instance.id) && instance.service === "database", + ); + const databaseObservation = database?.observation; + if (databaseObservation?.config.service !== "database") return { - descriptor: inspection.descriptor, - id: validId, - projectRoot: inspection.descriptor.projectRoot, - inspection, + status: "unavailable" as const, + message: "Configuration could not be compared without the saved database observation.", }; - } - const found = yield* catchStackError( - api.findStack({ projectRoot, ...(name === undefined ? {} : { name }) }), - ); - if (Option.isNone(found)) - return yield* new StackCommandStatusError({ - reason: "not-found", - message: "No managed stack exists for the selected project.", - suggestion: "Run supabase stack start first.", - }); - return { descriptor: found.value, id: found.value.id, projectRoot: found.value.projectRoot }; - }); + const loaded = yield* loadStackConfig(projectRoot); + const creations = yield* loaded.creations(stackId, { + jwtSecret: databaseObservation.config.config.jwtSecret, + }); + return compareConfig(creations, observed, members); + }).pipe( + Effect.catchTag("StackConfigError", (error) => + Effect.succeed({ + status: "unavailable" as const, + message: `Project configuration could not be compared: ${error.message}`, + }), + ), + ); export const stackStatus = Effect.fn("experimental.stack.status")(function* ( flags: StackStatusFlags, @@ -178,18 +407,18 @@ export const stackStatus = Effect.fn("experimental.stack.status")(function* ( const body = Effect.gen(function* () { const output = yield* Output; const settings = yield* CommandSettings; + const path = yield* Path.Path; + const api = yield* StackApi; const outputFlag = yield* Effect.serviceOption(OutputFlag); yield* rejectStackOutput(outputFlag).pipe( - Effect.mapError((error) => - Option.isSome(outputFlag) && Option.getOrUndefined(outputFlag.value) === "env" - ? new StackCommandStatusError({ - reason: "flags", - message: error.message, - suggestion: - "Use --env to export connection variables; add --output-format json for a variable map.", - cause: error, - }) - : mapTargetError(error), + Effect.mapError( + (error) => + new StackCommandStatusError({ + reason: "flags", + message: error.message, + suggestion: "Use --output-format json, --output-format text, or --env.", + cause: error, + }), ), ); yield* validateStackTarget({ @@ -202,61 +431,91 @@ export const stackStatus = Effect.fn("experimental.stack.status")(function* ( message: "--override-name requires --env.", }); const envNames = yield* stackEnvOverrides(flags.overrideName); - const target = yield* findDescriptor( + const target = yield* findTarget( settings.workdir, Option.getOrUndefined(flags.stack), Option.getOrUndefined(flags.stackId), ); - const api = yield* StackApi; + const locations = { + stateRoot: path.join(settings.supabaseHome, "stacks"), + cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), + }; + const stack = yield* api + .open({ ...locations, id: target.id }) + .pipe(Effect.mapError(mapStackError)); + const observed = yield* observe(stack, target.owner); if (flags.env) { - const stack = yield* catchStackError(api.openStack(target.id)); - const status = yield* catchStackError(stack.status); - if (status.lifecycle !== "running") + const memberIds = new Set(observed.members.map(({ id }) => id)); + const database = observed.observed.find( + ({ instance }) => memberIds.has(instance.id) && instance.service === "database", + ); + const databaseObservation = database?.observation; + if ( + target.owner === "unavailable" || + database === undefined || + databaseObservation === undefined || + databaseObservation.lifecycle !== "running" + ) return yield* new StackCommandStatusError({ reason: "lifecycle", - message: "The stack must be running to export connection variables.", + message: "The stack owner or primary database is unavailable for environment export.", suggestion: "Run supabase stack start first.", }); - const credentials = yield* catchStackError(stack.credentials); - const values = stackEnvValues(status, credentials, envNames); + const databaseConfig = databaseObservation.config; + if (databaseConfig.service !== "database") + return yield* new StackCommandStatusError({ + reason: "lifecycle", + message: "The primary database configuration is unavailable for environment export.", + suggestion: "Run supabase stack start first.", + }); + const jwtSecret = Redacted.value(databaseConfig.config.jwtSecret); + const sql = databaseObservation.endpoints.find(({ name }) => name === "sql"); + const databaseUrl = + sql === undefined + ? undefined + : toPostgresURL({ + host: sql.host, + port: sql.port, + user: "supabase_admin", + password: Redacted.value(databaseConfig.config.databasePassword), + database: "postgres", + }); + const services = observed.observed.map(serviceReport); + const endpoints = { + ...(endpointFor(services, observed.members, "rest", "http") === undefined + ? {} + : { api: endpointFor(services, observed.members, "rest", "http") }), + ...(endpointFor(services, observed.members, "studio", "http") === undefined + ? {} + : { studio: endpointFor(services, observed.members, "studio", "http") }), + ...(endpointFor(services, observed.members, "mail", "http") === undefined + ? {} + : { mailUi: endpointFor(services, observed.members, "mail", "http") }), + }; + const values = stackEnvValues( + { endpoints, jwtSecret }, + databaseUrl === undefined ? {} : { databaseUrl }, + envNames, + ); if (output.format === "text") yield* output.raw(yield* encodeStackEnv(values)); else yield* output.result(values); return; } - const loaded = yield* loadStackConfig(target.projectRoot).pipe( - Effect.map((config) => ({ config, warning: undefined })), - Effect.catchTag("StackConfigError", () => - Effect.succeed({ config: undefined, warning: configUnavailableWarning }), - ), + const config = yield* configDrift( + target.definition.identity.projectRoot, + target.definition.id, + observed.observed, + observed.members, + ); + const report = reportFor( + target.definition, + target.owner, + observed.observed, + observed.members, + config, ); - const comparison = - loaded.config === undefined - ? target.inspection === undefined - ? yield* catchStackError(api.inspectStack(target.id)).pipe(Effect.map(comparedInspection)) - : { inspection: target.inspection } - : yield* api.inspectStack(target.id, { config: loaded.config }).pipe( - Effect.map(comparedInspection), - Effect.catchTags({ - InvalidStackConfigError: (error) => - Effect.succeed({ - inspection: undefined, - warning: configComparisonWarning(error.message), - }), - StackVersionUnsupportedError: (error) => - Effect.succeed({ - inspection: undefined, - warning: configComparisonWarning(error.message), - }), - }), - catchStackError, - ); - const inspection = - comparison.inspection === undefined - ? (target.inspection ?? (yield* catchStackError(api.inspectStack(target.id)))) - : comparison.inspection; - const inspectionWarning = loaded.warning ?? comparison.warning; - if (output.format === "text") yield* output.raw(render(inspection, inspectionWarning)); - else yield* output.success("", payload(inspection, inspectionWarning)); + if (output.format === "text") yield* output.raw(render(report)); + else yield* output.success("", report); }); return yield* body.pipe(Effect.ensuring(telemetryState.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 65cdb5f045..d84606cd98 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -1,874 +1,553 @@ import { BunServices } from "@effect/platform-bun"; -import { describe, expect, it } from "@effect/vitest"; -import { parse as parseDotenv } from "dotenv"; -import { - Cause, - Effect, - FileSystem, - Exit, - Layer, - Option, - Path, - Redacted, - Schema, - Stream, -} from "effect"; -import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; -import { CliOutput, Command } from "effect/unstable/cli"; -import { - InvalidStackConfigError, - StackNotFoundError, - StackNotRunningError, - StackIdSchema, - StackStateFormatUnsupportedError, - type EffectStack, - type StackInspection, - type StackStatus, -} from "@supabase/stack/effect"; +import { expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, FileSystem, Layer, Option, Redacted, Stream } from "effect"; +import { type Observation, type ServiceCreation, StackError } from "@supabase/stack/effect"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; -import { GLOBAL_OUTPUT_FORMATS, OutputFlag } from "../../../../command-internal/global-flags.ts"; -import { - actionability, - ErrorActionabilityId, -} from "../../../../shared/telemetry/error-actionability.ts"; -import { StackApi } from "../stack.shared.ts"; +import { runtimeInfoLayer } from "../../../../shared/runtime/runtime-info.layer.ts"; +import { StackApi, StackTargetResolver } from "../stack.shared.ts"; +import type { StackStatusFlags } from "./status.command.ts"; import { stackStatus } from "./status.handler.ts"; -import { stackStatusCommand } from "./status.command.ts"; -import { textCliOutputFormatter } from "../../../../shared/output/text-formatter.ts"; -const id = StackIdSchema.make("a".repeat(64)); -const capabilityNames = [ - "database", - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", -] as const; -const flags = (stack = Option.none(), stackId = Option.none()) => ({ - stack, - stackId, +const stackId = "a".repeat(64); +type StatusOutputFormat = "text" | "json" | "stream-json"; +type OpenedStack = Effect.Success>; +type StackInstance = Effect.Success[number]; +const jwtSecret = "status-test-jwt-secret-with-at-least-32-chars"; +const database: ServiceCreation = { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(jwtSecret), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: 54322 } }, +}; +const rest: ServiceCreation = { + service: "rest", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: 54321 } }, +}; +const flags = (input?: Partial): StackStatusFlags => ({ + stack: Option.none(), + stackId: Option.none(), env: false, - overrideName: [] as string[], + overrideName: [], + ...input, +}); + +const makeObservation = ( + id: string, + config: ServiceCreation, + input: Partial = {}, +): Observation => ({ + id, + endpoints: [], + config, + lifecycle: "stopped", + health: undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 1, + wakeEnabled: true, + registered: true, + ...input, +}); + +const makeService = (input: { + readonly id: string; + readonly creation: ServiceCreation; + readonly observation?: Observation; + readonly statusCalls: { value: number }; + readonly statusError?: StackError; + readonly credentials?: Readonly>; + readonly rejectCredentials?: boolean; +}): StackInstance => ({ + id: input.id, + service: input.creation.service, + start: Effect.die("unused"), + ready: Effect.die("unused"), + stop: Effect.die("unused"), + restart: () => Effect.die("unused"), + destroy: Effect.die("unused"), + prepare: Effect.die("unused"), + status: Effect.suspend(() => { + input.statusCalls.value += 1; + if (input.statusError !== undefined) return Effect.fail(input.statusError); + return input.observation === undefined + ? Effect.die("status must not run") + : Effect.succeed(input.observation); + }), + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => + input.rejectCredentials + ? Effect.die("credentials must not run") + : Effect.succeed(input.credentials ?? {}), + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: Effect.die("unused"), }); -const makeStatus = ( - stackId: typeof id, - desiredLifecycle: StackStatus["desiredLifecycle"] = "running", -): StackStatus => ({ +const makeStack = ( + services: ReadonlyArray, + members: ReadonlyArray<{ readonly id: string; readonly activation: "eager" | "lazy" }>, +): OpenedStack => ({ id: stackId, - lifecycle: "running", - desiredLifecycle, - runtime: { kind: "native" }, - endpoints: { - api: { protocol: "http", address: "127.0.0.1", port: 54321, url: "http://127.0.0.1:54321" }, + services: { + create: (_creation) => Effect.die("unused"), + get: (_id) => Effect.die("unused"), + list: Effect.succeed([...services]), + }, + composition: { + supabase: (_services, _options) => Effect.die("unused"), + configure: (_config) => Effect.die("unused"), + describe: Effect.succeed({ members: [...members], dependencies: [] }), + start: Effect.die("unused"), + stop: Effect.die("unused"), + restart: Effect.die("unused"), + }, + stop: Effect.die("unused"), + destroy: Effect.die("unused"), + tools: { + run: (_tool, _options) => Effect.die("unused"), }, - versions: {}, - capabilities: capabilityNames.map((name) => ({ - name, - activation: "lazy" as const, - state: "dormant" as const, - })), - artifacts: [], }); -const runStatus = (options: { - readonly config?: "valid" | "missing" | "invalid"; - readonly owner?: StackInspection["owner"]; - readonly status?: StackStatus; - readonly drift?: StackInspection["configDrift"]; - readonly flags?: ReturnType; - readonly compareFailure?: "typed" | "defect"; - readonly missingTarget?: boolean; - readonly legacyOutput?: (typeof GLOBAL_OUTPUT_FORMATS)[number]; - readonly outputFormat?: "text" | "json" | "stream-json"; - readonly credentialFailure?: boolean; - readonly storageCredentials?: boolean; - readonly authDisabled?: boolean; +const runStatus = (input: { + readonly services: ReadonlyArray; + readonly members?: ReadonlyArray<{ readonly id: string; readonly activation: "eager" | "lazy" }>; + readonly reachable?: boolean; + readonly outputFormat?: StatusOutputFormat; + readonly config?: "missing" | "invalid" | "explicit"; + readonly flags?: StackStatusFlags; }) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-status-" }); - const projectRoot = path.join(root, "project"); - yield* fs.makeDirectory(path.join(projectRoot, "supabase"), { recursive: true }); - if (options.config !== "missing") + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-status-" }); + if (input.config === "invalid") { + yield* fs.makeDirectory(`${root}/supabase`); + yield* fs.writeFileString(`${root}/supabase/config.toml`, 'project_id = "broken\n[auth\n'); + } + if (input.config === "explicit") { + yield* fs.makeDirectory(`${root}/supabase`); yield* fs.writeFileString( - path.join(projectRoot, "supabase", "config.toml"), - options.config === "invalid" - ? 'project_id = "ok"\n\n[auth]\njwt_secret = "FAKE_STATUS_SECRET\n' - : 'project_id = "status-test"\n\n[auth]\njwt_secret = "candidate-secret"\n', + `${root}/supabase/config.toml`, + 'project_id = "status-test"\n[db]\nport = 54322\n', ); - const descriptor = { - id, - projectRoot, - name: "feature-a", - branchContext: "ordinary-workspace", - runtime: { kind: "native" as const }, - desiredLifecycle: "running" as const, - }; - const inspection: StackInspection = { - descriptor, - owner: options.owner ?? "running", - ...(options.status === undefined ? {} : { status: options.status }), - ...(options.drift === undefined ? {} : { configDrift: options.drift }), + } + const projectRoot = root; + const stack = makeStack( + input.services, + input.members ?? input.services.map(({ id }) => ({ id, activation: "lazy" as const })), + ); + const definition = { + id: stackId, + identity: { + projectRoot, + branchContext: "status-branch", + stackName: "status-stack", + }, + runtime: "native" as const, + instances: input.services.map(({ id, service }) => ({ + id, + creation: service === "database" ? database : rest, + })), + composition: { members: input.members ?? [], dependencies: [] }, + ports: [], }; - const out = mockOutput({ format: options.outputFormat ?? "text" }); - const telemetry = mockTelemetryStateTracked(); - const findInputs: unknown[] = []; - const inspectInputs: unknown[] = []; const api = Layer.succeed(StackApi, { - createStack: () => Effect.die("create must not run"), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - findStack: (input) => { - findInputs.push(input); - return Effect.succeed(options.missingTarget ? Option.none() : Option.some(descriptor)); - }, - openStack: (openId) => - Effect.succeed({ - id: openId, - status: Effect.succeed(options.status ?? makeStatus(id)), - credentials: - options.credentialFailure === true - ? Effect.fail( - new StackNotRunningError({ stackId: id, message: "Stack is not running" }), - ) - : Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:p%40ss@127.0.0.1:54322/postgres"), - password: Redacted.make("p@ss"), + create: () => Effect.die("create must not run"), + open: () => Effect.succeed(stack), + discover: () => + Effect.succeed([ + { + definition, + host: + input.reachable === false + ? undefined + : { + stackId, + identity: definition.identity, + pid: 123, + port: 4567, }, - ...(options.authDisabled === true - ? {} - : { - api: { - anonJwt: "anon-token", - serviceRoleJwt: Redacted.make("service-role-token"), - publishableKey: "sb_publishable_test", - secretKey: Redacted.make("sb_secret_test"), - }, - }), - ...(options.storageCredentials === true - ? { - storage: { - endpoint: "http://127.0.0.1:54321/storage/v1/s3", - region: "local", - accessKeyId: "storage-access", - secretAccessKey: Redacted.make("storage-secret"), - }, - } - : {}), - }), - prepare: () => Effect.die("unused"), - start: () => Effect.die("unused"), - stop: Effect.die("unused"), - destroy: Effect.die("unused"), - resetDatabase: Effect.die("unused"), - logs: () => Effect.die("unused"), - followLogs: () => Stream.empty, - } satisfies EffectStack), - inspectStack: (_stackId, inspectOptions) => { - inspectInputs.push(inspectOptions); - if (options.missingTarget === true) - return Effect.fail(new StackNotFoundError({ message: "stack id not found" })); - if (inspectOptions?.config !== undefined && options.compareFailure === "typed") - return Effect.fail( - new InvalidStackConfigError({ message: "candidate config is invalid" }), - ); - if (inspectOptions?.config !== undefined && options.compareFailure === "defect") - return Effect.die("comparison defect"); - return Effect.succeed(inspection); - }, + }, + ]), + resolveIdentity: () => Effect.succeed(definition.identity), }); + const resolver = Layer.succeed(StackTargetResolver, { + resolve: (target) => + Effect.succeed({ + projectRoot: target.projectRoot, + id: stackId, + runtime: "native" as const, + }), + }); + const out = mockOutput({ format: input.outputFormat ?? "text" }); + const telemetry = mockTelemetryStateTracked(); const layer = Layer.mergeAll( + api, + resolver, out.layer, telemetry.layer, - api, - mockCommandSettings({ workdir: root }), - ...(options.legacyOutput === undefined - ? [] - : [Layer.succeed(OutputFlag, Option.some(options.legacyOutput))]), + mockCommandSettings({ workdir: projectRoot, supabaseHome: root }), BunServices.layer, runtimeInfoLayer, ); - const effect = stackStatus(options.flags ?? flags()).pipe(Effect.provide(layer)); - return { effect, out, findInputs, inspectInputs, projectRoot, root }; + const effect = stackStatus(input.flags ?? flags()).pipe(Effect.provide(layer)); + return { effect, out, root }; }).pipe(Effect.provide(BunServices.layer)); -const withRunStatus = ( - options: Parameters[0], - test: (run: Effect.Success>) => Effect.Effect, -) => runStatus(options).pipe(Effect.flatMap(test)); - -describe("stack status", () => { - it.effect( - "reports configured identity, dormant readiness, endpoint, drift, and target config", - () => { - return withRunStatus( - { - status: makeStatus(id), - drift: { status: "changed", paths: ["definition.listeners.api.port"] }, - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.findInputs).toEqual([{ projectRoot: expect.any(String) }]); - expect(run.inspectInputs).toHaveLength(1); - expect(run.inspectInputs[0]).toEqual({ config: expect.any(Object) }); - expect(run.out.stdoutText).toContain("Runtime: native"); - expect(run.out.stdoutText).toContain("Readiness: dormant"); - expect(run.out.stdoutText).toContain("http://127.0.0.1:54321"); - expect(run.out.stdoutText).toContain("definition.listeners.api.port"); - expect(run.out.stdoutText).not.toContain("candidate-secret"); - }), - ), - ), - ); - }, - ); - - it.effect("forwards a named stack target with the settings project root", () => { - return withRunStatus( - { flags: flags(Option.some("feature-a")), status: makeStatus(id) }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.findInputs).toEqual([{ projectRoot: run.root, name: "feature-a" }]); - }), - ), - ), - ); - }); - - it.effect("uses the persisted project root for an explicit id from another cwd", () => { - return withRunStatus( - { flags: flags(Option.none(), Option.some(id)), status: makeStatus(id) }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.inspectInputs).toHaveLength(2); - expect(run.inspectInputs[1]).toEqual({ config: expect.any(Object) }); - }), - ), - ), - ); - }); - - it.effect("reuses the explicit id inspection when config is invalid", () => { - return withRunStatus( - { - config: "invalid", - flags: flags(Option.none(), Option.some(id)), - status: makeStatus(id), - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.inspectInputs).toHaveLength(1); - expect(run.inspectInputs[0]).toBeUndefined(); - }), - ), - ), - ); - }); - - it.effect("compares an absent config.toml against default settings like stack start", () => { - return withRunStatus( - { - config: "missing", - flags: flags(Option.none(), Option.some(id)), - status: makeStatus(id), - drift: { status: "unchanged", paths: [] }, - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.inspectInputs).toHaveLength(2); - expect(run.inspectInputs[1]).toEqual({ config: expect.any(Object) }); - expect(run.out.stdoutText).toContain("Config drift: unchanged"); - expect(run.out.stdoutText).not.toContain("Config warning"); - }), - ), - ), - ); - }); - - it.effect("reports stopped and unreachable stacks without claiming live readiness", () => { - return withRunStatus({ owner: "absent" }, (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.out.stdoutText).toContain("Lifecycle: unavailable"); - expect(run.out.stdoutText).toContain("Desired lifecycle: running"); - expect(run.out.stdoutText).toContain("Readiness: unknown"); - }), - ), - ), - ); - }); - - it.effect("does not claim ready when a running stack has stopped capabilities", () => { - const base = makeStatus(id); - return withRunStatus( - { - status: { - ...base, - capabilities: base.capabilities.map((capability, index) => - index === 0 ? { ...capability, state: "stopped" as const } : capability, - ), - }, - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => expect(run.out.stdoutText).toContain("Readiness: stopped")), - ), - ), - ); - }); - - it.effect("reports a retiring capability as stopping in text and JSON", () => { - const base = makeStatus(id); - const status = { - ...base, - capabilities: base.capabilities.map((capability) => - capability.name === "rest" ? { ...capability, state: "stopping" as const } : capability, - ), - }; - return Effect.all([runStatus({ status }), runStatus({ status, outputFormat: "json" })]).pipe( - Effect.flatMap(([text, json]) => - Effect.all([text.effect, json.effect]).pipe( - Effect.tap(() => - Effect.sync(() => { - expect(text.out.stdoutText).toContain("Readiness: stopping"); - const success = json.out.messages.find((message) => message.type === "success"); - expect(success?.data).toMatchObject({ readiness: "stopping" }); - }), - ), - ), - ), - ); - }); - - it.effect("reports failed capability diagnostics and targeted recovery in text and JSON", () => { - const base = makeStatus(id); - const status: StackStatus = { - ...base, - capabilities: base.capabilities.map((capability) => - capability.name === "rest" - ? { - ...capability, - state: "failed" as const, - error: "Unable to remove REST workload", - } - : capability, - ), - recovery: { - operation: "stop", - message: "Cleanup is incomplete; stop and start the stack to retry it.", - }, +it.live("reports observed lifecycle and health without requesting credentials", () => + Effect.gen(function* () { + const databaseCalls = { value: 0 }; + const restCalls = { value: 0 }; + const authCalls = { value: 0 }; + const auth: ServiceCreation = { + service: "auth", + config: { databaseUrl: "postgresql://placeholder", jwtSecret }, + endpoints: { http: { port: 54325 } }, }; - return Effect.all([runStatus({ status }), runStatus({ status, outputFormat: "json" })]).pipe( - Effect.flatMap(([text, json]) => - Effect.all([text.effect, json.effect]).pipe( - Effect.tap(() => - Effect.sync(() => { - expect(text.out.stdoutText).toContain( - "rest: failed — Unable to remove REST workload", - ); - expect(text.out.stdoutText).toContain( - `supabase stack stop --stack-id ${id} && supabase stack start --stack-id ${id}`, - ); - const success = json.out.messages.find((message) => message.type === "success"); - expect(success?.data).toMatchObject({ recovery: status.recovery }); - expect(success?.data).toMatchObject({ - capabilities: expect.arrayContaining([ - expect.objectContaining({ - name: "rest", - state: "failed", - error: "Unable to remove REST workload", - }), - ]), - }); - }), - ), - ), - ), - ); - }); - - it.effect("emits the structured unavailable inspection for invalid config", () => { - return withRunStatus( - { - config: "invalid", - flags: flags(Option.none(), Option.some(id)), - outputFormat: "json", - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.out.stdoutText).toBe(""); - const success = run.out.messages.find((message) => message.type === "success"); - expect(success?.data).toMatchObject({ - identity: { - id, - name: "feature-a", - project_root: run.projectRoot, - branch_context: "ordinary-workspace", - }, - owner: "running", - readiness: "unknown", - lifecycle: null, - desired_lifecycle: "running", - config_drift: { - status: "unavailable", - message: - "Project configuration could not be loaded; fix it before checking drift.", - }, - }); - }), - ), - ), - ); - }); - - it.effect("uses the live desired lifecycle consistently in text and JSON", () => { - return Effect.all([ - runStatus({ status: makeStatus(id, "stopped") }), - runStatus({ status: makeStatus(id, "stopped"), outputFormat: "json" }), - ]).pipe( - Effect.flatMap(([text, json]) => - Effect.all([text.effect, json.effect]).pipe( - Effect.tap(() => - Effect.sync(() => { - expect(text.out.stdoutText).toContain("Desired lifecycle: stopped"); - const success = json.out.messages.find((message) => message.type === "success"); - expect(success?.data).toMatchObject({ desired_lifecycle: "stopped" }); - }), - ), - ), - ), - ); - }); - - it.effect("reports unavailable drift for invalid config and keeps inspection", () => { - return Effect.all([ - runStatus({ config: "invalid", status: makeStatus(id) }), - runStatus({ - config: "invalid", - status: makeStatus(id), - outputFormat: "json", + const services = [ + makeService({ + id: "database-id", + creation: database, + statusCalls: databaseCalls, + observation: makeObservation("database-id", database, { + lifecycle: "running", + health: "healthy", + wakeEnabled: false, + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: 54322 }], + }), }), - ]).pipe( - Effect.flatMap(([invalid, invalidJson]) => - Effect.all([invalid.effect, invalidJson.effect]).pipe( - Effect.tap(() => - Effect.sync(() => { - expect(invalid.out.stdoutText).toContain("Config drift: unavailable"); - expect(invalid.out.stdoutText).not.toContain("FAKE_STATUS_SECRET"); - const success = invalidJson.out.messages.find( - (message) => message.type === "success", - ); - expect(success?.data).toMatchObject({ - config_drift: { - status: "unavailable", - message: - "Project configuration could not be loaded; fix it before checking drift.", - }, - }); - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- JSON.stringify checks all fields for leaked secrets; schema encoding could omit unexpected fields - expect(JSON.stringify(success?.data)).not.toContain("FAKE_STATUS_SECRET"); - }), - ), - ), - ), - ); - }); - - it.effect("points an empty current context to the start command", () => { - return withRunStatus({ missingTarget: true }, (run) => - run.effect.pipe( - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.suggestion).toBe("Run supabase stack start first."); - expect(run.inspectInputs).toEqual([]); - }), - ), - ), - ); - }); - - it.effect("gives actionable guidance when an explicit stack id is missing", () => { - return withRunStatus( - { - flags: flags(Option.none(), Option.some(id)), - missingTarget: true, - }, - (run) => - run.effect.pipe( - Effect.exit, - Effect.tap((exit) => - Effect.sync(() => { - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const error = Cause.findErrorOption(exit.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value.suggestion).toContain("existing --stack-id"); - expect(error.value[ErrorActionabilityId]).toEqual(actionability.provideFlags); - } - } - }), - ), - ), - ); - }); - - it.effect("falls back only for typed comparison errors and preserves defects", () => { - return Effect.all([ - runStatus({ compareFailure: "typed", status: makeStatus(id) }), - runStatus({ - compareFailure: "typed", - status: makeStatus(id), - outputFormat: "json", + makeService({ + id: "rest-id", + creation: rest, + statusCalls: restCalls, + observation: makeObservation("rest-id", rest, { + endpoints: [{ name: "http", protocol: "http", host: "127.0.0.1", port: 54321 }], + }), }), - runStatus({ compareFailure: "defect", status: makeStatus(id) }), - ]).pipe( - Effect.flatMap(([typed, typedJson, defect]) => - Effect.gen(function* () { - yield* typed.effect; - expect(typed.inspectInputs).toHaveLength(2); - expect(typed.out.stdoutText).toContain("Config drift: unavailable"); - expect(typed.out.stdoutText).toContain( - "Config warning: Project configuration could not be compared: candidate config is invalid", - ); - yield* typedJson.effect; - const success = typedJson.out.messages.find((message) => message.type === "success"); - expect(success?.data).toMatchObject({ - config_drift: { - status: "unavailable", - message: "Project configuration could not be compared: candidate config is invalid", - }, - }); - const exit = yield* defect.effect.pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - expect(defect.inspectInputs).toHaveLength(1); + makeService({ + id: "auth-id", + creation: auth, + statusCalls: authCalls, + observation: makeObservation("auth-id", auth, { + lifecycle: "running", + health: "unhealthy", }), - ), - ); - }); - - it.effect("rejects invalid flags and legacy output before discovery", () => { - return Effect.all([ - runStatus({ flags: flags(Option.some("feature-a"), Option.some(id)) }), - runStatus({ legacyOutput: "json" }), - ]).pipe( - Effect.flatMap(([invalid, legacy]) => - Effect.gen(function* () { - expect(Exit.isFailure(yield* invalid.effect.pipe(Effect.exit))).toBe(true); - expect(Exit.isFailure(yield* legacy.effect.pipe(Effect.exit))).toBe(true); - expect(invalid.findInputs).toHaveLength(0); - expect(legacy.findInputs).toHaveLength(0); + }), + ]; + const run = yield* runStatus({ services, reachable: true }); + yield* run.effect; + expect(run.out.stdoutText).toContain("Owner: reachable"); + expect(run.out.stdoutText).toContain("database (database-id): running"); + expect(run.out.stdoutText).toContain("rest (rest-id): sleeping"); + expect(run.out.stdoutText).toContain("auth (auth-id): unhealthy"); + expect(run.out.stdoutText).toContain("health=unhealthy"); + expect(run.out.stdoutText).toContain("Readiness: unhealthy"); + expect(databaseCalls.value).toBe(1); + expect(restCalls.value).toBe(1); + expect(authCalls.value).toBe(1); + }), +); + +it.live("preserves service status failures and marks aggregate readiness unavailable", () => + Effect.gen(function* () { + const run = yield* runStatus({ + services: [ + makeService({ + id: "database-id", + creation: database, + statusCalls: { value: 0 }, + statusError: new StackError({ + operation: "status", + message: "owner disconnected while observing database", + }), }), - ), - ); - }); + ], + reachable: true, + outputFormat: "json", + }); + yield* run.effect; + const result = run.out.messages.find((message) => message.type === "success")?.data; + expect(result).toMatchObject({ + owner: "reachable", + lifecycle: null, + readiness: "unavailable", + services: [{ error: "owner disconnected while observing database" }], + }); + }), +); - it.effect("rejects the legacy -o env form with a pointer to --env", () => { - return withRunStatus({ legacyOutput: "env" }, (run) => - run.effect.pipe( - Effect.flip, - Effect.tap((error) => - Effect.sync(() => { - expect(error.suggestion).toContain("--env"); - expect(run.findInputs).toHaveLength(0); - }), - ), - ), - ); - }); +it.live("does not label a requested stop as an unexpected process exit", () => + Effect.gen(function* () { + const stopped = makeObservation("database-id", database, { + lifecycle: "stopped", + wakeEnabled: false, + exit: Exit.fail({ _tag: "ServiceError", operation: "exit", message: "SIGTERM" }), + }); + const unexpected = makeObservation("rest-id", rest, { + lifecycle: "stopped", + wakeEnabled: false, + error: { _tag: "ServiceError", operation: "exit", message: "exit code 1" }, + }); + const run = yield* runStatus({ + services: [ + makeService({ + id: "database-id", + creation: database, + statusCalls: { value: 0 }, + observation: stopped, + }), + makeService({ + id: "rest-id", + creation: rest, + statusCalls: { value: 0 }, + observation: unexpected, + }), + ], + reachable: true, + outputFormat: "json", + }); + yield* run.effect; + const result = run.out.messages.find((message) => message.type === "success")?.data; + expect(result).toMatchObject({ + services: [ + { id: "database-id", state: "stopped" }, + { id: "rest-id", state: "exited" }, + ], + }); + }), +); - it.effect("does not retry discovery failures", () => { - return withRunStatus({}, (run) => { - const telemetry = mockTelemetryStateTracked(); - const discovery = Layer.succeed(StackApi, { - createStack: () => Effect.die("create must not run"), - discoverStacks: () => Effect.succeed({ stacks: [], errors: [] }), - findStack: () => - Effect.fail(new StackStateFormatUnsupportedError({ message: "discovery failed" })), - openStack: () => Effect.die("open must not run"), - inspectStack: () => Effect.die("inspect must not run"), - }); - const effect = stackStatus(flags()).pipe( - Effect.provide( - Layer.mergeAll( - run.out.layer, - telemetry.layer, - discovery, - mockCommandSettings({ workdir: run.projectRoot }), - BunServices.layer, - runtimeInfoLayer, - ), - ), - Effect.exit, - ); - return effect.pipe( - Effect.tap((exit) => - Effect.sync(() => { - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const error = Cause.findErrorOption(exit.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) - expect(error.value[ErrorActionabilityId]).toEqual(actionability.invalidConfig); - } +it.live("reports stopped readiness without treating unbound endpoints as drift", () => + Effect.gen(function* () { + const run = yield* runStatus({ + services: [ + makeService({ + id: "database-id", + creation: database, + statusCalls: { value: 0 }, + observation: makeObservation("database-id", database, { + lifecycle: "stopped", + health: undefined, + wakeEnabled: false, }), - ), - ); + }), + ], + reachable: true, + outputFormat: "json", + }); + yield* run.effect; + const result = run.out.messages.find((message) => message.type === "success")?.data; + expect(result).toMatchObject({ + readiness: "stopped", + config_drift: { status: "unchanged" }, }); - }); + }), +); - it.live("parses stack name and stack id through the command", () => { - let parsed: { stack: Option.Option; stackId: Option.Option } | undefined; - const command = stackStatusCommand.pipe( - Command.withHandler((parsedFlags) => - Effect.sync(() => { - parsed = { stack: parsedFlags.stack, stackId: parsedFlags.stackId }; +it.live("does not report port drift when a stopped service has no live binding", () => + Effect.gen(function* () { + const run = yield* runStatus({ + services: [ + makeService({ + id: "database-id", + creation: database, + statusCalls: { value: 0 }, + observation: makeObservation("database-id", database, { + lifecycle: "stopped", + wakeEnabled: false, + }), }), - ), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })(["--stack", "feature-a"]); - expect(parsed).toEqual({ stack: Option.some("feature-a"), stackId: Option.none() }); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); + ], + config: "explicit", + reachable: true, + outputFormat: "json", + }); + yield* run.effect; + const result = run.out.messages.find((message) => message.type === "success")?.data; + expect(result).toMatchObject({ config_drift: { status: "unchanged" } }); + }), +); - it.live("parses env selection and repeated CSV variable overrides", () => { - let input: { env: boolean; overrideName: ReadonlyArray } | undefined; - const command = stackStatusCommand.pipe( - Command.withHandler((parsedFlags) => - Effect.sync(() => { - input = { env: parsedFlags.env, overrideName: parsedFlags.overrideName }; +it.live("reports changed explicit ports even while a service is stopped", () => + Effect.gen(function* () { + const changed = { ...database, endpoints: { sql: { port: 54329 } } }; + const run = yield* runStatus({ + services: [ + makeService({ + id: "database-id", + creation: changed, + statusCalls: { value: 0 }, + observation: makeObservation("database-id", changed, { + lifecycle: "stopped", + wakeEnabled: false, + }), }), - ), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--env", - "--override-name", - "API_URL=APP_URL,ANON_KEY=APP_KEY", - "--override-name", - "DB_URL=DATABASE_URL", - ]); - expect(input?.env).toBe(true); - expect(input?.overrideName).toEqual([ - "API_URL=APP_URL", - "ANON_KEY=APP_KEY", - "DB_URL=DATABASE_URL", - ]); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); - - it.effect("exports the running stack credentials as dotenv with renamed variables", () => { - return withRunStatus( - { - config: "invalid", - flags: { ...flags(), env: true, overrideName: ["API_URL=NEXT_PUBLIC_SUPABASE_URL"] }, - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(parseDotenv(run.out.stdoutText)).toEqual({ - NEXT_PUBLIC_SUPABASE_URL: "http://127.0.0.1:54321", - DB_URL: "postgresql://postgres:p%40ss@127.0.0.1:54322/postgres", - ANON_KEY: "anon-token", - SERVICE_ROLE_KEY: "service-role-token", - PUBLISHABLE_KEY: "sb_publishable_test", - SECRET_KEY: "sb_secret_test", - }); - expect(run.inspectInputs).toHaveLength(0); - }), - ), - ), - ); - }); - - const exportedVariables = { - API_URL: "http://127.0.0.1:54321", - DB_URL: "postgresql://postgres:p%40ss@127.0.0.1:54322/postgres", - ANON_KEY: "anon-token", - SERVICE_ROLE_KEY: "service-role-token", - PUBLISHABLE_KEY: "sb_publishable_test", - SECRET_KEY: "sb_secret_test", - }; - const VariableMapJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)); + ], + config: "explicit", + reachable: true, + outputFormat: "json", + }); + yield* run.effect; + const result = run.out.messages.find((message) => message.type === "success")?.data; + expect(result).toMatchObject({ + config_drift: { status: "changed", paths: ["services.database.endpoints.sql"] }, + }); + }), +); - it.effect("exports a bare variable map in json", () => { - return withRunStatus({ flags: { ...flags(), env: true }, outputFormat: "json" }, (run) => - Effect.gen(function* () { - yield* run.effect; - const variables = yield* Schema.decodeEffect(VariableMapJson)(run.out.stdoutText.trim()); - expect(variables).toEqual(exportedVariables); - expect(run.out.messages).toEqual([]); +it.live("reports unavailable owner and does not query service status", () => + Effect.gen(function* () { + const statusCalls = { value: 0 }; + const services = [ + makeService({ + id: "database-id", + creation: database, + statusCalls, }), - ); - }); - - it.effect("exports a variable map as a stream-json result event", () => { - return withRunStatus({ flags: { ...flags(), env: true }, outputFormat: "stream-json" }, (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - const results = run.out.events.flatMap((event) => - event.type === "result" ? [event.data] : [], - ); - expect(results).toEqual([exportedVariables]); - expect(run.out.stdoutText).toBe(""); - }), - ), - ), - ); - }); - - it.effect("exports optional service URLs and storage credentials only when available", () => { - const status: StackStatus = { - ...makeStatus(id), - endpoints: { - studio: { - protocol: "http", - address: "127.0.0.1", - port: 54323, - url: "http://127.0.0.1:54323", - }, - mailUi: { - protocol: "http", - address: "127.0.0.1", - port: 54324, - url: "http://127.0.0.1:54324", + ]; + const run = yield* runStatus({ services, reachable: false }); + yield* run.effect; + expect(run.out.stdoutText).toContain("Owner: unavailable"); + expect(run.out.stdoutText).toContain("Lifecycle: unavailable"); + expect(run.out.stdoutText).toContain("database (database-id): unavailable"); + expect(statusCalls.value).toBe(0); + }), +); + +it.live("rejects environment export when the owner is unavailable", () => + Effect.gen(function* () { + const run = yield* runStatus({ + services: [makeService({ id: "database-id", creation: database, statusCalls: { value: 0 } })], + reachable: false, + flags: flags({ env: true }), + }); + const exit = yield* run.effect.pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(exit.cause.reasons.every(Cause.isFailReason)).toBe(true); + const error = exit.cause.reasons.find(Cause.isFailReason)?.error; + expect(error).toBeDefined(); + if (error !== undefined) expect(error.reason).toBe("lifecycle"); + } + }), +); + +it.live("exports host variables and JWTs only for a running database", () => + Effect.gen(function* () { + const services = [ + makeService({ + id: "database-id", + creation: database, + statusCalls: { value: 0 }, + observation: makeObservation("database-id", database, { + lifecycle: "running", + health: "healthy", + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: 54322 }], + }), + rejectCredentials: true, + credentials: { + databaseUrl: "postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres", }, - }, - }; - return withRunStatus( - { flags: { ...flags(), env: true }, status, storageCredentials: true }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - const values = parseDotenv(run.out.stdoutText); - expect(values.API_URL).toBeUndefined(); - expect(values).toMatchObject({ - STUDIO_URL: "http://127.0.0.1:54323", - INBUCKET_URL: "http://127.0.0.1:54324", - S3_PROTOCOL_ACCESS_KEY_SECRET: "storage-secret", - S3_PROTOCOL_REGION: "local", - }); - }), - ), - ), - ); - }); - - it.effect("exports a database-only stack without inventing API credentials", () => { - return withRunStatus( - { - flags: { ...flags(), env: true }, - status: { ...makeStatus(id), endpoints: {} }, - authDisabled: true, - }, - (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(parseDotenv(run.out.stdoutText)).toEqual({ - DB_URL: "postgresql://postgres:p%40ss@127.0.0.1:54322/postgres", - }); - }), - ), - ), - ); - }); - - it.effect("keeps ordinary status independent of credentials and free of secrets", () => { - return withRunStatus({ status: makeStatus(id), credentialFailure: true }, (run) => - run.effect.pipe( - Effect.tap(() => - Effect.sync(() => { - expect(run.out.stdoutText).toContain("Lifecycle: running"); - expect(run.out.stdoutText).not.toContain("sb_secret_test"); - }), - ), - ), + }), + makeService({ + id: "rest-id", + creation: rest, + statusCalls: { value: 0 }, + observation: makeObservation("rest-id", rest, { + lifecycle: "running", + health: "healthy", + endpoints: [{ name: "http", protocol: "http", host: "127.0.0.1", port: 54321 }], + }), + }), + ]; + const run = yield* runStatus({ + services, + reachable: true, + flags: flags({ env: true }), + }); + yield* run.effect; + expect(run.out.stdoutText).toContain( + "DB_URL='postgresql://supabase_admin:postgres@127.0.0.1:54322/postgres?connect_timeout=10'", ); - }); + expect(run.out.stdoutText).toContain("API_URL='http://127.0.0.1:54321'"); + expect(run.out.stdoutText).toContain("ANON_KEY="); + expect(run.out.stdoutText).toContain("SERVICE_ROLE_KEY="); + }), +); - it.effect("rejects invalid or colliding variable renames before discovery", () => { - const cases: ReadonlyArray>> = [ - { overrideName: ["API_URL=APP_URL"] }, - { env: true, overrideName: ["UNKNOWN=APP_URL"] }, - { env: true, overrideName: ["API_URL=NOT-VALID"] }, - { env: true, overrideName: ["API_URL=DB_URL"] }, - { env: true, overrideName: ["API_URL"] }, - { env: true, overrideName: ["API_URL=A=B"] }, - { env: true, overrideName: ["API_URL=A", "API_URL=B"] }, - ]; - return Effect.forEach(cases, (overrides) => { - return withRunStatus({ flags: { ...flags(), ...overrides } }, (run) => - run.effect.pipe( - Effect.exit, - Effect.tap((exit) => - Effect.sync(() => { - expect(Exit.isFailure(exit)).toBe(true); - expect(run.findInputs).toHaveLength(0); - expect(run.out.stdoutText).toBe(""); - }), - ), - ), - ); +it.live("uses only the composition database for environment export", () => + Effect.gen(function* () { + const shadow = makeObservation("shadow-db", database, { + lifecycle: "running", + health: "healthy", + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: 60000 }], + }); + const primary = makeObservation("primary-db", database, { + lifecycle: "running", + health: "healthy", + endpoints: [{ name: "sql", protocol: "tcp", host: "127.0.0.1", port: 54322 }], + }); + const run = yield* runStatus({ + services: [ + makeService({ + id: "shadow-db", + creation: database, + statusCalls: { value: 0 }, + observation: shadow, + }), + makeService({ + id: "primary-db", + creation: database, + statusCalls: { value: 0 }, + observation: primary, + }), + ], + members: [{ id: "primary-db", activation: "eager" }], + reachable: true, + flags: flags({ env: true }), }); - }); + yield* run.effect; + expect(run.out.stdoutText).toContain("127.0.0.1:54322"); + expect(run.out.stdoutText).not.toContain("127.0.0.1:60000"); + }), +); - it.effect("exports no partial secrets when the stack is stopped or credentials fail", () => { - return Effect.all([ - runStatus({ - flags: { ...flags(), env: true }, - status: { ...makeStatus(id), lifecycle: "stopped" }, - }), - runStatus({ - flags: { ...flags(), env: true }, - credentialFailure: true, - }), - ]).pipe( - Effect.flatMap(([stopped, failedCredentials]) => - Effect.gen(function* () { - const stoppedError = yield* stopped.effect.pipe(Effect.flip); - expect(stoppedError.reason).toBe("lifecycle"); - expect(stoppedError[ErrorActionabilityId]).toEqual(actionability.startStack); - expect(stopped.out.stdoutText).toBe(""); - const failedExit = yield* failedCredentials.effect.pipe(Effect.exit); - expect(Exit.isFailure(failedExit)).toBe(true); - expect(failedCredentials.out.stdoutText).toBe(""); +it.live("keeps status usable with malformed project configuration", () => + Effect.gen(function* () { + const services = [ + makeService({ + id: "database-id", + creation: database, + statusCalls: { value: 0 }, + observation: makeObservation("database-id", database, { + lifecycle: "running", + health: "healthy", }), - ), - ); - }); -}); + }), + ]; + const run = yield* runStatus({ + services, + config: "invalid", + reachable: true, + outputFormat: "json", + }); + yield* run.effect; + expect(run.out.stdoutText).toBe(""); + expect(run.out.messages.find((message) => message.type === "success")?.data).toMatchObject({ + owner: "reachable", + config_drift: { status: "unavailable" }, + }); + }), +); diff --git a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md index e64671c987..df78baab24 100644 --- a/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/stop/SIDE_EFFECTS.md @@ -1,48 +1,40 @@ # `supabase stack stop` -The command is available only when the `experimental.stack` feature flag is enabled. The -top-level `supabase stop` command uses this handler when the same flag is enabled. - -This command stops the managed stack identified by the current project, an optional `--stack` -name, or `--stack-id`. With `--all`, it discovers every readable managed stack and attempts each -stop while preserving persistent state and data volumes. `--all` cannot be combined with a named -stack or `--stack-id`; the command never destroys stacks. - -Durable stack state lives under `$SUPABASE_HOME/managed/stacks//` -(`~/.supabase/managed/stacks//` by default). A stop that does not finish within 60s -fails with still-running capabilities and reports that the owner stop continues in the background. - -## Files read and written - -The stack package reads and updates its durable state under `` -and the selected stack's lifecycle state. The CLI reads its normal workdir settings. Command -routing may read `supabase/config.toml` or `supabase/config.json` to select the experimental -backend; once the stack handler is selected, it does not load project configuration. Set -`SUPABASE_EXPERIMENTAL_STACK=1` to stop a stack addressed with `--stack-id` when project -configuration is missing or invalid. Implicit and named stacks still depend on workdir discovery, so removing an -ancestor config can change which stack is selected; use an explicit `--workdir` when needed. - -No project files, credentials, or runtime configuration files are written. The package owns -the supervisor teardown and state transition; the CLI does not remove containers, volumes, -or stack state itself. If persisted state says the stack is running but its owner is -unreachable, the package may launch a short-lived Supervisor to arbitrate teardown before -returning. That process is package-owned and is not managed directly by the CLI. - -## Output and telemetry - -Text mode reports the selected stack and stopped outcome. A successful single-stack JSON response is -`{ "found": true, "id": "", "lifecycle": "stopped", "message": "" }`; when no current -stack exists it is `{ "found": false, "message": "No managed stack found for this context." }`. -A successful bulk JSON response is `{ "stopped": ["", ...], "message": "" }`. -Stream-json wraps the same payload in its standard result event. Bulk mode attempts all readable -stacks, warns for unreadable entries, and reports stopped, failed, and skipped counts with per-stack -details. It exits nonzero when an entry is skipped or a stop fails. -The `--all` flag is presence-sensitive for target validation, so `--all=false` still conflicts with -`--stack` and `--stack-id`; `--all=false` alone uses single-stack mode. -Registry-root enumeration failures remain fatal. Exit status is `0` for a successful stop or no -current stack, `1` for a missing named stack or any typed stop failure, and `130` if the command is -interrupted before the stop completes. Standard command instrumentation records command -metadata; stack data and credentials are not emitted as telemetry properties. - -Telemetry state is flushed to `/telemetry.json` -after both successful and failed command runs. +Stops the selected managed stack's entire namespace, including standalone +services and attached jobs. Definitions, data, and assigned ports remain saved. +The experimental top-level `supabase stop` alias delegates to this handler. + +## Selection and output + +Select the current project/branch/name, `--stack `, or `--stack-id `. +`--stack` and `--stack-id` are mutually exclusive. `--all` cannot be combined +with either selector. Explicit legacy `-o/--output` is rejected; use +`--output-format` instead. + +Discovery probes saved owners before shutdown. If no owner is reachable, the +command reports `No owner is reachable; workload state is unavailable.` It +neither starts an owner nor claims workloads have stopped. If an owner disappears +after that preflight, the shutdown error remains a failure. + +Text confirms each successful shutdown and identifies each unavailable owner. +JSON and stream-json success data contain `stopped` and `unavailable` ID arrays. +A missing default selection reports `found: false`; an unknown explicit name or +ID fails. `--all` attempts every selected reachable owner and reports failures +with their IDs. Corrupt registry state fails discovery without partial results. + +## Files and network + +Reads saved state under `/stacks//` and the +current project/Git identity when no explicit ID is supplied. Shared routing and +settings may read project configuration and the selected profile. Communicates +only with local owner endpoints; no hosted API requests. Shutdown updates owned +runtime resources but preserves persistent instance definitions, data and port +claims. No caller-owned upload files are removed. + +## Exit codes and telemetry + +Exit 0 on successful shutdown, absent default selection, or unavailable owner; +1 for invalid flags, selection/registry failure, or shutdown failure; 130 on +interruption. The command wrapper retains standard command telemetry, with no +custom events. Telemetry flushes on success and failure to +`/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts index dc2ef35ce2..c76b9730d1 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.handler.ts @@ -1,76 +1,44 @@ -import { Effect, Match, Option, Result } from "effect"; -import { - type StackDescriptor, - type OpenStackError, - type StackDiscoveryError, - type StackStopError as ApiStackStopError, -} from "@supabase/stack/effect"; +import { Effect, Option, Path, Result } from "effect"; +import type { StackError } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { CommandSettings } from "../../../../config/command-settings.service.ts"; import { TelemetryState } from "../../../../telemetry/telemetry-state.service.ts"; import { StackApi, - StackTargetError, + StackTargetResolver, rejectStackOutput, - validateStackId, validateStackTarget, } from "../stack.shared.ts"; import type { StackStopFlags } from "./stop.command.ts"; import { StackCommandStopError } from "./stop.errors.ts"; -const mapTargetError = (error: StackTargetError) => +const stopError = (cause: { readonly message: string }) => new StackCommandStopError({ - reason: error.reason, - message: error.message, - ...(error.suggestion === undefined ? {} : { suggestion: error.suggestion }), - cause: error, - }); - -const stopError = (error: StackDiscoveryError | OpenStackError | ApiStackStopError) => { - const classification = Match.value(error).pipe( - Match.tag("StackNotFoundError", "InvalidStackIdentityError", () => ({ - reason: "flags" as const, - })), - Match.tag("StackOwnershipConflictError", () => ({ - reason: "unknown" as const, - suggestion: - "Retry the stack stop; if it remains owned, rerun with --debug and inspect cleanup diagnostics.", - })), - Match.tag("StackLifecycleConflictError", "StackUpgradeRequiredError", () => ({ - reason: "lifecycle" as const, - })), - Match.tag( - "StackStateFormatUnsupportedError", - "InvalidProjectRootError", - "StackStateInvalidError", - () => ({ reason: "invalid-config" as const }), - ), - Match.tag("StackRuntimeMismatchError", () => ({ reason: "unknown" as const })), - Match.tag("StackCleanupError", () => ({ - reason: "unknown" as const, - suggestion: "Retry the stack stop with --debug and inspect cleanup diagnostics.", - })), - Match.exhaustive, - ); - return new StackCommandStopError({ - ...classification, - message: error.message, - cause: error, + reason: "invalid-config", + message: cause.message, + cause, }); -}; - -const stoppedPayload = (id: StackDescriptor["id"]) => ({ found: true, id, lifecycle: "stopped" }); export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: StackStopFlags) { - const telemetryState = yield* TelemetryState; - const body = Effect.gen(function* () { + const telemetry = yield* TelemetryState; + return yield* Effect.gen(function* () { const output = yield* Output; const settings = yield* CommandSettings; - const stackApi = yield* StackApi; - const outputFlag = yield* Effect.serviceOption(OutputFlag); - const stopAll = Option.getOrElse(flags.all, () => false); - yield* rejectStackOutput(outputFlag).pipe(Effect.mapError(mapTargetError)); + const path = yield* Path.Path; + const api = yield* StackApi; + const resolver = yield* StackTargetResolver; + yield* rejectStackOutput(yield* Effect.serviceOption(OutputFlag)).pipe( + Effect.mapError( + (cause) => + new StackCommandStopError({ + reason: cause.reason, + message: cause.message, + ...(cause.suggestion === undefined ? {} : { suggestion: cause.suggestion }), + cause, + }), + ), + ); if (Option.isSome(flags.all) && (Option.isSome(flags.stack) || Option.isSome(flags.stackId))) return yield* new StackCommandStopError({ reason: "flags", @@ -79,110 +47,99 @@ export const stackStop = Effect.fn("experimental.stack.stop")(function* (flags: yield* validateStackTarget({ stack: Option.getOrUndefined(flags.stack), stackId: Option.getOrUndefined(flags.stackId), - }).pipe(Effect.mapError(mapTargetError)); - - if (stopAll) { - const discovered = yield* stackApi.discoverStacks().pipe(Effect.mapError(stopError)); - for (const issue of discovered.errors) - yield* output.warn(`Skipping managed stack: ${issue.error.message}`); - const stopping = yield* output.task( - `Stopping ${discovered.stacks.length} managed stack(s)...`, - ); - const results = yield* Effect.forEach( - discovered.stacks, - (descriptor) => - stackApi.openStack(descriptor.id).pipe( - Effect.flatMap((stack) => stack.stop), - Effect.result, - Effect.map((result) => ({ descriptor, result })), - ), - { concurrency: 1 }, - ); - const failed = results.flatMap(({ descriptor, result }) => - Result.isFailure(result) ? [{ descriptor, error: result.failure }] : [], - ); - if (failed.length > 0 || discovered.errors.length > 0) { - const message = `Stopped ${discovered.stacks.length - failed.length} managed stack(s); failed ${failed.length}; skipped ${discovered.errors.length}`; - const detail = [ - ...failed.map( - ({ descriptor, error }) => - `Failed to stop managed stack ${descriptor.id}: ${error.message}`, - ), - ...discovered.errors.map(({ error }) => `Skipped managed stack: ${error.message}`), - ].join("\n"); - yield* stopping.fail(message); - const classifications = [ - ...failed.map(({ error }) => stopError(error)), - ...discovered.errors.map(({ error }) => stopError(error)), - ]; - const firstClassification = classifications[0]; - const reason = - firstClassification !== undefined && - classifications.every( - (classification) => classification.reason === firstClassification.reason, - ) - ? firstClassification.reason - : "unknown"; - const suggestion = - firstClassification?.suggestion !== undefined && - classifications.every( - (classification) => classification.suggestion === firstClassification.suggestion, - ) - ? firstClassification.suggestion - : undefined; - return yield* new StackCommandStopError({ - reason, - message, - detail, - ...(suggestion === undefined ? {} : { suggestion }), - cause: { failures: failed, discovery: discovered.errors }, - }); - } - yield* stopping.clear(); - if (output.format === "text") - yield* output.raw(`Stopped ${discovered.stacks.length} managed stack(s).\n`); - else yield* output.success("", { stopped: discovered.stacks.map(({ id }) => id) }); - return; - } - - const id = Option.isSome(flags.stackId) ? flags.stackId.value : undefined; - const targetOption = - id === undefined - ? yield* stackApi - .findStack({ - projectRoot: settings.workdir, - ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), - }) - .pipe(Effect.mapError(stopError)) - : yield* validateStackId(id).pipe( - Effect.mapError(mapTargetError), - Effect.map((validId) => - Option.some({ - id: validId, - projectRoot: settings.workdir, - }), + }).pipe( + Effect.mapError( + (cause) => + new StackCommandStopError({ + reason: cause.reason, + message: cause.message, + ...(cause.suggestion === undefined ? {} : { suggestion: cause.suggestion }), + cause, + }), + ), + ); + const all = Option.getOrElse(flags.all, () => false); + const target = all + ? undefined + : yield* resolver + .resolve({ + projectRoot: settings.workdir, + ...(Option.isSome(flags.stack) ? { name: flags.stack.value } : {}), + ...(Option.isSome(flags.stackId) ? { id: flags.stackId.value } : {}), + runtime: "auto", + }) + .pipe( + Effect.mapError( + (cause) => + new StackCommandStopError({ + reason: cause.reason, + message: cause.message, + ...(cause.suggestion === undefined ? {} : { suggestion: cause.suggestion }), + cause, + }), ), ); - if (Option.isNone(targetOption)) { + if (!all && target?.id === undefined) { if (Option.isSome(flags.stack)) return yield* new StackCommandStopError({ reason: "flags", message: `No managed stack named "${flags.stack.value}" was found for this project.`, - suggestion: "Choose an existing --stack name or omit --stack for the current project.", }); yield* output.success("No managed stack found for this context.", { found: false }); return; } - const target = targetOption.value; - const stack = yield* stackApi.openStack(target.id).pipe(Effect.mapError(stopError)); - const stopping = yield* output.task(`Stopping stack ${target.id}...`); - yield* stack.stop.pipe( - Effect.tapError((error) => stopping.fail(error.message)), - Effect.tap(() => stopping.clear()), - Effect.mapError(stopError), + const locations = { + stateRoot: path.join(settings.supabaseHome, "stacks"), + cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), + }; + const discovered = yield* api.discover(locations).pipe(Effect.mapError(stopError)); + const selected = all + ? discovered + : discovered.filter(({ definition }) => definition.id === target?.id); + if (!all && selected.length === 0) + return yield* new StackCommandStopError({ + reason: "flags", + message: "The selected stack no longer exists.", + }); + const results = yield* Effect.forEach( + selected, + ({ + definition, + host, + }): Effect.Effect<{ + readonly id: string; + readonly result: Result.Result<"stopped" | "unavailable", StackError>; + }> => + host === undefined + ? Effect.succeed({ id: definition.id, result: Result.succeed("unavailable" as const) }) + : api.open({ ...locations, id: definition.id }).pipe( + Effect.flatMap((stack) => stack.stop), + Effect.as("stopped" as const), + Effect.result, + Effect.map((result) => ({ id: definition.id, result })), + ), ); - if (output.format === "text") yield* output.raw(`Stack ${target.id} stopped.\n`); - else yield* output.success("", stoppedPayload(target.id)); - }); - return yield* body.pipe(Effect.ensuring(telemetryState.flush)); + const failures = results.flatMap(({ id, result }) => + Result.isFailure(result) ? [{ id, error: result.failure }] : [], + ); + if (failures.length > 0) + return yield* new StackCommandStopError({ + reason: "unknown", + message: `Failed to stop ${failures.length} managed stack(s).`, + detail: failures.map(({ id, error }) => `${id}: ${error.message}`).join("\n"), + cause: failures, + }); + const stopped = results.flatMap(({ id, result }) => + Result.isSuccess(result) && result.success === "stopped" ? [id] : [], + ); + const unavailable = results.flatMap(({ id, result }) => + Result.isSuccess(result) && result.success === "unavailable" ? [id] : [], + ); + if (output.format === "text") { + for (const id of stopped) yield* output.raw(`Stack ${id} stopped.\n`); + for (const id of unavailable) + yield* output.raw(`Stack ${id}: No owner is reachable; workload state is unavailable.\n`); + if (results.length === 0) yield* output.raw("No managed stacks found.\n"); + } else yield* output.success("", { stopped, unavailable }); + }).pipe(Effect.ensuring(telemetry.flush)); }); diff --git a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts index 139da02e70..d3a2ce6a7f 100644 --- a/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stop/stop.integration.test.ts @@ -1,688 +1,114 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Effect, Layer, Option, Stream } from "effect"; -import { CliOutput, Command } from "effect/unstable/cli"; -import { - InvalidStackIdentityError, - StackCleanupError, - StackIdSchema, - StackNotFoundError, - StackOwnershipConflictError, - StackStateFormatUnsupportedError, - StackStateInvalidError, - StackUpgradeRequiredError, -} from "@supabase/stack/effect"; -import type { - EffectStack, - OpenStackError, - StackDiscoveryError, - StackDescriptor, - StackStatus, - StackStopError as ApiStackStopError, -} from "@supabase/stack/effect"; -import { mockOutput, mockProcessControl } from "../../../../../tests/helpers/mocks.ts"; +import { Effect, FileSystem, Layer, Option, Path, Ref } from "effect"; +import { StackError } from "@supabase/stack/effect"; +import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; import { mockCommandSettings, mockTelemetryStateTracked, } from "../../../../../tests/helpers/command-mocks.ts"; -import { OutputFlag } from "../../../../command-internal/global-flags.ts"; -import { - actionability, - ErrorActionabilityId, -} from "../../../../shared/telemetry/error-actionability.ts"; -import { StackApi } from "../stack.shared.ts"; -import { textCliOutputFormatter } from "../../../../shared/output/text-formatter.ts"; +import { StackApi, stackApiLayer, stackTargetResolverLayer } from "../stack.shared.ts"; import { stackStop } from "./stop.handler.ts"; -import { StackCommandStopError } from "./stop.errors.ts"; -import { stackStopCommand } from "./stop.command.ts"; -import { withJsonErrorHandling } from "../../../../shared/output/json-error-handling.ts"; - -const status = (id: string): StackStatus => ({ - id: StackIdSchema.make(id), - lifecycle: "running", - desiredLifecycle: "running", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: [], - artifacts: [], -}); -const flags = ( - overrides: Partial[0]> = {}, -): Parameters[0] => ({ +const flags = (id?: string) => ({ all: Option.none(), stack: Option.none(), - stackId: Option.none(), - ...overrides, + stackId: Option.fromUndefinedOr(id), }); - -function setup(opts: { - root: string; - format?: "text" | "json" | "stream-json"; - found?: { id: string; name?: string }; - stop?: Effect.Effect; - openFailure?: OpenStackError; - findFailure?: StackDiscoveryError; - discoveryFailure?: StackDiscoveryError; - discovered?: { - readonly stacks: ReadonlyArray; - readonly errors: ReadonlyArray<{ - readonly id: StackDescriptor["id"]; - readonly error: StackDiscoveryError; - }>; - }; -}) { - const out = mockOutput({ format: opts.format }); +const live = Layer.provideMerge(stackApiLayer, BunServices.layer); +const fixture = Effect.fn("StackStopTest.fixture")(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-stop-" }); + const api = yield* StackApi; + const locations = { stateRoot: path.join(root, "stacks"), cacheRoot: path.join(root, "cache") }; + const stack = yield* api.create({ ...locations, projectRoot: root, runtime: "native" }); + const output = mockOutput(); const telemetry = mockTelemetryStateTracked(); - const state = { - findInputs: [] as Array<{ projectRoot: string; name?: string }>, - openedIds: [] as string[], - stopCalls: 0, - destroyCalled: false, - }; - const id = opts.found?.id ?? "a".repeat(64); - const stack = { - id: StackIdSchema.make(id), - status: Effect.succeed(status(id)), - credentials: Effect.die("unused"), - prepare: () => Effect.die("unused"), - start: () => Effect.die("unused"), - stop: - opts.stop ?? - Effect.sync(() => { - state.stopCalls += 1; - }), - destroy: Effect.sync(() => { - state.destroyCalled = true; - }), - resetDatabase: Effect.die("unused"), - logs: () => Effect.die("unused"), - followLogs: () => Stream.empty, - } satisfies EffectStack; - const descriptor = opts.found - ? { - id: stack.id, - projectRoot: opts.root, - name: opts.found.name ?? "feature-a", - branchContext: "ordinary-workspace", - runtime: { kind: "native" as const }, - desiredLifecycle: "running" as const, - } - : undefined; + const settings = mockCommandSettings({ workdir: root, supabaseHome: root }); const layer = Layer.mergeAll( - out.layer, - mockProcessControl().layer, + output.layer, telemetry.layer, - mockCommandSettings({ workdir: opts.root }), - Layer.succeed(StackApi, { - createStack: () => Effect.die("must not create"), - findStack: (input) => - Effect.sync(() => { - state.findInputs.push(input); - return descriptor === undefined ? Option.none() : Option.some(descriptor); - }).pipe( - Effect.flatMap((value) => - opts.findFailure === undefined ? Effect.succeed(value) : Effect.fail(opts.findFailure), - ), - ), - openStack: (stackId) => { - if (opts.openFailure !== undefined) return Effect.fail(opts.openFailure); - return Effect.sync(() => { - state.openedIds.push(stackId); - return stack; - }); - }, - inspectStack: () => Effect.die("must not inspect"), - discoverStacks: () => - opts.discoveryFailure === undefined - ? Effect.succeed({ - stacks: opts.discovered?.stacks ?? [], - errors: opts.discovered?.errors ?? [], - }) - : Effect.fail(opts.discoveryFailure), - }), - BunServices.layer, + settings, + stackTargetResolverLayer.pipe(Layer.provide(settings)), ); - return { layer, out, state, telemetry }; -} + return { root, api, locations, stack, output, telemetry, layer }; +}); describe("stack stop", () => { - it.effect("stops every discovered stack when --all is selected", () => { - const root = "/tmp/supabase-stack-stop-all"; - const id = "b".repeat(64); - const setupResult = setup({ - root, - found: { id, name: "feature-a" }, - discovered: { - stacks: [ - { - id: StackIdSchema.make(id), - projectRoot: root, - name: "feature-a", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - { - id: StackIdSchema.make("c".repeat(64)), - projectRoot: root, - name: "feature-b", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - ], - errors: [], - }, - }); - return Effect.gen(function* () { - yield* stackStop(flags({ all: Option.some(true) })); - expect(setupResult.state.openedIds).toEqual([id, "c".repeat(64)]); - expect(setupResult.state.stopCalls).toBe(2); - expect(setupResult.state.destroyCalled).toBe(false); - expect(setupResult.out.stdoutText).toContain("Stopped 2"); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("continues attempting every stack after a stop failure", () => { - const root = "/tmp/supabase-stack-stop-all-failure"; - const first = "d".repeat(64); - const second = "e".repeat(64); - const setupResult = setup({ - root, - found: { id: first }, - stop: Effect.fail(new StackCleanupError({ message: "stop failed" })), - discovered: { - stacks: [ - { - id: StackIdSchema.make(first), - projectRoot: root, - name: "first", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - { - id: StackIdSchema.make(second), - projectRoot: root, - name: "second", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - ], - errors: [], - }, - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ all: Option.some(true) })).pipe(Effect.flip); - expect(failure.message).toContain("failed 2"); - expect(setupResult.state.openedIds).toEqual([first, second]); - expect(setupResult.state.destroyCalled).toBe(false); - expect(failure.detail).toBe( - `Failed to stop managed stack ${first}: stop failed\nFailed to stop managed stack ${second}: stop failed`, + it.live("stops an owner while preserving standalone definitions", () => + Effect.gen(function* () { + const f = yield* fixture(); + const stopped = yield* Ref.make(false); + yield* Effect.acquireUseRelease( + f.stack.services.create({ service: "mail", config: {}, endpoints: {} }), + (mail) => + Effect.gen(function* () { + expect((yield* f.api.discover(f.locations))[0]?.host).toBeDefined(); + yield* stackStop(flags(f.stack.id)).pipe(Effect.provide(f.layer)); + yield* Ref.set(stopped, true); + expect((yield* f.stack.services.list).map(({ id }) => id)).toEqual([mail.id]); + expect(f.output.stdoutText).toContain(`Stack ${f.stack.id} stopped.`); + expect(f.telemetry.flushed).toBe(true); + }), + () => Ref.get(stopped).pipe(Effect.flatMap((done) => (done ? Effect.void : f.stack.stop))), ); - expect(failure.reason).toBe("unknown"); - expect(failure.suggestion).toContain("--debug"); - }).pipe(Effect.provide(setupResult.layer)); - }); + }).pipe(Effect.provide(live)), + ); - it.effect("reports corrupt discovery entries while stopping healthy stacks", () => { - const root = "/tmp/supabase-stack-stop-all-corrupt"; - const healthy = "f".repeat(64); - const corrupt = "1".repeat(64); - const setupResult = setup({ - root, - found: { id: healthy }, - discovered: { - stacks: [ - { - id: StackIdSchema.make(healthy), - projectRoot: root, - name: "healthy", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - ], - errors: [ - { - id: StackIdSchema.make(corrupt), - error: new StackStateInvalidError({ - message: `Failed to read managed stack ${corrupt}: corrupt state`, - }), - }, - ], - }, - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ all: Option.some(true) })).pipe(Effect.flip); - expect(failure.message).toBe("Stopped 1 managed stack(s); failed 0; skipped 1"); - expect(failure.detail).toBe( - `Skipped managed stack: Failed to read managed stack ${corrupt}: corrupt state`, - ); - expect(setupResult.state.stopCalls).toBe(1); - expect(setupResult.state.destroyCalled).toBe(false); - expect(setupResult.out.messages).toEqual( - expect.arrayContaining([expect.objectContaining({ type: "warn" })]), + it.live("stop all preserves an offline registry without claiming stopped workloads", () => + Effect.gen(function* () { + const f = yield* fixture(); + yield* f.api.create({ + ...f.locations, + projectRoot: f.root, + name: "second", + runtime: "native", + }); + yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); + yield* stackStop({ ...flags(), all: Option.some(true) }).pipe(Effect.provide(f.layer)); + expect(f.output.stdoutText.match(/workload state is unavailable/g)).toHaveLength(4); + expect(f.output.stdoutText).not.toContain("stopped."); + expect((yield* f.api.discover(f.locations)).every(({ host }) => host === undefined)).toBe( + true, ); - }).pipe(Effect.provide(setupResult.layer)); - }); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); - it.effect("preserves bulk failure details through JSON error handling", () => { - const failed = "2".repeat(64); - const skipped = "3".repeat(64); - const setupResult = setup({ - root: "/tmp/supabase-stack-stop-json", - format: "json", - found: { id: failed }, - stop: Effect.fail(new StackCleanupError({ message: "cleanup failed" })), - discovered: { - stacks: [ - { - id: StackIdSchema.make(failed), - projectRoot: "/tmp/supabase-stack-stop-json", - name: "failed", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - ], - errors: [ - { - id: StackIdSchema.make(skipped), - error: new StackStateInvalidError({ - message: `Failed to read managed stack ${skipped}: invalid state`, + it.live("surfaces a shutdown failure after a successful owner preflight", () => + Effect.gen(function* () { + const f = yield* fixture(); + const saved = (yield* f.api.discover(f.locations))[0]; + if (saved === undefined) return yield* Effect.die("created stack missing"); + const failedApi = Layer.succeed( + StackApi, + StackApi.of({ + ...f.api, + discover: () => + Effect.succeed([ + { + ...saved, + host: { stackId: f.stack.id, identity: saved.definition.identity, pid: 1, port: 1 }, + }, + ]), + open: () => + Effect.succeed({ + ...f.stack, + stop: Effect.fail( + new StackError({ operation: "shutdown", message: "owner disconnected" }), + ), }), - }, - ], - }, - }); - return Effect.gen(function* () { - yield* stackStop(flags({ all: Option.some(true) })).pipe(withJsonErrorHandling); - expect(setupResult.out.failures).toEqual([ - expect.objectContaining({ - message: "Stopped 0 managed stack(s); failed 1; skipped 1", - detail: `Failed to stop managed stack ${failed}: cleanup failed\nSkipped managed stack: Failed to read managed stack ${skipped}: invalid state`, }), - ]); - expect(setupResult.state.openedIds).toEqual([failed]); - expect(setupResult.state.destroyCalled).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("fails before opening any stack when registry discovery fails", () => { - const setupResult = setup({ - root: "/tmp/supabase-stack-stop-all-discovery-failure", - discoveryFailure: new StackStateInvalidError({ message: "registry is unreadable" }), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ all: Option.some(true) })).pipe(Effect.flip); - expect(failure.message).toContain("registry is unreadable"); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.destroyCalled).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("treats an empty registry as a successful bulk no-op", () => { - const setupResult = setup({ root: "/tmp/supabase-stack-stop-all-empty" }); - return Effect.gen(function* () { - yield* stackStop(flags({ all: Option.some(true) })); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.stopCalls).toBe(0); - expect(setupResult.state.destroyCalled).toBe(false); - expect(setupResult.out.stdoutText).toContain("Stopped 0 managed stack(s)."); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("rejects bulk stop target combinations", () => { - const setupResult = setup({ root: "/tmp/supabase-stack-stop-conflict" }); - return Effect.gen(function* () { - const failure = yield* stackStop( - flags({ all: Option.some(true), stack: Option.some("feature-a") }), - ).pipe(Effect.flip); - expect(failure.message).toContain("cannot be combined"); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("rejects an explicit false --all with a stack target", () => { - const setupResult = setup({ root: "/tmp/supabase-stack-stop-explicit-false" }); - return Effect.gen(function* () { - const failure = yield* stackStop( - flags({ all: Option.some(false), stack: Option.some("feature-a") }), - ).pipe(Effect.flip); - expect(failure.message).toContain("cannot be combined"); - expect(setupResult.state.findInputs).toEqual([]); - expect(setupResult.state.openedIds).toEqual([]); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("uses unknown classification when bulk failures disagree", () => { - const failed = "6".repeat(64); - const skipped = "5".repeat(64); - const setupResult = setup({ - root: "/tmp/supabase-stack-stop-mixed", - found: { id: failed }, - stop: Effect.fail(new StackCleanupError({ message: "cleanup failed" })), - discovered: { - stacks: [ - { - id: StackIdSchema.make(failed), - projectRoot: "/tmp/supabase-stack-stop-mixed", - name: "failed", - branchContext: "ordinary-workspace", - runtime: { kind: "native" }, - desiredLifecycle: "running", - }, - ], - errors: [ - { - id: StackIdSchema.make(skipped), - error: new StackStateInvalidError({ - message: `Failed to read managed stack ${skipped}: corrupt state`, - }), - }, - ], - }, - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ all: Option.some(true) })).pipe(Effect.flip); - expect(failure.reason).toBe("unknown"); - expect(failure.suggestion).toBeUndefined(); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("stops a named stack without calling destroy", () => { - const root = "/tmp/supabase-stack-stop"; - const setupResult = setup({ - root, - found: { id: "a".repeat(64), name: "feature-a" }, - }); - return Effect.gen(function* () { - yield* stackStop(flags({ stack: Option.some("feature-a") })); - expect(setupResult.state.findInputs).toEqual([{ projectRoot: root, name: "feature-a" }]); - expect(setupResult.state.openedIds).toEqual(["a".repeat(64)]); - expect(setupResult.state.stopCalls).toBe(1); - expect(setupResult.state.destroyCalled).toBe(false); - expect(setupResult.out.stdoutText).toContain("stopped"); - expect(setupResult.telemetry.flushed).toBe(true); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("opens an explicit id without discovering a stack", () => { - const root = "/tmp/supabase-stack-stop-id"; - const id = "c".repeat(64); - const setupResult = setup({ root, found: { id } }); - return Effect.gen(function* () { - yield* stackStop(flags({ stackId: Option.some(id) })); - expect(setupResult.state.findInputs).toEqual([]); - expect(setupResult.state.openedIds).toEqual([id]); - expect(setupResult.state.stopCalls).toBe(1); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("invokes stop twice without calling destroy", () => { - const root = "/tmp/supabase-stack-stop-repeat"; - const setupResult = setup({ root, found: { id: "d".repeat(64) } }); - return Effect.gen(function* () { - yield* stackStop(flags()); - yield* stackStop(flags()); - expect(setupResult.state.stopCalls).toBe(2); - expect(setupResult.state.destroyCalled).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("classifies an addressed missing stack as actionable flags", () => { - const root = "/tmp/supabase-stack-stop-open-missing"; - const setupResult = setup({ - root, - found: { id: "e".repeat(64) }, - openFailure: new StackNotFoundError({ message: "Stack state was not found" }), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ stackId: Option.some("e".repeat(64)) })).pipe( - Effect.flip, ); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(setupResult.state.stopCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("emits a self-describing JSON stopped result", () => { - const root = "/tmp/supabase-stack-stop-json"; - const setupResult = setup({ root, found: { id: "f".repeat(64) } }); - const output = mockOutput({ format: "json" }); - return Effect.gen(function* () { - yield* stackStop(flags()); - expect(output.messages.find((message) => message.type === "success")?.data).toEqual({ - found: true, - id: "f".repeat(64), - lifecycle: "stopped", - }); - }).pipe(Effect.provide(Layer.mergeAll(setupResult.layer, output.layer))); - }); - - it.effect("reports a missing named stack without opening or stopping anything", () => { - const root = "/tmp/supabase-stack-stop-named-missing"; - const setupResult = setup({ root }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ stack: Option.some("missing") })).pipe(Effect.flip); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(setupResult.state.findInputs).toEqual([{ projectRoot: root, name: "missing" }]); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.stopCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("classifies invalid stack names as actionable flags", () => { - const root = "/tmp/supabase-stack-stop-invalid-name"; - const setupResult = setup({ - root, - findFailure: new InvalidStackIdentityError({ message: "The stack name must not be blank" }), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ stack: Option.some("") })).pipe(Effect.flip); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.stopCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("rejects malformed ids before opening or stopping anything", () => { - const root = "/tmp/supabase-stack-stop-malformed"; - const setupResult = setup({ root }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags({ stackId: Option.some("invalid") })).pipe( + const error = yield* stackStop(flags(f.stack.id)).pipe( + Effect.provide(Layer.provideMerge(f.layer, failedApi)), Effect.flip, ); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(setupResult.state.findInputs).toEqual([]); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.stopCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("reports when no current stack exists", () => { - const root = "/tmp/supabase-stack-stop-missing"; - const setupResult = setup({ root }); - return Effect.gen(function* () { - yield* stackStop(flags()); - expect( - setupResult.out.messages.some((message) => - message.message.includes("No managed stack found"), - ), - ).toBe(true); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.stopCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("rejects mutually exclusive targets before discovering a stack", () => { - const setupResult = setup({ root: "/tmp/supabase-stack-stop-mutex" }); - return Effect.gen(function* () { - const targetFailure = yield* stackStop( - flags({ stack: Option.some("feature-a"), stackId: Option.some("a".repeat(64)) }), - ).pipe(Effect.flip); - expect(targetFailure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - expect(targetFailure.message).toContain("cannot be used together"); - expect(setupResult.state.findInputs).toEqual([]); - expect(setupResult.state.openedIds).toEqual([]); - expect(setupResult.state.stopCalls).toBe(0); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("does not report success when package stop fails", () => { - const root = "/tmp/supabase-stack-stop-failure"; - const setupResult = setup({ - root, - found: { id: "b".repeat(64) }, - stop: Effect.fail(new StackStateInvalidError({ message: "stop failed" })), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags()).pipe(Effect.flip); - expect(failure).toBeInstanceOf(StackCommandStopError); - expect(setupResult.out.messages.some((message) => message.type === "success")).toBe(false); - expect(setupResult.telemetry.flushed).toBe(true); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("classifies an ownership conflict as unknown with retry guidance", () => { - const root = "/tmp/supabase-stack-stop-conflict"; - const ownershipConflict = new StackOwnershipConflictError({ message: "stack is owned" }); - const setupResult = setup({ - root, - found: { id: "7".repeat(64) }, - stop: Effect.fail(ownershipConflict), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags()).pipe(Effect.flip); - expect(failure.reason).toBe("unknown"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.unknown); - expect(failure.suggestion).toBe( - "Retry the stack stop; if it remains owned, rerun with --debug and inspect cleanup diagnostics.", - ); - expect(failure.message).toBe(ownershipConflict.message); - expect(failure.cause).toBe(ownershipConflict); - expect(setupResult.state.destroyCalled).toBe(false); - expect(setupResult.out.messages.some((message) => message.type === "success")).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("classifies persisted state format failures as invalid config", () => { - const root = "/tmp/supabase-stack-stop-format"; - const setupResult = setup({ - root, - found: { id: "8".repeat(64) }, - openFailure: new StackStateFormatUnsupportedError({ - format: "future", - message: "Unsupported stack state format", - }), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags()).pipe(Effect.flip); - expect(failure.reason).toBe("invalid-config"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.invalidConfig); - expect(setupResult.state.stopCalls).toBe(0); - expect(setupResult.out.messages.some((message) => message.type === "success")).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("classifies stack upgrade requirements as lifecycle failures", () => { - const root = "/tmp/supabase-stack-stop-upgrade"; - const setupResult = setup({ - root, - found: { id: "9".repeat(64) }, - openFailure: new StackUpgradeRequiredError({ - expectedRelease: "next", - actualRelease: "current", - message: "Stack upgrade required", - }), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags()).pipe(Effect.flip); - expect(failure.reason).toBe("lifecycle"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.invalidConfig); - expect(setupResult.state.stopCalls).toBe(0); - expect(setupResult.out.messages.some((message) => message.type === "success")).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("classifies cleanup failures as unknown actionability with debug guidance", () => { - const root = "/tmp/supabase-stack-stop-cleanup"; - const setupResult = setup({ - root, - found: { id: "a".repeat(64) }, - stop: Effect.fail(new StackCleanupError({ message: "cleanup failed" })), - }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags()).pipe(Effect.flip); - expect(failure.reason).toBe("unknown"); - expect(failure[ErrorActionabilityId]).toEqual(actionability.unknown); - expect(failure.suggestion).toContain("--debug"); - expect(setupResult.state.openedIds).toEqual(["a".repeat(64)]); - expect(setupResult.state.destroyCalled).toBe(false); - expect(setupResult.out.messages.some((message) => message.type === "success")).toBe(false); - }).pipe(Effect.provide(setupResult.layer)); - }); - - it.effect("rejects the output flag with actionable guidance", () => { - const root = "/tmp/supabase-stack-stop-output"; - const setupResult = setup({ root }); - return Effect.gen(function* () { - const failure = yield* stackStop(flags()).pipe(Effect.flip); - expect(failure).toBeInstanceOf(StackCommandStopError); - expect(failure[ErrorActionabilityId]).toEqual(actionability.provideFlags); - }).pipe( - Effect.provide( - Layer.mergeAll(setupResult.layer, Layer.succeed(OutputFlag, Option.some("json"))), - ), - ); - }); -}); - -describe("stack stop parser", () => { - it.live("passes a stack name to the handler", () => { - let parsed: Option.Option | undefined; - const command = stackStopCommand.pipe( - Command.withHandler((flags) => Effect.sync(() => (parsed = flags.stack))), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })(["--stack", "feature-a"]); - expect(parsed).toEqual(Option.some("feature-a")); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); - - it.live("preserves explicit false for --all alongside a target", () => { - let parsed: Parameters[0] | undefined; - const command = stackStopCommand.pipe( - Command.withHandler((flags) => Effect.sync(() => (parsed = flags))), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })([ - "--all=false", - "--stack", - "feature-a", - ]); - expect(parsed?.all).toEqual(Option.some(false)); - expect(parsed?.stack).toEqual(Option.some("feature-a")); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); - - it.live("parses bare --all as true", () => { - let parsed: Parameters[0] | undefined; - const command = stackStopCommand.pipe( - Command.withHandler((flags) => Effect.sync(() => (parsed = flags))), - ); - return Effect.gen(function* () { - yield* Command.runWith(command, { version: "0.0.0-test" })(["--all"]); - expect(parsed?.all).toEqual(Option.some(true)); - }).pipe( - Effect.provide(Layer.mergeAll(BunServices.layer, CliOutput.layer(textCliOutputFormatter()))), - ); - }); + expect(error.reason).toBe("unknown"); + expect(error.detail).toContain(`${f.stack.id}: owner disconnected`); + expect(f.output.stdoutText).toBe(""); + expect(f.telemetry.flushed).toBe(true); + }).pipe(Effect.provide(live)), + ); }); diff --git a/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md b/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md index 0015111a41..11db2544ed 100644 --- a/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md @@ -100,10 +100,25 @@ Long-running raw log / error output only; there is no final success payload obje ### `--output-format stream-json` -Long-running raw log / error events only; there is no terminal `result` event on success. +The legacy backend emits long-running raw log / error events with no terminal `result` event. +Stack mode emits a readiness `result` containing the instance ID and URL, then live log events. ## Notes +When `SUPABASE_EXPERIMENTAL_STACK=1` selects the Stack backend, the command opens the saved +project stack and serves its Functions service. A composed Functions service is started or woken +and readied in place; the stack composition and its other services remain available on shutdown. +If Functions is excluded from the composition, a temporary standalone Functions instance is +created under the same stack owner and destroyed with the command scope. `--env-file` and +`--no-verify-jwt` apply only for the command lifetime and restore the saved composed configuration +when the owner remains reachable. Import-map and inspector flags are rejected before any stack +mutation. Stack mode writes no project files and does not stop or restart the whole composition. +It reads the shared Functions `.env` when creating a temporary service, or the explicit `--env-file`; +per-function `.env` files are not read, and reserved `SUPABASE_*` entries are ignored. +For Stack mode, TOML `edge_runtime.secrets` overrides the shared `.env`; an explicit +`--env-file` overrides matching shared secrets while retaining other saved values. +Per-function TOML `env` remains the final override for that function. + - Any legacy Function name positional arguments are accepted and ignored. The command always serves every discovered Function, preserving invocations such as `supabase functions serve `. diff --git a/apps/cli/src/commands/functions/serve/serve.command.ts b/apps/cli/src/commands/functions/serve/serve.command.ts index 2591620fe8..e7fd48e098 100644 --- a/apps/cli/src/commands/functions/serve/serve.command.ts +++ b/apps/cli/src/commands/functions/serve/serve.command.ts @@ -10,6 +10,7 @@ import { debugLoggerLayer } from "../../../command-internal/debug-logger.layer.t import { withCommandTelemetry } from "../../../telemetry/command-telemetry.ts"; import { telemetryStateLayer } from "../../../telemetry/telemetry-state.layer.ts"; import { functionsServe } from "./serve.handler.ts"; +import { stackApiLayer } from "../../../command-internal/stack-api.ts"; const cliSettings = commandSettingsLayer.pipe(Layer.provide(debugLoggerLayer)); const functionsServeRuntimeLayer = Layer.mergeAll( @@ -19,6 +20,7 @@ const functionsServeRuntimeLayer = Layer.mergeAll( debugLoggerLayer, telemetryStateLayer, commandRuntimeLayer(["functions", "serve"]), + stackApiLayer, ); const config = { diff --git a/apps/cli/src/commands/functions/serve/serve.errors.ts b/apps/cli/src/commands/functions/serve/serve.errors.ts new file mode 100644 index 0000000000..1c6ef725f1 --- /dev/null +++ b/apps/cli/src/commands/functions/serve/serve.errors.ts @@ -0,0 +1,26 @@ +import { Data } from "effect"; +import { + actionability, + type CliErrorActionabilityDeclaration, + ErrorActionabilityId, +} from "../../../shared/telemetry/error-actionability.ts"; + +export class FunctionsServeStackError extends Data.TaggedError("FunctionsServeStackError")<{ + readonly reason: "flags" | "invalid-config" | "lifecycle" | "runtime"; + readonly message: string; + readonly suggestion?: string; + readonly cause?: unknown; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + switch (this.reason) { + case "flags": + return actionability.provideFlags; + case "invalid-config": + return actionability.invalidConfig; + case "lifecycle": + return actionability.startStack; + case "runtime": + return actionability.unknown; + } + } +} diff --git a/apps/cli/src/commands/functions/serve/serve.handler.ts b/apps/cli/src/commands/functions/serve/serve.handler.ts index 70f810d0f2..85b5f51b18 100644 --- a/apps/cli/src/commands/functions/serve/serve.handler.ts +++ b/apps/cli/src/commands/functions/serve/serve.handler.ts @@ -6,8 +6,12 @@ import { DebugFlag, NetworkIdFlag } from "../../../command-internal/global-flags import { RuntimeInfo } from "../../../shared/runtime/runtime-info.service.ts"; import { TelemetryState } from "../../../telemetry/telemetry-state.service.ts"; import { serveFunctions, type FunctionsServeFlags } from "../../../shared/functions/serve.ts"; +import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; +import { functionsServeStack } from "./serve.stack.handler.ts"; export const functionsServe = Effect.fn("functions.serve")(function* (flags: FunctionsServeFlags) { + const backend = yield* currentStackBackend; + if (backend.kind === "stack") return yield* functionsServeStack(flags); const cliSettings = yield* CommandSettings; const runtimeInfo = yield* RuntimeInfo; const telemetryState = yield* TelemetryState; diff --git a/apps/cli/src/commands/functions/serve/serve.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.integration.test.ts index c36dbf99ae..d389d53520 100644 --- a/apps/cli/src/commands/functions/serve/serve.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.integration.test.ts @@ -37,6 +37,7 @@ import { mockRuntimeInfo, } from "../../../../tests/helpers/mocks.ts"; import { CommandSettings } from "../../../config/command-settings.service.ts"; +import { StackApi } from "../../../command-internal/stack-api.ts"; import { functionsGoConfigCompat } from "../../../command-internal/functions-go-config.ts"; import { DebugFlag, NetworkIdFlag } from "../../../command-internal/global-flags.ts"; import { FileWatcher, type FileWatchEvent } from "../../../shared/runtime/file-watcher.service.ts"; @@ -431,6 +432,7 @@ function setupServe(options: SetupOptions = {}) { childSpawner.layer, Layer.succeed(DebugFlag, options.debug ?? false), Layer.succeed(NetworkIdFlag, options.networkId ?? Option.none()), + Layer.mock(StackApi, {}), ); return { layer, out, telemetry, processControl, fileWatcher, childSpawner }; diff --git a/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts new file mode 100644 index 0000000000..f8aa23148e --- /dev/null +++ b/apps/cli/src/commands/functions/serve/serve.stack.e2e.test.ts @@ -0,0 +1,281 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { create as createStack } from "@supabase/stack/effect"; +import { Data, Effect, FileSystem, Layer, Path, Redacted, Schema } from "effect"; +import { FetchHttpClient, HttpClient } from "effect/unstable/http"; +import { homedir, tmpdir } from "node:os"; + +import { spawnSupabase } from "../../../../tests/helpers/cli.ts"; +import { bundleStackFunctionsServeMainTemplate } from "../../../command-internal/stack-functions-bundler.ts"; +import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; + +const jwtSecret = "functions-serve-stack-e2e-secret-at-least-32-characters"; +const nativeSupported = + (process.platform === "linux" && (process.arch === "x64" || process.arch === "arm64")) || + (process.platform === "darwin" && process.arch === "arm64"); + +class FunctionsServeE2eError extends Data.TaggedError("FunctionsServeE2eError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const Payload = Schema.Struct({ + value: Schema.NullOr(Schema.String), + anon: Schema.Boolean, + service: Schema.Boolean, + databaseUrl: Schema.NullOr(Schema.String), +}); + +const fixture = Effect.fn("FunctionsServeE2e.fixture")(function* ( + runtime: "native" | "docker", + included: boolean, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // Linux Edge Runtime overlays /tmp with a private worker filesystem, so native functions need a visible host path. + const root = yield* fs.makeTempDirectoryScoped({ + ...(runtime === "native" && process.platform === "linux" ? { directory: homedir() } : {}), + prefix: `functions-serve-${runtime}-`, + }); + const home = yield* fs.makeTempDirectoryScoped({ prefix: "functions-serve-home-" }); + const functionsRoot = path.join(root, "supabase", "functions"); + yield* fs.makeDirectory(path.join(functionsRoot, "hello"), { recursive: true }); + yield* fs.writeFileString( + path.join(root, "supabase", "config.toml"), + 'project_id = "functions-serve-e2e"\n[experimental]\nstack = true\n[db]\nmajor_version = 17\n[edge_runtime]\nenabled = true\n', + ); + yield* fs.writeFileString( + path.join(functionsRoot, "hello", "index.ts"), + `Deno.serve(() => Response.json({ + value: Deno.env.get("CUSTOM_VALUE") ?? null, + anon: Boolean(Deno.env.get("SUPABASE_ANON_KEY")), + service: Boolean(Deno.env.get("SUPABASE_SERVICE_ROLE_KEY")), + databaseUrl: Deno.env.get("SUPABASE_DB_URL") ?? null, + }));`, + ); + yield* fs.writeFileString(path.join(root, "override.env"), "CUSTOM_VALUE=overridden\n"); + yield* fs.makeDirectory(path.join(home, "cache"), { recursive: true }); + const artifacts = path.join(tmpdir(), "supabase-stack-artifacts"); + yield* fs.makeDirectory(artifacts, { recursive: true }); + yield* fs.symlink(artifacts, path.join(home, "cache", "stack")); + const stack = yield* createStack({ + projectRoot: root, + stateRoot: path.join(home, "stacks"), + cacheRoot: path.join(home, "cache", "stack"), + runtime, + }); + yield* Effect.addFinalizer(() => + stack.destroy.pipe( + Effect.catch((cause) => + Effect.die(new FunctionsServeE2eError({ message: "stack cleanup failed", cause })), + ), + ), + ); + const bootstrap = yield* bundleStackFunctionsServeMainTemplate(); + yield* stack.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(jwtSecret), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { + service: "rest", + config: { databaseUrl: "postgresql://placeholder", jwtSecret }, + endpoints: { http: { port: "auto" } }, + }, + ...(included + ? [ + { + service: "functions" as const, + config: { + functionsRoot, + bootstrap, + jwtSecret, + verifyJwt: true, + env: { CUSTOM_VALUE: "original" }, + }, + endpoints: { http: { port: "auto" as const } }, + }, + ] + : []), + ]); + yield* stack.composition.start; + const services = yield* stack.services.list; + const rest = services.find((instance) => instance.service === "rest"); + const database = services.find((instance) => instance.service === "database"); + if (rest === undefined || database === undefined) + return yield* Effect.die("fixture services missing"); + const apiUrl = (yield* rest.credentials()).apiUrl; + const databaseUrl = (yield* database.credentials({ from: "runtime" })).databaseUrl; + if (apiUrl === undefined || databaseUrl === undefined) + return yield* Effect.die("fixture URLs missing"); + if (!included) { + const httpPort = (yield* rest.status).endpoints.find(({ name }) => name === "http")?.port; + if (httpPort === undefined) return yield* Effect.die("API port missing"); + yield* stack.services.create({ + service: "functions", + config: { + functionsRoot, + bootstrap, + jwtSecret, + verifyJwt: true, + env: { CUSTOM_VALUE: "excluded" }, + }, + endpoints: { http: { port: httpPort } }, + }); + } + return { root, home, stack, services: yield* stack.services.list, apiUrl, databaseUrl }; +}); + +const serve = Effect.fn("FunctionsServeE2e.serve")(function* ( + root: string, + home: string, + flags: ReadonlyArray = [], +) { + const child = yield* Effect.acquireRelease( + Effect.try({ + try: () => + spawnSupabase(["functions", "serve", ...flags], { + cwd: root, + home, + env: { SUPABASE_EXPERIMENTAL_STACK: "1" }, + exitTimeoutMs: 120_000, + }), + catch: (cause) => new FunctionsServeE2eError({ message: "CLI spawn failed", cause }), + }), + (process) => Effect.sync(() => process.releaseOwned({ successOptOut: false })), + ); + yield* Effect.tryPromise({ + try: () => child.waitForOutput(/Serving Functions at /, 120_000), + catch: (cause) => new FunctionsServeE2eError({ message: "serve did not become ready", cause }), + }); + return child; +}); + +const interrupt = Effect.fn("FunctionsServeE2e.interrupt")(function* ( + child: ReturnType, +) { + yield* Effect.sync(() => child.kill("SIGINT")); + const exit = yield* child.exitEffect(120_000); + expect(exit.exitCode, `stdout:\n${child.stdout()}\nstderr:\n${child.stderr()}`).toBe(0); +}); + +const invoke = Effect.fn("FunctionsServeE2e.invoke")(function* ( + apiUrl: string, + authenticated: boolean, +) { + const http = yield* HttpClient.HttpClient; + return yield* http.get(`${apiUrl}/functions/v1/hello`, { + headers: authenticated ? { Authorization: `Bearer ${generateGoJwt(jwtSecret, "anon")}` } : {}, + }); +}); + +const payload = Effect.fn("FunctionsServeE2e.payload")(function* ( + apiUrl: string, + authenticated: boolean, +) { + const response = yield* invoke(apiUrl, authenticated); + expect(response.status).toBe(200); + return yield* Schema.decodeUnknownEffect(Payload)(yield* response.json); +}); + +const layer = Layer.mergeAll(BunServices.layer, FetchHttpClient.layer); + +describe("functions serve (stack e2e)", () => { + for (const runtime of ["native", "docker"] as const) { + if (runtime === "native" && !nativeSupported) continue; + it.live( + `attaches and restores temporary overrides on the ${runtime} Functions member`, + () => + Effect.gen(function* () { + const { root, home, stack, services, apiUrl, databaseUrl } = yield* fixture( + runtime, + true, + ); + const functions = services.find((instance) => instance.service === "functions"); + if (functions?.service !== "functions") return yield* Effect.die("Functions missing"); + const saved = yield* functions.status; + expect(saved.lifecycle).toBe("stopped"); + expect(saved.wakeEnabled).toBe(true); + const composition = yield* stack.composition.describe; + const plain = yield* serve(root, home); + const active = yield* functions.status; + expect(yield* payload(apiUrl, true)).toEqual({ + value: "original", + anon: true, + service: true, + databaseUrl, + }); + const fs = yield* FileSystem.FileSystem; + const sourcePath = `${root}/supabase/functions/hello/index.ts`; + const source = yield* fs.readFileString(sourcePath); + yield* fs.writeFileString( + sourcePath, + source.replace('"CUSTOM_VALUE"', '"MISSING_VALUE"'), + ); + expect((yield* payload(apiUrl, true)).value).toBeNull(); + yield* fs.writeFileString(sourcePath, source); + yield* interrupt(plain); + expect((yield* functions.status).launchId).toBe(active.launchId); + expect((yield* invoke(apiUrl, false)).status).toBe(401); + + const overridden = yield* serve(root, home, [ + "--no-verify-jwt", + "--env-file", + "override.env", + ]); + expect(yield* payload(apiUrl, false)).toEqual({ + value: "overridden", + anon: true, + service: true, + databaseUrl, + }); + yield* interrupt(overridden); + expect((yield* functions.status).config).toEqual(saved.config); + expect((yield* invoke(apiUrl, false)).status).toBe(401); + expect((yield* payload(apiUrl, true)).value).toBe("original"); + expect(yield* stack.composition.describe).toEqual(composition); + }).pipe(Effect.provide(layer)), + { timeout: 360_000 }, + ); + + it.live( + `removes temporary ${runtime} Functions and keeps it excluded on Ctrl-C`, + () => + Effect.gen(function* () { + const { root, home, stack, services, apiUrl, databaseUrl } = yield* fixture( + runtime, + false, + ); + const composition = yield* stack.composition.describe; + const database = services.find((instance) => instance.service === "database"); + if (database === undefined) return yield* Effect.die("Database missing"); + const databaseLaunch = (yield* database.status).launchId; + const child = yield* serve(root, home); + expect(yield* payload(apiUrl, true)).toEqual({ + value: null, + anon: true, + service: true, + databaseUrl, + }); + expect(yield* stack.composition.describe).toEqual(composition); + expect(yield* stack.services.list).toHaveLength(services.length + 1); + yield* interrupt(child); + expect((yield* stack.services.list).map(({ id }) => id).sort()).toEqual( + services.map(({ id }) => id).sort(), + ); + expect(yield* stack.composition.describe).toEqual(composition); + expect((yield* database.status).launchId).toBe(databaseLaunch); + expect((yield* invoke(apiUrl, false)).status).toBe(404); + const http = yield* HttpClient.HttpClient; + expect((yield* http.get(`${apiUrl}/rest/v1/`)).status).toBe(200); + }).pipe(Effect.provide(layer)), + { timeout: 360_000 }, + ); + } +}); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.handler.ts b/apps/cli/src/commands/functions/serve/serve.stack.handler.ts new file mode 100644 index 0000000000..bfa117538a --- /dev/null +++ b/apps/cli/src/commands/functions/serve/serve.stack.handler.ts @@ -0,0 +1,275 @@ +import type { ServiceCreation, Stack } from "@supabase/stack/effect"; +import { DateTime, Effect, Equal, Fiber, Option, Path, Stream, type Scope } from "effect"; +import { Output } from "../../../shared/output/output.service.ts"; +import { CommandSettings } from "../../../config/command-settings.service.ts"; +import { RuntimeInfo } from "../../../shared/runtime/runtime-info.service.ts"; +import { ProcessControl } from "../../../shared/runtime/process-control.service.ts"; +import { loadStackConfig, StackConfigError } from "../../../command-internal/stack-config.ts"; +import { + readStackFunctionsEnv, + StackFunctionsEnvError, +} from "../../../command-internal/stack-functions-env.ts"; +import { LocalDbRunningError } from "../../../command-internal/db-bootstrap/local-db-running.ts"; +import { stackOpenReadyProject } from "../../../command-internal/stack-local-database.ts"; +import { OutputFlag } from "../../../command-internal/global-flags.ts"; +import { TelemetryState } from "../../../telemetry/telemetry-state.service.ts"; +import type { FunctionsServeFlags } from "../../../shared/functions/serve.ts"; +import { FunctionsServeStackError } from "./serve.errors.ts"; + +type Instance = Effect.Success>; +type FunctionsInstance = Extract; +type FunctionsCreation = Extract; + +const runtimeError = (cause: { readonly message: string }) => + cause instanceof FunctionsServeStackError + ? cause + : new FunctionsServeStackError({ + reason: + cause instanceof StackFunctionsEnvError || cause instanceof StackConfigError + ? "invalid-config" + : cause instanceof LocalDbRunningError + ? "lifecycle" + : "runtime", + message: cause.message, + cause, + }); +const invalidConfig = (message: string) => + new FunctionsServeStackError({ reason: "invalid-config", message }); + +const follow = Effect.fn("functions.serve.follow")(function* ( + instance: FunctionsInstance, + launch: Effect.Effect, +) { + const output = yield* Output; + const logs = yield* instance.logs.pipe( + Stream.groupByKey((entry) => entry.stream), + Stream.flatMap( + ([channel, entries]) => + entries.pipe( + Stream.map(({ bytes }) => bytes), + Stream.decodeText, + Stream.splitLines, + Stream.map((line) => ({ channel, line })), + ), + { concurrency: 2 }, + ), + Stream.runForEach( + Effect.fn(function* ({ channel, line }) { + const timestamp = DateTime.formatIso(yield* DateTime.now); + yield* output.format === "stream-json" + ? output.event({ + type: "log-entry", + timestamp, + source: "live", + service: "functions", + instance_id: instance.id, + stream: channel, + line, + }) + : output.raw(`${line}\n`, channel); + }), + ), + Effect.forkScoped({ startImmediately: true }), + ); + yield* launch; + yield* instance.ready; + const url = (yield* instance.credentials()).url; + if (url === undefined) return yield* invalidConfig("Functions has no public HTTP endpoint."); + if (output.format === "stream-json") yield* output.result({ instance_id: instance.id, url }); + else yield* output.raw(`Serving Functions at ${url}\n`); + yield* Effect.raceFirst( + Fiber.join(logs).pipe( + Effect.andThen( + Effect.fail( + new FunctionsServeStackError({ + reason: "runtime", + message: "The Functions log stream ended.", + }), + ), + ), + ), + instance.followStatus.pipe( + Stream.runForEach((status) => { + if (status.currentOperation === "restart") return Effect.void; + return status.lifecycle === "stopped" || + status.health === "unhealthy" || + status.error !== undefined + ? Effect.fail( + new FunctionsServeStackError({ + reason: "runtime", + message: + status.error?.message ?? "The Functions service stopped or became unhealthy.", + suggestion: "Run supabase start to make Functions available again.", + }), + ) + : Effect.void; + }), + ), + ); +}); + +const session = Effect.fn("functions.serve.session")(function* (flags: FunctionsServeFlags) { + const output = yield* Output; + const processControl = yield* ProcessControl; + const settings = yield* CommandSettings; + const runtime = yield* RuntimeInfo; + const path = yield* Path.Path; + const legacyOutput = yield* OutputFlag; + if (output.format === "json" || Option.isSome(legacyOutput)) + return yield* new FunctionsServeStackError({ + reason: "flags", + message: "Functions serve requires text or stream-json output.", + suggestion: "Use --output-format stream-json for structured live output.", + }); + if ( + Option.isSome(flags.importMap) || + flags.inspect || + Option.isSome(flags.inspectMode) || + flags.inspectMain + ) + return yield* new FunctionsServeStackError({ + reason: "flags", + message: "Import-map and inspector flags are not supported by experimental Functions serve.", + suggestion: "Remove --import-map, --inspect, --inspect-mode, and --inspect-main.", + }); + const opened = yield* stackOpenReadyProject; + if (Option.isNone(opened)) + return yield* new FunctionsServeStackError({ + reason: "lifecycle", + message: "The local stack is not running.", + suggestion: "Run supabase start first.", + }); + const { stack, database } = opened.value; + const composition = yield* stack.composition.describe; + const instances = yield* stack.services.list; + const members = instances.filter(({ id }) => + composition.members.some((member) => member.id === id), + ); + const existing = members.find((instance) => instance.service === "functions"); + const envOverride = Option.isSome(flags.envFile) + ? yield* readStackFunctionsEnv(path.resolve(runtime.cwd, flags.envFile.value), false) + : undefined; + const cleanupWarning = (message: string, error: { readonly message: string }) => + output + .raw(`${message}: ${error.message}\n`, "stderr") + .pipe(Effect.andThen(processControl.setExitCode(1))); + + if (existing?.service === "functions") { + const before = yield* existing.status; + if (before.config.service !== "functions") + return yield* invalidConfig("Invalid saved Functions configuration."); + const saved = before.config.config; + const desired = { + ...saved, + ...(envOverride === undefined ? {} : { env: { ...saved.env, ...envOverride } }), + ...(Option.isSome(flags.noVerifyJwt) ? { verifyJwt: !flags.noVerifyJwt.value } : {}), + }; + const changed = !Equal.equals(saved, desired); + const launch = changed + ? Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const mutation = yield* restore(existing.restart({ config: desired })).pipe( + Effect.forkScoped(), + ); + // A submitted restart must settle before cleanup reads its persisted configuration. + yield* Effect.addFinalizer(() => + Fiber.await(mutation).pipe( + Effect.andThen(existing.status), + Effect.flatMap((current) => + Equal.equals(current.config.config, desired) || + (Equal.equals(current.config.config, saved) && current.lifecycle === "stopped") + ? existing.restart({ config: saved }) + : Effect.void, + ), + Effect.catch((error) => + cleanupWarning("Failed to restore Functions configuration", error), + ), + ), + ); + return yield* restore(Fiber.join(mutation)); + }), + ) + : existing.start; + yield* follow(existing, launch); + return; + } + + for (const instance of instances) { + if (instance.service !== "functions") continue; + const status = yield* instance.status; + if (status.lifecycle !== "stopped" || status.wakeEnabled) + return yield* new FunctionsServeStackError({ + reason: "lifecycle", + message: `Standalone Functions instance ${instance.id} is already active.`, + suggestion: "Stop the stack before starting another Functions serving session.", + }); + } + const databaseStatus = yield* database.status; + if (databaseStatus.config.service !== "database") + return yield* invalidConfig("Invalid saved database configuration."); + const config = yield* loadStackConfig(settings.workdir); + const creations = yield* config.creations(stack.id, { + jwtSecret: databaseStatus.config.config.jwtSecret, + }); + const source = creations.find( + (creation): creation is FunctionsCreation => creation.service === "functions", + ); + if (source === undefined) + return yield* invalidConfig( + "Enable edge_runtime in supabase/config.toml before serving Functions.", + ); + const databaseUrl = (yield* database.credentials({ from: "runtime" })).databaseUrl; + if (databaseUrl === undefined) return yield* invalidConfig("The database has no runtime URL."); + const apiSource = [...members, ...instances].find( + (instance) => + instance.service === "rest" || instance.service === "auth" || instance.service === "storage", + ); + const apiStatus = apiSource === undefined ? undefined : yield* apiSource.status; + const savedPort = apiStatus?.endpoints.find(({ name }) => name === "http")?.port; + const requestedPort = source.endpoints?.http?.port; + const port = + savedPort ?? (typeof requestedPort === "number" ? requestedPort : config.source.api.port); + const databaseAddress = URL.parse(databaseUrl); + if (databaseAddress === null) return yield* invalidConfig("Invalid runtime database URL."); + const apiUrl = + apiSource === undefined + ? `http://${databaseAddress.hostname}:${port}` + : (yield* apiSource.credentials({ from: "runtime" })).apiUrl; + if (apiUrl === undefined) return yield* invalidConfig("The stack has no runtime API URL."); + const env = + envOverride ?? (yield* readStackFunctionsEnv(`${source.config.functionsRoot}/.env`, true)); + const creation: FunctionsCreation = { + ...source, + config: { + ...source.config, + apiUrl, + databaseUrl, + env: { ...env, ...source.config.env, ...envOverride }, + ...(Option.isSome(flags.noVerifyJwt) ? { verifyJwt: !flags.noVerifyJwt.value } : {}), + }, + endpoints: { ...source.endpoints, http: { port } }, + }; + const temporary = yield* Effect.acquireRelease(stack.services.create(creation), (instance) => + instance.status.pipe( + Effect.andThen(instance.destroy), + Effect.catch((error) => cleanupWarning("Failed to remove temporary Functions", error)), + ), + ); + yield* follow(temporary, temporary.start); +}); + +export const functionsServeStack = Effect.fn("functions.serve.stack")(function* ( + flags: FunctionsServeFlags, +) { + const telemetry = yield* TelemetryState; + yield* Effect.scoped( + Effect.gen(function* () { + const control = yield* ProcessControl; + yield* control.holdSignals(["SIGINT", "SIGTERM"]); + const shutdown = yield* control + .awaitSignal() + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Effect.raceFirst(Fiber.join(shutdown).pipe(Effect.asVoid), session(flags)); + }), + ).pipe(Effect.mapError(runtimeError), Effect.ensuring(telemetry.flush)); +}); diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts new file mode 100644 index 0000000000..5e66b5f4da --- /dev/null +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -0,0 +1,412 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { + Cause, + Deferred, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Option, + Redacted, + Scope, + Stream, +} from "effect"; +import { + StackError, + type Observation, + type ServiceCreation, + type Stack, +} from "@supabase/stack/effect"; +import { StackApi } from "../../../command-internal/stack-api.ts"; +import { OutputFlag } from "../../../command-internal/global-flags.ts"; +import { mockCommandSettings } from "../../../../tests/helpers/command-mocks.ts"; +import { + mockOutput, + mockProcessControl, + mockRuntimeInfo, +} from "../../../../tests/helpers/mocks.ts"; +import { TelemetryState } from "../../../telemetry/telemetry-state.service.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { functionsServeStack } from "./serve.stack.handler.ts"; + +type DatabaseInstance = Extract< + Effect.Success>, + { service: "database" } +>; +type FunctionsInstance = Extract< + Effect.Success>, + { service: "functions" } +>; + +const functionsConfig = (): Extract["config"] => ({ + functionsRoot: "/project/supabase/functions", + bootstrap: "export default {};", + apiUrl: "http://127.0.0.1:54321", + databaseUrl: "postgresql://postgres@127.0.0.1:54322/postgres", + jwtSecret: "jwt-secret", + verifyJwt: true, + env: { SHARED: "saved", RETAINED: "retained" }, +}); + +const databaseConfig: Extract["config"] = { + version: "15", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make("jwt-secret"), + jwtExpiry: 3600, +}; + +const observation = ( + id: string, + config: ServiceCreation, + overrides: Partial = {}, +): Observation => ({ + id, + config, + endpoints: [{ name: "http", protocol: "http", host: "127.0.0.1", port: 54321 }], + lifecycle: "running", + health: "healthy", + currentOperation: undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + launchId: 1, + intentRevision: 1, + wakeEnabled: false, + registered: true, + ...overrides, +}); + +const flags = (overrides: Partial[0]> = {}) => ({ + noVerifyJwt: Option.none(), + envFile: Option.none(), + importMap: Option.none(), + inspect: false, + inspectMode: Option.none<"run" | "brk" | "wait">(), + inspectMain: false, + all: true, + ...overrides, +}); + +const fixture = ( + options: { + readonly failRestartOn?: number; + readonly restartGate?: { + readonly entered: Deferred.Deferred; + readonly release: Deferred.Deferred; + readonly mutationDone: Deferred.Deferred; + }; + } = {}, +) => + Effect.gen(function* () { + const ownerScope = yield* Scope.make(); + yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); + const signal = yield* Deferred.make(); + const started = yield* Deferred.make(); + const signalControl = mockProcessControl({ + awaitSignal: Deferred.await(signal).pipe(Effect.as("SIGINT" as const)), + }); + const output = mockOutput({ format: "text", interactive: false }); + const telemetry = Layer.succeed(TelemetryState, { + flush: Effect.void, + stitchLogin: () => Effect.void, + resetIdentity: Effect.void, + clearDistinctId: Effect.void, + }); + let currentFunctions = functionsConfig(); + let restartCount = 0; + let destroyed = false; + const databaseCreation: Extract = { + service: "database", + config: databaseConfig, + endpoints: { sql: { port: 54322 } }, + }; + const functionsCreation = (): Extract => ({ + service: "functions", + config: currentFunctions, + endpoints: { http: { port: 54321 } }, + }); + const database: DatabaseInstance = { + id: "database", + service: "database" as const, + status: Effect.succeed(observation("database", databaseCreation)), + credentials: () => + Effect.succeed({ databaseUrl: "postgresql://postgres@127.0.0.1:54322/postgres" }), + start: Effect.void, + ready: Effect.void, + stop: Effect.void, + restart: () => Effect.void, + destroy: Effect.void, + prepare: Effect.void, + resetData: Effect.void, + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + followStatus: Stream.never, + logs: Stream.never, + }; + const functions: FunctionsInstance = { + id: "functions", + service: "functions" as const, + status: Effect.sync(() => observation("functions", functionsCreation())), + credentials: () => Effect.succeed({ url: "http://127.0.0.1:54321" }), + start: Deferred.succeed(started, undefined), + ready: Effect.void, + stop: Effect.void, + restart: (input?: Parameters[0]) => + Effect.gen(function* () { + restartCount += 1; + const first = restartCount === 1; + if (first && options.restartGate !== undefined) { + const gate = options.restartGate; + yield* Deferred.succeed(gate.entered, undefined); + const pending = yield* Effect.forkIn( + Effect.gen(function* () { + yield* Deferred.await(gate.release); + if (input !== undefined) currentFunctions = input.config; + yield* Deferred.succeed(gate.mutationDone, undefined); + }), + ownerScope, + { startImmediately: true }, + ); + yield* Fiber.join(pending); + return; + } + if (input !== undefined) currentFunctions = input.config; + if (restartCount === options.failRestartOn) + return yield* new StackError({ operation: "restart", message: "restart failed" }); + yield* Deferred.succeed(started, undefined); + }), + destroy: Effect.sync(() => { + destroyed = true; + }), + prepare: Effect.void, + followStatus: Stream.never, + logs: Stream.never, + }; + const stack = { + id: "a".repeat(64), + services: { + list: Effect.succeed([database, functions]), + get: (id: string) => Effect.succeed(id === "database" ? database : functions), + create: () => Effect.die("unused"), + }, + composition: { + describe: Effect.succeed({ + members: [ + { id: "database", activation: "eager" as const }, + { id: "functions", activation: "eager" as const }, + ], + dependencies: [], + }), + supabase: () => Effect.die("unused"), + configure: () => Effect.die("unused"), + start: Effect.die("unused"), + stop: Effect.die("unused"), + restart: Effect.die("unused"), + }, + stop: Effect.die("unused"), + destroy: Effect.die("unused"), + tools: { run: () => Effect.die("unused") }, + } satisfies Stack; + const identity = { projectRoot: "/project", branchContext: "main", stackName: "default" }; + const apiService = { + create: () => Effect.die("unused"), + open: () => Effect.succeed(stack), + discover: () => + Effect.succeed([ + { + definition: { + id: stack.id, + identity, + runtime: "native", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }, + host: undefined, + }, + ]), + resolveIdentity: () => Effect.succeed(identity), + } satisfies StackApi["Service"]; + const api = Layer.succeed(StackApi, apiService); + const layer = Layer.mergeAll( + BunServices.layer, + api, + mockCommandSettings({ workdir: "/project", supabaseHome: "/home" }), + mockRuntimeInfo({ cwd: "/project", homeDir: "/home" }), + output.layer, + signalControl.layer, + telemetry, + stackBackendLayer("stack"), + Layer.succeed(OutputFlag, Option.none()), + ); + return { + layer, + signal, + started, + output, + signalControl, + get currentFunctions() { + return currentFunctions; + }, + get restartCount() { + return restartCount; + }, + get destroyed() { + return destroyed; + }, + }; + }); + +describe("experimental Stack Functions serve", () => { + it.live("restores an overridden composed Functions config on SIGINT", () => + Effect.gen(function* () { + const state = yield* fixture(); + const run = yield* functionsServeStack(flags({ noVerifyJwt: Option.some(true) })).pipe( + Effect.provide(state.layer), + Effect.forkChild, + ); + yield* Deferred.await(state.started); + expect(state.currentFunctions.verifyJwt).toBe(false); + yield* Deferred.succeed(state.signal, undefined); + yield* Fiber.join(run); + expect(state.currentFunctions.verifyJwt).toBe(true); + expect(state.restartCount).toBe(2); + }), + ); + + it.live("merges an explicit env file with saved secrets and restores them on SIGINT", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "functions-env-merge-" }); + const file = `${root}/override.env`; + yield* fs.writeFileString(file, "SHARED=override\nADDED=new\n"); + const state = yield* fixture(); + const run = yield* functionsServeStack(flags({ envFile: Option.some(file) })).pipe( + Effect.provide(state.layer), + Effect.forkChild, + ); + yield* Deferred.await(state.started); + expect(state.currentFunctions.env).toEqual({ + SHARED: "override", + RETAINED: "retained", + ADDED: "new", + }); + yield* Deferred.succeed(state.signal, undefined); + yield* Fiber.join(run); + expect(state.currentFunctions.env).toEqual(functionsConfig().env); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("rejects unsupported flags before reading or mutating the stack", () => + Effect.gen(function* () { + const state = yield* fixture(); + const exit = yield* functionsServeStack(flags({ inspect: true })).pipe( + Effect.provide(state.layer), + Effect.exit, + ); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + const failure = Cause.findErrorOption(exit.cause); + expect(Option.isSome(failure)).toBe(true); + if (Option.isSome(failure)) expect(failure.value.reason).toBe("flags"); + } + expect(state.restartCount).toBe(0); + expect(state.destroyed).toBe(false); + }), + ); + + it.live("restores the saved config when applying an override fails", () => + Effect.gen(function* () { + const state = yield* fixture({ failRestartOn: 1 }); + const exit = yield* functionsServeStack(flags({ noVerifyJwt: Option.some(true) })).pipe( + Effect.provide(state.layer), + Effect.exit, + ); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + const failure = Cause.findErrorOption(exit.cause); + expect(Option.isSome(failure)).toBe(true); + if (Option.isSome(failure)) expect(failure.value.message).toContain("restart failed"); + } + expect(state.currentFunctions.verifyJwt).toBe(true); + expect(state.restartCount).toBe(2); + }), + ); + + it.live("reports a restore failure and sets the command exit code", () => + Effect.gen(function* () { + const state = yield* fixture({ failRestartOn: 2 }); + const run = yield* functionsServeStack(flags({ noVerifyJwt: Option.some(true) })).pipe( + Effect.provide(state.layer), + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(state.started); + yield* Deferred.succeed(state.signal, undefined); + const result = yield* Fiber.await(run); + expect(Exit.isSuccess(result)).toBe(true); + expect(state.signalControl.exitCode).toBe(1); + expect( + state.output.rawChunks.some( + ({ text, stream }) => stream === "stderr" && text.includes("Failed to restore"), + ), + ).toBe(true); + }), + ); + it.live("rejects multiline environment values before restarting Functions", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "functions-env-" }); + const file = `${root}/override.env`; + yield* fs.writeFileString(file, 'CUSTOM_VALUE="first\nsecond"\n'); + const state = yield* fixture(); + const error = yield* functionsServeStack(flags({ envFile: Option.some(file) })).pipe( + Effect.provide(state.layer), + Effect.flip, + ); + expect(error.reason).toBe("invalid-config"); + expect(error.message).toContain("Multiline"); + expect(state.currentFunctions).toEqual(functionsConfig()); + expect(state.restartCount).toBe(0); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("rejects an invalid environment key before restarting Functions", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "functions-env-key-" }); + const file = `${root}/override.env`; + yield* fs.writeFileString(file, "INVALID.KEY=value\n"); + const state = yield* fixture(); + const error = yield* functionsServeStack(flags({ envFile: Option.some(file) })).pipe( + Effect.provide(state.layer), + Effect.flip, + ); + expect(error.message).toContain("Environment names"); + expect(state.currentFunctions).toEqual(functionsConfig()); + expect(state.restartCount).toBe(0); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("restores config when SIGINT arrives during an in-flight restart", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const mutationDone = yield* Deferred.make(); + const state = yield* fixture({ restartGate: { entered, release, mutationDone } }); + const run = yield* functionsServeStack(flags({ noVerifyJwt: Option.some(true) })).pipe( + Effect.provide(state.layer), + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.await(entered); + yield* Deferred.succeed(state.signal, undefined); + yield* Effect.yieldNow; + yield* Deferred.succeed(release, undefined); + yield* Deferred.await(mutationDone); + const result = yield* Fiber.await(run); + expect(Exit.isSuccess(result)).toBe(true); + expect(state.currentFunctions).toEqual(functionsConfig()); + }), + ); +}); diff --git a/apps/cli/src/commands/migration/migration.layers.ts b/apps/cli/src/commands/migration/migration.layers.ts index a599f2f9be..e9cebc6e59 100644 --- a/apps/cli/src/commands/migration/migration.layers.ts +++ b/apps/cli/src/commands/migration/migration.layers.ts @@ -13,7 +13,6 @@ import { linkedDbResolverRuntimeLayer } from "../../command-internal/management- import { telemetryStateLayer } from "../../telemetry/telemetry-state.layer.ts"; import { stackApiLayer } from "../../command-internal/stack-api.ts"; import { bundledPostgresClientLayer } from "../../command-internal/bundled-postgres-client.ts"; -import { ephemeralPostgresLayer } from "../../command-internal/stack-shadow.ts"; import { stackCatalogSetupLayer } from "../../command-internal/stack-catalog-setup.ts"; const cliSettings = commandSettingsLayer.pipe(Layer.provide(debugLoggerLayer)); @@ -69,7 +68,6 @@ export const migrationSquashRuntimeLayer = Layer.mergeAll( httpClient, debugLoggerLayer, stackApiLayer, - bundledPostgresClientLayer, - ephemeralPostgresLayer, + bundledPostgresClientLayer.pipe(Layer.provide(dockerRunLayer), Layer.provide(cliSettings)), stackCatalogSetupLayer, ); diff --git a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md index c3c13cfe3d..6f6e176dcf 100644 --- a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md @@ -7,9 +7,13 @@ full schema into the target file, and deleting the merged files — then either suggests `migration repair` (local target) or prompts to baseline the remote migration-history table to match. -When `[experimental].stack` is on, the shadow is `EphemeralPostgres` under -`$SUPABASE_HOME/managed/ephemeral-postgres//` (`~/.supabase/managed/…` by default). -On the stack backend, squash dumps through catalog `pg_dump` (native artifact or a one-shot container of the same image). There is no PATH fallback. +When `[experimental].stack` is on, each shadow is a fresh database in an invocation-owned, +unique temporary stack namespace. The command applies the catalog and project migrations as needed, +then destroys its namespace when the Effect scope closes. Shadow baselines are not cached; +the snapshot cache described below belongs to the legacy container backend. Native artifacts +are shared through `$SUPABASE_HOME/cache/stack`; shadow state and data use the normal stack registry, so `stack list` and `stack destroy` can +find a shadow left by an abrupt CLI exit. Each shadow owns a unique temporary project root +and uses an automatically assigned port; `db.shadow_port` applies only to the legacy backend. Schema dumps use the namespace's catalog `pg_dump` tool and runtime database address. ## Files Read diff --git a/apps/cli/src/commands/migration/squash/squash.handler.ts b/apps/cli/src/commands/migration/squash/squash.handler.ts index 9966cdba33..e9e2e461b1 100644 --- a/apps/cli/src/commands/migration/squash/squash.handler.ts +++ b/apps/cli/src/commands/migration/squash/squash.handler.ts @@ -1,7 +1,6 @@ -import { Effect, FileSystem, Option, Path, Predicate } from "effect"; +import { Effect, FileSystem, Option, Path } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerType } from "effect/unstable/process/ChildProcessSpawner"; -import { resolveEphemeralPostgresRelease } from "@supabase/stack/effect"; import { cobraMutuallyExclusiveErrorMessage } from "../../../shared/cli/cobra-flag-groups.ts"; import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; @@ -42,16 +41,11 @@ import type { ResolvedDbConfig } from "../../../command-internal/db-config.types import { DbConnection, type PgConnInput } from "../../../command-internal/db-connection.service.ts"; import { resolveDbTargetFlags } from "../../../command-internal/db-target-flags.ts"; import { DebugLogger } from "../../../command-internal/debug-logger.service.ts"; -import { DockerRunError } from "../../../command-internal/docker-run.errors.ts"; import { errorMessage, relativizeErrorMessage } from "../../../command-internal/error-message.ts"; import { currentStackBackend } from "../../../command-internal/stack-backend.ts"; import { stackWithShadowDatabase } from "../../../command-internal/stack-shadow.ts"; -import { - dumpConnForHostClient, - rewriteDumpHostForToolContainer, - toolContainerUsesHostNetwork, -} from "../../../command-internal/postgres-client.run.ts"; -import { bundledPostgresClientRuntime } from "../../../command-internal/bundled-postgres-client.ts"; +import { parseConnectionString } from "../../../command-internal/db-config.parse.ts"; + import { applyMigrations, MigrationApplyError } from "../../../command-internal/migration-apply.ts"; import { INSERT_MIGRATION_VERSION, @@ -122,7 +116,6 @@ const squashMigrations = Effect.fnUntraced(function* ( const image = localInputs.bootstrapConfig.postgresImage; if (stackBackend) { - const runtimeInfo = yield* RuntimeInfo; return yield* stackWithShadowDatabase(shadowInput, (handle) => Effect.scoped( Effect.gen(function* () { @@ -133,32 +126,18 @@ const squashMigrations = Effect.fnUntraced(function* ( password: toml.password, database: "postgres", }; - const networkIdFlag = yield* NetworkIdFlag; - const networkId = Option.getOrUndefined(networkIdFlag); - const dumpUsesHostNetwork = toolContainerUsesHostNetwork(networkId); - const dumpRuntime = - bundledPostgresClientRuntime(handle.runtime, runtimeInfo.platform, runtimeInfo.arch) ?? - handle.runtime; - const release = yield* resolveEphemeralPostgresRelease(handle.ephemeral.version).pipe( - Effect.orElseSucceed(() => undefined), - ); - const image = release?.image ?? localInputs.bootstrapConfig.postgresImage; + const credentials = yield* handle.database.credentials({ from: "runtime" }); + const dumpConn = parseConnectionString(credentials.databaseUrl ?? ""); + if (dumpConn === undefined) + return yield* new MigrationSquashDumpError({ + message: "Shadow database tool URL is unavailable", + }); const dumpClient = { - kind: "bundled" as const, + kind: "stack" as const, + stack: handle.stack, command: "pg_dump" as const, - version: handle.ephemeral.version, - runtime: dumpRuntime, + major: toml.majorVersion === 15 ? (15 as const) : (17 as const), }; - const dumpConn: PgConnInput = - dumpRuntime.kind === "native" - ? dumpConnForHostClient(stackConn) - : { - ...stackConn, - host: rewriteDumpHostForToolContainer(handle.host, { - platform: runtimeInfo.platform, - usesHostNetwork: dumpUsesHostNetwork, - }), - }; const session = yield* connectShadowDatabase(stackConn); const before = yield* squashDumpSchemaToString({ image, @@ -221,20 +200,6 @@ const squashMigrations = Effect.fnUntraced(function* ( }), ); }), - ).pipe( - Effect.catchIf( - (error): error is DockerRunError => - Predicate.isTagged(error, "DockerRunError") && - handle.runtime.kind === "native" && - runtimeInfo.platform === "win32", - (error) => - Effect.fail( - new MigrationSquashDumpError({ - message: error.message, - suggestion: "Install Docker Desktop to squash a native stack on Windows.", - }), - ), - ), ), ); } diff --git a/apps/cli/src/commands/migration/squash/squash.integration.test.ts b/apps/cli/src/commands/migration/squash/squash.integration.test.ts index 80ff3930e6..31f99f4aa5 100644 --- a/apps/cli/src/commands/migration/squash/squash.integration.test.ts +++ b/apps/cli/src/commands/migration/squash/squash.integration.test.ts @@ -1,3 +1,4 @@ +import { unusedStackServices } from "../../../../tests/helpers/unused-stack.ts"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { @@ -339,6 +340,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { }); const baseLayer = Layer.mergeAll( + unusedStackServices, // Listed first so every fake service layer below overrides it; Layer.mergeAll is // last-wins on a shared service. BunServices.layer, diff --git a/apps/cli/src/commands/migration/squash/squash.native.integration.test.ts b/apps/cli/src/commands/migration/squash/squash.native.integration.test.ts new file mode 100644 index 0000000000..282fefcbb1 --- /dev/null +++ b/apps/cli/src/commands/migration/squash/squash.native.integration.test.ts @@ -0,0 +1,170 @@ +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, FileSystem, Layer, Option, Path } from "effect"; +import { FetchHttpClient } from "effect/unstable/http"; + +import { + mockCommandSettings, + mockLinkedProjectCacheTracked, + mockTelemetryStateTracked, +} from "../../../../tests/helpers/command-mocks.ts"; +import { mockOutput, mockRuntimeInfo, mockStdin } from "../../../../tests/helpers/mocks.ts"; +import { CliArgs } from "../../../shared/cli/cli-args.service.ts"; +import { + DebugFlag, + DnsResolverFlag, + ExperimentalFlag, + NetworkIdFlag, + YesFlag, +} from "../../../command-internal/global-flags.ts"; +import { DbConfigResolver } from "../../../command-internal/db-config.service.ts"; +import { dbConnectionLayer } from "../../../command-internal/db-connection.layer.ts"; +import { DebugLogger } from "../../../command-internal/debug-logger.service.ts"; +import { DockerRun } from "../../../command-internal/docker-run.service.ts"; +import { StackApi, stackApiLayer } from "../../../command-internal/stack-api.ts"; +import { stackBackendLayer } from "../../../command-internal/stack-backend.ts"; +import { stackCatalogSetupLayer } from "../../../command-internal/stack-catalog-setup.ts"; +import { BundledPostgresClient } from "../../../command-internal/bundled-postgres-client.ts"; +import { ProjectRefResolver } from "../../../config/project-ref.service.ts"; +import { migrationSquash } from "./squash.handler.ts"; +import type { MigrationSquashFlags } from "./squash.command.ts"; + +const runtimes = ["native", "docker"] as const; +const liveStackApi = stackApiLayer.pipe(Layer.provide(BunServices.layer)); + +const flags: MigrationSquashFlags = { + version: Option.none(), + dbUrl: Option.none(), + linked: false, + local: true, + password: Option.none(), + projectRef: Option.none(), +}; + +const failureMessage = (exit: Exit.Exit): string => { + if (Exit.isSuccess(exit)) return ""; + const error = Cause.findErrorOption(exit.cause); + return Option.isSome(error) && error.value instanceof Error + ? error.value.message + : String(exit.cause); +}; + +const testLayer = (root: string) => + Layer.mergeAll( + BunServices.layer, + FetchHttpClient.layer, + liveStackApi, + mockCommandSettings({ workdir: root, supabaseHome: root }), + mockOutput().layer, + mockTelemetryStateTracked().layer, + mockLinkedProjectCacheTracked().layer, + mockRuntimeInfo(), + Layer.succeed(CliArgs, { args: [] }), + Layer.succeed(DbConfigResolver, { + resolve: () => + Effect.succeed({ + conn: { + host: "127.0.0.1", + port: 54322, + user: "postgres", + password: "postgres", + database: "postgres", + }, + isLocal: true, + }), + resolvePoolerFallback: () => Effect.succeed(Option.none()), + }), + Layer.succeed(DebugFlag, false), + Layer.succeed(DnsResolverFlag, "native"), + Layer.succeed(ExperimentalFlag, false), + Layer.succeed(NetworkIdFlag, Option.none()), + Layer.succeed(YesFlag, true), + Layer.succeed(DebugLogger, { debug: () => Effect.void, http: () => Effect.void }), + Layer.succeed(ProjectRefResolver, { + resolve: () => Effect.succeed("abcdefghijklmnopqrst"), + resolveForLink: () => Effect.succeed("abcdefghijklmnopqrst"), + resolveOptional: () => Effect.succeed(Option.some("abcdefghijklmnopqrst")), + loadProjectRef: () => Effect.succeed("abcdefghijklmnopqrst"), + promptProjectRef: () => Effect.succeed("abcdefghijklmnopqrst"), + }), + mockStdin(false), + Layer.succeed(BundledPostgresClient, { run: () => Effect.die("unused") }), + Layer.succeed(DockerRun, { + run: () => Effect.die("unused"), + runCapture: () => Effect.die("unused"), + runStream: () => Effect.die("unused"), + }), + dbConnectionLayer, + stackCatalogSetupLayer, + stackBackendLayer("stack"), + ); + +describe("managed migration squash", { timeout: 180_000 }, () => { + for (const runtime of runtimes) { + it.live(`${runtime} squashes through an owned shadow stack`, () => + Effect.scoped( + Effect.gen(function* () { + const { root, first, second } = yield* Effect.provide( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "cli-migration-squash-" }); + const migrations = path.join(root, "supabase", "migrations"); + yield* fs.makeDirectory(migrations, { recursive: true }); + yield* fs.writeFileString( + path.join(root, "supabase", "config.toml"), + "[db]\nmajor_version = 17\n[auth]\nenabled = false\n[storage]\nenabled = false\n[realtime]\nenabled = false\n[api]\nenabled = false\n", + ); + const first = path.join(migrations, "20260919000000_first.sql"); + const second = path.join(migrations, "20260919000001_second.sql"); + yield* fs.writeFileString(first, "create table public.squash_probe(value text);\n"); + yield* fs.writeFileString( + second, + "insert into public.squash_probe(value) values ('squashed');\n", + ); + return { root, first, second }; + }), + Layer.mergeAll(BunServices.layer, FetchHttpClient.layer), + ); + + yield* Effect.provide( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const api = yield* StackApi; + const current = yield* api.create({ + projectRoot: root, + stateRoot: path.join(root, "stacks"), + cacheRoot: path.join(root, "cache"), + runtime, + }); + yield* Effect.addFinalizer(() => + current.destroy.pipe( + Effect.catch((cause) => + Effect.die(`failed to destroy test stack ${current.id}: ${cause.message}`), + ), + ), + ); + const before = yield* current.composition.describe; + expect(before.members).toHaveLength(0); + + const exit = yield* migrationSquash(flags).pipe(Effect.exit); + if (Exit.isFailure(exit)) { + return yield* Effect.die(`managed squash failed: ${failureMessage(exit)}`); + } + + expect(yield* fs.exists(first)).toBe(false); + const squashed = yield* fs.readFileString(second); + expect(squashed).toContain("squash_probe"); + const after = yield* current.composition.describe; + expect(after.members).toHaveLength(0); + const saved = yield* api.discover({ stateRoot: path.join(root, "stacks") }); + expect(saved.map(({ definition }) => definition.id)).toEqual([current.id]); + }), + testLayer(root), + ); + }), + ), + ); + } +}); diff --git a/apps/cli/src/commands/pull/pull.integration.test.ts b/apps/cli/src/commands/pull/pull.integration.test.ts index e7aefc2c06..634f7649b1 100644 --- a/apps/cli/src/commands/pull/pull.integration.test.ts +++ b/apps/cli/src/commands/pull/pull.integration.test.ts @@ -39,6 +39,7 @@ import { useShadowCacheDisabled, useTempWorkdir, } from "../../../tests/helpers/command-mocks.ts"; +import { unusedStackServices } from "../../../tests/helpers/unused-stack.ts"; import { commandRuntimeLayer } from "../../shared/runtime/command-runtime.layer.ts"; import { machineErrorContextLayer } from "../../shared/output/machine-error-context.layer.ts"; import { jsonOutputLayer, streamJsonOutputLayer } from "../../shared/output/output.layer.ts"; @@ -691,6 +692,7 @@ function setup(opts: SetupOpts = {}) { Layer.succeed(DnsResolverFlag, "native"), Layer.succeed(NetworkIdFlag, Option.none()), Layer.succeed(CliArgs, { args: [] }), + unusedStackServices, // Listed after `buildTestRuntime` so it overrides the real spawner `BunServices.layer` // provides (last-wins). spawner.layer, diff --git a/apps/cli/src/commands/seed/buckets/SIDE_EFFECTS.md b/apps/cli/src/commands/seed/buckets/SIDE_EFFECTS.md index ced9ca5690..86947dfcca 100644 --- a/apps/cli/src/commands/seed/buckets/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/seed/buckets/SIDE_EFFECTS.md @@ -40,10 +40,11 @@ values are resolved with their `SUPABASE_API_*` shell/dotenv overrides applied first, so the gateway targets the same port/scheme the stack was actually brought up on (#6452). -**Local (stack backend):** the selected stack's API gateway URL -(`status.endpoints.api.url`) + `/storage/v1/...`; the credential is the stack's -service-role JWT read from the stack's credentials. `[api]`/`[api.tls]` fields and their -`SUPABASE_API_*` overrides are not consulted. See Notes for the capability-state gate. +**Local (stack backend):** the Storage endpoint comes from the selected composition's +Storage member observation, and the service-role JWT is generated from the primary database's +observed JWT secret. `[api]`/`[api.tls]` fields and their `SUPABASE_API_*` overrides are not +consulted. The stack is opened without launching the owner or any service member. See Notes +for the capability-state gate. **Remote (`--linked`):** `https://.` (default host: `supabase.co`). @@ -113,7 +114,7 @@ Analytics bucket routes (`/storage/v1/iceberg/...`) are only reached when | `1` | unreadable `objects_path` (filesystem error during walk/upload) | | `1` | `--project-ref` set without `--linked` (see Notes) | | `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | -| `1` | stack backend: stack not registered/running, missing API endpoint or credentials, or the stack API is unavailable (`StackStorageUnavailableError`) | +| `1` | stack backend: stack is not registered/ready, has no primary database, or the stack API is unavailable (`StackStorageUnavailableError`); a missing Storage endpoint is a capability failure (`StackStorageCapabilityError`) | ## Telemetry Events Fired @@ -207,33 +208,20 @@ configured`, or a 404 on `ListVectorBuckets`), a WARNING is printed and object (`command-internal/detect-content-type.ts`), and only a generic `text/plain` result is refined by extension via a built-in MIME table (the host OS MIME database is not consulted; the deterministic built-in table is used instead). -- **Stack backend (local).** The Storage endpoint is the selected stack's API - gateway URL (`status.endpoints.api.url`) + `/storage/v1/...`; the credential is the - stack's service-role JWT read from the stack's credentials. The stack is located by - the project root (workdir realpath) via the `@supabase/stack` API, which reads stack - state under `SUPABASE_HOME`. `seed buckets` never creates a stack. All legacy `[api]`/ - `[api.tls]`/`SUPABASE_API_*`/`SUPABASE_AUTH_{JWT_SECRET,SERVICE_ROLE_KEY}` config - validation above (including the `<16`-char secret and TLS cert/key-pairing checks) - is skipped on this path, even when there is nothing configured to seed. -- **Stack backend — capability policy.** Storage `disabled` (e.g. - `stack start -x storage`) errors with `StackStorageCapabilityError` ("Storage is - disabled for this stack.") with guidance to enable `[storage]`, then run - `supabase stack stop` followed by `supabase stack start` without `-x storage`; - `failed`/`stopped` raises the same error class ("Storage failed to start for this - stack"/"Storage is stopped for this stack."), with the capability error appended - when present, and guidance to run `supabase stack restart`. - `dormant`/`starting`/`ready`/`stopping` all proceed — the - gateway activates a lazily-configured Storage on the first request and holds that - request, and wakes a stopping Storage once its cleanup completes; there is no - client-side polling. A stack that is not registered, not running, missing - its API endpoint or credentials, or whose stack API is unavailable errors with - `StackStorageUnavailableError` and guidance to run `supabase stack status` or - `supabase stack restart` (or `supabase start` when the stack was never configured). A - stack-gateway 502/503 during Storage activation is reported as - `StackStorageCapabilityError` with guidance to run `supabase stack logs` then - `supabase stack restart`, instead of a raw status body — for a `--local` target only; - a `--linked` failure passes through unchanged. No HTTP request is sent when Storage is - disabled or the stack is not running. +- **Stack backend (local).** The Storage endpoint comes from the selected composition's + Storage member observation, and the service-role JWT is generated from the primary database's + observed JWT secret. `seed buckets` opens saved state without launching the owner or any + service member. All legacy `[api]`/`[api.tls]`/`SUPABASE_API_*`/`SUPABASE_AUTH_{JWT_SECRET,SERVICE_ROLE_KEY}` + config validation above (including the `<16`-char secret and TLS cert/key-pairing checks) is + skipped on this path, even when there is nothing configured to seed. +- **Stack backend — capability policy.** Storage excluded from the composition errors with + `StackStorageCapabilityError` and guidance to enable `[storage]`, then run `supabase start` + without `--exclude storage`. Failed or stopped Storage is unusable; dormant, starting, ready, + and stopping Storage may proceed because the first request can wake a lazy member. The primary + database must be running and healthy before Storage requests are attempted. A stack that is + not registered, has no primary database composition member, is not ready, or has no Storage + HTTP endpoint errors with `StackStorageUnavailableError` or `StackStorageCapabilityError`. + No HTTP request is sent when Storage is disabled or the primary database is not ready. - **Secrets.** The stack's service-role JWT is never printed or logged; error messages contain only lifecycle/capability state text. - **Local Kong TLS (legacy backend only).** When `[api.tls] enabled = true` for a local stack, the diff --git a/apps/cli/src/commands/seed/buckets/buckets.integration.test.ts b/apps/cli/src/commands/seed/buckets/buckets.integration.test.ts index a284fe89d1..d1dd7650f2 100644 --- a/apps/cli/src/commands/seed/buckets/buckets.integration.test.ts +++ b/apps/cli/src/commands/seed/buckets/buckets.integration.test.ts @@ -38,8 +38,10 @@ import { } from "../../../command-internal/stack-storage.ts"; import { buildStorageStackApi, + STORAGE_TEST_JWT_SECRET, type SetupStorageStackApiOptions, } from "../../../../tests/helpers/storage.ts"; +import { unusedStackServices } from "../../../../tests/helpers/unused-stack.ts"; interface MockRoute { readonly method: string; @@ -227,6 +229,7 @@ const setupSeedBuckets = Effect.fnUntraced(function* ( make: CommandPlatformApi.pipe(Effect.provide(managementApi.layer)), }), linkedCache.layer, + unusedStackServices, ...(opts.stackBackend === true ? [stackBackendLayer("stack"), stackApi.layer] : []), ); @@ -2844,7 +2847,6 @@ describe("stack backend", () => { toml: '[storage.buckets.images]\npublic = true\nobjects_path = "./assets"\n', files: { "supabase/assets/a.txt": "hello" }, stackBackend: true, - stackApi: { apiEndpoint: "http://127.0.0.1:59999", serviceRoleJwt: "stack-jwt" }, routes: [ { method: "GET", match: "/storage/v1/bucket", body: [] }, { method: "POST", match: "/storage/v1/bucket", body: { name: "images" } }, @@ -2855,7 +2857,11 @@ describe("stack backend", () => { expect(Exit.isSuccess(exit)).toBe(true); expect(requests.length).toBeGreaterThan(0); expect(requests.every((r) => r.url.startsWith("http://127.0.0.1:59999"))).toBe(true); - expect(requests.every((r) => r.headers["apikey"] === "stack-jwt")).toBe(true); + expect( + requests.every( + (r) => r.headers["apikey"] === generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role"), + ), + ).toBe(true); expect( requests.some((r) => r.method === "POST" && r.url.endsWith("/storage/v1/bucket")), ).toBe(true); @@ -2913,7 +2919,7 @@ describe("stack backend", () => { (error) => error instanceof StackStorageCapabilityError, ); expect(capability).toBeInstanceOf(StackStorageCapabilityError); - expect(capability?.suggestion).toContain("-x storage"); + expect(capability?.suggestion).toContain("--exclude storage"); expect(requests).toHaveLength(0); }).pipe(Effect.provide(BunServices.layer)), ); diff --git a/apps/cli/src/commands/start/start.command.ts b/apps/cli/src/commands/start/start.command.ts index 7eb36de6a8..1047256290 100644 --- a/apps/cli/src/commands/start/start.command.ts +++ b/apps/cli/src/commands/start/start.command.ts @@ -13,6 +13,7 @@ import { dbConnectionLayer } from "../../command-internal/db-connection.layer.ts import { debugLoggerLayer } from "../../command-internal/debug-logger.layer.ts"; import { dockerRunLayer } from "../../command-internal/docker-run.layer.ts"; import { identityStitchLayer } from "../../command-internal/identity-stitch.ts"; +import { stackApiLayer } from "../../command-internal/stack-api.ts"; import { stringSliceFlag } from "../../command-internal/string-slice-flag.ts"; import { telemetryStateLayer } from "../../telemetry/telemetry-state.layer.ts"; import { withCommandTelemetry } from "../../telemetry/command-telemetry.ts"; @@ -75,7 +76,9 @@ const startRuntimeLayer = Layer.mergeAll( dbConnectionLayer, httpClient, platformApiFactory, - // `stdinLayer` satisfies `promptYesNo`'s `Stdin` requirement (seed-buckets runs + // Shared bucket seeding requires StackApi even when legacy routing is selected. + stackApiLayer, + // `stdinLayer` satisfies `promptYesNo`'s `Stdin` requirement (seed-buckets runs with // `promptless`), so `start` never reads a piped line at runtime — type requirements only. stdinLayer, ); diff --git a/apps/cli/src/commands/start/start.integration.test.ts b/apps/cli/src/commands/start/start.integration.test.ts index 10b291e610..23c2f964b2 100644 --- a/apps/cli/src/commands/start/start.integration.test.ts +++ b/apps/cli/src/commands/start/start.integration.test.ts @@ -36,6 +36,7 @@ import { sequentialExecBatch, withEnvVar, } from "../../../tests/helpers/command-mocks.ts"; +import { unusedStackServices } from "../../../tests/helpers/unused-stack.ts"; import { CliArgs } from "../../shared/cli/cli-args.service.ts"; import { classifyCliCauseActionability } from "../../shared/telemetry/error-actionability.ts"; import { @@ -491,6 +492,7 @@ function setup(opts: SetupOpts = {}) { const dbSession = fakeDbSession(); const layer = Layer.mergeAll( + unusedStackServices, BunServices.layer, out.layer, cliSettings, diff --git a/apps/cli/src/commands/start/start.string-slice-flags.integration.test.ts b/apps/cli/src/commands/start/start.string-slice-flags.integration.test.ts index 4fde3f0e64..9633110cd2 100644 --- a/apps/cli/src/commands/start/start.string-slice-flags.integration.test.ts +++ b/apps/cli/src/commands/start/start.string-slice-flags.integration.test.ts @@ -14,6 +14,7 @@ import { mockCommandPlatformApi, useTempWorkdir, } from "../../../tests/helpers/command-mocks.ts"; +import { unusedStackServices } from "../../../tests/helpers/unused-stack.ts"; import { startCommand } from "./start.command.ts"; // Malformed CSV aborts flag parsing before the handler runs, with pflag's exact @@ -41,6 +42,7 @@ function setup() { runtimeInfo: mockRuntimeInfo({ homeDir: tempRoot.current }), }); const layer = Layer.mergeAll( + unusedStackServices, runtime, CliOutput.layer(textCliOutputFormatter()), processEnvLayer({ SUPABASE_NO_KEYRING: "1" }), diff --git a/apps/cli/src/commands/storage/cp/SIDE_EFFECTS.md b/apps/cli/src/commands/storage/cp/SIDE_EFFECTS.md index 8f8cfab130..8189135c91 100644 --- a/apps/cli/src/commands/storage/cp/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/storage/cp/SIDE_EFFECTS.md @@ -56,15 +56,15 @@ backend — same roles as `storage ls`. ## Exit Codes -| Code | Condition | -| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | success | -| `1` | resolved `--workdir`/`SUPABASE_WORKDIR` doesn't exist or isn't a directory (`StorageWorkdirError`) — beats every other guard | -| `1` | an explicit `--workdir`/`SUPABASE_WORKDIR` on a LOCAL target holds no project config (`StorageMissingProjectConfigError`) | -| `1` | invalid/parse url, unsupported operation (local→local), copy-between-buckets, object-not-found (recursive download), file create/read failure, API non-2xx, network, auth, config parse | -| `1` | `--project-ref` set with `--local` (see Notes) | -| `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | -| `1` | stack backend: stack not registered/running, missing API endpoint or credentials, or the stack API is unavailable (`StackStorageUnavailableError`) | +| Code | Condition | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success | +| `1` | resolved `--workdir`/`SUPABASE_WORKDIR` doesn't exist or isn't a directory (`StorageWorkdirError`) — beats every other guard | +| `1` | an explicit `--workdir`/`SUPABASE_WORKDIR` on a LOCAL target holds no project config (`StorageMissingProjectConfigError`) | +| `1` | invalid/parse url, unsupported operation (local→local), copy-between-buckets, object-not-found (recursive download), file create/read failure, API non-2xx, network, auth, config parse | +| `1` | `--project-ref` set with `--local` (see Notes) | +| `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | +| `1` | stack backend: stack is not registered/ready, has no primary database, or the stack API is unavailable (`StackStorageUnavailableError`); a missing Storage endpoint is a capability failure (`StackStorageCapabilityError`) | ## Output @@ -106,30 +106,22 @@ backend — same roles as `storage ls`. - `--jobs`/`-j` bounds upload/download concurrency (default 1). - The runtime default for `--content-type` is `""` (empty ⇒ auto-detect). - Relative local paths resolve against the original cwd. -- **Stack backend (`--local`).** The Storage endpoint is the selected stack's API - gateway URL (`status.endpoints.api.url`) + `/storage/v1/...`; the credential is the - stack's service-role JWT read from the stack's credentials. The stack is located by - the project root (workdir realpath) via the `@supabase/stack` API, which reads stack - state under `SUPABASE_HOME`. `cp` never creates a stack. -- **Stack backend — capability policy.** Storage `disabled` (e.g. - `stack start -x storage`) errors with `StackStorageCapabilityError` ("Storage is - disabled for this stack.") with guidance to enable `[storage]`, then run - `supabase stack stop` followed by `supabase stack start` without `-x storage`; - `failed`/`stopped` raises the same error class ("Storage failed to start for this - stack"/"Storage is stopped for this stack."), with the capability error appended - when present, and guidance to run `supabase stack restart`. - `dormant`/`starting`/`ready`/`stopping` all proceed — the - gateway activates a lazily-configured Storage on the first request and holds that - request, and wakes a stopping Storage once its cleanup completes; there is no - client-side polling. A stack that is not registered, not running, missing - its API endpoint or credentials, or whose stack API is unavailable errors with - `StackStorageUnavailableError` and guidance to run `supabase stack status` or - `supabase stack restart` (or `supabase start` when the stack was never configured). A - stack-gateway 502/503 during Storage activation is reported as - `StackStorageCapabilityError` with guidance to run `supabase stack logs` then - `supabase stack restart`, instead of a raw status body — for a `--local` target only; - a `--linked` failure passes through unchanged. No HTTP request is sent when Storage is - disabled or the stack is not running. +- **Stack backend (`--local`).** The Storage endpoint comes from the selected composition's + Storage member observation, and the service-role JWT is generated from the primary database's + observed JWT secret. The stack is opened by project identity through the `@supabase/stack` API, + which reads stack state under `SUPABASE_HOME`; `cp` opens saved state without launching + the owner or any service member. +- **Stack backend — capability policy.** Storage excluded from the composition errors with + `StackStorageCapabilityError` and guidance to enable `[storage]`, then run `supabase start` + without `--exclude storage`. Failed or manually stopped Storage is unusable. Dormant, starting, and ready Storage + proceed; stopping Storage proceeds only when idle shutdown retains wake-up. The gateway + wakes lazy members on the first request. The primary database must be running and healthy + before Storage requests are attempted. A stack that is + not registered, has no primary database composition member, is not ready, or has no Storage + HTTP endpoint errors with `StackStorageUnavailableError` or `StackStorageCapabilityError`. + A stack-gateway 502/503 during Storage activation is reported as `StackStorageCapabilityError` + with guidance to inspect logs and restart; linked failures pass through unchanged. No HTTP + request is sent when Storage is disabled or the primary database is not ready. - **Secrets.** The stack's service-role JWT is never printed or logged; error messages contain only lifecycle/capability state text. - **Recursive download path traversal (accepted risk).** Recursive download writes diff --git a/apps/cli/src/commands/storage/cp/cp.integration.test.ts b/apps/cli/src/commands/storage/cp/cp.integration.test.ts index 3cc8484151..afe9b72e48 100644 --- a/apps/cli/src/commands/storage/cp/cp.integration.test.ts +++ b/apps/cli/src/commands/storage/cp/cp.integration.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, FileSystem, Option, Path } from "effect"; -import { setupStorage } from "../../../../tests/helpers/storage.ts"; +import { setupStorage, STORAGE_TEST_JWT_SECRET } from "../../../../tests/helpers/storage.ts"; import { VALID_REF, useTempWorkdir } from "../../../../tests/helpers/command-mocks.ts"; import { StackStorageCapabilityError } from "../../../command-internal/stack-storage.ts"; +import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; import { StorageUnsupportedOperationError } from "../storage.errors.ts"; import { storageCp } from "./cp.handler.ts"; import type { StorageCpFlags } from "./cp.command.ts"; @@ -653,7 +654,6 @@ describe("stack backend", () => { toml: 'project_id = "test"\n', local: true, stackBackend: true, - stackApi: { apiEndpoint: "http://127.0.0.1:59999", serviceRoleJwt: "stack-jwt" }, routes: [{ method: "POST", match: OBJECT("private/readme.md"), body: {} }], }); return Effect.gen(function* () { @@ -666,7 +666,9 @@ describe("stack backend", () => { expect(Exit.isSuccess(exit)).toBe(true); expect(requests).toHaveLength(1); expect(requests[0]?.url.startsWith("http://127.0.0.1:59999")).toBe(true); - expect(requests[0]?.headers["apikey"]).toBe("stack-jwt"); + expect(requests[0]?.headers["apikey"]).toBe( + generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role"), + ); }).pipe(Effect.provide(layer)); }); @@ -691,7 +693,7 @@ describe("stack backend", () => { (error) => error instanceof StackStorageCapabilityError, ); expect(capability).toBeInstanceOf(StackStorageCapabilityError); - expect(capability?.suggestion).toContain("-x storage"); + expect(capability?.suggestion).toContain("--exclude storage"); expect(requests).toHaveLength(0); }).pipe(Effect.provide(layer)); }, diff --git a/apps/cli/src/commands/storage/ls/SIDE_EFFECTS.md b/apps/cli/src/commands/storage/ls/SIDE_EFFECTS.md index b2a9e06b8b..a55de670a8 100644 --- a/apps/cli/src/commands/storage/ls/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/storage/ls/SIDE_EFFECTS.md @@ -54,15 +54,15 @@ Auth: `apikey` header always; `Authorization: Bearer ` unless the key is `s ## Exit Codes -| Code | Condition | -| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | success | -| `1` | resolved `--workdir`/`SUPABASE_WORKDIR` doesn't exist or isn't a directory (`StorageWorkdirError`) — beats every other guard | -| `1` | an explicit `--workdir`/`SUPABASE_WORKDIR` on a LOCAL target holds no project config (`StorageMissingProjectConfigError`) | -| `1` | invalid URL / url-parse error / API non-2xx / network / auth / config parse | -| `1` | `--project-ref` set with `--local` (see Notes) | -| `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | -| `1` | stack backend: stack not registered/running, missing API endpoint or credentials, or the stack API is unavailable (`StackStorageUnavailableError`) | +| Code | Condition | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success | +| `1` | resolved `--workdir`/`SUPABASE_WORKDIR` doesn't exist or isn't a directory (`StorageWorkdirError`) — beats every other guard | +| `1` | an explicit `--workdir`/`SUPABASE_WORKDIR` on a LOCAL target holds no project config (`StorageMissingProjectConfigError`) | +| `1` | invalid URL / url-parse error / API non-2xx / network / auth / config parse | +| `1` | `--project-ref` set with `--local` (see Notes) | +| `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | +| `1` | stack backend: stack is not registered/ready, has no primary database, or the stack API is unavailable (`StackStorageUnavailableError`); a missing Storage endpoint is a capability failure (`StackStorageCapabilityError`) | ## Output @@ -106,29 +106,21 @@ No custom storage telemetry events. unique tree-wide and `seed` already owns `linked`/`local`; the only behavioural cost is that they must follow the subcommand token (`storage ls --local`, not `storage --local ls`) — the same shape the `db` family uses. -- **Stack backend (`--local`).** The Storage endpoint is the selected stack's API - gateway URL (`status.endpoints.api.url`) + `/storage/v1/...`; the credential is the - stack's service-role JWT read from the stack's credentials. The stack is located by - the project root (workdir realpath) via the `@supabase/stack` API, which reads stack - state under `SUPABASE_HOME`. `ls` never creates a stack. -- **Stack backend — capability policy.** Storage `disabled` (e.g. - `stack start -x storage`) errors with `StackStorageCapabilityError` ("Storage is - disabled for this stack.") with guidance to enable `[storage]`, then run - `supabase stack stop` followed by `supabase stack start` without `-x storage`; - `failed`/`stopped` raises the same error class ("Storage failed to start for this - stack"/"Storage is stopped for this stack."), with the capability error appended - when present, and guidance to run `supabase stack restart`. - `dormant`/`starting`/`ready`/`stopping` all proceed — the - gateway activates a lazily-configured Storage on the first request and holds that - request, and wakes a stopping Storage once its cleanup completes; there is no - client-side polling. A stack that is not registered, not running, missing - its API endpoint or credentials, or whose stack API is unavailable errors with - `StackStorageUnavailableError` and guidance to run `supabase stack status` or - `supabase stack restart` (or `supabase start` when the stack was never configured). A - stack-gateway 502/503 during Storage activation is reported as - `StackStorageCapabilityError` with guidance to run `supabase stack logs` then - `supabase stack restart`, instead of a raw status body — for a `--local` target only; - a `--linked` failure passes through unchanged. No HTTP request is sent when Storage is - disabled or the stack is not running. +- **Stack backend (`--local`).** The Storage endpoint comes from the selected composition's + Storage member observation, and the service-role JWT is generated from the primary database's + observed JWT secret. The stack is opened by project identity through the `@supabase/stack` API, + which reads stack state under `SUPABASE_HOME`; `ls` opens saved state without launching + the owner or any service member. +- **Stack backend — capability policy.** Storage excluded from the composition errors with + `StackStorageCapabilityError` and guidance to enable `[storage]`, then run `supabase start` + without `--exclude storage`. Failed or manually stopped Storage is unusable. Dormant, starting, and ready Storage + proceed; stopping Storage proceeds only when idle shutdown retains wake-up. The gateway + wakes lazy members on the first request. The primary database must be running and healthy + before Storage requests are attempted. A stack that is + not registered, has no primary database composition member, is not ready, or has no Storage + HTTP endpoint errors with `StackStorageUnavailableError` or `StackStorageCapabilityError`. + A stack-gateway 502/503 during Storage activation is reported as `StackStorageCapabilityError` + with guidance to inspect logs and restart; linked failures pass through unchanged. No HTTP + request is sent when Storage is disabled or the primary database is not ready. - **Secrets.** The stack's service-role JWT is never printed or logged; error messages contain only lifecycle/capability state text. diff --git a/apps/cli/src/commands/storage/ls/ls.integration.test.ts b/apps/cli/src/commands/storage/ls/ls.integration.test.ts index a8fbb4290a..f3dad5fc62 100644 --- a/apps/cli/src/commands/storage/ls/ls.integration.test.ts +++ b/apps/cli/src/commands/storage/ls/ls.integration.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, FileSystem, Option, Path } from "effect"; import { VALID_REF, useTempWorkdir, withEnvVar } from "../../../../tests/helpers/command-mocks.ts"; -import { setupStorage } from "../../../../tests/helpers/storage.ts"; +import { setupStorage, STORAGE_TEST_JWT_SECRET } from "../../../../tests/helpers/storage.ts"; import { StackStorageCapabilityError, StackStorageUnavailableError, @@ -11,6 +11,7 @@ import { import { StorageGatewayStatusError } from "../../../command-internal/storage-gateway.errors.ts"; import { storageLs } from "./ls.handler.ts"; import type { StorageLsFlags } from "./ls.command.ts"; +import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; const BUCKET = "/storage/v1/bucket"; const LIST = (bucket: string) => `/storage/v1/object/list/${bucket}`; @@ -454,7 +455,6 @@ describe("stack backend", () => { toml: 'project_id = "test"\n[api]\nport = 65000\n', local: true, stackBackend: true, - stackApi: { apiEndpoint: "http://127.0.0.1:59999", serviceRoleJwt: "stack-jwt" }, routes: [{ method: "GET", match: BUCKET, body: [{ name: "test", id: "test" }] }], }); return Effect.gen(function* () { @@ -462,8 +462,9 @@ describe("stack backend", () => { expect(Exit.isSuccess(exit)).toBe(true); expect(requests).toHaveLength(1); expect(requests[0]?.url.startsWith("http://127.0.0.1:59999")).toBe(true); - expect(requests[0]?.headers["apikey"]).toBe("stack-jwt"); - expect(requests[0]?.headers["authorization"]).toBe("Bearer stack-jwt"); + const serviceRoleJwt = generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role"); + expect(requests[0]?.headers["apikey"]).toBe(serviceRoleJwt); + expect(requests[0]?.headers["authorization"]).toBe(`Bearer ${serviceRoleJwt}`); }); }); @@ -474,7 +475,6 @@ describe("stack backend", () => { toml: 'project_id = "test"\n[api]\nport = 65000\n', local: true, stackBackend: true, - stackApi: { apiEndpoint: "http://127.0.0.1:59999", serviceRoleJwt: "stack-jwt" }, files: { "supabase/.env": "SUPABASE_API_PORT=65001\n" + @@ -492,7 +492,9 @@ describe("stack backend", () => { expect(url).not.toContain("65001"); expect(url).not.toContain("legacy.invalid"); expect(url.startsWith("http://127.0.0.1:59999")).toBe(true); - expect(requests[0]?.headers["apikey"]).toBe("stack-jwt"); + expect(requests[0]?.headers["apikey"]).toBe( + generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role"), + ); }); }, ); @@ -513,7 +515,7 @@ describe("stack backend", () => { (error) => error instanceof StackStorageCapabilityError, ); expect(capability).toBeInstanceOf(StackStorageCapabilityError); - expect(capability?.suggestion).toContain("-x storage"); + expect(capability?.suggestion).toContain("--exclude storage"); expect(requests).toHaveLength(0); }); }, @@ -560,58 +562,55 @@ describe("stack backend", () => { }); }); - it.live( - "fails with StackStorageUnavailableError when the StackApi service is unavailable", - () => { - const { layer, requests } = setupStorage(tmp.current, { - toml: 'project_id = "test"\n', - local: true, - stackBackend: true, - stackApi: { present: false }, - }); - return Effect.gen(function* () { - const exit = yield* storageLs(lsFlags()).pipe(Effect.provide(layer), Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(Cause.pretty(exit.cause)).toContain("StackStorageUnavailableError"); - } - expect(requests).toHaveLength(0); - }); - }, - ); + it.live("fails when the required StackApi service is unavailable", () => { + const { layer, requests } = setupStorage(tmp.current, { + toml: 'project_id = "test"\n', + local: true, + stackBackend: true, + stackApi: { present: false }, + }); + return Effect.gen(function* () { + const exit = yield* storageLs(lsFlags()).pipe(Effect.provide(layer), Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Cause.pretty(exit.cause)).toContain("Service not found: supabase/stack/StackApi"); + } + expect(requests).toHaveLength(0); + }); + }); it.live( - "fails with StackStorageUnavailableError when the stack exposes no API credentials", + "derives Storage credentials from the primary database when Auth credentials are unavailable", () => { const { layer, requests } = setupStorage(tmp.current, { toml: 'project_id = "test"\n', local: true, stackBackend: true, - stackApi: { omitApiCredentials: true }, + routes: [{ method: "GET", match: BUCKET, body: [{ name: "test", id: "test" }] }], }); return Effect.gen(function* () { const exit = yield* storageLs(lsFlags()).pipe(Effect.provide(layer), Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - expect(Cause.pretty(exit.cause)).toContain("StackStorageUnavailableError"); - } - expect(requests).toHaveLength(0); + expect(Exit.isSuccess(exit)).toBe(true); + expect(requests).toHaveLength(1); + expect(requests[0]?.headers["apikey"]).toBe( + generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role"), + ); }); }, ); - it.live("fails with StackStorageUnavailableError when the stack exposes no api endpoint", () => { + it.live("fails with StackStorageCapabilityError when the stack exposes no api endpoint", () => { const { layer, requests } = setupStorage(tmp.current, { toml: 'project_id = "test"\n', local: true, stackBackend: true, - stackApi: { apiEndpoint: null }, + stackApi: { storageEndpoint: false }, }); return Effect.gen(function* () { const exit = yield* storageLs(lsFlags()).pipe(Effect.provide(layer), Effect.exit); expect(Exit.isFailure(exit)).toBe(true); if (Exit.isFailure(exit)) { - expect(Cause.pretty(exit.cause)).toContain("StackStorageUnavailableError"); + expect(Cause.pretty(exit.cause)).toContain("StackStorageCapabilityError"); } expect(requests).toHaveLength(0); }); @@ -650,6 +649,38 @@ describe("stack backend", () => { }); }); + it.live("proceeds to the gateway while Storage is stopping with wake retained", () => { + const { layer, out } = setupStorage(tmp.current, { + toml: 'project_id = "test"\n', + local: true, + stackBackend: true, + stackApi: { storageState: "stopping" }, + routes: [{ method: "GET", match: BUCKET, body: [{ name: "test", id: "test" }] }], + }); + return Effect.gen(function* () { + const exit = yield* storageLs(lsFlags()).pipe(Effect.provide(layer), Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + expect(out.stdoutText).toBe("test/\n"); + }); + }); + + it.live("fails when Storage is stopping after a manual stop disabled wake", () => { + const { layer, requests } = setupStorage(tmp.current, { + toml: 'project_id = "test"\n', + local: true, + stackBackend: true, + stackApi: { storageState: "stopping", storageWakeEnabled: false }, + }); + return Effect.gen(function* () { + const exit = yield* storageLs(lsFlags()).pipe(Effect.provide(layer), Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + expect(Cause.pretty(exit.cause)).toContain("Storage is stopped"); + } + expect(requests).toHaveLength(0); + }); + }); + it.live( "maps a gateway 503 (Storage still activating) to StackStorageCapabilityError under the stack backend", () => { @@ -749,7 +780,7 @@ describe("stack backend", () => { }, ); - it.live("suggests stopping and starting the stack when Storage is disabled", () => { + it.live("suggests starting without excluding Storage when it is disabled", () => { const { layer } = setupStorage(tmp.current, { toml: 'project_id = "test"\n', local: true, @@ -763,8 +794,7 @@ describe("stack backend", () => { (error) => error instanceof StackStorageCapabilityError, ); expect(capability).toBeInstanceOf(StackStorageCapabilityError); - expect(capability?.suggestion).toContain("supabase stack stop"); - expect(capability?.suggestion).toContain("supabase stack start without -x storage"); + expect(capability?.suggestion).toContain("supabase start without --exclude storage"); expect(capability?.suggestion).not.toContain("supabase stack restart"); }); }); diff --git a/apps/cli/src/commands/storage/mv/SIDE_EFFECTS.md b/apps/cli/src/commands/storage/mv/SIDE_EFFECTS.md index 3673d1b011..6303501305 100644 --- a/apps/cli/src/commands/storage/mv/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/storage/mv/SIDE_EFFECTS.md @@ -51,15 +51,15 @@ backend — same roles as `storage ls`. ## Exit Codes -| Code | Condition | -| ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | success | -| `1` | resolved `--workdir`/`SUPABASE_WORKDIR` doesn't exist or isn't a directory (`StorageWorkdirError`) — beats every other guard | -| `1` | an explicit `--workdir`/`SUPABASE_WORKDIR` on a LOCAL target holds no project config (`StorageMissingProjectConfigError`) | -| `1` | invalid/parse url, missing object path (both roots), cross-bucket move, object-not-found (recursive empty), API non-2xx, network, auth, config parse | -| `1` | `--project-ref` set with `--local` (see Notes) | -| `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | -| `1` | stack backend: stack not registered/running, missing API endpoint or credentials, or the stack API is unavailable (`StackStorageUnavailableError`) | +| Code | Condition | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | success | +| `1` | resolved `--workdir`/`SUPABASE_WORKDIR` doesn't exist or isn't a directory (`StorageWorkdirError`) — beats every other guard | +| `1` | an explicit `--workdir`/`SUPABASE_WORKDIR` on a LOCAL target holds no project config (`StorageMissingProjectConfigError`) | +| `1` | invalid/parse url, missing object path (both roots), cross-bucket move, object-not-found (recursive empty), API non-2xx, network, auth, config parse | +| `1` | `--project-ref` set with `--local` (see Notes) | +| `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | +| `1` | stack backend: stack is not registered/ready, has no primary database, or the stack API is unavailable (`StackStorageUnavailableError`); a missing Storage endpoint is a capability failure (`StackStorageCapabilityError`) | ## Output @@ -96,30 +96,22 @@ backend — same roles as `storage ls`. rather than a silently discarded flag. - Both `src` and `dst` must be `ss://` URLs (a stricter parse than the lenient one `cp` uses). -- **Stack backend (`--local`).** The Storage endpoint is the selected stack's API - gateway URL (`status.endpoints.api.url`) + `/storage/v1/...`; the credential is the - stack's service-role JWT read from the stack's credentials. The stack is located by - the project root (workdir realpath) via the `@supabase/stack` API, which reads stack - state under `SUPABASE_HOME`. `mv` never creates a stack. -- **Stack backend — capability policy.** Storage `disabled` (e.g. - `stack start -x storage`) errors with `StackStorageCapabilityError` ("Storage is - disabled for this stack.") with guidance to enable `[storage]`, then run - `supabase stack stop` followed by `supabase stack start` without `-x storage`; - `failed`/`stopped` raises the same error class ("Storage failed to start for this - stack"/"Storage is stopped for this stack."), with the capability error appended - when present, and guidance to run `supabase stack restart`. - `dormant`/`starting`/`ready`/`stopping` all proceed — the - gateway activates a lazily-configured Storage on the first request and holds that - request, and wakes a stopping Storage once its cleanup completes; there is no - client-side polling. A stack that is not registered, not running, missing - its API endpoint or credentials, or whose stack API is unavailable errors with - `StackStorageUnavailableError` and guidance to run `supabase stack status` or - `supabase stack restart` (or `supabase start` when the stack was never configured). A - stack-gateway 502/503 during Storage activation is reported as - `StackStorageCapabilityError` with guidance to run `supabase stack logs` then - `supabase stack restart`, instead of a raw status body — for a `--local` target only; - a `--linked` failure passes through unchanged. No HTTP request is sent when Storage is - disabled or the stack is not running. +- **Stack backend (`--local`).** The Storage endpoint comes from the selected composition's + Storage member observation, and the service-role JWT is generated from the primary database's + observed JWT secret. The stack is opened by project identity through the `@supabase/stack` API, + which reads stack state under `SUPABASE_HOME`; `mv` opens saved state without launching + the owner or any service member. +- **Stack backend — capability policy.** Storage excluded from the composition errors with + `StackStorageCapabilityError` and guidance to enable `[storage]`, then run `supabase start` + without `--exclude storage`. Failed or manually stopped Storage is unusable. Dormant, starting, and ready Storage + proceed; stopping Storage proceeds only when idle shutdown retains wake-up. The gateway + wakes lazy members on the first request. The primary database must be running and healthy + before Storage requests are attempted. A stack that is + not registered, has no primary database composition member, is not ready, or has no Storage + HTTP endpoint errors with `StackStorageUnavailableError` or `StackStorageCapabilityError`. + A stack-gateway 502/503 during Storage activation is reported as `StackStorageCapabilityError` + with guidance to inspect logs and restart; linked failures pass through unchanged. No HTTP + request is sent when Storage is disabled or the primary database is not ready. - **Secrets.** The stack's service-role JWT is never printed or logged; error messages contain only lifecycle/capability state text. - The cross-bucket and missing-path checks run before any network call. diff --git a/apps/cli/src/commands/storage/mv/mv.integration.test.ts b/apps/cli/src/commands/storage/mv/mv.integration.test.ts index 8ced83ab7c..eb1f67601c 100644 --- a/apps/cli/src/commands/storage/mv/mv.integration.test.ts +++ b/apps/cli/src/commands/storage/mv/mv.integration.test.ts @@ -2,7 +2,8 @@ import { describe, expect, it } from "@effect/vitest"; import { Cause, Effect, Exit, Option } from "effect"; import { StackStorageCapabilityError } from "../../../command-internal/stack-storage.ts"; -import { setupStorage } from "../../../../tests/helpers/storage.ts"; +import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; +import { setupStorage, STORAGE_TEST_JWT_SECRET } from "../../../../tests/helpers/storage.ts"; import { VALID_REF, useTempWorkdir } from "../../../../tests/helpers/command-mocks.ts"; import { storageMv } from "./mv.handler.ts"; import type { StorageMvFlags } from "./mv.command.ts"; @@ -402,7 +403,6 @@ describe("stack backend", () => { toml: 'project_id = "test"\n', local: true, stackBackend: true, - stackApi: { apiEndpoint: "http://127.0.0.1:59999", serviceRoleJwt: "stack-jwt" }, routes: [{ method: "POST", match: MOVE, body: { message: "Successfully moved" } }], }); return Effect.gen(function* () { @@ -413,7 +413,7 @@ describe("stack backend", () => { expect(out.stderrText).toContain("Successfully moved"); const move = requests.find((r) => r.url.includes(MOVE)); expect(move?.url.startsWith("http://127.0.0.1:59999")).toBe(true); - expect(move?.headers["apikey"]).toBe("stack-jwt"); + expect(move?.headers["apikey"]).toBe(generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role")); }); }); @@ -438,7 +438,7 @@ describe("stack backend", () => { .map((reason) => reason.error) .find((error) => error instanceof StackStorageCapabilityError); expect(capability).toBeDefined(); - expect(capability?.suggestion).toContain("-x storage"); + expect(capability?.suggestion).toContain("--exclude storage"); } expect(requests).toHaveLength(0); }); diff --git a/apps/cli/src/commands/storage/rm/SIDE_EFFECTS.md b/apps/cli/src/commands/storage/rm/SIDE_EFFECTS.md index bb89b92f2c..0080ac45df 100644 --- a/apps/cli/src/commands/storage/rm/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/storage/rm/SIDE_EFFECTS.md @@ -64,7 +64,7 @@ read from the shell env OR the project `.env`/`.env.local`/`.env.[.local]` | `1` | invalid/parse url, missing bucket (root path), missing `-r` flag (directory or no args), object-not-found (recursive empty prefix), API non-2xx, network, auth, config parse | | `1` | `--project-ref` set with `--local` (see Notes) | | `1` | stack backend: Storage disabled or its stack `failed`/`stopped` (`StackStorageCapabilityError`) | -| `1` | stack backend: stack not registered/running, missing API endpoint or credentials, or the stack API is unavailable (`StackStorageUnavailableError`) | +| `1` | stack backend: stack is not registered/ready, has no primary database, or the stack API is unavailable (`StackStorageUnavailableError`); a missing Storage endpoint is a capability failure (`StackStorageCapabilityError`) | ## Output @@ -120,29 +120,21 @@ Requires `--yes`/`SUPABASE_YES` on the same terms as `json`. - Explicit deletes are attempted first ("in case the paths resolve to extensionless files"); prefixes not returned as removed are then walked recursively when `-r` is set. - Object deletes are chunked at `DELETE_OBJECTS_LIMIT` (1000) per request. -- **Stack backend (`--local`).** The Storage endpoint is the selected stack's API - gateway URL (`status.endpoints.api.url`) + `/storage/v1/...`; the credential is the - stack's service-role JWT read from the stack's credentials. The stack is located by - the project root (workdir realpath) via the `@supabase/stack` API, which reads stack - state under `SUPABASE_HOME`. `rm` never creates a stack. -- **Stack backend — capability policy.** Storage `disabled` (e.g. - `stack start -x storage`) errors with `StackStorageCapabilityError` ("Storage is - disabled for this stack.") with guidance to enable `[storage]`, then run - `supabase stack stop` followed by `supabase stack start` without `-x storage`; - `failed`/`stopped` raises the same error class ("Storage failed to start for this - stack"/"Storage is stopped for this stack."), with the capability error appended - when present, and guidance to run `supabase stack restart`. - `dormant`/`starting`/`ready`/`stopping` all proceed — the - gateway activates a lazily-configured Storage on the first request and holds that - request, and wakes a stopping Storage once its cleanup completes; there is no - client-side polling. A stack that is not registered, not running, missing - its API endpoint or credentials, or whose stack API is unavailable errors with - `StackStorageUnavailableError` and guidance to run `supabase stack status` or - `supabase stack restart` (or `supabase start` when the stack was never configured). A - stack-gateway 502/503 during Storage activation is reported as - `StackStorageCapabilityError` with guidance to run `supabase stack logs` then - `supabase stack restart`, instead of a raw status body — for a `--local` target only; - a `--linked` failure passes through unchanged. No HTTP request is sent when Storage is - disabled or the stack is not running. +- **Stack backend (`--local`).** The Storage endpoint comes from the selected composition's + Storage member observation, and the service-role JWT is generated from the primary database's + observed JWT secret. The stack is opened by project identity through the `@supabase/stack` API, + which reads stack state under `SUPABASE_HOME`; `rm` opens saved state without launching + the owner or any service member. +- **Stack backend — capability policy.** Storage excluded from the composition errors with + `StackStorageCapabilityError` and guidance to enable `[storage]`, then run `supabase start` + without `--exclude storage`. Failed or manually stopped Storage is unusable. Dormant, starting, and ready Storage + proceed; stopping Storage proceeds only when idle shutdown retains wake-up. The gateway + wakes lazy members on the first request. The primary database must be running and healthy + before Storage requests are attempted. A stack that is + not registered, has no primary database composition member, is not ready, or has no Storage + HTTP endpoint errors with `StackStorageUnavailableError` or `StackStorageCapabilityError`. + A stack-gateway 502/503 during Storage activation is reported as `StackStorageCapabilityError` + with guidance to inspect logs and restart; linked failures pass through unchanged. No HTTP + request is sent when Storage is disabled or the primary database is not ready. - **Secrets.** The stack's service-role JWT is never printed or logged; error messages contain only lifecycle/capability state text. diff --git a/apps/cli/src/commands/storage/rm/rm.integration.test.ts b/apps/cli/src/commands/storage/rm/rm.integration.test.ts index b81ec1af6d..2ae7abf059 100644 --- a/apps/cli/src/commands/storage/rm/rm.integration.test.ts +++ b/apps/cli/src/commands/storage/rm/rm.integration.test.ts @@ -4,9 +4,10 @@ import { Cause, Effect, Exit, FileSystem, Option, Path } from "effect"; import { DbConfigLoadError } from "../../../command-internal/db-config.errors.ts"; import { StackStorageCapabilityError } from "../../../command-internal/stack-storage.ts"; +import { generateGoJwt } from "../../../command-internal/go-jwt.ts"; import { ProjectRefNotLinkedError } from "../../../config/project-ref.errors.ts"; import { StorageRmConfirmationRequiredError } from "../storage.errors.ts"; -import { setupStorage } from "../../../../tests/helpers/storage.ts"; +import { setupStorage, STORAGE_TEST_JWT_SECRET } from "../../../../tests/helpers/storage.ts"; import { VALID_REF, useTempWorkdir, withEnvVar } from "../../../../tests/helpers/command-mocks.ts"; import { storageRm } from "./rm.handler.ts"; @@ -870,7 +871,6 @@ describe("stack backend", () => { toml: 'project_id = "test"\n', local: true, stackBackend: true, - stackApi: { apiEndpoint: "http://127.0.0.1:59999", serviceRoleJwt: "stack-jwt" }, confirm: [true], routes: [{ method: "DELETE", match: DELETE_OBJECT("private"), body: [{ name: "a.pdf" }] }], }); @@ -887,7 +887,7 @@ describe("stack backend", () => { (r) => r.method === "DELETE" && r.url.includes(DELETE_OBJECT("private")), ); expect(del?.url.startsWith("http://127.0.0.1:59999")).toBe(true); - expect(del?.headers["apikey"]).toBe("stack-jwt"); + expect(del?.headers["apikey"]).toBe(generateGoJwt(STORAGE_TEST_JWT_SECRET, "service_role")); }); }); @@ -937,7 +937,7 @@ describe("stack backend", () => { .map((reason) => reason.error) .find((error) => error instanceof StackStorageCapabilityError); expect(capability).toBeDefined(); - expect(capability?.suggestion).toContain("-x storage"); + expect(capability?.suggestion).toContain("--exclude storage"); } expect(requests).toHaveLength(0); // The confirm prompt is a `--yes`-only bucket-deletion notice; disabled storage diff --git a/apps/cli/src/commands/test/db/SIDE_EFFECTS.md b/apps/cli/src/commands/test/db/SIDE_EFFECTS.md index 89039a727d..c870ff5185 100644 --- a/apps/cli/src/commands/test/db/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/test/db/SIDE_EFFECTS.md @@ -10,6 +10,7 @@ | `/supabase/.temp/pooler-url` | text | `--linked` pooler fallback only — the connection-pooler URL written by `supabase link` (read from this file rather than from config.toml) | | `~/.supabase/access-token` | text | `--linked` only, when `SUPABASE_ACCESS_TOKEN` unset | | `/supabase/.temp/project-ref` | text | `--linked` only, to resolve the ref — skipped when `--project-ref` (or `SUPABASE_PROJECT_ID`) is set | +| `/stacks/*/state.json` | JSON | managed `--local`: discover the project stack and open its saved definition | ## Files Written @@ -30,11 +31,11 @@ Compose `--local` / `--linked` / `--db-url` run one-shot `docker run --rm `, where the image is `supabase/pg_prove:3.36` resolved through the registry (`getRegistryImageUrl`): `SUPABASE_INTERNAL_IMAGE_REGISTRY` overrides the registry, `docker.io` pulls from Docker Hub unchanged, and the default is `public.ecr.aws/supabase/pg_prove:3.36`. -Stack prove uses catalog `pg_prove` via `runPostgresClient` (same path as dump): native prepends `artifact/bin`; container or no-native-artifact platforms run a one-shot of the digest-pinned catalog Postgres image. That pin is not rewritten through `SUPABASE_INTERNAL_IMAGE_REGISTRY`. Missing `pg_prove` fails closed. There is no PATH fallback. +Managed stack `--local` prove runs the catalog `pg_prove` tool through the owning stack runtime with runtime-facing host and port, preserving the CLI resolver’s database user, password, and database. External `--db-url`/`--linked` prove uses the catalog Postgres client: native prepends `artifact/bin`; container or no-native-artifact platforms run a one-shot of the digest-pinned catalog Postgres image. That pin is not rewritten through `SUPABASE_INTERNAL_IMAGE_REGISTRY`. Missing `pg_prove` fails closed. There is no PATH fallback. - `-v ::ro` for each test path on Compose and on catalog container prove. A path that is a **file** is mounted via its **containing directory** (not the lone file) so that psql `\ir`/`\i` includes — which resolve relative to the test file's own directory — find their sibling files inside the container (CLI-1139). Directory paths are mounted as-is. Mounts are deduped by container target, so multiple files in the same directory produce a single `-v`. The full file path is still passed to `pg_prove`, so only the requested file runs. Native prove reads host paths directly (no bind). - `--security-opt label:disable` on Compose and catalog container prove (omitted in Bitbucket Pipelines) -- `--network supabase_network_` (Compose `--local`) with env `PGHOST=db PGPORT=5432`, or `--network host` (db-url / linked, and every stack-backend prove) with the published host/port. Stack `--local` never uses `PGHOST=db`. Native stack prove rewrites only local loopback to `127.0.0.1`; remote native keeps the resolved host. Catalog container prove uses the dump host rewrite (`host.docker.internal`, except Linux host-network loopback). `` is sanitized (`sanitizeProjectId`), so an invalid configured value (e.g. `"my project"`) joins the same network the local stack created +- `--network supabase_network_` (Compose `--local`) with env `PGHOST=db PGPORT=5432`. External `--db-url`/`--linked` runs use host networking with the resolved host and port. Managed stack proves use the owner-managed tool with runtime credentials and its runtime-specific attachment; they do not use the Compose network or `PGHOST=db`. `` is sanitized (`sanitizeProjectId`), so an invalid configured value (e.g. `"my project"`) joins the same network the local stack created - `-e PGHOST/PGPORT/PGUSER/PGPASSWORD/PGDATABASE` - cmd `pg_prove --ext .pg --ext .sql -r [--verbose]` (`--verbose` when `--debug`) @@ -59,14 +60,14 @@ Stack prove uses catalog `pg_prove` via `runPostgresClient` (same path as dump): ## Exit Codes -| Code | Condition | -| ---- | ------------------------------------------------------------------------------------------------------------------------------------ | -| `0` | all pgTAP tests pass | -| `1` | `pg_prove` exits non-zero (test failures) — `error running container: exit N`, or `error running pg_prove: exit N` on a native stack | -| `1` | `pg_prove` ran no tests (`Result: NOTESTS`) — `no pgTAP tests found in `; Go exits `0` here | -| `1` | `--db-url` / `--linked` / `--local` set together (mutually exclusive) | -| `1` | database connection failure / pgTAP enable failure / docker failure / `--linked` auth or IPv6 errors | -| `1` | `--project-ref` set with a resolved target other than linked (see Notes) | +| Code | Condition | +| ---- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | all pgTAP tests pass | +| `1` | `pg_prove` exits non-zero (test failures) — `error running container: exit N`, or `error running pg_prove: exit N` on a managed stack or native external client | +| `1` | `pg_prove` ran no tests (`Result: NOTESTS`) — `no pgTAP tests found in `; Go exits `0` here | +| `1` | `--db-url` / `--linked` / `--local` set together (mutually exclusive) | +| `1` | database connection failure / pgTAP enable failure / docker failure / `--linked` auth or IPv6 errors | +| `1` | `--project-ref` set with a resolved target other than linked (see Notes) | ## Telemetry Events Fired @@ -78,7 +79,7 @@ Stack prove uses catalog `pg_prove` via `runPostgresClient` (same path as dump): command's flag config and handler verbatim, but records `command: "db test"` instead, since each command's own telemetry wrapper records its own invocation path even though the underlying handler is the same function. Driven by -`commandPath` in `../../../shared/test-db.layers.ts`'s +`commandPath` in `../../../command-internal/test-db.layers.ts`'s `testDbRuntimeLayer` factory — each `.command.ts` passes its own actual invocation path. @@ -104,7 +105,7 @@ command (exit 1). ## Notes - Native TypeScript port (Phase 1+); no Go proxy. Hidden command. -- Stack `test db` uses catalog `pg_prove` (same `runPostgresClient` path as dump) for native and container runtimes. Compose stays on `supabase/pg_prove:3.36`. There is no PATH fallback or host major-version check. +- Stack `test db` runs catalog `pg_prove` through the owning stack runtime with runtime credentials for native and container runtimes. External targets use the catalog Postgres client. Compose stays on `supabase/pg_prove:3.36`. There is no PATH fallback or host major-version check. - **`--project-ref`** (TS-only, no Go equivalent on any user-facing command; shared verbatim by `db test` via `testDbConfig`) overrides ONLY the linked-ref resolution used for the connection (flag > `SUPABASE_PROJECT_ID` > diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index f70502d2d3..66a5498a1e 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -1,13 +1,10 @@ #!/usr/bin/env bun import { + runHostProcessIfDispatched, runNativeProcessIfDispatched, - runSupervisorProcessIfDispatched, -} from "@supabase/stack/internal/supervisor"; +} from "@supabase/stack/internal/dispatch"; const argv = process.argv.slice(2); -if ( - !(await runSupervisorProcessIfDispatched(argv)) && - !(await runNativeProcessIfDispatched(argv)) -) { +if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispatched(argv))) { await import("./cli/main.ts"); } diff --git a/apps/cli/src/shared/init/project-init.templates.ts b/apps/cli/src/shared/init/project-init.templates.ts index 683a44fc3a..7d9a266f2c 100644 --- a/apps/cli/src/shared/init/project-init.templates.ts +++ b/apps/cli/src/shared/init/project-init.templates.ts @@ -502,7 +502,10 @@ function applyExperimentalStackInitTemplate(source: string): string { for (const block of STACK_INIT_OMITTED_PORT_BLOCKS) { next = next.replace(block, ""); } - return next; + return next.replace( + /^(openai_api_key|s3_host|s3_region|s3_access_key|s3_secret_key) = /gm, + "# $&", + ); } export function renderCliConfigTemplate( diff --git a/apps/cli/src/shared/output/types.ts b/apps/cli/src/shared/output/types.ts index 6c935aa289..e6eb28cbb9 100644 --- a/apps/cli/src/shared/output/types.ts +++ b/apps/cli/src/shared/output/types.ts @@ -9,6 +9,7 @@ export type StreamEvent = } | { readonly type: "log-entry"; + readonly instance_id?: string; readonly timestamp: string; readonly service: string; readonly stream: "stdout" | "stderr" | "internal"; diff --git a/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts b/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts index 830acf41c8..691615937a 100644 --- a/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts +++ b/apps/cli/src/shared/runtime/stack-e2e-cleanup.unit.test.ts @@ -216,7 +216,7 @@ describe("stack e2e cleanup manager", () => { captureSnapshot: () => ({ managedStacksRootExists: true, documentFiles: [], - stackDirs: ["/tmp/home/managed/stacks/stack-id"], + stackDirs: ["/tmp/home/stacks/stack-id"], trackedPids: [], }), }), diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index f21549be84..49a2bd1fe7 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -267,6 +267,7 @@ FunctionsNewFileExistsError FunctionsNewInvalidSlugError FunctionsNewWorkdirError FunctionsNewWriteError +FunctionsServeStackError GenBearerJwtConfigParseError GenBearerJwtDecodeError GenBearerJwtKeyNotFoundError @@ -505,8 +506,11 @@ SsoUpdateMetadataFileError SsoUpdateNetworkError SsoUpdateNotFoundError SsoUpdateUnexpectedStatusError +StackBootstrapError StackCatalogSetupError StackConfigError +StackFunctionsBundleError +StackFunctionsEnvError StackNativeEngineError StackRoutingError StackRuntimeUnavailableError diff --git a/apps/cli/tests/helpers/stack-e2e-cleanup.ts b/apps/cli/tests/helpers/stack-e2e-cleanup.ts index ff4afaaacf..24854055a6 100644 --- a/apps/cli/tests/helpers/stack-e2e-cleanup.ts +++ b/apps/cli/tests/helpers/stack-e2e-cleanup.ts @@ -286,7 +286,7 @@ async function cleanupProject( function captureSnapshot(projectDir: string, homeDir?: string): StackRuntimeSnapshot { const normalized = normalizeDir(projectDir); const managedStacksRoot = - homeDir === undefined ? undefined : path.join(normalizeDir(homeDir), "managed", "stacks"); + homeDir === undefined ? undefined : path.join(normalizeDir(homeDir), "stacks"); if (managedStacksRoot === undefined || !existsSync(managedStacksRoot)) { return { managedStacksRootExists: false, diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 5b9226e19d..6db2d9d2e9 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -5,14 +5,12 @@ import { BunServices } from "@effect/platform-bun"; import { Effect, Layer, Option, Redacted, Stream } from "effect"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; -import { - CAPABILITY_NAMES, - InvalidProjectRootError, - StackIdSchema, - StackNotFoundError, - type CapabilityState, - type EffectStack, - type StackLifecycle, +import type { + DatabaseInstance, + Observation, + ServiceCreation, + ServiceInstance, + Stack, } from "@supabase/stack/effect"; import { CliArgs } from "../../src/shared/cli/cli-args.service.ts"; @@ -25,6 +23,7 @@ import { StackApi } from "../../src/command-internal/stack-api.ts"; import { stackBackendLayer } from "../../src/command-internal/stack-backend.ts"; import type { OutputFormat } from "../../src/shared/output/types.ts"; import { mockOutput, mockRuntimeInfo, mockStdin, mockTty } from "./mocks.ts"; +import { unusedStackServices } from "./unused-stack.ts"; import { VALID_REF, jsonResponse, @@ -35,28 +34,15 @@ import { mockTelemetryStateTracked, } from "./command-mocks.ts"; -/** - * One Storage gateway / Management API mock route. - * - * Routes match in registration order and are consumed once matched, so pagination - * or recursive flows need one route per expected call unless `persist` is set. - * `when` narrows a match by the parsed request body. - */ interface StorageRoute { readonly method: string; - /** Substring matched against the request URL. */ readonly match: string; readonly status?: number; - /** Response JSON body. */ readonly body?: unknown; - /** Raw (non-JSON) response body, for streamed object downloads. */ readonly rawBody?: string; - /** When true, fail with a transport error instead of responding. */ readonly transport?: boolean; readonly transportDescription?: string; - /** Narrow the match by the parsed request body. */ readonly when?: (reqBody: unknown) => boolean; - /** Keep this route matchable on every call instead of consuming it. */ readonly persist?: boolean; } @@ -72,158 +58,223 @@ export interface SetupStorageOptions { readonly routes?: ReadonlyArray; readonly files?: Readonly>; readonly format?: OutputFormat; - /** Routing is driven by the `local` field on each command's flags, not here. */ readonly local?: boolean; readonly yes?: boolean; readonly confirm?: ReadonlyArray; readonly promptConfirmFail?: boolean; - /** stdin interactivity; defaults to a TTY so `confirm`-driven tests reach clack. */ readonly stdinIsTty?: boolean; - /** Piped (non-TTY) stdin answers, one consumed per confirmation prompt. */ readonly pipedAnswers?: ReadonlyArray; - /** Raw argv seen by the handler (e.g. to exercise an explicit `--yes=false`). */ readonly cliArgs?: ReadonlyArray; - /** Project ref returned by the resolver for the linked path. */ readonly projectRef?: string; - /** api-keys list returned by the Management API mock (linked path). */ readonly apiKeys?: ReadonlyArray<{ name: string; api_key?: string | null; type?: string | null; secret_jwt_template?: Record | null; }>; - /** When true, `loadProjectRef` fails with `ProjectRefNotLinkedError`. */ readonly linkedFails?: boolean; - /** cliSettings.explicitWorkdir override — true iff --workdir/SUPABASE_WORKDIR was set verbatim. */ readonly explicitWorkdir?: boolean; - /** Selects the `stack` backend (`StackBackendContext`); omitted/false keeps the legacy default. */ readonly stackBackend?: boolean; - /** Configures the fake `StackApi` consulted by the stack backend's credential resolution. */ readonly stackApi?: SetupStorageStackApiOptions; } export interface SetupStorageStackApiOptions { - /** Whether the `StackApi` service is provided at all. Defaults to `true`. */ readonly present?: boolean; - /** Whether `findStack` resolves a descriptor for the workdir. Defaults to `true`. */ readonly found?: boolean; - readonly lifecycle?: StackLifecycle; - /** The stack's `api` gateway endpoint URL, or `null` to omit it entirely. */ - readonly apiEndpoint?: string | null; - readonly storageState?: CapabilityState; + readonly lifecycle?: "running" | "starting" | "stopping" | "stopped"; + readonly storageEndpoint?: boolean; + readonly storageWakeEnabled?: boolean; + readonly storageState?: + | "dormant" + | "starting" + | "ready" + | "stopping" + | "stopped" + | "failed" + | "disabled"; readonly storageError?: string; - readonly serviceRoleJwt?: string; - /** Omits `credentials.api` from the fake stack's credentials effect. */ - readonly omitApiCredentials?: boolean; - /** Makes `findStack` fail with `InvalidProjectRootError({ message })`. */ - readonly findStackFails?: string; - /** Makes `status` fail with `StackNotFoundError({ message })`. */ - readonly statusFails?: string; } -const STORAGE_STACK_ID = StackIdSchema.make("e".repeat(64)); +const STORAGE_STACK_ID = "e".repeat(64); +export const STORAGE_TEST_JWT_SECRET = "storage-test-jwt-secret-with-at-least-32-chars"; +const databaseCreation: Extract = { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(STORAGE_TEST_JWT_SECRET), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: 54329 } }, +}; +const storageCreation: Extract = { + service: "storage", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: STORAGE_TEST_JWT_SECRET, + filePath: "/tmp/storage", + }, + endpoints: { http: { port: 59999 } }, +}; + +const observation = ( + id: string, + config: ServiceCreation, + lifecycle: "running" | "starting" | "stopping" | "stopped", + health: "starting" | "healthy" | "unhealthy" | undefined, + wakeEnabled: boolean, + endpoint?: { + readonly name: string; + readonly protocol: "tcp" | "http"; + readonly host: string; + readonly port: number; + }, + error?: string, +): Observation => ({ + id, + config, + endpoints: endpoint === undefined ? [] : [endpoint], + lifecycle, + health, + error: + error === undefined ? undefined : { _tag: "ServiceError", operation: "status", message: error }, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 1, + wakeEnabled, + registered: true, +}); + +const instance = ( + id: string, + creation: Extract, + status: Effect.Effect, +): ServiceInstance => ({ + id, + service: creation.service, + start: Effect.void, + ready: Effect.void, + stop: Effect.void, + restart: () => Effect.void, + destroy: Effect.void, + prepare: Effect.void, + status, + followStatus: Stream.empty, + logs: Stream.empty, + credentials: () => Effect.succeed({}), +}); -/** - * Builds the fake `EffectStack`/`StackApi` consulted by `resolveStorageCredentials` under the - * stack backend. Every method besides `findStack`/`openStack`/`status`/`credentials` dies — - * `storage`/`seed buckets` never call them. - */ export function buildStorageStackApi( workdir: string, opts: SetupStorageStackApiOptions | undefined, ) { const options = opts ?? {}; - const apiEndpointUrl = - options.apiEndpoint === undefined ? "http://127.0.0.1:59999" : options.apiEndpoint; - const lifecycle = options.lifecycle ?? "running"; const storageState = options.storageState ?? "dormant"; - const serviceRoleJwt = options.serviceRoleJwt ?? "stack-service-role-jwt"; - const findStackCalls: Array<{ readonly projectRoot: string }> = []; - const unused = Effect.die("unused in storage stack-backend tests"); - const unusedFn = () => unused; - - const stack: EffectStack = { + const lifecycle = options.lifecycle ?? "running"; + const storageEnabled = storageState !== "disabled"; + const storageLifecycle = + storageState === "disabled" + ? "stopped" + : storageState === "dormant" + ? "stopped" + : storageState === "ready" || storageState === "failed" + ? "running" + : storageState; + const storageHealth = + storageState === "ready" || storageState === "dormant" + ? "healthy" + : storageState === "failed" + ? "unhealthy" + : storageState === "starting" + ? "starting" + : undefined; + const storageWakeEnabled = + options.storageWakeEnabled ?? (storageState === "dormant" || storageState === "stopping"); + const dbObservation = observation( + "database-id", + databaseCreation, + lifecycle, + lifecycle === "running" ? "healthy" : undefined, + false, + { name: "sql", protocol: "tcp", host: "127.0.0.1", port: 54329 }, + undefined, + ); + const storageObservation = observation( + "storage-id", + storageCreation, + storageLifecycle, + storageHealth, + storageWakeEnabled, + options.storageEndpoint === false + ? undefined + : { name: "http", protocol: "http", host: "127.0.0.1", port: 59999 }, + options.storageError, + ); + const database = { + ...instance("database-id", databaseCreation, Effect.succeed(dbObservation)), + exportSnapshot: () => Effect.die("unused"), + restoreSnapshot: () => Effect.die("unused"), + resetData: Effect.die("unused"), + } satisfies DatabaseInstance; + const storage = instance("storage-id", storageCreation, Effect.succeed(storageObservation)); + const members = storageEnabled + ? [ + { id: database.id, activation: "eager" as const }, + { id: storage.id, activation: "lazy" as const }, + ] + : [{ id: database.id, activation: "eager" as const }]; + const stack: Stack = { id: STORAGE_STACK_ID, - status: - options.statusFails !== undefined - ? Effect.fail(new StackNotFoundError({ message: options.statusFails })) - : Effect.succeed({ - id: STORAGE_STACK_ID, - lifecycle, - desiredLifecycle: lifecycle === "running" ? "running" : "stopped", - runtime: { kind: "native" }, - endpoints: - apiEndpointUrl === null - ? {} - : { - api: { - protocol: "http" as const, - address: "127.0.0.1", - port: 59999, - url: apiEndpointUrl, - }, - }, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "database" ? ("eager" as const) : ("lazy" as const), - state: name === "storage" ? storageState : ("ready" as const), - error: name === "storage" ? options.storageError : undefined, - })), - artifacts: [], - }), - credentials: Effect.succeed({ - database: { - url: Redacted.make("postgresql://postgres:postgres@127.0.0.1:54329/postgres"), - password: Redacted.make("postgres"), - }, - api: - options.omitApiCredentials === true - ? undefined - : { - publishableKey: "anon", - secretKey: Redacted.make("secret"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make(serviceRoleJwt), - }, - }), - prepare: unusedFn, - start: unusedFn, - stop: unused, - destroy: unused, - resetDatabase: unused, - logs: unusedFn, - followLogs: () => Stream.empty, + services: { + create: () => Effect.die("unused"), + get: (id: string) => + id === database.id ? Effect.succeed(database) : Effect.succeed(storage), + list: Effect.succeed(storageEnabled ? [database, storage] : [database]), + }, + composition: { + supabase: () => Effect.die("unused"), + configure: () => Effect.die("unused"), + describe: Effect.succeed({ members, dependencies: [] }), + start: Effect.die("unused"), + stop: Effect.die("unused"), + restart: Effect.die("unused"), + }, + stop: Effect.die("unused"), + destroy: Effect.die("unused"), + tools: { run: () => Effect.die("unused") }, }; - + const definition = { + id: STORAGE_STACK_ID, + identity: { projectRoot: workdir, branchContext: "main", stackName: "default" }, + runtime: "native" as const, + instances: [ + { id: database.id, creation: databaseCreation }, + ...(storageEnabled ? [{ id: storage.id, creation: storageCreation }] : []), + ], + composition: { members, dependencies: [] }, + ports: [], + }; + const findStackCalls: Array<{ readonly projectRoot: string }> = []; const layer = options.present === false - ? Layer.empty + ? Layer.succeed(StackApi, { + create: () => Effect.die("Service not found: supabase/stack/StackApi"), + open: () => Effect.die("Service not found: supabase/stack/StackApi"), + discover: () => Effect.die("Service not found: supabase/stack/StackApi"), + resolveIdentity: () => Effect.die("Service not found: supabase/stack/StackApi"), + }) : Layer.succeed(StackApi, { - createStack: unusedFn, - findStack: (input: { readonly projectRoot: string }) => { - findStackCalls.push({ projectRoot: input.projectRoot }); - if (options.findStackFails !== undefined) { - return Effect.fail(new InvalidProjectRootError({ message: options.findStackFails })); - } - return Effect.succeed( - options.found === false - ? Option.none() - : Option.some({ - id: STORAGE_STACK_ID, - projectRoot: workdir, - name: "default", - branchContext: "main", - runtime: { kind: "native" as const }, - desiredLifecycle: "running" as const, - }), - ); + create: () => Effect.die("unused"), + resolveIdentity: () => Effect.succeed(definition.identity), + discover: () => + Effect.succeed(options.found === false ? [] : [{ definition, host: undefined }]), + open: () => { + findStackCalls.push({ projectRoot: workdir }); + return Effect.succeed(stack); }, - openStack: () => Effect.succeed(stack), - discoverStacks: unusedFn, - inspectStack: unusedFn, }); - return { layer, stackCalls: { findStack: findStackCalls } }; } @@ -367,7 +418,9 @@ export function setupStorage(workdir: string, opts: SetupStorageOptions) { mockRuntimeInfo({ cwd: workdir }), // `resolveYes` scans the raw argv for an explicit `--yes=false`. Layer.succeed(CliArgs, { args: opts.cliArgs ?? [] }), - ...(opts.stackBackend === true ? [stackBackendLayer("stack"), stackApi.layer] : []), + unusedStackServices, + stackApi.layer, + ...(opts.stackBackend === true ? [stackBackendLayer("stack")] : []), ); return { layer, out, requests, telemetry, linkedCache, stackCalls: stackApi.stackCalls }; diff --git a/apps/cli/tests/helpers/unused-stack.ts b/apps/cli/tests/helpers/unused-stack.ts new file mode 100644 index 0000000000..fdbf9ba205 --- /dev/null +++ b/apps/cli/tests/helpers/unused-stack.ts @@ -0,0 +1,15 @@ +import { Effect, Layer } from "effect"; +import { StackApi } from "../../src/command-internal/stack-api.ts"; +import { StackCatalogSetup } from "../../src/command-internal/stack-catalog-setup.ts"; + +const unused = () => Effect.die("Stack services must not run in this legacy-backend scenario"); + +export const unusedStackServices = Layer.mergeAll( + Layer.succeed(StackApi, { + create: unused, + open: unused, + discover: unused, + resolveIdentity: unused, + }), + Layer.succeed(StackCatalogSetup, { apply: unused }), +); diff --git a/docs/adr/0017-simplified-managed-stack-architecture.md b/docs/adr/0017-simplified-managed-stack-architecture.md index 0f4bd8f084..d8fd09fea1 100644 --- a/docs/adr/0017-simplified-managed-stack-architecture.md +++ b/docs/adr/0017-simplified-managed-stack-architecture.md @@ -1,8 +1,11 @@ # 0017. Simplified managed stack architecture -**Status**: accepted +**Status**: superseded by [the Stack package architecture](../../packages/stack/ARCHITECTURE.md) **Date**: 2026-08-17 +> This decision record describes an earlier managed-stack architecture. The current package +> architecture and lifecycle contract are maintained in `packages/stack/ARCHITECTURE.md`. + ## Decision Managed local-stack coordination has one implementation and one durable state diff --git a/docs/output.md b/docs/output.md index 228e51bb49..a38ea854ff 100644 --- a/docs/output.md +++ b/docs/output.md @@ -44,11 +44,11 @@ yield* output.fail({ code: "InvalidTokenError", message: "Bad token format" }) ## How It Works -`stream-json` commands emit newline-delimited JSON events. Bounded stack logs and compute logs -reads emit one `result` event containing their structured output. Commands that follow -line-oriented output emit `log-entry` events as entries arrive. A log-entry event has -`type: "log-entry"`, `timestamp`, `service`, `stream`, `line`, and `source`; stack logs use -`stream: "stdout" | "stderr" | "internal"` and `source: "history" | "live"`. +`stream-json` commands emit newline-delimited JSON events. Bounded compute log +reads emit one `result` event. Stack logs stream live `log-entry` events with +`timestamp`, `service`, `instance_id`, `stream`, `line`, and `source: "live"`. +Stack log streams are `stdout` or `stderr`; other emitters may include history +or internal messages and may omit `instance_id`. Each format has its own layer implementation. The root command provides the appropriate layer based on the `--output-format` flag: diff --git a/knip.jsonc b/knip.jsonc index 2d52beafc1..e370cf9d14 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -59,7 +59,9 @@ "entry": ["scripts/*.ts"], }, "packages/stack": { - // TODO(CLI-2423): knip does not credit the ws import in src/public/whole-stack.e2e.test.ts. + // Source-mode NativeProcess resolves this standalone launcher by URL. + "entry": ["src/runtime/native-launcher.ts"], + // Used by the raw WebSocket integration fixture; Knip excludes that test-only import. "ignoreDependencies": ["@types/ws", "ws"], }, }, diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md new file mode 100644 index 0000000000..63da409ade --- /dev/null +++ b/packages/stack/ARCHITECTURE.md @@ -0,0 +1,737 @@ +# Stack package architecture + +This document defines the stack package architecture. The package implements the service graph, lifecycle, composition, proxy, tools, and detached owner described here. CLI integration is a separate consumer concern. + +## Core model + +Use **a graph of service instances, with a small serialized lifecycle for each instance**. Keep application readiness separate from that lifecycle. Add a proxy that starts services on public traffic and sleeps them on public inactivity. Run finite tools with the same identity, artifacts and execution primitives, without treating them as services. + +The critical simplification is: + +> Starting a process and waiting for that process to become healthy are different operations. + +A service finishes its current lifecycle transition before accepting another lifecycle signal. There is no cancellation of one transition by another, ownership handoff, special pending-stop state, or general desired-state reconciliation engine. + +Preserve the useful identity and instance use cases while replacing the underlying machinery. The contract intentionally stays small and behavioral. + +## The mental model: a service runs itself; the orchestrator connects services + +Establish the individual service lifecycle first, then compose it through a dependency-aware orchestrator. Identity and resource ownership are foundations for both. The orchestrator uses the same service operations for one service or a whole stack; it does not implement a second lifecycle. + +```mermaid +flowchart TB + Handle["Public service or stack handle"] --> Orchestrator + Proxy["Proxy: traffic and idle signals"] --> Orchestrator + Composition["Stack composition
dependency edges and input wiring"] --> Orchestrator + Orchestrator["Orchestrator
validate graph, order operations, await readiness"] --> Executor["Individual service executor
serialize its own lifecycle transitions"] + Executor --> Backend["Native or container backend
launch, observe, stop, remove"] + Context["Shared stack context
identity, owned data, sticky ports"] -.-> Orchestrator + Context -.-> Executor +``` + +**Dependency declarations and input wiring both belong to stack composition, enforced by the orchestrator.** REST's definition accepts a database URL. It does not declare a database-service dependency or know where that URL comes from. Composition can wire the URL from a managed database instance and declare its readiness edge, or supply an external URL without a managed lifecycle dependency. The orchestrator resolves that wiring, enforces the declared ordering, and passes ordinary configuration values to REST. + +A URL implies a connection requirement, not ownership of another service. Do not infer stack graph edges by inspecting URLs. Keep the managed producer reference, readiness requirement and consumer input mapping together in the composition declaration; the service does not maintain a parallel dependency list. + +| Boundary | Knows | Does not decide | +| ----------------- | ------------------------------------------------------------------------ | ------------------------------------------------------- | +| Recipe/definition | Typed configuration inputs, such as a database URL; own runtime behavior | Where inputs originate or which stack services must run | +| Stack composition | Concrete dependency edges and how values populate service inputs | Runtime lifecycle implementation | +| Orchestrator | Graph, input resolution, readiness requirements, selected operations | How PostgreSQL launches or a container stops | +| Service executor | Own state, processes, readiness, data and resolved inputs | Dependency ordering or other services' lifecycle | + +Calling public `service.start()` or `service.stop()` still goes through the orchestrator. The executor is internal, so callers cannot accidentally bypass graph checks. Sleep is an internal proxy policy over the same stop/start mechanics; it has no public service method or RPC endpoint. + +Each long-running service is an individually identified instance with its own executor, lifecycle and health. There is no capability layer, hidden service group or aggregate service handle. Composition connects instances; it does not hide their identities. + +## Implementation organization + +Keep the implementation Effect V4 from the domain inward. Promise is the outer facade for package consumers and the boundary for foreign APIs; adapt a foreign Promise once at its leaf with typed errors. Host-owned execution fibers own admitted transitions, while callers may cancel only their wait. Use bounded streams and backpressure for tool and log transport. + +Organize by cohesive responsibilities. The package shape is: + +- `src/` modules: the instance executor, orchestrator, owner, detached host, networking, persistence, and RPC boundary. +- `services/`: one definition per service, owning its configuration, endpoints, launch settings, and readiness. `Catalog.ts` validates and dispatches creation; `Recipe.ts` defines their contract and `ProcessRecipe.ts` shares process mechanics. +- `composition/Supabase.ts`: default Supabase membership, dependency edges, and input wiring. +- `host/`: endpoint projections, tool execution, and tool attachment transport. +- `runtime/`: native and container adapters. +- `Tools.ts`: public finite-tool descriptors. +- `effect.ts`: Effect-facing composition and services. +- `index.ts`: Promise-facing public boundary. + +This is navigational guidance, not a required file scaffold. Split modules when a responsibility needs it; avoid one folder or interface per operation. Keep service definitions narrow, with graph edges and input wiring in composition. Do not introduce capabilities, projections, recovery journals, reservations, public sleep APIs, or extra lifecycle states to force this shape. + +Application services use Effect `Context.Service` and `Layer.effect`; consumers obtain their dependencies from the Effect context. Each owner receives an isolated orchestrator graph. Tools share the host lifetime alongside the owner. Individual executors, recipes, and process handles remain scoped resources because a stack owns multiple independently identified instances. + +## 1. Follow Compose's useful separation + +Docker separates container lifecycle from health. A container can be running while its health is still starting. Compose can wait for a dependency's health before starting its dependent, but stop does not wait for the application to become healthy. [Docker startup documentation](https://docs.docker.com/compose/how-tos/startup-order/), [Compose stop implementation](https://github.com/docker/compose/blob/main/pkg/compose/stop.go), [engine stop implementation](https://github.com/moby/moby/blob/master/daemon/stop.go). + +Use that separation for both backends: + +### Chart 1: one service instance's lifecycle + +```mermaid +stateDiagram-v2 + [*] --> Stopped: register + Stopped --> Starting: start or authorized wake + Starting --> Running: process launched + Starting --> Stopped: launch failed and cleanup confirmed + Running --> Stopping: stop, proxy idle sleep, destroy, or unexpected exit + Stopping --> Stopped: cleanup confirmed + Stopped --> [*]: destroy owned data and registration + + note right of Starting + The next lifecycle signal waits. + Preparation already completed. + end note + note right of Running + Start transition is complete. + Health is observed separately. + end note + note left of Stopped + Data and public port assignments + survive until explicit destruction. + end note +``` + +The final marker means the registration no longer exists. `destroy` of a running instance follows stop, then removal, within the admitted operation. If runtime cleanup fails, remain stopping with the current process/exit observation and cleanup error; the next stop/restart retries the remaining cleanup. Do not report a dead process as running or unfinished cleanup as stopped. Failed data removal leaves the registration present. Restart composes stop and launch; it does not add a new stable state. + +### Chart 2: health does not hold the lifecycle gate + +```mermaid +flowchart LR + Launch["Launch completes"] --> Waiting["Lifecycle: running
Health: starting"] + Waiting -->|readiness succeeds| Healthy["Lifecycle: running
Health: healthy"] + Waiting -->|readiness fails or times out| Unhealthy["Lifecycle: running
Health: unhealthy"] + Waiting -->|ordinary stop| Stopping["Lifecycle: stopping"] + Healthy -->|ordinary stop| Stopping + Unhealthy -->|ordinary stop| Stopping + Stopping -->|cleanup confirmed| Stopped["Lifecycle: stopped"] +``` + +Only running + healthy satisfies a readiness wait. A health failure records an error and leaves the process running; it neither locks the lifecycle nor automatically stops the process. + +**For one service instance, running means its process has launched. It does not mean the application is ready.** Every instance exposes its own observation. Separately observe health: starting, healthy, or unhealthy. A failed launch records its error; unsuccessful cleanup must not be reported as a successful stop. Destroy removes the registration only after stopped resources and owned data are safely removed. + +Example: PostgreSQL launches, so its start transition finishes in running. Its initialization and health are still pending. A stop request can now run the ordinary stopping transition; there is no start transition to interrupt. A request arriving during actual launch waits for launch to settle first. + +Dependency waits and health waits never hold a lifecycle transition open. Artifact preparation completes before the start transition is admitted, as a shared cache operation whose callers may cancel their own wait. Downloads and image pulls cannot run inside the serialized launch transition. Bounded native pre-launch steps such as initdb remain part of launch. Actual launch and stop work have bounded success/failure paths. A timeout is the current transition settling through its own failure handling, not another command pre-empting it. + +## 2. Ownership, instances and composition + +| Concept | Contains | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Stack | Identity, complete owned-instance registry, runtime choice, security, resource namespace | +| Composition | Explicit member selection, dependency edges, input wiring and eager/lazy activation policy | +| Service recipe | Typed configuration inputs, native/container launch specifications, readiness and optional storage operations; no stack graph knowledge | +| Service instance | Immutable ID, recipe/configuration, runtime handle, lifecycle and health observations; graph relationships belong to stack composition | +| Tool job | One finite artifact-backed execution belonging to the stack | + +The default Supabase composition is a factory that registers, selects and connects instances. Registration establishes ownership; membership establishes participation in the default application. Registering or starting another instance never implicitly adds it to that composition. A shadow database is an ordinary database instance with its own ID, password, ports and data, owned by the same stack but managed individually. + +```mermaid +flowchart TB + subgraph Namespace["Stack identity and owned resources"] + subgraph Application["Default application composition"] + DB["Primary database"] --> REST["REST"] + DB --> Auth["Auth"] + end + ShadowA["Standalone database A"] + ShadowB["Standalone database B"] + Tool["Temporary tool invocation"] + end +``` + +A composition is an explicit selection over the same instance registry and graph, not another StackHost or lifecycle engine. With no declared dependencies, starting a standalone database selects only that instance and its internal owned resources. Dependency checks still use the shared graph when edges exist. Copying an initialization profile from the primary is setup input, not a permanent runtime dependency. + +### Proposed standalone instance API + +This sketch retains the existing `services.create` shape while making creation independent of default composition membership: + +```ts +const shadow = await stack.services.create({ + service: "database", + config: { + version: "17", + endpoints: { sql: { port: "auto" } }, + }, +}); + +try { + await shadow.start(); + await shadow.ready(); + // Run the migration or comparison against this instance. +} finally { + await shadow.destroy(); +} +``` + +### Typing the creation API + +Use a discriminated union of creation definitions. `DatabaseDefinition` is configuration input, while `DatabaseInstance` is the live handle returned by creation. Neither type needs a runtime class hierarchy. The following abbreviated types illustrate the public contract; the actual configuration types come from each service module's validated schema. + +```ts +type DatabaseDefinition = { + service: "database"; + config: DatabaseConfig; +}; + +type RestDefinition = { + service: "rest"; + config: RestConfig; +}; + +type ServiceDefinition = DatabaseDefinition | RestDefinition; + +type ServiceInstances = { + database: DatabaseInstance; + rest: RestInstance; +}; + +interface ServiceCollection { + create( + input: Input, + ): Promise; +} +``` + +The `service` literal selects the permitted configuration and the returned handle. An inline call with `service: "database"` infers `DatabaseInstance`; it needs neither a generic argument nor `as const`. For a definition saved in a variable, use `satisfies DatabaseDefinition` to validate it while preserving the discriminator. A definition whose kind is only known at runtime returns the corresponding union of handles. Keep kind/config pairs correlated as a discriminated union, rather than giving `service` and `config` independent unions. As the catalog grows, the union and instance mapping can be derived from one typed catalog rather than duplicated. + +Use a service-to-config map and mapped creation union for this typing while keeping dependency requirements in composition rather than individual service definitions. Runtime schema validation remains necessary at persisted/control boundaries; TypeScript alone does not validate received data. + +Creation assigns a unique instance identity and isolated data directory; endpoint allocation belongs to that instance and stays sticky until destruction. The same call can create a second independent database. No `shadow` service type or special lifecycle is required. Launch and health remain separate. The caller owns the decision to destroy temporary instances; closing a handle is not destruction. Existing initialization/snapshot options still apply where the CLI needs a prepared database. + +Make operation scope explicit in the proposed API: + +| Operation | Selection | +| ---------------------------------------- | ----------------------------------------------------------------------------- | +| `instance.start/stop/restart/destroy()` | That instance, subject to the ordinary graph rules | +| `stack.composition.start/stop/restart()` | Default composition members; start also includes their declared prerequisites | +| `stack.stop()` | All owned service instances, including standalone instances | +| `stack.destroy()` | All owned resources, including standalone instances and tool jobs | + +Namespace stop also cancels and settles attached jobs before StackHost shutdown; composition stop does not reserve services on behalf of tools. Namespace destruction performs shutdown before owned-data removal. The CLI's full stop uses namespace scope; restarting the application composition does not restart shadows. Composition startup never resurrects a standalone instance. Membership and eager/lazy activation are separate: eager means start when the containing composition is started, not include every registered eager instance. The existing stack-wide start convenience, if retained, delegates to default composition startup only. + +For example, REST binds to the primary database; a shadow database has no relationship to it unless the caller asks to copy its initialization profile. Functions may run without PostgreSQL or Auth. An API URL in configuration is not automatically a hard lifecycle dependency. + +For example, Studio and pg-meta are separate service instances. Each has its own ID, configuration, lifecycle and health. Composition declares the relevant edges explicitly. Storage and imgproxy, or Logflare and Vector, follow the same rule when selected; there is no parent handle that starts private children or computes combined health. + +```mermaid +flowchart LR + Database["Database instance"] -->|ready before launch| PgMeta["pg-meta instance"] + PgMeta -->|ready before launch| Studio["Studio instance"] +``` + +The `service: "database"` discriminator selects a service definition and its configuration type; it does not name a group or identify a unique instance. Two databases have the same definition kind and different instance IDs. Operations and dependency edges address instance IDs. Short-lived initialization helpers remain implementation details of an operation; they do not justify a hidden hierarchy of long-running services. + +## 3. One lifecycle implementation, one graph planner + +An instance has one serial transition executor. Requests wait until the current transition settles, then revalidate against the resulting state. An already-satisfied request does no runtime work; an explicit stop still updates intent and disables wake as described below. A waiting caller can cancel its wait; an admitted transition belongs to the StackHost and finishes independently of that caller. + +### Interruption and command concurrency + +An executing instance operation cannot be interrupted or replaced by another command. Start, stop, restart, destroy, export and restore use the same serial execution gate. Internal proxy-triggered sleep also uses that gate. Incoming commands wait, revalidate against the resulting state, then execute. An instance's restart keeps that gate across stop and launch; readiness observation happens afterward. There is no priority-stop path or ownership handoff. + +```text +start executing → stop requested → stop waits +start settles → stop executes → stopped +``` + +Cancellation has a limited meaning at each boundary: + +| Caller activity | Effect of cancellation | +| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | +| Preparing artifacts or waiting for execution admission | Abandon this request before its lifecycle operation begins; shared preparation may continue for other callers | +| Waiting for an executing instance operation | Stop waiting; the StackHost finishes the operation and its cleanup | +| Waiting for readiness or following logs | End the observation without changing lifecycle | +| Running an attached tool invocation | Terminate and clean up that invocation | + +A service is stopped by an explicit stop command, not by a disconnected caller. Launch completes at running, so stop during health-starting uses the ordinary stop operation. If the launch ends while a caller awaits readiness, that observation fails rather than attaching to a later launch. + +A pending orchestration plan is different from an executing transition: stopping REST while its start request waits for database readiness must prevent the old request from later launching REST. The intent revision described below handles this one stale-request check. It does not interrupt an admitted operation. + +Independent instances execute concurrently. Admission and dependency validation use short coordination sections; downloads, readiness probes, process operations and snapshots do not hold a global execution lock. Each operation settles through success or its own timeout/failure handling and cleanup. Process crashes and owner loss remain possible; report unavailable observations or cleanup errors rather than claiming success. Crash recovery is outside this design. This ownership rule does not require one blanket uninterruptible region around all Effect code. + +### Chart 3: dependency awareness lives in the orchestrator + +Here stack composition has wired the primary database’s connection URL into REST configuration and declared a readiness dependency. The caller composes `start` and `ready`, as a CLI command that needs a usable service would. The service executors never call each other. With an external database URL, the managed database branch is absent. + +```mermaid +sequenceDiagram + participant Caller + participant O as Orchestrator + participant DB as Database executor + participant REST as REST executor + + Caller->>O: Start REST and wait for readiness + O->>O: Read composition edge and input mapping + O->>DB: Start + DB-->>O: Running; health still starting + Note over DB: Launch transition has finished + O->>DB: Await readiness of this launch + Note over O,DB: Observation only; lifecycle remains available + DB-->>O: Healthy + O->>O: Revalidate target intent and graph + O->>REST: Start with ordinary database URL input + REST-->>O: Running; health still starting + O->>REST: Await readiness of this launch + REST-->>O: Healthy + O-->>Caller: REST ready +``` + +If the prerequisite stops or readiness fails, the wait fails and REST is not launched. The wait does not reserve a long-running start transition. Deliberate stop still goes through the orchestrator's graph checks. + +The orchestrator performs a few graph operations: + +| Operation | Rule | +| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Start | Start declared prerequisite instances; await their health; launch the selected instance | +| Stop | Admit only when every dependent is stopped with wake disabled, regardless of batch selection; composition stops dependents first | +| Restart | Individual restart follows the same stop check, then launches; composition restart stops in reverse dependency order, then starts in forward order | +| Sleep | Require proxy inactivity and no running/starting dependent | +| Destroy | Reject while any registered dependent still references the instance; composition/namespace destruction removes dependents first, then performs ordinary stop and owned-data removal | + +For stop/restart, a running, starting, stopping or wake-armed dependent blocks prerequisite shutdown until that dependent has stopped and settled. Batch membership never bypasses this check. For sleep, a dependent that is already sleeping does not prevent prerequisite sleep. These are admission checks under the same short graph coordination boundary, not long-lived reservations. + +Validate composition references, input/output types and cycles when registering or changing the graph. Wiring uses only a source instance ID and named output mapped to a target instance ID and named configuration input. The service catalog declares those input/output schemas. There is no expression language, arbitrary callback or general JSON-path evaluator. Resolve bindings into ordinary configuration before handing it to the service. Validate each service’s configuration separately from that graph. Bind references to immutable instance IDs, not names. Derive ordering from this one graph; do not maintain separate capability and instance lifecycle graphs. + +**Readiness waiting is observation, not a transition.** If PostgreSQL stops while REST is waiting for its health, the wait fails and REST does not launch. It does not restart PostgreSQL or continue using a stale ready result. Readiness is associated with the exact runtime launch. + +Before launching a dependent, atomically check its prerequisites, verify its own intent revision is unchanged since the request began, and mark its transition starting. Explicit stop advances that revision and disables wake even if no process exists yet; an older request still preparing artifacts or waiting for dependencies cannot launch afterward. This revision fences unadmitted work; it never interrupts an executing transition. This closes the race between dependency validation and another command stopping the prerequisite. Do not reserve an entire composition or acquire a set of batch locks. Each instance operation waits for its own gate and rechecks the current graph and observations on admission. If another caller starts a dependent between two batch steps, stopping the prerequisite is rejected and the batch reports partial results. A stale start plan cannot resume after another operation stops or replaces its prerequisites. + +Independent instances still execute concurrently. A shadow startup or snapshot must not block Functions restart. Serialize only short in-memory admission decisions and durable writes; never hold the graph decision boundary while downloading, starting processes, probing health, archiving data or writing files. Gateway traffic on unrelated ready instances must not wait behind a state write. + +Composition and namespace methods select instances and call these same operations. Default composition start first assigns/reuses and binds every configured public listener needed by its selected members, including lazy members, and registers their routes. It then launches eager members and arms lazy routes; it does not select unrelated standalone instances. Successful start returns member observations with their public endpoints; credentials reads render usable connection strings from these assignments without starting lazy backends. It awaits readiness for every launched eager member, including leaves. A failed launch or readiness check makes the operation return an error with per-instance outcomes; a running-but-unhealthy instance remains running, already completed steps are not rolled back, and blocked dependents are not launched. Armed lazy members need not launch unless required by an eager member. + +After binding the shared API listener, default composition setup uses the endpoint renderer to populate ordinary configuration values: Studio receives `apiUrl` and `publicApiUrl`, Auth receives `externalApiUrl`, and Functions receives `apiUrl`. When the database has a configured SQL endpoint, Functions also receives its rendered `databaseUrl`; this saved value is refreshed when callers recompose the composition, and does not create a database readiness dependency for Function execution. These values are saved in each instance's configuration alongside the saved listener assignment; individual starts and normal host reopening consume that configuration. This is concrete default setup code, not declarative input wiring or a public listener-binding API. The endpoint renderer owns the host/runtime reachability rules; custom configurations supply plain URLs through the same inputs. URL values do not imply graph edges. + +Composition restart is two passes: stop the selected instances in reverse dependency order, then perform composition start in forward dependency order according to eager/lazy policy. It is not a loop of individual `restart()` calls. If the stop pass fails, report partial results without beginning the start pass. A cancelled composition request abandons steps not yet executing; admitted instance operations settle and already launched instances remain. Readiness does not hold an instance gate or a composition-wide reservation. Explicit empty selection is a no-op; there is no generic rollback engine. + +## 4. Recipes own runtime details + +A recipe receives ordinary typed configuration and a context for its own resources. A database URL is just an input value: the recipe does not receive its producer’s identity, another service handle, or the stack graph. It does not receive the entire persisted stack document. The orchestrator owns resolving managed outputs into these configuration values. + +The backend provides mechanical operations: launch, observe exit/logs, stop and remove exact resources. Native and container implementations differ in commands, mounts, network setup and cleanup, but share the lifecycle contract. Keep one runtime choice per stack; mixed backends and automatic tool-runtime fallback are unnecessary for the current scope. Preparation validates that the selected service/tool artifact exists for that runtime and platform, and reports an unsupported-platform/artifact error otherwise. A native stack requires native artifacts; Windows native support is not implied by a fallback branch in the current CLI. + +A successful launch returns an owned runtime handle with readiness observation and cleanup. One shared readiness program belongs to each runtime launch and has a bounded outcome. Readiness timeout or initialization failure leaves the process running with unhealthy status and an actionable error. It does not automatically stop or restart the process. Stop remains available, and traffic/dependent launches do not treat unhealthy as ready. Initialization that requires a running process belongs to that runtime session, not the start transition. For PostgreSQL, healthy means required catalog initialization and credential reconciliation have completed—not merely that TCP accepts a connection. Stopping the running instance closes that session, settling its health probes and initialization helpers as ordinary resource cleanup. There are not two competing lifecycle operations. + +The database keeps only the initialization information needed to reuse its data on a normal stop/start. Mark initialization complete only after it succeeds. If an ordinary stop interrupts application initialization, the recipe must either support a safe retry or report that initialization is incomplete; do not add a generic persisted operation journal or recovery workflow. + +Unexpected process exit is observed by the living host. Immediately disable forwarding and automatic wake for that instance and record its exit result. The executor uses the ordinary serialized stop/remove cleanup for the exact owned runtime handle, reaching stopped only after cleanup succeeds. Stop cannot be a no-op while runtime resources remain; start/restart must finish that cleanup before launching a replacement. If exit occurs during an executing transition, that transition settles and the same gate orders any remaining cleanup; no transition is pre-empted. Old-handle events cannot affect a replacement. This adds no Exited lifecycle state and no cleanup scanner: the host already owns the handle and its exit observation. + +There is **no automatic restart, continuous dependency shutdown cascade, or self-healing controller**. A crashed service stays stopped until explicit start/restart. A wake operation may use a running prerequisite or launch a stopped prerequisite whose wake route is armed; an explicitly stopped or unexpectedly exited prerequisite makes wake fail. Only an explicit start may launch those prerequisites. Explicit composition startup can arm initially lazy instances, and sleep retains that permission; unexpected exit removes it. Existing dependents may observe connection failures, as they can with Compose. Deliberate lifecycle changes still obey graph checks. + +## 5. Sleep stays simple + +**Public traffic through the proxy is the only activity signal.** Sleep is triggered internally by proxy inactivity, never by a public client command. Do not track private connections, background SQL, or inferred workload activity. Add a public sleep operation only if a concrete consumer needs it later. + +- A request or open public stream/connection counts as activity; idle HTTP keep-alive does not. +- The idle timer and wake route belong to one instance. Sleep stops that instance after the ordinary dependency checks, retains its data and keeps its wake route armed. It does not implicitly stop companion instances. Instances without a supported public wake route remain running until explicitly stopped; no group sleep mechanism is introduced. +- The next request starts the instance through the orchestrator, ensuring its declared prerequisites are ready, then waits for its own health before forwarding. +- Explicit stop disables wake. Traffic never reverses it. +- If a prerequisite's sleep is blocked by a running dependent, recheck its existing idle eligibility when that dependent sleeps/stops. No new activity heuristic is needed. + +Sleep is therefore ordinary stop mechanics plus an armed wake route. There is no separate sleep runtime. Activity acquisition and sleep admission must make one atomic in-memory decision so sleep cannot race a request. A request that arrives after sleep was admitted waits for it to finish, then wakes the instance. + +### Chart 4: sleep, stop and destroy have different retained state + +These are outcomes of the same local stop mechanics after graph checks. “Sleeping” describes a stopped instance with wake enabled, not another backend state. + +```mermaid +flowchart TB + Halted["Instance stopped
Cleanup confirmed"] + Halted -->|internal idle sleep| Sleeping["Sleeping
Wake route armed
Data retained
Public port assignments retained"] + Halted -->|stop| Stopped["Stopped
Wake disabled
Data retained
Public port assignments retained"] + Halted -->|destroy| Delete["Remove proven-owned data
and registration"] + Delete --> Removed["Removed
Owned endpoint claims released"] + Sleeping -->|public traffic| Wake["Orchestrator starts dependency closure
Waits for health, then forwards"] + Stopped -->|explicit start| Start["Orchestrator starts selected services
Reuses saved public ports"] +``` + +Sleep retains the public listener needed to wake. Whole-stack stop closes listeners while preserving saved assignments; the next start rebinds the same ports. Destruction releases only the removed instance's owned assignments, not unrelated services' shared listeners or claims. + +Composition validation permits lazy activation only for instances with a configured public wake endpoint. A route-less prerequisite, such as pg-meta without its own public endpoint, is eager; it is not implicitly armed through a dependent. This avoids a second wake-permission mechanism. Internal sleep is available only for instances with a supported public wake route and an enabled idle policy. Database and Functions need no automatic idle timer by default. Functions inspector access remains possible while health is starting; ordinary application traffic waits for healthy. + +## 6. Tools share execution, not service lifecycle + +Provide a stack-scoped finite runner for concrete CLI needs such as `pg_dump` and `psql`. Each invocation has a job ID; temp files, logs and runtime resources belong to that stack. Jobs reuse artifact selection and native/container process execution. They have exit results and byte streams, not healthchecks, wake policy or service registrations. + +For `pg_dump` and `psql`, container mode launches a temporary tool container from a compatible PostgreSQL image, overriding its entrypoint to run the client command. It may reuse the database service's image, but it creates a separate container without starting another database server or mounting the server's data directory. Native mode launches the corresponding binary from the selected native artifact. The tool definition supplies these two executable forms; the runner supplies stack ownership, arguments, environment, streams and cleanup. Docker supports entrypoint overrides and automatic removal for this execution shape. [Docker run reference](https://docs.docker.com/engine/containers/run/). + +| Invocation | Native runtime | Container runtime | +| ----------------------------- | -------------------------------------------- | ------------------------------------------------------------------------ | +| Run `pg_dump` | Spawn the selected client binary | Create an attached temporary container running `pg_dump` | +| Connect to a managed database | Resolved public proxy endpoint | The same public proxy endpoint, addressed from the container network | +| Return the result | Stream stdout/stderr and collect exit status | Stream stdout/stderr, collect exit status and remove the owned container | + +The caller supplies ordinary arguments and environment values, including any connection string. The tool runner treats them as opaque: it does not resolve service references, infer dependencies or rewrite URLs. The caller can obtain execution-reachable connection values through `service.credentials({ from: "runtime" })`; `from: "host"` is the default for ordinary host clients. This extends the existing read operation rather than adding tool-specific inputs. The networking/endpoint renderer supplies these concrete values; `localhost` inside a tool container is not generally the host. Managed tool traffic uses the public proxy so the existing wake/activity rules apply. Tools publish no listening ports. Dump output streams to the CLI's destination; temporary files, when needed, use the stack/job directory. Use a client version compatible with the target; matching the managed database's selected major is the simple default. [PostgreSQL client compatibility](https://www.postgresql.org/docs/17/app-pgdump.html). + +Managed-local execution uses this temporary-container pattern under stack ownership. Executing a command inside an existing service container is unnecessary for these network clients and need not become a second public tool mechanism. + +For the managed local backend, the StackHost owns jobs so it can clean up exact resources after client disconnection. Attached-job cancellation stops that job, not a service transition. Explicit StackHost shutdown cancels and settles attached jobs; finishing the last job does not automatically retire the host. Use bounded, backpressured stdin/stdout/stderr transport; do not buffer entire dumps or confuse output EOF with successful exit. + +Tools do not participate in the service dependency graph. Traffic to a public endpoint wakes a sleeping service through the proxy, exactly as traffic from any other client does. An explicitly stopped service remains stopped. There are no tool-specific readiness checks or lifecycle locks: an explicit stop or restart can disrupt the connection and the tool reports its ordinary error. Internal bootstrap helpers use the same low-level runner under their parent service operation. + +Migrate **managed-local** dump/reset execution first. Linked/remote and legacy compose CLI paths remain outside this package's lifecycle; do not create a stack or reserve ports just to run their tools. An explicitly supplied stack handle may still run a job against an external URL. The identity claim applies to everything executed through that stack. + +CLI policy remains CLI policy: SQL scripts, migrations, seeds, hosted targets, output files and command flags. The caller chooses a compatible client version; the runner resolves that version to its executable artifact and owns execution. + +### Proposed public tool API + +Expose one awaited `stack.tools.run` operation. The Promise facade below has an Effect counterpart for CLI consumers; both use the same owner-side runner. + +```ts +import { postgres } from "@supabase/stack/tools"; + +// Connection values are plain data, rendered for the stack runtime. +const { databaseUrl } = await database.credentials({ from: "runtime" }); +const result = await stack.tools.run(postgres.pgDump({ major: 17 }), { + args: ["--dbname", databaseUrl, "--schema-only", "--no-owner"], + stdout: (bytes) => destination.write(bytes), + stderr: (bytes) => diagnostics.write(bytes), + signal, +}); + +// result: { jobId, exitCode } +``` + +`postgres.pgDump({ major })` describes the selected client package and how to launch its native and container forms. The caller chooses the version; 17 is illustrative. The stack's runtime chooses the execution form. The invocation supplies ordinary arguments, environment and byte streams. Neither descriptor nor invocation declares managed service dependencies. + +`databaseUrl` is a plain string. The same invocation can target the primary database, a shadow database or an external database without changing the tool definition or the runner. The proxy handles any wake-up caused by connecting to a sleeping managed service. A URL obtained for container execution is already reachable from that container; endpoint address selection remains outside the generic tool runner. The orchestrator uses the same endpoint renderer when supplying connection values to service instances. It handles the host gateway and the listener binding needed to reach it, not just hostname substitution, and returns an error if the selected networking configuration cannot reach the endpoint. Caller-supplied external URLs remain unchanged. + +Arguments are passed as an argument vector, not interpreted as shell text. Stdin accepts an optional asynchronous byte stream; stdout/stderr callbacks receive bytes and may return Promises, which the runner awaits for backpressure. All examples use byte sinks whose writes await capacity. `psql` uses the same operation with `postgres.psql({ major })`, optionally supplying stdin. Callers supply stdout/stderr sinks explicitly. Neither facade collects unbounded output into a return value. + +The operation settles after process exit, output delivery and owned-resource cleanup. A nonzero tool exit is returned in `exitCode` for CLI-specific handling; preparation, transport, sink and cleanup failures reject with typed execution errors. Cancellation stops and cleans up the attached job and reports cancellation. The StackHost assigns `jobId` and owns the resources. No public job registry, job healthcheck, persistent tool service or separate launch/attach/wait sequence is needed for these use cases. + +## 7. StackHost owns lifetime; components own behavior + +Use `StackHost` for the detached process that owns one stack identity. It hosts the service executors, composition orchestrator, proxy, tool runner and shared resources. A standalone instance needs this owner and its executor without needing composition scheduling. Tools use the runner without entering the service dependency graph. + +The host acquires exclusive ownership, constructs the components, exposes Effect RPC, delegates requests and coordinates explicit shutdown. Domain behavior remains in the components: the host does not understand PostgreSQL archives, decide dependency order or implement another service state machine. Existing dependency checks still apply when individual operations target instances with declared graph relationships. + +```mermaid +flowchart TB + Client["CLI or package handles"] --> RPC["Effect RPC endpoint"] + subgraph Host["StackHost — one detached process per stack identity"] + RPC --> Composition["Composition orchestrator"] + RPC --> Instances["Service instance executors"] + RPC --> Tools["Tool runner"] + Composition --> Instances + Proxy["Public proxy"] --> Composition + Resources["Shared ownership, ports and metadata"] + end +``` + +Keep Effect RPC as the transport initially. Its handlers should mostly delegate to the same operations used by internal components. Composition startup calls the executors directly, not RPC back into its own host. Replacing RPC with handwritten messages would still require framing, validation, errors and stream transport; that replacement is not part of this simplification. + +### Proposed RPC surface + +| Area | Methods | Responsibility | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| Host | `host.describe`, `host.stop`, `host.destroy` | Inspect the stack; stop or destroy everything it owns | +| Registry | `services.create`, `services.get`, `services.list` | Create and locate instances | +| Instance lifecycle | `service.prepare`, `service.start`, `service.stop`, `service.restart`, `service.destroy` | Operate one instance | +| Instance observation | `service.status`, `service.ready`, `service.followStatus`, `service.logs`, `service.followLogs`, `service.credentials` | Read state, await health and inspect outputs | +| Database snapshots | `database.exportSnapshot`, `database.restoreSnapshot` | Database-specific storage operations | +| Composition | `composition.configure`, `composition.describe`, `composition.start`, `composition.stop`, `composition.restart` | Define and operate the application selection and dependencies | +| Tools | `tools.run`, `tools.writeStdin`, `tools.closeStdin` | Execute an attached command with streamed input/output | + +These are proposed wire names. Public `shadow.start()` maps to `service.start({ id })`; `stack.stop()` maps to `host.stop`. Public `stack.composition.stop()` leaves the host and independent instances available. `composition.configure` sends validated declarative membership, edges and input wiring, not executable callbacks. Configuration changes use the same graph and lifecycle admission rules as other mutations. + +### Host lifecycle + +Launch the host when an operation needs a live owner. Once started, it remains alive until namespace stop or destruction. Client disconnection, completion of a tool, or stopping the final individual instance does not cause automatic retirement. The accepted tradeoff is one resident process for an opened stack until explicitly stopped, even if all its service instances are stopped. + +```mermaid +stateDiagram-v2 + [*] --> Starting: operation needs a host + Starting --> Serving: ownership acquired, metadata loaded, RPC ready + Starting --> Exited: startup fails or another owner wins + Serving --> Serving: clients connect or disconnect + Serving --> Serving: instances start, stop or sleep + Serving --> Draining: host.stop or host.destroy + Draining --> Exited: operations settled and cleanup complete + Draining --> Serving: cleanup failed; report error and retain ownership + Exited --> [*] +``` + +During Starting, acquire the exclusive stack lease, load the instance definitions and saved resources needed for normal restart, construct components and open the control endpoint. The lease is held by an operating-system locking primitive whose ownership ends with the process; an on-disk PID/endpoint file is only discovery metadata and never proves an active owner. A competing launcher connects to the winning owner. Startup does not replay interrupted operations or scan for orphaned resources. Existing runtime or port conflicts are reported rather than automatically adopted or removed, including leftover containers from a previous host. No resource adoption or orphan cleanup is added after owner death. + +During Serving, keep the owner alive independently of callers. Sleeping instances still need its public listeners. This is process lifetime management, not automatic service restart or continuous reconciliation. + +Where the platform delivers SIGTERM or SIGINT to the host, treat it as the same graceful shutdown request as `host.stop`. Repeated shutdown requests join that shutdown; they do not pre-empt executing transitions. Forced termination remains outside the graceful-shutdown guarantee. + +During Draining: + +1. Close admission to new mutations and proxy wake requests. +2. Reject queued work that has not begun. +3. Let executing instance operations settle. +4. Cancel and settle attached tools and stop owned services through their existing operations. +5. For destruction, remove proven-owned data and metadata after shutdown. +6. Send the outcome, close the control endpoint and release ownership. + +Public whole-stack `stop` and `destroy` complete only after acknowledged cleanup and confirmed owner-process exit. The client captures the live owner PID from the validated identity endpoint or readiness handshake, completes and closes the shutdown RPC, then performs bounded process-existence checks. An absent PID confirms exit; a permission-denied probe remains inconclusive until the deadline. Failure to confirm exit is a `shutdown-exit` error carrying the PID in its message, even when workload cleanup has already succeeded. This does not require persisted PID records or forceful termination. Caller cancellation ends its wait without cancelling admitted owner cleanup. + +Callers must not start or restart the same stack concurrently with whole-stack shutdown. In particular, replacing an owner between identity lookup and the shutdown request is outside this guarantee. Parallel stacks with separate identities remain independent. Client disposal and Effect scope closure do not implicitly stop a detached stack; disposable fixtures register explicit destruction. + +On cleanup failure, retain the host so callers can inspect the current observations and error. Returning to Serving does not undo completed cleanup. Unexpected host death is outside the supported normal stop/start lifecycle: there is no automatic recovery, orphan reconciliation or resumption of interrupted operations. Leftover resources may require manual cleanup. A lost control response is reported as uncertain; do not blindly retry a mutation. + +### Request lifetime is separate from execution lifetime + +Once an instance operation starts executing, the host owns it until settlement. An RPC disconnect ends the caller's wait without interrupting that operation. Status and log subscriptions end with their connection. Requests not yet admitted remain cancellable as described in the concurrency section. + +```mermaid +sequenceDiagram + participant Client + participant RPC as RPC handler + participant Executor as Host-owned executor + Client->>RPC: service.start(id) + RPC->>Executor: Submit start + Executor->>Executor: Admit and execute launch + Client--xRPC: Client disconnects + Note over RPC: Caller stops waiting + Note over Executor: Executing operation continues + Executor->>Executor: Record running or failure + Client->>RPC: Reconnect; service.status(id) + RPC->>Executor: Read observation + Executor-->>RPC: Current state + RPC-->>Client: Current state +``` + +A lost response does not prove failure. While the host lives, callers can inspect its instance observations and in-memory operation result; never blindly repeat creation or restoration. No operation journal or result history survives host death, and there is no command replay or exactly-once execution promise. + +Attached tools have a separate contract. `tools.run` streams `started(jobId)`, stdout/stderr chunks and a terminal exit result. Optional input arrives through bounded `tools.writeStdin` calls and `tools.closeStdin`. These calls are restricted to the originating invocation/session. Cancelling or losing the execution stream terminates and cleans up that job. The public `stack.tools.run()` wrapper handles this exchange behind its stdin/stdout interface; no separate public launch/attach/wait workflow is required. The terminal success response follows output delivery and cleanup, with execution/cleanup failures reported as errors. + +## 8. Keep safety infrastructure at its boundary + +### Instance observations are the runtime source of truth + +Each instance owns one observable runtime state. Its executor updates lifecycle and current operation; runtime observations update health, exits and errors. `service.status`, `service.followStatus`, dependency readiness checks and CLI presentation consume those same observations. Endpoint availability comes from the component that owns the listener. The StackHost transports these observations without maintaining another lifecycle copy. + +```mermaid +flowchart LR + Executor["Executor transitions"] --> Instance["Instance observation"] + Runtime["Health and process events"] --> Instance + Instance --> RPC["status and followStatus"] + Instance --> Dependencies["Orchestrator readiness checks"] + RPC --> CLI["CLI display"] +``` + +A composition returns the observations of its members. Remove the separate capability status and aggregate stack lifecycle models, their conversion rules and their offline reconstruction. No shared summary model is required. The StackHost's Serving/Draining state describes only the owner process and remains independent of instance observations. + +### Persist only what normal stop/start needs + +Retain stack/instance identity, service configuration and selected versions, composition membership/wiring, credentials, public port assignments and the service-specific information needed to reopen the data. Derive stack and instance data locations from the state root and saved identities rather than persisting duplicate paths. Keep the data itself across stop. Shared port claims remain necessary for stickiness across parallel stacks. The active host's exclusive lease prevents competing owners; it is not a crash-recovery subsystem. + +Do not persist health, runtime lifecycle projections, transition progress, command queues, operation results or recovery checkpoints. A disconnected caller can reconnect to the same living host; resuming after host death is not part of the contract. Incomplete operations and leftover resources after a crash may require manual cleanup, without automatic deletion of valuable data. + +Some foundations are not optional, but they do not need to dominate the service model: + +| Foundation | Keep | +| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| Identity | Stack and instance IDs; deterministic names and paths for their resources | +| Ownership | One detached StackHost and exclusive lease; client handles do not own service lifetime | +| Ports | Persist public port assignments across stop/start; reuse the same ports, including initially automatic assignments; never silently relocate | +| Data | Stop retains data; destroy removes only proven owned data | +| Snapshots | DB-specific stopped export, compatible empty-target restore, complete archive publication and credential reconciliation | + +### Port allocation and endpoint ownership + +The shared allocator manages physical public listeners. A service definition declares its endpoints; composition may map an HTTP route to a shared listener. The service implementation does not search for public host ports. A requested port is a number or `"auto"`. + +| Listener owner | Allocation key | Example | +| -------------- | ---------------------------------------- | ---------------------------------------------------------------------- | +| Instance | `(stack ID, instance ID, endpoint name)` | Database `sql`, Functions `inspector` | +| Stack | `(stack ID, listener name)` | Shared `api` listener serving REST, Auth, Storage and Functions routes | + +A shared listener is a networking resource, not a capability or service group. Composition maps each route to an individual instance ID. Traffic is attributed to the matched route's instance for wake and idle decisions; there is no aggregate service lifecycle. + +| Address | Purpose | Lifetime | +| ------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | +| Public | Stable endpoint exposed through a dedicated listener or shared route | Assignment survives stop/start until its owner is destroyed or explicitly reconfigured | +| Backend | Address where the current process/container can be reached | May change on each launch | + +**Reservation happens during startup endpoint setup, before URLs are returned—not on the first request.** `supabase start` and programmatic composition startup perform the same sequence: assign or reuse all configured public ports for the selected composition, bind the proxy listeners, register the routes, then apply eager/lazy activation. Shared listeners are bound once even if every route is lazy. An individual explicit `instance.start()` likewise prepares its endpoints before launching. `services.create()` alone records endpoint intentions and does not promise a bound port for `"auto"`. + +A lazy database already has a bound SQL proxy listener and a concrete connection URL; lazy REST already has its path on the bound shared API listener. Their backend processes may still be absent. The first request launches the needed backends, waits for readiness and forwards through the already published endpoint. Backend address allocation can wait until that launch; public port assignment cannot. If a required listener cannot bind, startup returns an error with partial outcomes rather than reporting successful startup with unusable URLs. + +A host-wide allocation registry coordinates claims across parallel stacks, including stopped instances and stacks. Before first listener activation, obtain an assignment and bind it. Automatic allocation excludes managed claims and tries to bind an available candidate. Persist the assignment before advertising it and retain that socket; do not probe then release it. Later activations bind the saved port. Fixed requests obey the same ownership and binding checks. + +```mermaid +flowchart LR + Client["Client or dependent service"] --> Listener["Stable public listener
Instance-owned or stack-owned"] + Listener --> Route["Route to one instance
Wake and traffic attribution"] + Route --> Backend["Current backend address"] + Allocator["Shared allocator
Persisted listener claims"] -.-> Listener + Runtime["Native or container runtime"] -.-> Backend +``` + +| Operation | Dedicated instance listener | Route on a shared listener | +| ---------------------- | --------------------------------------------------------------- | ---------------------------------------------------------- | +| Start | Bind saved port and connect backend | Enable this route and bind the saved shared port if needed | +| Restart | Retain assignment and update target | Retain shared assignment and update this route's target | +| Sleep | Keep listener for wake | Keep route armed for wake | +| Stop | Close listener; retain claim | Disable this route; leave other routes alone | +| Instance destroy | Remove endpoint and release its claim | Remove this route; retain the stack-owned listener claim | +| Namespace stop/destroy | Close all listeners; stop retains claims, destroy releases them | Same rule | + +A shared listener stays bound while any route is running or armed. When no route needs it, its socket may close while its stack-owned claim remains. Removing the final route does not release that claim; namespace destruction or explicit listener reconfiguration does. Sleeping routes still need the listener. + +**Public port stickiness is a retained contract.** Sleep/wake, stop/start, restart and host reopening preserve every established assignment, including the shared API port. Other managed stacks cannot take retained claims. An unrelated process may occupy a saved port while its socket is closed; the next activation reports a conflict and preserves the assignment, never silently relocating it. A destroyed and recreated instance has a new identity and receives a new dedicated allocation. + +Standalone and composed instances use the same allocator. Two shadow databases have separate IDs, data and dedicated listeners. Composition membership does not change listener ownership. The runtime reports the backend address; the proxy updates its target without altering the public assignment. + +The endpoint renderer produces host-facing or stack-runtime-facing connection values for `service.credentials({ from: "host" | "runtime" })` and for composition wiring. This is ordinary networking configuration, not a dependency-aware tool API. In container mode, both the rendered address and configured listener reachability must work from that network; report unsupported network configurations before executing the dependent/tool. + +| Owner | Responsibility | +| ------------------ | ----------------------------------------------------------------------------------------------------- | +| Service definition | Declare endpoint names and workload behavior | +| Composition | Map shared HTTP routes to individual instances | +| Allocator | Assign and persist listener addresses and claims | +| Runtime | Report the launched workload's backend address | +| Networking/proxy | Render reachable connection values, bind listeners, route traffic and attribute activity to instances | + +Mutable files live under the stack namespace; container resources carry equivalent identity labels. Shared immutable artifact caches and host-wide port coordination are justified exceptions. User-requested exports can live at their chosen destination. + +The StackHost serializes updates to saved instance definitions, composition wiring and resource assignments. Lifecycle, health, active operations, runtime handles and errors remain in the live instance observation. There is no durable lifecycle/operation journal, projected capability state or duplicate stack lifecycle state. + +Persistence supports reopening normally stopped instances, not reconstructing interrupted execution after owner loss. Do not infer current runtime state from saved configuration. When the host is absent or unreachable, expose the saved definitions and ports separately from unavailable live observations. Crash recovery, automatic orphan cleanup, resource adoption, interrupted-operation replay and private-format migration machinery are outside scope. + +### Durable stack layout + +The state root is the stack registry root. Each stack keeps one state document and its owned runtime data together: + +```text +//state.json +//data//... +``` + +Registry updates use an OS-backed lock through a private `node:sqlite` connection to +`/.registry-lock.sqlite`. Each `withLock` call opens its own connection, disables +SQLite busy waiting, and retries `BEGIN IMMEDIATE` contention for up to five seconds through +Effect. Closing the scoped connection releases the lock, including on cancellation or failure; +process death also releases ownership. The file stays in place and is accessed only through +SQLite. No tables, state records, or WAL are created there. Saved stack data remains in JSON; +the lock does not make multi-file operations transactional or recover interrupted operations. +This uses the built-in SQLite API available in the pinned Bun runtime and modern Node.js. + +The artifact cache is independent and shared across stacks. Normal stop preserves the stack directory and service data. Destroy removes the state document and proven-owned, empty parents; caller-owned paths such as Storage uploads remain untouched. + +### Snapshots belong to the database instance + +Snapshots are initially supported only for `database`, including the shadow-baseline cache use case. Start with that concrete case. + +Expose `exportSnapshot` and `restoreSnapshot` on `DatabaseInstance` only. The common service handle retains lifecycle, health and logs; REST and other instance types do not expose unsupported snapshot methods. Keep a database-specific snapshot descriptor. No generic snapshot provider registry, mandatory storage interface or whole-composition snapshot is needed until a second concrete use case requires one. + +```ts +interface DatabaseInstance extends ServiceInstance { + readonly service: "database"; + exportSnapshot(options: { destination: string }): Promise; + restoreSnapshot(options: { source: string }): Promise; +} + +// `baseline` has already been initialized; `shadow` is a fresh instance. +await baseline.stop(); +await baseline.exportSnapshot({ destination: archivePath }); + +await shadow.restoreSnapshot({ source: archivePath }); +await shadow.start(); +await shadow.ready(); +``` + +The database implementation owns the snapshot format, PostgreSQL data selection, compatibility validation, initialization metadata and credential reconciliation. It uses native filesystem operations or container volume/helper operations through the runtime backend. The orchestrator knows only admission, instance ownership and operation settlement; it never needs to understand PostgreSQL archive contents. + +Keep the contract narrow: + +- Export requires confirmed stopped, initialized data. The caller explicitly stops first; snapshotting does not secretly stop dependents or restart services. +- Restore requires a confirmed stopped instance with empty data. Validate archive safety, format, artifact/runtime compatibility and initialization profile before installing restored data. Reject a nonempty target; there is no overwrite option. +- Both operations occupy the instance's existing serial operation gate and leave lifecycle stopped. Queued start, destroy or another storage operation waits for settlement and revalidates. No new lifecycle states are necessary; the observable pending operation identifies snapshot work. An armed wake route is not a substitute for explicit stop. +- Export publishes only a completed archive. Validate and stage restoration before treating the target as usable. Normal failures clean up this invocation's temporary resources and report any cleanup failure; incomplete restoration is never reported as success. Do not add persisted recovery phases or an operation journal. +- Restore transfers compatible database contents, not the source instance's identity, public port claims or composition membership. The target retains its own data location and configuration, with database-specific credentials reconciled before readiness. + +These are physical database snapshots for the cache use case. A `pg_dump` invocation remains an ordinary client tool for logical exports. CLI code owns cache keys, eviction, migrations and the decision to fall back to rebuilding a baseline. The snapshot API does not acquire those policies. + +### Resetting database data + +`DatabaseInstance.resetData` removes the selected database instance's owned data and initialization metadata while retaining its registration, configuration, composition bindings, and public port assignments. It is database-specific, alongside snapshot export and restore; other service types do not expose a reset operation. + +The caller must stop the database with wake disabled first. Reset runs through the same serialized storage-operation gate as snapshots and leaves the database stopped. Deletion uses the database's ownership checks and runtime-specific filesystem handling. The next normal start initializes a fresh PostgreSQL cluster using the retained configuration. Reset does not apply project migrations or seeds, stop other services, or resume the composition; those decisions belong to the CLI. It adds no lifecycle state or persisted recovery phase. + +## 9. What changes for the CLI + +Make lifecycle and health explicit in status instead of forcing them into the old single `phase` enum. Update repo consumers together; do not add a historical compatibility projection. A presentation label can be derived as follows: + +| Runtime observation | Display/consumer meaning | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Stopped, wake disabled, no exit error | Stopped | +| Stopped with armed wake route | Sleeping | +| Launch transition executing | Starting process | +| Running, health starting | Running; waiting for readiness | +| Running, healthy | Ready | +| Running, unhealthy | Unhealthy; process still running | +| Stopping | Stopping | +| Process exited | Exit result is visible while ordinary cleanup settles, then stopped; report unexpected exit as failure | +| Stop cleanup failed | Runtime cleanup remains stopping with current process/exit observation and error; stop/restart retries cleanup | + +Functions serve observes lifecycle and health separately and reports an unhealthy or stopped runtime. DB/storage readiness checks require running and healthy. The current operation remains observable in memory, but shadow finalizers no longer poll it: `destroy()` queues behind an executing snapshot and runs after settlement. Do not persist it for crash recovery. Health observations are attached to the current launch; they cannot make a replacement healthy or failed. + +The public surface stays recognizable: create/open/discover, services create/get/list, lifecycle, credentials, status/logs and preparation. Promise stays an outer adapter over Effect. The important API clarification is **launch versus readiness**. + +Use one explicit contract: `start` completes launch; `ready` waits for the current launch's readiness. For an individual instance, `restart` is serialized stop plus launch, followed by a separate readiness wait where the consumer needs it. Composition orders these ordinary instance operations without a second lifecycle state machine. Default composition start composes the two for its eager members and their dependencies. No readiness wait holds the lifecycle gate. + +Update actual consumers together: + +| Consumer | Required sequence | +| -------------------- | -------------------------------------------------------------------------------------------------------- | +| DB start and shadows | Launch → await ready → SQL/migrations | +| Functions serve | Launch/restart → await ready → follow logs and status; inspector flags are unsupported | +| Proxy wake | Launch if sleeping → await ready → forward | +| DB reset/cache | Stop selected dependents → stopped snapshot/restore or DB config work → launch/ready → resume dependents | +| Tools | Select identity/artifact → execute/stream → await exit and cleanup | + +A Functions runtime exit must reach `followStatus` so serve can report it. + +### Established CLI integration boundaries + +The Stack package remains the owner of identity semantics. It canonicalizes `projectRoot`, resolves the Git branch context (or ordinary-workspace fallback), and validates the stack name in [`Identity.ts`](./src/identity/Identity.ts); it also owns `deriveStackId` from that complete tuple. The CLI currently calls `resolveStackIdentity` through the internal [`identity` entrypoint](./src/identity/Identity.ts), then uses the result when matching `discover` records for status and related read operations. Resolving identity is read-only and does not create a stack. A follow-up recommendation is to expose an equivalent public, read-only `resolveIdentity` operation so the CLI need not import an internal entrypoint; this is a recommended public API, not an existing export. + +When `stack start` includes Functions, the CLI uses the existing package export `@supabase/stack/internal/functions/serve-main`, whose source is [`serve.main.ts`](./src/functions/serve.main.ts), as the bootstrap entrypoint for esbuild bundling in [`stack-functions-bundler.ts`](../../apps/cli/src/command-internal/stack-functions-bundler.ts). The stack-backed `functions serve` command uses the same bootstrap when it creates a temporary Functions instance. Configured embedded templates may satisfy the same bootstrap input before bundling is needed. + +The CLI owns the foreground `functions serve` session. It attaches to an existing composition member and leaves it available on exit. Supported explicit overrides replace its configuration for the session, then restore it on normal cleanup. If Functions is excluded, the CLI creates and later destroys one standalone instance without changing composition. The package needs no session or recovery API: ordinary create, start, restart, status, logs, and destroy suffice. Functions accepts custom environment values and a database URL; its recipe derives default keys, while the composer supplies the runtime database URL without a dependency edge. See the [command lifecycle](../../apps/cli/docs/stack-commands.md) for supported flags and cleanup limits. + +PostgreSQL artifact knowledge remains in Stack and is exposed through [`postgres-artifact.ts`](./src/internal/postgres-artifact.ts), including catalog resolution and native artifact preparation and verification. A remote `db dump --db-url` can use those existing helpers and run without creating a local or dummy stack: the CLI owns the external process or container execution, as shown by [`bundled-postgres-client.ts`](../../apps/cli/src/command-internal/bundled-postgres-client.ts), while managed jobs continue to use `stack.tools.run`. + +Changing internal and public-to-repository contracts is acceptable when callers are updated; preserving valuable data is still required. Keep per-instance configuration replacement through `service.restart({ config })`. Validate the candidate configuration and prepare its artifacts before stopping the existing runtime; invalid input must leave it running. The admitted restart then performs ordinary stop and launch without waiting for application health inside the gate. Adding or removing a companion means explicitly adding or removing an ordinary instance and updating composition edges. Validate the graph using the same rules as registration; do not implement private-child expansion or group replacement logic. Defer live shared configuration changes and config-bearing whole-stack restart. + +## 10. What we can delete or consolidate + +| Current complexity | Replacement | +| -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| Start-to-stop ownership handoffs and supersession | Serial transitions; health wait outside transition | +| Lifecycle decisions spread across phase maps, batch claims, traffic and journals | One instance record and transition executor; one short graph admission boundary | +| Capabilities and hidden service groups | Remove; each service has an instance ID and its own lifecycle/health | +| Global runtime specifications reading whole stack state | Cohesive recipes with narrow inputs | +| Durable operation journals and crash reconciliation | Remove; persist only normal stop/start definitions and resources | +| Separate capability/stack lifecycle projections | Observe instance state directly; composition returns its member observations | +| Separate whole-stack behavior | Selection over the same graph operations | +| Managed-local CLI tool ownership branches | Shared stack job execution | + +Before the package is considered complete or integrated with the final CLI, remove superseded package capability, supervisor, projection and journal implementations together with tests and exports that only served those implementations. The final package has one implementation path and no compatibility facade or parallel lifecycle implementation. + +During this replacement, the in-repository CLI may temporarily fail to compile against the new package. Update its consumers in the final integration step before repository-wide acceptance; do not retain old package exports to keep that intermediate state compiling. + +Do not replace every existing file with a new abstraction. A recipe is an ordinary module, a graph is ordinary data, a transition is a bounded Effect, and readiness is observation of an owned runtime. No actor framework, event-sourced history, generic workflow engine, or automatic recovery scheduler is needed. + +## 11. Acceptance scenarios + +Verify through consumed integration flows: + +1. Studio and pg-meta appear as individually addressable instances with separate lifecycle/health; no capability or hidden group handle exists. REST accepts the same configuration shape for a managed database URL or an external URL; only composition carries a managed readiness edge. Launch finishes before health; stop during health waiting uses ordinary stop. Stop during actual launch waits for launch settlement. No lifecycle signal pre-empts a transition. +2. Dependency readiness gates dependent launch; stopping a prerequisite invalidates an outstanding wait and a stale plan cannot restart it. Composition restart stops dependents first, then starts prerequisites first. Racing an individual start with composition stop yields a safe partial failure rather than bypassing a dependency check or holding a batch reservation. +3. Register two standalone databases, then start/restart the default composition: neither standalone instance is implicitly started or restarted. Namespace stop/destroy still includes both. Two stacks and multiple databases have separate ports/data; a shadow operation does not block Functions restart. Record every public endpoint, stop the stack, start another parallel stack, then reopen/start the original: every original public port is unchanged. Occupying one saved port with an unrelated listener instead produces a conflict without rewriting its assignment. +4. Start a composition containing lazy database and REST instances: startup returns concrete URLs with their proxy listeners bound and their backend processes still absent. Connecting to the returned database/REST URLs launches the appropriate instances without changing the URLs. No public or RPC sleep operation is exposed. Proxy inactivity can trigger internal sleep; public traffic prevents it; sleep retains wake; explicit stop disables wake; prerequisite idle eligibility is rechecked after dependent sleep. +5. Unexpected exit disables wake and triggers ordinary cleanup of the owned runtime before another launch; Functions can explicitly restart afterward. Traffic cannot restart an exited prerequisite. A stale exit cannot damage a replacement. +6. Normal snapshot failures do not publish partial archives or overwrite existing data. Client disconnection leaves admitted snapshot work owned by the live host. Host-crash recovery is not an acceptance requirement; unavailable observations must not be presented as current running/healthy/stopped state. +7. Container-mode service wiring and dumps receive a reachable runtime-facing database URL as plain data. Dumps stream with bounded buffering and exact cleanup; stackless commands create no managed stack. Stopping/destroying REST leaves Auth on the shared API port working; shared port claims survive removal of the final route. +8. Namespace stop settles executing work and leaves no live runtime resources; failed cleanup cannot report successful stop. Disconnecting a client does not stop admitted lifecycle/snapshot operations. Stopping the last individual instance or finishing the last tool does not retire the StackHost; namespace stop/destroy does. diff --git a/packages/stack/README.md b/packages/stack/README.md index 405e2a4045..80c6b01f5a 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -1,178 +1,182 @@ # `@supabase/stack` -The local Supabase stack runtime. Its public API is a greenfield, -Effect-native managed runtime; implementation modules are private to the -package. - -The package runs independently of the CLI and accepts normalized `StackConfig` values; -it does not load `config.toml`. CLI command wiring, configuration translation, and -presentation belong to the M5 integration work in separate PRs. - -The supported entrypoints are: - -- `@supabase/stack` — Promise facade -- `@supabase/stack/effect` — Effect-native API -- `@supabase/stack/testing` — test helpers - -Stacks are managed identities: closing a handle does not stop a running stack. Creating or opening a -handle starts nothing, and a stopped stack retains no Supervisor, workload, container, network, or -listener. Status and retained logs remain available directly from durable state while stopped; a -later start on the same handle launches a fresh Supervisor. -With no configuration override, all capabilities and their companion workloads (including -imgproxy and Vector) are enabled, PostgreSQL is the only eager capability, and every other -capability is lazy. Starting the stack therefore launches only -PostgreSQL by default; capabilities configured as eager join its startup dependency closure. -The remaining lazy capabilities activate through the stack's listeners on demand for the current -running session. - -Lazy REST, Auth, Realtime, Studio, and pooler capabilities stop after 60 seconds without traffic -by default. Traffic means an active request or stream; idle HTTP keep-alive sockets do not keep a -service running, while open WebSocket or TCP connections do. Configure a different positive -timeout, or disable traffic stopping for a capability, with `idleTimeoutSeconds`: +An Effect-based local Supabase runtime with individually identified services, explicit composition, and a detached owner process. See [the architecture](ARCHITECTURE.md) for lifecycle and ownership rules. + +The package accepts service configuration directly. Loading CLI configuration, migrations, seeds, and command presentation belong to the CLI. + +## Standalone services + +The Promise entrypoint accepts plain configuration: ```ts -await stack.start({ +import { create, postgres } from "@supabase/stack"; + +const stack = await create({ + projectRoot: process.cwd(), + runtime: "native", // or "docker" / "podman" + stateRoot: "/tmp/example-stack/state", + cacheRoot: "/tmp/example-stack/cache", +}); + +const database = await stack.services.create({ + service: "database", config: { - capabilities: { - rest: { idleTimeoutSeconds: 120 }, - realtime: { idleTimeoutSeconds: false }, + version: "17", + databasePassword: "local-password", + jwtSecret: "local-development-secret-at-least-32-characters", + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, +}); + +await database.start(); // process launched +await database.ready(); // initialization and health completed +const { databaseUrl } = await database.credentials({ from: "runtime" }); + +if (databaseUrl === undefined) throw new Error("Database endpoint missing"); +await stack.tools.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "--command", "SELECT 1"], + stdout: (bytes) => { + process.stdout.write(bytes); + }, + stderr: (bytes) => { + process.stderr.write(bytes); + }, +}); + +await database.stop(); +await database.exportSnapshot("/tmp/example-stack/baseline.tar"); +await stack.close(); // disconnects this client +``` + +`start` and `ready` are separate operations. `restart({ config })` replaces recipe configuration while retaining the instance identity and endpoint intentions. A health failure leaves a launched process running and observable; it does not prevent `stop`. Database snapshots require a stopped instance with wake disabled. Other service handles have no snapshot methods. + +Creating a service records its definition. Configured public ports are bound during startup and retained across normal stop/start and owner reopening. An occupied saved port reports a conflict instead of moving. Omitted public endpoints are not exposed. + +Native public listeners bind to loopback. Docker and Podman public proxies bind all interfaces so services inside the container network can reach them; those listeners are reachable from the LAN according to the host firewall. + +Functions configuration requires bootstrap source. Database versions belong in `config.version`; other recipes accept an optional top-level artifact `version`. + +On Linux, native Functions project files must be outside `/tmp`: Edge Runtime uses a private filesystem at that path. Docker and Podman mount project files at a separate runtime path. + +`open({ id, stateRoot, cacheRoot })` reconnects to a saved stack. The package stores the stack document at `//state.json` and service data at `//data/`. `discover({ stateRoot })` lists saved definitions and port assignments separately from live-owner availability. Offline definitions are not live lifecycle observations. + +The stack owns database, Functions bootstrap, and tool-job directories below its data directory. Storage uploads remain at the caller-supplied Storage `filePath` and are preserved when the stack is destroyed; the caller owns that directory. + +Without an explicit database root key, PostgreSQL generates one in its data directory. The key survives stop/reopen and is removed with the database data on reset or destroy. + +## Composition and operation scope + +Registering a service does not add it to the application composition. For a fresh composition, `composition.supabase` registers the selected recipes and supplies their standard bindings: + +```ts +const services = await stack.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: "local-password", + jwtSecret: "local-development-secret-at-least-32-characters", + jwtExpiry: 3600, }, + endpoints: { sql: { port: "auto" } }, }, + { + service: "rest", + config: { databaseUrl: "postgresql://configured-by-composition" }, + endpoints: { http: { port: "auto" } }, + }, +]); +await stack.composition.start(); +``` + +The factory accepts one instance of each selected recipe, binds its configured public endpoints, and wires managed inputs such as REST's database URL. When the database SQL endpoint is configured, Functions receives the saved database URL as an ordinary input too; recompose the composition after rotating database credentials to refresh that value. This binding does not make Functions wait for database readiness. Database is eager; Functions are lazy without an idle timeout; other public services are lazy with a 60-second idle timeout. Services without public endpoints are eager. A managed URL binding requires its producer's endpoint to be configured. The factory also supplies ordinary host/runtime API URLs to Auth, Studio, and Functions without adding dependencies from those URLs. It rejects an already configured composition. + +The whole-stack E2E suite covers the default lazy lifecycle and reopen, all-eager startup, idle and wake, and parallel stack isolation for native and Docker runtimes. Run one runtime with `pnpm --filter @supabase/stack test:e2e:run src/whole-stack.native.e2e.test.ts` or the corresponding Docker test file. + +For multiple instances or custom dependencies, configure members and bindings explicitly instead: + +```ts +const rest = await stack.services.create({ + service: "rest", + config: { databaseUrl: "postgresql://configured-by-composition" }, + endpoints: { http: { port: "auto" } }, +}); + +await stack.composition.configure({ + members: [ + { id: database.id, activation: "eager" }, + { id: rest.id, activation: "lazy", idleMillis: 60_000 }, + ], + dependencies: [ + { + from: database.id, + to: rest.id, + bindings: [{ output: "authenticatorUrl", input: "databaseUrl" }], + }, + ], }); +await stack.composition.start(); ``` -To change `idleTimeoutSeconds` on a running stack, call `stop()` and then `start()` with the updated -configuration. - -Stacks saved before idle stopping retain their previous policy: missing timeout values are read as -`false`. Restarting with the saved definition preserves that policy. To adopt the current defaults, -stop the stack and start it with the project configuration. Status can report changed effective -defaults even when the project file is unchanged; a stack still marked running must be stopped -before those defaults can be applied. - -Eager capabilities never auto-stop; an explicit timeout on an eager capability is ignored and its -effective timeout is `false`. PostgreSQL, Storage, Functions, Mail, and Analytics do not accept -idle timeout configuration. Studio and its `pg-meta` companion are stopped and started together. -Dependency protection keeps required dependencies available while a capability is running. -Stopping preserves listeners and data, and the next request wakes the lazy capability and restarts -its workloads. -Retirement cleanup is fail closed: if removal is unproven, the stack remains stopping and new -activation is rejected. An unproven cleanup marks participating capabilities as failed and fences -new activation across the stack. This is an operation-level result; the workload ledger tracks -resources still requiring removal. Unrelated healthy capabilities retain their observations. An -explicit stop retries the retained ledger. A failed committed destroy remains destroying and -accepts only a destroy retry; successful cleanup is required before the managed state is removed. -Status exposes the required recovery operation (`stop` or `destroy`) and its reason in `recovery`. -Participating capability failure states describe incomplete cleanup, including shared listener -cleanup; they do not imply that each workload process failed. - -The Effect API's `excludeStackCapabilities` helper disables requested optional capabilities and -their dependents in an in-memory config. Excluding `rest` also disables `studio`; excluding -`analytics` does not. The database remains required. The project config is unchanged, and runtime listeners are -created only for enabled capability routes. - -Native workloads have a two-minute readiness budget to allow cold starts to load shared libraries; -container workloads retain a 30-second budget, and PostgreSQL uses its configured `health_timeout`. -Each readiness probe returns immediately when its endpoint becomes healthy. - -Artifact preparation is controlled independently from capability activation through the optional -top-level `preparation` setting: +Bindings supply ordinary configuration values; a URL alone never creates a dependency. An independent REST instance can instead receive an external database URL. Lazy members receive public listeners before their processes start. Public traffic starts an armed instance and waits for health; inspector connections can attach before health succeeds. Explicit stop disables wake. An individual restart runs the instance explicitly; restart the composition to reapply its lazy policy. + +- Instance methods affect that instance, subject to dependency checks. +- Composition methods affect selected members; startup also includes declared prerequisites. +- `stack.stop()` stops every owned instance and attached tool and returns after confirming owner exit. It requires a live owner; an unavailable owner cannot confirm cleanup. +- `stack.destroy()` additionally removes owned data and registrations, and also waits for owner exit. +- `stack.close()` disposes the client and invalidates its active observation iterators. Stopping the last instance leaves the owner available. + +Exit confirmation is bounded. If cleanup is acknowledged but owner exit cannot be confirmed, the operation fails with `operation: "shutdown-exit"` and the owner PID in the message. A failed or cancelled call does not guarantee that teardown has completed. Do not start or restart the same stack concurrently with whole-stack shutdown; separate stacks remain independent. + +Disposable test stacks need explicit teardown; closing a client does not own their lifetime: ```ts -await stack.start({ config: { preparation: "on-demand" } }); +try { + // Exercise the stack. +} finally { + try { + await stack.destroy(); + } finally { + await stack.close(); + } +} ``` -The default `"background"` mode prepares all enabled lazy artifacts after PostgreSQL has started, -without launching those services. A single Supervisor-owned background operation prepares the -finite selected workload set concurrently, streams downloads through hashing and decompression, -is canceled and awaited by `stop()` or `destroy()`, and keeps completed cache entries. -`"on-demand"` skips that background work for callers that want full lazy preparation. In either -mode, activating a lazy service prepares its requested dependency closure concurrently, while -explicit `stack.prepare(...)` remains available as a cache-only warmup. Eager capabilities remain -independent of this preparation policy. The setting is persisted with the stack definition and -survives `openStack()` and restart. Changing it for a running stack follows the existing -stop-before-change configuration rule. - -Running status includes an artifacts array for the current session. Each entry identifies a -workload and capability and reports `queued`, `preparing`, `downloading`, `ready`, or -`failed`; `failed` includes an error message and can be retried by activating the capability again. -`preparing` covers validation, verification, and extraction around the transfer. During stopping -or destroying, status may retain active preparation until teardown clears it. Once stopped, status -reports an empty array even when completed artifacts remain in the cache. - -The Functions inspector can be exposed through its own loopback listener. Set the Edge Runtime -mode and enable that listener together; the resulting `functionsInspector` endpoint forwards the -runtime's `/json/list` and WebSocket inspector paths. +Each service exposes `status`, `followStatus`, `logs`, and `credentials`. Observations include the currently bound public endpoints, including listeners for sleeping services. Credentials default to host addressing. Use `from: "runtime"` for a URL passed to a service or tool container. + +## Effect consumers + +The Effect entrypoint exposes the same operations as Effects and Streams. Database secrets use `Redacted` in the Effect configuration: ```ts -await stack.start({ - config: { - capabilities: { functions: { settings: { inspector: { mode: "run", main: true } } } }, - listeners: { functionsInspector: { enabled: true, address: "127.0.0.1", port: 9223 } }, - }, +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { Effect, Layer } from "effect"; +import * as Stack from "@supabase/stack/effect"; + +const program = Effect.gen(function* () { + const stack = yield* Stack.open({ id, stateRoot, cacheRoot }); + const database = yield* stack.services.get(databaseId); + yield* database.start; + yield* database.ready; + return yield* database.status; }); -const inspector = (await stack.status()).endpoints.functionsInspector; + +await Effect.runPromise( + program.pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); +``` + +Cancelling an admitted lifecycle caller ends its wait; the owner finishes the operation. Cancelling an attached tool ends that job and cleans up its resources. Tool input and output stream with backpressure; the result contains a job ID and exit code, not collected output. Promise tool sinks should return a Promise when the destination requires waiting for capacity. + +For disposable Effect test fixtures, `acquireUseRelease` runs teardown on failure and interruption while retaining typed cleanup errors: + +```ts +const test = Effect.acquireUseRelease( + Stack.create(options), + (stack) => exerciseStack(stack), + (stack) => stack.destroy, +); ``` -Native runtimes bind a private inspector port on loopback. Container runtimes bind port `9229` -inside the service and publish that private port to the configured loopback listener. Connect your -debugger through `inspector.url`; the workload's private inspector port stays local to the stack. - -Explicit `stack.prepare(...)` accepts a synchronous `onProgress` callback for -caller-owned preparation. It receives the same phase values, including `ready` when an artifact is -available while its capability remains dormant. This transfer-local callback is not reconstructed by -a separate status request. - -The package's end-to-end contract is exercised through the same public Stack API in native and -Docker modes. It begins from the PostgreSQL-only default, progressively activates every service with -realistic traffic, and verifies stop/start cycles, stable ports, and persistent data. The -CLI is not involved in these runtime tests. - -When `runtime` is omitted for a new stack, the package selects Docker when the Docker client is -installed and its daemon is reachable, and native otherwise. An installed client with an -unreachable daemon selects native, and the Effect handle carries a `dockerFallbackNotice` explaining -that the choice is persisted; switching to Docker later requires destroying the stack or choosing a -new stack name. The Promise facade does not expose the notice. -Native is refused when the process runs as uid 0. Existing stacks reuse their persisted runtime -without probing; native, Docker, and Podman preferences remain explicit when supplied, and an -explicit Docker runtime does not fall back. Podman is supported only on local Linux hosts. - -Stack identity is the length-delimited SHA-256 tuple of the canonical project root, Git branch -context (or `ordinary-workspace` outside Git), and stack name. Separate worktree roots, branches, -projects in a monorepo, and named stacks therefore receive separate managed state. Identity -resolution is read-only; moving a project creates a new identity. - -`createTestStack` gives each test stack a unique temporary project root and identity while sharing -the managed state root used by ordinary package callers. It uses the same runtime selection as -`createStack`: a reachable Docker daemon selects Docker, and anything else selects native. Pass -`runtime: { kind: "native" }` or an explicit container runtime for reproducible test environments. -Automatic ports therefore -coordinate across all default callers. Helper project roots and identities remain isolated; a -temporary test stack is excluded from listings scoped to another project root but appears in an -unfiltered package `listStacks()` result. A failed destroy retains the affected project root and -managed state for recovery. - -Callers can warm selected native artifacts or container images with `stack.prepare(...)` while a -stack is stopped or running; explicit preparation is cache-only and cancellation does not affect -completed entries. -Each capability may opt into eager activation in `StackConfig`; omitted settings keep every -non-PostgreSQL capability lazy. Prepared artifacts are not automatically pruned. `followLogs(...)` -provides filterable live entries through a stateless client-polled cursor. - -`resetDatabase()` wipes Postgres data only: identity, ports, secrets, logs, and storage volumes -stay. The database is started and bootstrapped before return. Applying migrations, declarative -schemas, and seeds remains the caller's responsibility. The runtime bootstrap only reconciles the -`_realtime` schema owner, closed database role passwords, and JWT settings in one transaction; the -slim database artifact owns its initialization and migrations. - -`createEphemeralPostgres` is a scoped, Supervisor-free Postgres cluster for schema tooling. It uses -the same catalog artifact and bootstrap as a stack database, is not registered in `listStacks` / -`discoverStacks`, and destroys its data directory or volume when the Effect scope closes. The -Promise facade returns a handle with explicit `destroy()`. Callers own migrations and PGDATA -cache keys. `exportPgData` is valid only while the cluster is stopped; native and container snapshots -are not interchangeable. - -`runPostgresClient` prepares that same catalog pin and runs caller argv (`bash -c` dump scripts, -`pg_prove`, …) without starting Postgres. Native prepends `artifact/bin` to `PATH`; container is a -one-shot `docker|podman run --rm`. It is not an `EffectStack` method, so linked dump can prepare -tools without a running stack. +The owner supports normal stop/start persistence. Unexpected owner death does not trigger resource adoption, orphan removal, or interrupted-operation recovery. CLI integration is maintained separately from this package. diff --git a/packages/stack/package.json b/packages/stack/package.json index c98ce1e480..ef95e90bae 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -6,8 +6,11 @@ "exports": { ".": "./src/index.ts", "./effect": "./src/effect.ts", - "./testing": "./src/testing.ts", - "./internal/supervisor": "./src/internal/supervisor-process.ts" + "./tools": "./src/Tools.ts", + "./internal/dispatch": "./src/internal/dispatch.ts", + "./internal/identity": "./src/identity/Identity.ts", + "./internal/functions/serve-main": "./src/functions/serve.main.ts", + "./internal/postgres-artifact": "./src/internal/postgres-artifact.ts" }, "scripts": { "test": "pnpm run test:unit && pnpm run test:integration", diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts new file mode 100644 index 0000000000..1f7ed92735 --- /dev/null +++ b/packages/stack/src/Artifacts.ts @@ -0,0 +1,266 @@ +import { Data, Effect, Path } from "effect"; +import { makeArtifactStore } from "./preparation/ArtifactStore.ts"; +import { + makeSlimServicesSource, + type SlimServicesArtifact, +} from "./preparation/SlimServicesSource.ts"; + +type NativeTarget = SlimServicesArtifact["target"]; + +export type ServiceKind = + | "database" + | "rest" + | "auth" + | "realtime" + | "storage" + | "imgproxy" + | "functions" + | "studio" + | "pgmeta" + | "mail" + | "analytics" + | "vector" + | "pooler"; + +export class ArtifactError extends Data.TaggedError("ArtifactError")<{ + readonly message: string; + readonly service?: string; + readonly version?: string; + readonly platform?: string; + readonly cause?: unknown; +}> {} + +interface ArtifactResolution { + readonly service: ServiceKind; + readonly version: string; + readonly image: string; + readonly executablePath: string; + readonly requiredRuntimePaths: ReadonlyArray; +} + +export interface PreparedNativeArtifact { + readonly service: ServiceKind; + readonly version: string; + readonly root: string; + readonly executable: string; +} + +interface ArtifactDefinition { + readonly sourceService: string; + readonly defaultVersion: string; + readonly images: Readonly>; + readonly requiredRuntimePaths: ReadonlyArray; + readonly executablePath: string; +} + +const definition = ( + sourceService: string, + defaultVersion: string, + image: string, + executablePath: string, + requiredRuntimePaths: ReadonlyArray = [executablePath], + additionalImages: Readonly> = {}, +): ArtifactDefinition => ({ + sourceService, + defaultVersion, + images: { [defaultVersion]: image, ...additionalImages }, + requiredRuntimePaths, + executablePath, +}); + +const definitions: Readonly> = { + database: definition( + "postgres", + "17.6.1.173", + "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + "bin/supabase-postgres-start", + ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], + { + "15.14.1.173": + "ghcr.io/supabase/cli/postgres:15.14.1.173@sha256:b7d210fa3bca26568fa20448e3bec092bd4f8da6a5af5029597639d38e7d896e", + }, + ), + rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), + auth: definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth"), + realtime: definition( + "realtime", + "v2.134.5", + "ghcr.io/supabase/cli/realtime:v2.134.5@sha256:7fb53cc6987085d739c7d161608505ed138d1895df884c3bdc2147cef444138a", + "bin/server", + ["bin/server", "bin/prepare"], + ), + storage: definition( + "storage", + "v1.73.0", + "ghcr.io/supabase/cli/storage:v1.73.0@sha256:69590a75f916837641976d4018e5ead7c7d2c2305312d9bfb06d86aec8fb1cdd", + "bin/storage", + ["bin/storage", "bin/prepare"], + ), + imgproxy: definition( + "imgproxy", + "v3.8.0", + "ghcr.io/supabase/cli/imgproxy:v3.8.0", + "bin/imgproxy", + ), + functions: definition( + "edge-runtime", + "v1.76.2", + "ghcr.io/supabase/cli/edge-runtime:v1.76.2", + "bin/edge-runtime", + ), + studio: definition( + "studio", + "2026.09.04-sha-5a67366", + "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9823a31668028f1846e87331bc21598d9cd74bcaa1466c72dab58c33c9c82720", + "bin/studio", + ), + pgmeta: definition( + "pgmeta", + "v0.99.0", + "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:90de2dcf03ac548ae2d1d3e71b3cd10bde4c627572720a42e4c3946b7090292e", + "bin/pgmeta", + ), + mail: definition("mailpit", "v1.30.2", "ghcr.io/supabase/cli/mailpit:v1.30.2", "bin/mailpit"), + analytics: definition( + "analytics", + "v1.50.9", + "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:7db85cc6cb0cdeb4b71f2fadb49c0f9197bea0492daf7896f6ae69edad76d28e", + "bin/logflare", + ["bin/logflare", "bin/prepare"], + ), + vector: definition("vector", "0.53.0", "ghcr.io/supabase/cli/vector:0.53.0", "bin/vector", [ + "bin/vector", + "share/doc/vector/config/vector.yaml", + ]), + pooler: definition( + "pooler", + "v2.9.12", + "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:12bb9dcb7ddace79bee173ccb7327c6646af2236679f3bd932a86b3a06479aac", + "bin/server", + ["bin/server", "bin/prepare", "bin/provision-tenant"], + ), +}; + +const targetForPlatform = (platform: { + readonly os: string; + readonly arch: string; +}): NativeTarget | undefined => { + if (platform.os === "darwin" && platform.arch === "arm64") return "darwin-arm64"; + if (platform.os === "linux" && platform.arch === "x64") return "linux-amd64"; + if (platform.os === "linux" && platform.arch === "arm64") return "linux-arm64"; + return undefined; +}; + +const platformText = (platform: { readonly os: string; readonly arch: string }): string => + `${platform.os}/${platform.arch}`; + +const errorMessage = (cause: unknown): string => + cause instanceof Error ? cause.message : typeof cause === "string" ? cause : String(cause); + +const artifactFor = ( + service: ServiceKind, + resolved: ArtifactResolution, + target: NativeTarget, +): SlimServicesArtifact => { + const sourceService = definitions[service].sourceService; + const releaseTag = `${sourceService}-${resolved.version}`; + const assetName = `${releaseTag}-${target}`; + const base = `https://github.com/supabase/slim-services/releases/download/${releaseTag}`; + return { + provider: "supabase/slim-services", + service: sourceService, + version: resolved.version, + releaseTag, + target, + archive: "tar.zst", + assetName, + downloadUrl: `${base}/${assetName}.tar.zst`, + manifestUrl: `${base}/${assetName}.manifest.json`, + checksumUrl: `${base}/SHA256SUMS`, + requiredRuntimePaths: resolved.requiredRuntimePaths, + executablePath: resolved.executablePath, + }; +}; + +export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* (request: { + readonly service: ServiceKind; + readonly version?: string; +}) { + if (!Object.hasOwn(definitions, request.service)) + return yield* new ArtifactError({ message: `Unknown service kind: ${request.service}` }); + const selected = definitions[request.service]; + const version = request.version ?? selected.defaultVersion; + const image = Object.entries(selected.images).find(([candidate]) => candidate === version)?.[1]; + if (image === undefined) + return yield* new ArtifactError({ + message: `Unsupported ${request.service} artifact version: ${version}`, + service: request.service, + version, + }); + return { + service: request.service, + version, + image, + executablePath: selected.executablePath, + requiredRuntimePaths: selected.requiredRuntimePaths, + }; +}); + +/** Resolves a PostgreSQL major alias against the pinned database artifacts. */ +export const postgresVersion = (version: string): string => + Object.keys(definitions.database.images).find( + (candidate) => candidate.split(".")[0] === version, + ) ?? version; + +const artifactKey = (artifact: SlimServicesArtifact): string => + `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; + +export const prepareNativeArtifact = Effect.fn("Artifacts.prepareNativeArtifact")(function* ( + request: { readonly service: ServiceKind; readonly version?: string }, + cacheRoot: string, + platform: { readonly os: string; readonly arch: string } = { + os: process.platform, + arch: process.arch, + }, +) { + const resolved = yield* resolveArtifact(request); + return yield* Effect.gen(function* () { + const target = targetForPlatform(platform); + if (target === undefined) + return yield* new ArtifactError({ + message: `Native artifacts are unsupported on ${platformText(platform)}`, + service: request.service, + version: resolved.version, + platform: platformText(platform), + }); + const sourceArtifact = artifactFor(request.service, resolved, target); + const key = artifactKey(sourceArtifact); + const source = makeSlimServicesSource((candidate) => + candidate.key === key ? sourceArtifact : undefined, + ); + const store = yield* makeArtifactStore({ cacheRoot, source }); + const prepared = yield* store.prepare({ + key, + requiredRuntimePaths: resolved.requiredRuntimePaths, + executablePath: resolved.executablePath, + }); + const path = yield* Path.Path; + return { + service: resolved.service, + version: resolved.version, + root: prepared.path, + executable: path.join(prepared.path, resolved.executablePath), + }; + }).pipe( + Effect.mapError((cause) => + cause instanceof ArtifactError + ? cause + : new ArtifactError({ + message: `Unable to prepare ${request.service} artifact: ${errorMessage(cause)}`, + service: request.service, + version: resolved.version, + cause, + }), + ), + ); +}); diff --git a/packages/stack/src/HostProcess.integration.test.ts b/packages/stack/src/HostProcess.integration.test.ts new file mode 100644 index 0000000000..4f88e9c57a --- /dev/null +++ b/packages/stack/src/HostProcess.integration.test.ts @@ -0,0 +1,359 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + Context, + Data, + Effect, + FileSystem, + Fiber, + Layer, + Option, + Path, + Schema, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; +import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- integration verifies exact-port reopening. +import * as Net from "node:net"; +import { fileURLToPath } from "node:url"; +import { HostEndpoint, connectHost, launchHost } from "./HostProcess.ts"; +import * as State from "./State.ts"; + +class ProcessTestError extends Data.TaggedError("ProcessTestError")<{ readonly message: string }> {} + +const makeTestState = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); + +type ChildHandle = { + readonly child: ChildProcessSpawner.ChildProcessHandle; + closed: boolean; + closeCode: number | null; + closeSignal: NodeJS.Signals | null; +}; + +const spawnChild = (args: ReadonlyArray, stdio: Array<"ignore" | "pipe">) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make(process.execPath, args, { + cwd: process.cwd(), + detached: true, + stdin: stdio[0], + stdout: stdio[1], + stderr: stdio[2], + ...(stdio[3] === "pipe" ? { additionalFds: { fd3: { type: "output" as const } } } : {}), + forceKillAfter: "2 seconds", + }), + ); + const handle: ChildHandle = { child, closed: false, closeCode: null, closeSignal: null }; + return handle; + }); + +const waitForClose = (handle: ChildHandle) => + handle.closed + ? Effect.succeed({ code: handle.closeCode, signal: handle.closeSignal }) + : handle.child.exitCode.pipe( + Effect.map((code) => { + handle.closed = true; + handle.closeCode = Number(code); + return { code: handle.closeCode, signal: null }; + }), + Effect.catch(() => { + handle.closed = true; + handle.closeCode = null; + handle.closeSignal = null; + return Effect.succeed({ code: null, signal: null }); + }), + ); + +const waitForLine = (handle: ChildHandle, streamIndex: 1 | 3, label: string) => + (streamIndex === 1 ? handle.child.stdout : handle.child.getOutputFd(3)).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runHead, + Effect.flatMap( + Option.match({ + onNone: () => + Effect.fail(new ProcessTestError({ message: `${label} exited before readiness` })), + onSome: Effect.succeed, + }), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => Effect.fail(new ProcessTestError({ message: `${label} readiness timed out` })), + }), + ); + +const waitForReady = (handle: ChildHandle) => + waitForLine(handle, 3, "Host").pipe( + Effect.flatMap((line) => + Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ type: Schema.Literal("ready"), endpoint: HostEndpoint }), + ), + )(line).pipe( + Effect.map((message) => message.endpoint), + Effect.mapError((cause) => new ProcessTestError({ message: String(cause) })), + ), + ), + ); + +const waitForLauncherEndpoint = (handle: ChildHandle) => + waitForLine(handle, 1, "Launcher").pipe( + Effect.flatMap((line) => + Schema.decodeEffect(Schema.fromJsonString(HostEndpoint))(line).pipe( + Effect.mapError((cause) => new ProcessTestError({ message: String(cause) })), + ), + ), + ); + +const stopChild = (handle: ChildHandle) => + Effect.exit( + Effect.gen(function* () { + if (!handle.closed) + yield* handle.child.kill({ killSignal: "SIGTERM", forceKillAfter: "2 seconds" }); + yield* waitForClose(handle); + }), + ).pipe(Effect.asVoid); + +const bestEffortShutdown = (client: HttpClient.HttpClient, endpoint: HostEndpoint) => + Effect.exit( + client + .execute( + HttpClientRequest.post(`http://127.0.0.1:${endpoint.port}/shutdown`).pipe( + HttpClientRequest.setHeader("connection", "close"), + ), + ) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)), + ).pipe(Effect.asVoid); + +const bestEffortShutdownByState = ( + client: HttpClient.HttpClient, + state: State.Interface, + stackId: string, +) => + Effect.exit( + connectHost(state, stackId).pipe( + Effect.flatMap((endpoint) => bestEffortShutdown(client, endpoint)), + ), + ).pipe(Effect.asVoid); + +const bindExact = (port: number) => + Effect.callback((resume) => { + const server = Net.createServer(); + const onListening = () => resume(Effect.succeed(server)); + const onError = (cause: Error) => + resume(Effect.fail(new ProcessTestError({ message: cause.message }))); + server.once("listening", onListening); + server.once("error", onError); + server.listen({ host: "127.0.0.1", port }); + return Effect.sync(() => { + server.off("listening", onListening); + server.off("error", onError); + }); + }); + +const closeServer = (server: Net.Server) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + }); + +const waitForMarker = (marker: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.watch(path.dirname(marker)).pipe( + Stream.filter((event) => event.path === path.basename(marker)), + Stream.runHead, + Effect.flatMap( + Option.match({ + onNone: () => Effect.fail(new ProcessTestError({ message: "marker watcher ended" })), + onSome: Effect.succeed, + }), + ), + ); + return yield* fs.readFileString(marker).pipe( + Effect.map(Number), + Effect.mapError(() => new ProcessTestError({ message: "marker read failed" })), + ); + }).pipe( + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => Effect.fail(new ProcessTestError({ message: "slow fixture did not start" })), + }), + ); + +it.live("launches one detached owner and attaches competing launchers", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "host-process-" }); + const state = yield* makeTestState(root); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "local" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const entrypoint = fileURLToPath( + new URL("../tests/host-process-fixture.ts", import.meta.url), + ); + const options = { + stateRoot: root, + cacheRoot: root, + stackId: "stack", + entrypoint, + }; + const client = yield* HttpClient.HttpClient; + yield* Effect.gen(function* () { + const [first, second] = yield* Effect.all( + [launchHost(state, options), launchHost(state, options)], + { concurrency: 2 }, + ); + expect(second.port).toBe(first.port); + expect(second.pid).toBe(first.pid); + }).pipe(Effect.ensuring(bestEffortShutdownByState(client, state, "stack"))); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("lets an external launcher attach, then reopens the owner port after shutdown", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "host-process-launcher-" }); + const state = yield* makeTestState(root); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "local" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const entrypoint = fileURLToPath( + new URL("../tests/host-process-fixture.ts", import.meta.url), + ); + const args = [entrypoint, root, root, "stack"]; + const owner = yield* Effect.acquireRelease( + spawnChild(args, ["ignore", "ignore", "ignore", "pipe"]), + stopChild, + ); + const endpoint = yield* waitForReady(owner); + const observedEndpoint = yield* connectHost(state, "stack"); + const client = yield* HttpClient.HttpClient; + expect(observedEndpoint.pid).toBe(endpoint.pid); + const ownerExitFiber = yield* waitForClose(owner).pipe(Effect.forkScoped); + yield* Effect.gen(function* () { + const response = yield* client.execute( + HttpClientRequest.post(`http://127.0.0.1:${endpoint.port}/shutdown`).pipe( + HttpClientRequest.setHeader("connection", "close"), + ), + ); + yield* HttpClientResponse.filterStatusOk(response); + const ownerExit = yield* Fiber.join(ownerExitFiber); + owner.closed = true; + owner.closeCode = ownerExit.code; + owner.closeSignal = ownerExit.signal; + expect(ownerExit.signal).toBeNull(); + }).pipe(Effect.ensuring(bestEffortShutdown(client, endpoint))); + yield* Effect.scoped( + Effect.gen(function* () { + const reopened = yield* Effect.acquireRelease(bindExact(endpoint.port), closeServer); + expect(reopened.listening).toBe(true); + }), + ); + const relaunched = yield* Effect.acquireRelease( + launchHost(state, { stateRoot: root, cacheRoot: root, stackId: "stack", entrypoint }).pipe( + Effect.provide(FetchHttpClient.layer), + ), + (next) => bestEffortShutdown(client, next), + ); + expect(relaunched.port).toBe(endpoint.port); + expect(relaunched.pid).not.toBe(endpoint.pid); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "starts an owner through an external launcher and keeps it reachable after launcher exit", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: "host-process-external-launcher-", + }); + const state = yield* makeTestState(root); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "local" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const entrypoint = fileURLToPath( + new URL("../tests/host-process-fixture.ts", import.meta.url), + ); + const client = yield* HttpClient.HttpClient; + yield* Effect.gen(function* () { + const args = [entrypoint, root, root, "stack", "launcher", entrypoint]; + const launcher = yield* Effect.acquireRelease( + spawnChild(args, ["ignore", "pipe", "pipe", "ignore"]), + stopChild, + ); + const endpoint = yield* waitForLauncherEndpoint(launcher); + const launcherExit = yield* waitForClose(launcher); + expect(launcherExit.code).toBe(0); + const connected = yield* connectHost(state, "stack"); + expect(connected).toEqual(endpoint); + }).pipe(Effect.ensuring(bestEffortShutdownByState(client, state, "stack"))); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("terminates a detached child when readiness is interrupted", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "host-process-cancel-" }); + const state = yield* makeTestState(root); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "slow-handshake" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const entrypoint = fileURLToPath( + new URL("../tests/host-process-fixture.ts", import.meta.url), + ); + const marker = path.join(root, "slow-handshake.pid"); + const markerReady = yield* waitForMarker(marker).pipe( + Effect.forkChild({ startImmediately: true }), + ); + const launch = yield* launchHost(state, { + stateRoot: root, + cacheRoot: root, + stackId: "stack", + entrypoint, + }).pipe(Effect.forkChild); + const pid = yield* Fiber.join(markerReady); + expect(Number.isInteger(pid) && pid > 0).toBe(true); + expect(() => process.kill(pid, 0)).not.toThrow(); + yield* Fiber.interrupt(launch); + expect(() => process.kill(pid, 0)).toThrow(); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/HostProcess.ts b/packages/stack/src/HostProcess.ts new file mode 100644 index 0000000000..77b49aff1b --- /dev/null +++ b/packages/stack/src/HostProcess.ts @@ -0,0 +1,340 @@ +import { NodeHttpServer } from "@effect/platform-node"; +import { Data, Effect, Duration, Option, Schedule, Schema, Scope, Stream } from "effect"; +import * as HttpServer from "effect/unstable/http/HttpServer"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- NodeHttpServer.make requires a native server factory. +import * as Http from "node:http"; +import { HOST_PROCESS_DISPATCH_SENTINEL, isBunVirtualPath } from "./internal/dispatch-markers.ts"; +import { makePorts, PortError } from "./Ports.ts"; +import type * as State from "./State.ts"; + +export class HostProcessError extends Data.TaggedError("HostProcessError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; + readonly reason?: HostFailureReason; +}> {} +type HostFailureReason = + | "missing-control-listener" + | "connection-failure" + | "bind-conflict" + | "invalid-owner-pid" + | "owner-exit-pending" + | "owner-exit-probe"; +const HostIdentity = Schema.Struct({ + projectRoot: Schema.String, + branchContext: Schema.String, + stackName: Schema.String, +}); +interface HostIdentity extends Schema.Schema.Type {} +export const HostEndpoint = Schema.Struct({ + stackId: Schema.String, + identity: HostIdentity, + pid: Schema.Int, + port: Schema.Int, +}); +export interface HostEndpoint extends Schema.Schema.Type {} +const error = (operation: string, cause: unknown, reason?: HostFailureReason) => + new HostProcessError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + ...(reason === undefined ? {} : { reason }), + }); + +export const acquireHost = Effect.fn("HostProcess.acquireHost")(function* ( + state: State.Interface, + stackId: string, +): Effect.fn.Return< + { + readonly port: number; + readonly server: HttpServer.HttpServer["Service"]; + readonly closeConnections: Effect.Effect; + }, + HostProcessError | PortError | State.StateError, + Scope.Scope | import("effect").Crypto.Crypto +> { + if ((yield* state.read(stackId)) === undefined) + return yield* error("acquire", "Stack is not registered"); + const ports = yield* makePorts(state); + let rawServer: Http.Server | undefined; + const acquired = yield* ports.acquire( + { stackId, key: "control", host: "127.0.0.1", port: "auto" }, + (host, port) => { + const server = Http.createServer(); + rawServer = server; + return NodeHttpServer.make(() => server, { host, port }).pipe( + Effect.mapError( + (cause) => + new PortError({ key: "control", message: "Cannot bind control listener", cause }), + ), + ); + }, + ); + return { + port: acquired.port, + server: acquired.listener, + closeConnections: Effect.sync(() => { + rawServer?.closeAllConnections(); + rawServer?.closeIdleConnections(); + }), + }; +}); + +const endpointFromResponse = Effect.fn("HostProcess.endpointFromResponse")(function* ( + state: State.Interface, + stackId: string, + port: number, +) { + const stack = yield* state.read(stackId); + if (stack === undefined) return yield* error("connect", "Stack is not registered"); + const client = yield* HttpClient.HttpClient; + const response = yield* client + .execute(HttpClientRequest.get(`http://127.0.0.1:${port}/identity`)) + .pipe( + Effect.timeoutOrElse({ + duration: Duration.seconds(2), + orElse: () => + Effect.fail( + error( + "connect", + "Timed out connecting to the host control listener", + "connection-failure", + ), + ), + }), + Effect.mapError((cause) => + cause instanceof HostProcessError ? cause : error("connect", cause), + ), + ); + yield* HttpClientResponse.filterStatusOk(response).pipe( + Effect.mapError((cause) => error("connect", cause)), + ); + const remote = yield* HttpClientResponse.schemaBodyJson(HostEndpoint)(response).pipe( + Effect.mapError((cause) => error("connect", cause)), + ); + if ( + remote.port !== port || + remote.stackId !== stackId || + remote.identity.projectRoot !== stack.identity.projectRoot || + remote.identity.branchContext !== stack.identity.branchContext || + remote.identity.stackName !== stack.identity.stackName + ) + return yield* error("connect", "Control listener belongs to another stack"); + return remote; +}); + +export const connectHost = Effect.fn("HostProcess.connectHost")(function* ( + state: State.Interface, + stackId: string, +): Effect.fn.Return { + const stack = yield* state.read(stackId); + if (stack === undefined) return yield* error("connect", "Stack is not registered"); + const claim = stack.ports.find((entry) => entry.key === "control"); + if (claim === undefined) + return yield* error("connect", "Stack has no control listener", "missing-control-listener"); + return yield* endpointFromResponse(state, stackId, claim.port); +}); + +export interface LaunchOptions { + readonly stateRoot: string; + readonly cacheRoot: string; + readonly stackId: string; + readonly entrypoint?: string; +} +const readyLine = Schema.Union([ + Schema.Struct({ type: Schema.Literal("ready"), endpoint: HostEndpoint }), + Schema.Struct({ + type: Schema.Literal("error"), + message: Schema.String, + reason: Schema.optionalKey(Schema.Literal("bind-conflict")), + }), +]); +const spawnDetached = Effect.fn("HostProcess.spawnDetached")(function* ( + options: LaunchOptions, + entrypoint: string, +) { + return yield* Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner + .spawn( + ChildProcess.make( + process.execPath, + [entrypoint, options.stateRoot, options.cacheRoot, options.stackId], + { + cwd: process.cwd(), + detached: true, + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + additionalFds: { fd3: { type: "output" } }, + forceKillAfter: "2 seconds", + }, + ), + ) + .pipe(Effect.mapError((cause) => error("startup", cause))); + const readiness = child.getOutputFd(3).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runHead, + Effect.flatMap( + Option.match({ + onNone: () => Effect.fail(error("startup", "Host readiness stream closed")), + onSome: Effect.succeed, + }), + ), + Effect.mapError((cause) => error("startup", cause)), + ); + return yield* readiness.pipe( + Effect.timeout(Duration.seconds(30)), + Effect.mapError((cause) => error("startup", cause)), + Effect.flatMap((line) => + Schema.decodeEffect(Schema.fromJsonString(readyLine))(line).pipe( + Effect.mapError((cause) => error("startup", cause)), + Effect.flatMap((decoded) => { + if (decoded.type === "error") + return Effect.fail(error("startup", decoded.message, decoded.reason)); + if (decoded.endpoint.stackId !== options.stackId) + return Effect.fail( + error("startup", "Host readiness identity does not match the requested stack"), + ); + return child.unref.pipe( + Effect.mapError((cause) => error("startup", cause)), + Effect.as(decoded.endpoint), + ); + }), + ), + ), + ); + }), + ); +}); + +const causeCode = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null) return undefined; + if ("code" in cause && typeof cause.code === "string") return cause.code; + if ("cause" in cause) return causeCode(cause.cause); + return undefined; +}; +const isConnectionFailure = (failure: HostProcessError) => { + const code = causeCode(failure.cause); + return ( + failure.reason === "connection-failure" || + code === "ECONNREFUSED" || + code === "ConnectionRefused" || + code === "ECONNRESET" || + code === "ETIMEDOUT" || + code === "UND_ERR_CONNECT_TIMEOUT" + ); +}; +const isBindConflict = (failure: HostProcessError) => { + const code = causeCode(failure.cause); + return failure.reason === "bind-conflict" || code === "EADDRINUSE"; +}; + +export type OwnerExitProbeResult = + | { readonly state: "absent" } + | { readonly state: "present" } + | { readonly state: "inconclusive"; readonly code: "EPERM" }; +export type OwnerExitProbe = (pid: number) => Effect.Effect; + +/** Probes the captured owner PID with signal 0. */ +const probeOwnerExit: OwnerExitProbe = Effect.fn("HostProcess.probeOwnerExit")(function* (pid) { + return yield* Effect.try({ + try: () => { + process.kill(pid, 0); + return { state: "present" } as const; + }, + catch: (cause) => + new HostProcessError({ + operation: "shutdown-exit", + message: `Shutdown acknowledged, but probing owner process ${pid} failed: ${String(cause)}`, + reason: "owner-exit-probe", + cause, + }), + }).pipe( + Effect.catch((failure): Effect.Effect => { + const code = causeCode(failure.cause); + if (code === "ESRCH") return Effect.succeed({ state: "absent" } as const); + if (code === "EPERM") return Effect.succeed({ state: "inconclusive", code } as const); + return Effect.fail(failure); + }), + ); +}); + +/** Waits until the captured owner PID is absent from the process table. */ +export const waitForOwnerExit = Effect.fn("HostProcess.waitForOwnerExit")(function* ( + pid: number, + probe: OwnerExitProbe = probeOwnerExit, +) { + if (!Number.isSafeInteger(pid) || pid <= 0) + return yield* error( + "shutdown-exit", + `Owner endpoint returned an invalid PID: ${pid}`, + "invalid-owner-pid", + ); + const check = probe(pid).pipe( + Effect.flatMap((result) => + result.state === "absent" + ? Effect.void + : Effect.fail( + error( + "shutdown-exit", + result.state === "present" + ? `Owner shutdown acknowledgement completed, but process ${pid} is still running` + : `Owner shutdown acknowledgement completed, but process ${pid} is inaccessible (${result.code}); exit is inconclusive`, + "owner-exit-pending", + ), + ), + ), + ); + return yield* check.pipe( + Effect.retry({ + schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "5 seconds" })), + while: (failure) => failure.reason === "owner-exit-pending", + }), + ); +}); + +export const launchHost = Effect.fn("HostProcess.launchHost")(function* ( + state: State.Interface, + options: LaunchOptions, +): Effect.fn.Return< + HostEndpoint, + HostProcessError | State.StateError, + | Scope.Scope + | HttpClient.HttpClient + | import("effect").Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner +> { + const existing = yield* connectHost(state, options.stackId).pipe( + Effect.map(Option.some), + Effect.catchTag("HostProcessError", (failure) => + failure.reason === "missing-control-listener" || isConnectionFailure(failure) + ? Effect.succeed(Option.none()) + : Effect.fail(failure), + ), + ); + if (Option.isSome(existing)) return existing.value; + return yield* spawnDetached( + options, + options.entrypoint ?? hostEntrypointFor(import.meta.url), + ).pipe( + Effect.catchTag("HostProcessError", (failure) => + isBindConflict(failure) + ? connectHost(state, options.stackId).pipe(Effect.mapError(() => failure)) + : Effect.fail(failure), + ), + ); +}); + +const hostEntrypointFor = (moduleUrl: string): string => { + if (isBunVirtualPath(moduleUrl)) return HOST_PROCESS_DISPATCH_SENTINEL; + const sourceEntrypoint = fileURLToPath(new URL("./internal/host-process.ts", moduleUrl)); + return isBunVirtualPath(sourceEntrypoint) ? HOST_PROCESS_DISPATCH_SENTINEL : sourceEntrypoint; +}; diff --git a/packages/stack/src/HostProcess.unit.test.ts b/packages/stack/src/HostProcess.unit.test.ts new file mode 100644 index 0000000000..96a8c78067 --- /dev/null +++ b/packages/stack/src/HostProcess.unit.test.ts @@ -0,0 +1,121 @@ +import { expect, it } from "@effect/vitest"; +import { Cause, Deferred, Effect, Exit, Fiber, Ref } from "effect"; +import * as TestClock from "effect/testing/TestClock"; +import { + HostProcessError, + type OwnerExitProbe, + type OwnerExitProbeResult, + waitForOwnerExit, +} from "./HostProcess.ts"; + +const pending = (state: "present" | "inconclusive"): OwnerExitProbeResult => + state === "present" ? { state } : { state, code: "EPERM" }; + +it.effect("waits for the owner to disappear after an acknowledged shutdown", () => + Effect.gen(function* () { + const attempts = yield* Ref.make(0); + const probe: OwnerExitProbe = () => + Ref.getAndUpdate(attempts, (count) => count + 1).pipe( + Effect.map((count) => (count === 0 ? pending("present") : { state: "absent" })), + ); + const fiber = yield* waitForOwnerExit(123, probe).pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* TestClock.adjust("25 millis"); + yield* Fiber.join(fiber); + expect(yield* Ref.get(attempts)).toBe(2); + }).pipe(Effect.provide(TestClock.layer())), +); + +it.effect("retries an inconclusive EPERM probe until the owner is absent", () => + Effect.gen(function* () { + const attempts = yield* Ref.make(0); + const probe: OwnerExitProbe = () => + Ref.getAndUpdate(attempts, (count) => count + 1).pipe( + Effect.map((count) => (count < 2 ? pending("inconclusive") : { state: "absent" })), + ); + const fiber = yield* waitForOwnerExit(123, probe).pipe(Effect.forkChild); + yield* Effect.yieldNow; + yield* TestClock.adjust("50 millis"); + yield* Fiber.join(fiber); + expect(yield* Ref.get(attempts)).toBe(3); + }).pipe(Effect.provide(TestClock.layer())), +); + +it.effect("times out with the last present owner diagnostic", () => + Effect.gen(function* () { + const probe: OwnerExitProbe = () => Effect.succeed(pending("present")); + const fiber = yield* waitForOwnerExit(456, probe).pipe(Effect.flip, Effect.forkChild); + yield* Effect.yieldNow; + yield* TestClock.adjust("6 seconds"); + const failure = yield* Fiber.join(fiber); + expect(failure).toMatchObject({ + operation: "shutdown-exit", + reason: "owner-exit-pending", + message: "Owner shutdown acknowledgement completed, but process 456 is still running", + }); + }).pipe(Effect.provide(TestClock.layer())), +); + +it.effect("times out with the last EPERM diagnostic", () => + Effect.gen(function* () { + const probe: OwnerExitProbe = () => Effect.succeed(pending("inconclusive")); + const fiber = yield* waitForOwnerExit(789, probe).pipe(Effect.flip, Effect.forkChild); + yield* Effect.yieldNow; + yield* TestClock.adjust("6 seconds"); + const failure = yield* Fiber.join(fiber); + expect(failure).toMatchObject({ + operation: "shutdown-exit", + reason: "owner-exit-pending", + message: + "Owner shutdown acknowledgement completed, but process 789 is inaccessible (EPERM); exit is inconclusive", + }); + }).pipe(Effect.provide(TestClock.layer())), +); + +it.effect("propagates unexpected probe errors without retrying", () => + Effect.gen(function* () { + const failure = new HostProcessError({ + operation: "shutdown-exit", + reason: "owner-exit-probe", + message: "signal 0 failed with EINVAL", + }); + const attempts = yield* Ref.make(0); + const probe: OwnerExitProbe = () => + Ref.updateAndGet(attempts, (count) => count + 1).pipe( + Effect.flatMap(() => Effect.fail(failure)), + ); + const received = yield* waitForOwnerExit(321, probe).pipe(Effect.flip); + expect(received).toBe(failure); + expect(yield* Ref.get(attempts)).toBe(1); + }), +); + +it.effect("cancels the local owner wait", () => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const probe: OwnerExitProbe = () => Effect.succeed(pending("present")); + const signalProbe: OwnerExitProbe = () => + Deferred.succeed(started, undefined).pipe(Effect.andThen(probe(654))); + const fiber = yield* waitForOwnerExit(654, signalProbe).pipe(Effect.forkChild); + yield* Deferred.await(started); + const exit = yield* Fiber.interrupt(fiber).pipe(Effect.andThen(Fiber.await(fiber))); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) expect(Cause.hasInterrupts(exit.cause)).toBe(true); + }).pipe(Effect.provide(TestClock.layer())), +); + +it.effect("rejects an invalid owner PID before probing", () => + Effect.gen(function* () { + const attempts = yield* Ref.make(0); + const probe: OwnerExitProbe = () => + Ref.updateAndGet(attempts, (count) => count + 1).pipe( + Effect.as({ state: "absent" } as const), + ); + const failure = yield* waitForOwnerExit(0, probe).pipe(Effect.flip); + expect(failure).toMatchObject({ + operation: "shutdown-exit", + reason: "invalid-owner-pid", + }); + expect(yield* Ref.get(attempts)).toBe(0); + }), +); diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts new file mode 100644 index 0000000000..88ab3eca80 --- /dev/null +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -0,0 +1,343 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Data, Deferred, Effect, Fiber, Layer } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { createServer, type Server, type ServerResponse } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw server fixture. +import { Socket } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw disconnect fixture. +// oxlint-disable-next-line effecttsgo/node-builtin-import -- raw WebSocket upgrade fixture. +import { WebSocket, WebSocketServer } from "ws"; +import { ProxyError } from "./Proxy.ts"; +import { makeHttpProxy, type HttpRoute } from "./HttpProxy.ts"; + +const listen = (server: Server) => + Effect.acquireRelease( + Effect.callback<{ host: string; port: number }, HttpProxyTestError>((resume) => { + const onError = (cause: Error) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))); + server.once("error", onError); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (address === null || typeof address === "string") + return resume(Effect.fail(new HttpProxyTestError({ message: "No address" }))); + resume(Effect.succeed({ host: "127.0.0.1", port: address.port })); + }); + return Effect.sync(() => server.off("error", onError)); + }), + () => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + return Effect.void; + }), + ); + +class HttpProxyTestError extends Data.TaggedError("HttpProxyTestError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const request = (port: number, path: string, body: Uint8Array) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const response = yield* client.execute( + HttpClientRequest.post(`http://127.0.0.1:${port}${path}`).pipe( + HttpClientRequest.bodyUint8Array(body), + ), + ); + return { status: response.status, body: new Uint8Array(yield* response.arrayBuffer) }; + }); + +it.live("routes streamed HTTP bodies and releases target activity after the response", () => + Effect.scoped( + Effect.gen(function* () { + const body = new Uint8Array(2 * 1024 * 1024).fill(71); + const backend = createServer((_incoming, outgoing) => { + _incoming.resume(); + outgoing.writeHead(200); + outgoing.write(body); + }); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + const acquired = yield* Deferred.make(); + const released = yield* Deferred.make(); + const route: HttpRoute = { + id: "echo", + prefix: "/api", + target: Effect.acquireRelease( + Deferred.succeed(acquired, undefined).pipe(Effect.as(backendAddress)), + () => Deferred.succeed(released, undefined), + ), + }; + yield* proxy.setRoutes([route]); + const heldResponseFiber = yield* Effect.callback( + (resume) => { + const onRequest = (_incoming: unknown, outgoing: ServerResponse) => + resume(Effect.succeed(outgoing)); + backend.once("request", onRequest); + return Effect.sync(() => backend.off("request", onRequest)); + }, + ).pipe(Effect.forkScoped); + const responseFiber = yield* request(proxy.port, "/api/echo", body).pipe( + Effect.provide(NodeHttpClient.layerNodeHttp), + Effect.forkScoped, + ); + yield* Deferred.await(acquired); + const heldResponse = yield* Fiber.join(heldResponseFiber); + expect(yield* Deferred.isDone(released)).toBe(false); + heldResponse?.end(body); + const response = yield* Fiber.join(responseFiber); + expect(response.status).toBe(200); + expect(response.body.byteLength).toBe(body.byteLength * 2); + expect(response.body.every((value) => value === 71)).toBe(true); + yield* Deferred.await(released); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("keeps the retained listener and remaining route after one route is removed", () => + Effect.scoped( + Effect.gen(function* () { + const first = createServer((_request, response) => response.end("first")); + const second = createServer((request, response) => response.end(request.url)); + const firstAddress = yield* listen(first); + const secondAddress = yield* listen(second); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + const route = (id: string, prefix: string, address: typeof firstAddress): HttpRoute => ({ + id, + prefix, + upstreamPrefix: "/", + target: Effect.succeed(address), + }); + yield* proxy.setRoutes([ + route("first", "/one", firstAddress), + route("second", "/two", secondAddress), + ]); + const before = yield* request(proxy.port, "/one", new Uint8Array()).pipe( + Effect.provide(NodeHttpClient.layerNodeHttp), + ); + expect(new TextDecoder().decode(before.body)).toBe("first"); + yield* proxy.setRoutes([route("second", "/two", secondAddress)]); + const after = yield* request(proxy.port, "/two?query=1", new Uint8Array()).pipe( + Effect.provide(NodeHttpClient.layerNodeHttp), + ); + expect(new TextDecoder().decode(after.body)).toBe("/?query=1"); + const removed = yield* request(proxy.port, "/one", new Uint8Array()).pipe( + Effect.provide(NodeHttpClient.layerNodeHttp), + ); + expect(removed.status).toBe(404); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("interrupts target acquisition when a waiting client disconnects", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer((_request, response) => response.end("unused")); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + const acquired = yield* Deferred.make(); + const released = yield* Deferred.make(); + const gate = yield* Deferred.make(); + yield* proxy.setRoutes([ + { + id: "waiting", + prefix: "/wait", + target: Effect.gen(function* () { + const address = yield* Effect.acquireRelease( + Deferred.succeed(acquired, undefined).pipe(Effect.as(backendAddress)), + () => Deferred.succeed(released, undefined), + ); + yield* Deferred.await(gate); + return address; + }), + }, + ]); + const client = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + const onConnect = () => resume(Effect.succeed(socket)); + const onError = (cause: Error) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))); + socket.once("connect", onConnect); + socket.once("error", onError); + socket.connect(proxy.port, proxy.host); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("error", onError); + }); + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + yield* Effect.sync(() => + client.write("GET /wait HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"), + ); + yield* Deferred.await(acquired).pipe(Effect.timeout("5 seconds")); + yield* Effect.sync(() => client.destroy()); + yield* Deferred.await(released); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("forwards raw WebSocket upgrades, subprotocols, and echo frames", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer(); + const sockets = new WebSocketServer({ + server: backend, + handleProtocols: (protocols) => (protocols.has("chat") ? "chat" : false), + }); + let upstreamHost: string | undefined; + sockets.on("connection", (socket, request) => { + upstreamHost = request.headers.host; + socket.on("message", (message) => socket.send(message)); + }); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { + id: "ws", + prefix: "/socket", + upstreamHost: "realtime-dev", + target: Effect.succeed(backendAddress), + }, + ]); + const result = yield* Effect.callback< + { protocol: string; message: string }, + HttpProxyTestError + >((resume) => { + const client = new WebSocket(`ws://127.0.0.1:${proxy.port}/socket`, ["chat"]); + client.once("open", () => client.send("hello")); + client.once("message", (message) => { + const text = Array.isArray(message) + ? Buffer.concat(message).toString() + : Buffer.isBuffer(message) + ? message.toString() + : new TextDecoder().decode(message); + resume(Effect.succeed({ protocol: client.protocol, message: text })); + client.close(); + }); + client.once("error", (cause) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))), + ); + return Effect.sync(() => client.close()); + }).pipe(Effect.timeout("10 seconds")); + expect(result.protocol).toBe("chat"); + expect(result.message).toBe("hello"); + expect(upstreamHost).toBe("realtime-dev"); + yield* Effect.callback((resume) => { + sockets.close((cause) => + cause === undefined + ? resume(Effect.void) + : resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))), + ); + return Effect.void; + }); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("overrides the upstream host for HTTP routes when configured", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer((request, response) => { + response.end(request.headers.host); + }); + const backendAddress = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { + id: "http", + prefix: "/api", + upstreamHost: "realtime-dev", + target: Effect.succeed(backendAddress), + }, + ]); + const response = yield* request(proxy.port, "/api/health", new Uint8Array()).pipe( + Effect.provide(NodeHttpClient.layerNodeHttp), + ); + expect(new TextDecoder().decode(response.body)).toBe("realtime-dev"); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("returns a gateway error when a managed target cannot become ready", () => + Effect.scoped( + Effect.gen(function* () { + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([ + { + id: "failed", + prefix: "/", + target: Effect.fail(new ProxyError({ message: "readiness failed" })), + }, + ]); + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://127.0.0.1:${proxy.port}/`); + expect(response.status).toBe(502); + expect(response.headers["access-control-allow-origin"]).toBe("*"); + expect(yield* response.text).toBe("Bad Gateway"); + }), + ).pipe(Effect.provide(Layer.merge(NodeHttpClient.layerNodeHttp, NodeServices.layer))), +); + +it.live("disconnects a pending upstream response when its client closes", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer(); + const address = yield* listen(backend); + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + yield* proxy.setRoutes([{ id: "pending", prefix: "/", target: Effect.succeed(address) }]); + yield* Effect.callback((resume) => { + const client = new Socket(); + client.on("error", (cause) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message }))), + ); + backend.once("request", (_request, response) => { + response.once("close", () => resume(Effect.void)); + client.destroy(); + }); + client.connect(proxy.port, "127.0.0.1", () => + client.write("GET / HTTP/1.1\r\nHost: localhost\r\n\r\n"), + ); + return Effect.sync(() => client.destroy()); + }).pipe(Effect.timeout("5 seconds")); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("releases a waiting WebSocket target when its client resets the connection", () => + Effect.scoped( + Effect.gen(function* () { + const proxy = yield* makeHttpProxy({ host: "127.0.0.1", port: 0 }); + const acquiring = yield* Deferred.make(); + const released = yield* Deferred.make(); + yield* proxy.setRoutes([ + { + id: "socket", + prefix: "/socket", + target: Effect.acquireRelease(Deferred.succeed(acquiring, undefined), () => + Deferred.succeed(released, undefined), + ).pipe(Effect.andThen(Effect.never)), + }, + ]); + const socket = yield* Effect.acquireRelease( + Effect.sync(() => new Socket()), + (value) => Effect.sync(() => value.destroy()), + ); + yield* Effect.callback((resume) => { + socket.on("error", (cause) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message }))), + ); + socket.connect(proxy.port, "127.0.0.1", () => { + socket.write( + "GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", + ); + resume(Effect.void); + }); + return Effect.void; + }); + yield* Deferred.await(acquiring); + yield* Effect.sync(() => socket.resetAndDestroy()); + yield* Deferred.await(released).pipe(Effect.timeout("5 seconds")); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts new file mode 100644 index 0000000000..a1a0e28bb5 --- /dev/null +++ b/packages/stack/src/HttpProxy.ts @@ -0,0 +1,354 @@ +import { Data, Effect, FiberSet, Ref, Scope } from "effect"; +import { PortError } from "./Ports.ts"; +import type { BackendAddress, ProxyError } from "./Proxy.ts"; +import { + createServer, + request as upstreamRequest, + type IncomingMessage, + type ServerResponse, +} from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw HTTP upgrade sockets preserve handshake bytes. +import { Socket } from "node:net"; +import type { Duplex } from "node:stream"; + +class HttpProxyError extends Data.TaggedError("HttpProxyError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface HttpRoute { + readonly id: string; + readonly prefix: string; + readonly target: Effect.Effect; + readonly upstreamPrefix?: string; + readonly upstreamHost?: string; +} + +export interface HttpProxy { + readonly host: string; + readonly port: number; + readonly setRoutes: (routes: ReadonlyArray) => Effect.Effect; +} + +const hopByHop = new Set([ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", +]); + +const errorFor = (cause: unknown) => + new HttpProxyError({ message: cause instanceof Error ? cause.message : String(cause), cause }); + +const headersFor = (headers: IncomingMessage["headers"]) => + Object.fromEntries( + Object.entries(headers).filter( + ([name, value]) => value !== undefined && !hopByHop.has(name.toLowerCase()), + ), + ); + +const upstreamHeadersFor = (headers: IncomingMessage["headers"], route: HttpRoute) => ({ + ...headersFor(headers), + ...(route.upstreamHost === undefined ? {} : { host: route.upstreamHost }), +}); + +const pathFor = (request: IncomingMessage, route: HttpRoute) => { + const input = request.url ?? "/"; + if (route.upstreamPrefix === undefined) return input; + const queryAt = input.indexOf("?"); + const pathname = queryAt < 0 ? input : input.slice(0, queryAt); + const query = queryAt < 0 ? "" : input.slice(queryAt); + const suffix = route.prefix === "/" ? pathname : pathname.slice(route.prefix.length); + return `${route.upstreamPrefix.replace(/\/$/u, "")}${suffix.startsWith("/") ? suffix : `/${suffix}`}${query}`; +}; + +const pathnameFor = (url: string) => url.split("?", 1)[0] || "/"; + +const routeFor = (url: string, routes: ReadonlyArray) => { + const pathname = pathnameFor(url); + return routes.find( + (route) => + pathname === route.prefix || route.prefix === "/" || pathname.startsWith(`${route.prefix}/`), + ); +}; + +const setCors = (response: ServerResponse, request: IncomingMessage) => { + response.setHeader("access-control-allow-origin", "*"); + response.setHeader("access-control-allow-methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS"); + response.setHeader( + "access-control-allow-headers", + request.headers["access-control-request-headers"] ?? "authorization,apikey,content-type", + ); +}; + +const disconnected = (request: IncomingMessage, response: ServerResponse) => + Effect.callback((resume) => { + const onAbort = () => resume(Effect.fail(errorFor("client disconnected"))); + const onRequestClose = () => { + if (!request.complete) onAbort(); + }; + const onSocketClose = () => { + if (!response.writableEnded) onAbort(); + }; + request.once("aborted", onAbort); + request.once("close", onRequestClose); + request.socket.once("close", onSocketClose); + response.once("close", onAbort); + if (request.aborted || request.socket.destroyed || response.destroyed) onAbort(); + return Effect.sync(() => { + request.off("aborted", onAbort); + request.off("close", onRequestClose); + request.socket.off("close", onSocketClose); + response.off("close", onAbort); + }); + }); + +const connectInterruptibly = Effect.fn("HttpProxy.connect")((address: BackendAddress) => + Effect.gen(function* () { + const socket = yield* Effect.acquireRelease( + Effect.sync(() => new Socket({ allowHalfOpen: true })), + (value) => Effect.sync(() => value.destroy()), + ); + yield* Effect.callback((resume) => { + const onConnect = () => resume(Effect.void); + const onError = (cause: Error) => resume(Effect.fail(errorFor(cause))); + socket.once("connect", onConnect); + socket.on("error", onError); + if ("path" in address) socket.connect(address.path); + else socket.connect(address.port, address.host); + return Effect.sync(() => { + socket.off("connect", onConnect); + }); + }).pipe(Effect.timeout("10 seconds")); + return socket; + }), +); + +const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( + (request: IncomingMessage, response: ServerResponse, route: HttpRoute) => + Effect.gen(function* () { + const backend = yield* Effect.raceFirst(route.target, disconnected(request, response)); + yield* Effect.callback((resume) => { + let outgoing: ReturnType | undefined; + let incoming: IncomingMessage | undefined; + let settled = false; + // Error listeners remain until collection because destroy may emit errors asynchronously. + const cleanup = () => { + request.off("aborted", onError); + response.off("close", onResponseClose); + response.off("finish", onFinish); + + incoming?.off("aborted", onError); + }; + const finish = (result: Effect.Effect) => { + if (settled) return; + settled = true; + cleanup(); + resume(result); + }; + const onError = (cause: Error) => { + if (settled) return; + outgoing?.destroy(); + incoming?.destroy(); + finish(Effect.fail(errorFor(cause))); + }; + const onFinish = () => finish(Effect.void); + const onResponseClose = () => { + if (!response.writableEnded) onError(new Error("client response closed")); + }; + outgoing = upstreamRequest( + { + host: "path" in backend ? undefined : backend.host, + port: "path" in backend ? undefined : backend.port, + socketPath: "path" in backend ? backend.path : undefined, + method: request.method, + path: pathFor(request, route), + headers: upstreamHeadersFor(request.headers, route), + }, + (value) => { + incoming = value; + value.once("aborted", onError); + value.on("error", onError); + response.once("finish", onFinish); + setCors(response, request); + response.statusCode = value.statusCode ?? 502; + for (const [name, header] of Object.entries(value.headers)) { + if (header !== undefined && !hopByHop.has(name.toLowerCase())) + response.setHeader(name, header); + } + value.pipe(response); + }, + ); + outgoing.on("error", onError); + request.once("aborted", onError); + response.once("close", onResponseClose); + request.pipe(outgoing); + return Effect.sync(() => { + settled = true; + cleanup(); + outgoing?.destroy(); + incoming?.destroy(); + }); + }); + }), +); + +const upgrade = Effect.fn("HttpProxy.upgrade")( + (request: IncomingMessage, client: Duplex, head: Buffer, route: HttpRoute) => + Effect.gen(function* () { + const backend = yield* Effect.raceFirst( + route.target, + Effect.callback((resume) => { + const onClose = () => resume(Effect.fail(errorFor("client disconnected"))); + client.once("close", onClose); + if (client.destroyed) onClose(); + return Effect.sync(() => client.off("close", onClose)); + }), + ); + const upstream = yield* connectInterruptibly(backend); + yield* Effect.callback((resume) => { + let settled = false; + const cleanup = () => { + client.off("close", onClose); + + upstream.off("close", onClose); + client.off("end", onClientEnd); + upstream.off("end", onUpstreamEnd); + }; + const finish = (result: Effect.Effect) => { + if (settled) return; + settled = true; + cleanup(); + resume(result); + }; + const onError = (cause: Error) => { + client.destroy(); + upstream.destroy(); + finish(Effect.fail(errorFor(cause))); + }; + const onClose = () => { + client.destroy(); + upstream.destroy(); + finish(Effect.void); + }; + const onClientEnd = () => upstream.end(); + const onUpstreamEnd = () => client.end(); + client.on("error", onError); + client.once("close", onClose); + upstream.on("error", onError); + upstream.once("close", onClose); + client.once("end", onClientEnd); + upstream.once("end", onUpstreamEnd); + const lines = [`${request.method ?? "GET"} ${pathFor(request, route)} HTTP/1.1`]; + for (const [name, value] of Object.entries(upstreamHeadersFor(request.headers, route))) { + if (value !== undefined) + lines.push(`${name}: ${Array.isArray(value) ? value.join(", ") : value}`); + } + lines.push("Connection: Upgrade", "Upgrade: websocket", "", ""); + upstream.write(lines.join("\r\n")); + if (head.length > 0) upstream.write(head); + client.pipe(upstream, { end: false }); + upstream.pipe(client, { end: false }); + return Effect.sync(() => { + settled = true; + cleanup(); + client.destroy(); + upstream.destroy(); + }); + }); + }), +); + +export const makeHttpProxy = (options: { + readonly host: string; + readonly port: number; +}): Effect.Effect => + Effect.gen(function* () { + const routes = yield* Ref.make>([]); + const runRequest = yield* FiberSet.makeRuntime(); + const sockets = new Set(); + const server = createServer((request, response) => { + runRequest( + Effect.scoped( + Effect.gen(function* () { + const route = yield* Ref.get(routes).pipe( + Effect.map((current) => routeFor(request.url ?? "/", current)), + ); + setCors(response, request); + if (request.method === "OPTIONS") { + response.statusCode = 204; + response.end(); + } else if (route === undefined) { + response.statusCode = 404; + response.end("Not Found"); + } else { + yield* proxyRequest(request, response, route).pipe( + Effect.catch(() => + Effect.sync(() => { + if (response.destroyed) return; + if (response.headersSent) response.destroy(); + else { + setCors(response, request); + response.statusCode = 502; + response.end("Bad Gateway"); + } + }), + ), + ); + } + }), + ), + ); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + server.on("upgrade", (request, socket, head) => { + socket.on("error", () => socket.destroy()); + runRequest( + Effect.scoped( + Effect.gen(function* () { + const route = yield* Ref.get(routes).pipe( + Effect.map((current) => routeFor(request.url ?? "/", current)), + ); + if (route === undefined) socket.destroy(); + else + yield* upgrade(request, socket, head, route).pipe( + Effect.catch(() => Effect.sync(() => socket.destroy())), + ); + }), + ), + ); + }); + yield* Effect.acquireRelease( + Effect.callback((resume) => { + const onError = (cause: Error) => + resume(Effect.fail(new PortError({ key: "http", message: cause.message, cause }))); + server.once("error", onError); + server.listen(options.port, options.host, () => resume(Effect.void)); + return Effect.sync(() => server.off("error", onError)); + }), + () => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + for (const socket of sockets) socket.destroy(); + return Effect.sync(() => server.closeAllConnections?.()); + }), + ); + const address = server.address(); + if (address === null || typeof address === "string") + return yield* new PortError({ key: "http", message: "HTTP listener has no address" }); + return { + host: address.address, + port: address.port, + setRoutes: (next: ReadonlyArray) => + Ref.set( + routes, + [...next].sort((a, b) => b.prefix.length - a.prefix.length), + ), + }; + }); diff --git a/packages/stack/src/Network.integration.test.ts b/packages/stack/src/Network.integration.test.ts new file mode 100644 index 0000000000..a5351f9a16 --- /dev/null +++ b/packages/stack/src/Network.integration.test.ts @@ -0,0 +1,322 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Data, Deferred, Effect, FileSystem, Layer, Path, Ref } from "effect"; +import * as Net from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- fixture retains an idle HTTP connection. +import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. +import { HttpClient } from "effect/unstable/http"; +import * as Network from "./Network.ts"; +import * as State from "./State.ts"; + +const makeTestState = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); +const makeTestNetwork = (options: { + readonly stackId: string; + readonly runtime: Network.NetworkRuntime; + readonly state: State.Interface; +}) => + Layer.build( + Network.layer({ stackId: options.stackId, runtime: options.runtime }).pipe( + Layer.provide(Layer.succeed(State.Service, options.state)), + ), + ).pipe(Effect.map((context) => Context.get(context, Network.Service))); + +const stack = (id: string, port: number | "auto") => ({ + id, + identity: { projectRoot: "/tmp", branchContext: "test", stackName: id }, + runtime: "native" as const, + instances: [], + composition: {}, + ports: port === "auto" ? [] : [{ key: "api", host: "127.0.0.1", port }], +}); + +class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} + +const backend = Effect.acquireRelease( + Effect.callback< + { host: string; port: number; server: ReturnType }, + FixtureError + >((resume) => { + const server = createServer((request, response) => { + response.end(`backend:${request.url ?? "/"}`); + }); + const onError = (cause: Error) => + resume(Effect.fail(new FixtureError({ message: cause.message }))); + server.once("error", onError); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (address === null || typeof address === "string") + resume(Effect.fail(new FixtureError({ message: "no address" }))); + else resume(Effect.succeed({ host: "127.0.0.1", port: address.port, server })); + }); + return Effect.sync(() => { + server.off("error", onError); + server.close(); + }); + }), + ({ server }) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + return Effect.void; + }), +); + +const request = (host: string, port: number, path: string) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://${host}:${port}${path}`); + return yield* response.text; + }).pipe(Effect.provide(NodeHttpClient.layerNodeHttp)); + +const endpoint = (target: { host: string; port: number }, enabled: Effect.Effect) => ({ + protocol: "http" as const, + port: "auto" as const, + backend: Effect.succeed(target), + enabled, +}); + +it.live("retains dedicated assignments across network reopen", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-retain-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const target = yield* backend; + const enabled = yield* Ref.make(true); + const first = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const firstNamespace = yield* first.register({ + id: "one", + endpoints: { api: endpoint(target, Ref.get(enabled)) }, + }); + yield* firstNamespace.bind; + const firstAddress = yield* firstNamespace.address("api", "host"); + expect(yield* request(firstAddress.host, firstAddress.port, "/one")).toBe("backend:/one"); + const saved = yield* state.read("stack"); + expect(saved?.ports).toHaveLength(1); + yield* Ref.set(enabled, false); + yield* firstNamespace.close; + yield* firstNamespace.bind; + expect((yield* firstNamespace.address("api", "host")).port).toBe(firstAddress.port); + expect(yield* request(firstAddress.host, firstAddress.port, "/restart")).toBe( + "backend:/restart", + ); + yield* firstNamespace.close; + const second = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const secondNamespace = yield* second.register({ + id: "one", + endpoints: { api: endpoint(target, Ref.get(enabled)) }, + }); + yield* secondNamespace.bind; + const secondAddress = yield* secondNamespace.address("api", "host"); + expect(secondAddress.port).toBe(firstAddress.port); + expect(yield* request(secondAddress.host, secondAddress.port, "/reopen")).toBe( + "backend:/reopen", + ); + yield* secondNamespace.release; + expect((yield* state.read("stack"))?.ports).toHaveLength(0); + expect(yield* fs.exists(path.join(root, "stack", "state.json"))).toBe(true); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("keeps shared routes independent and retains the shared claim", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-shared-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const target = yield* backend; + const firstEnabled = yield* Ref.make(true); + const secondEnabled = yield* Ref.make(true); + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const first = yield* network.register({ + id: "rest", + endpoints: { + api: { + ...endpoint(target, Ref.get(firstEnabled)), + shared: [ + { prefix: "/rest" }, + { prefix: "/realtime/v1/api", upstreamPrefix: "/api" }, + { prefix: "/realtime/v1", upstreamPrefix: "/socket" }, + ], + }, + }, + }); + const second = yield* network.register({ + id: "auth", + endpoints: { + api: { ...endpoint(target, Ref.get(secondEnabled)), shared: [{ prefix: "/auth" }] }, + }, + }); + yield* first.bind; + yield* second.bind; + const address = yield* first.address("api", "host"); + expect(yield* request(address.host, address.port, "/rest/v1")).toBe("backend:/rest/v1"); + expect(yield* request(address.host, address.port, "/realtime/v1/websocket?key=one")).toBe( + "backend:/socket/websocket?key=one", + ); + expect(yield* request(address.host, address.port, "/realtime/v1/api/health")).toBe( + "backend:/api/health", + ); + expect(yield* request(address.host, address.port, "/auth/v1")).toBe("backend:/auth/v1"); + yield* Ref.set(firstEnabled, false); + yield* first.close; + expect(yield* request(address.host, address.port, "/auth/v1")).toBe("backend:/auth/v1"); + yield* Ref.set(secondEnabled, false); + yield* second.close; + expect((yield* state.read("stack"))?.ports).toHaveLength(1); + yield* first.bind; + expect((yield* first.address("api", "host")).port).toBe(address.port); + expect(yield* request(address.host, address.port, "/rest/reopen")).toBe( + "backend:/rest/reopen", + ); + yield* first.close; + yield* second.release; + expect((yield* state.read("stack"))?.ports).toHaveLength(1); + expect(yield* fs.exists(root)).toBe(true); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("reports a dedicated port conflict without rewriting saved ownership", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-conflict-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("first", "auto")); + yield* state.save(stack("second", 20001)); + const target = yield* backend; + const firstEnabled = yield* Ref.make(true); + const firstNetwork = yield* makeTestNetwork({ stackId: "first", runtime: "native", state }); + const first = yield* firstNetwork.register({ + id: "db", + endpoints: { sql: { ...endpoint(target, Ref.get(firstEnabled)), protocol: "tcp" } }, + }); + yield* first.bind; + const firstPort = (yield* first.address("sql", "host")).port; + const secondEnabled = yield* Ref.make(true); + const secondNetwork = yield* makeTestNetwork({ stackId: "second", runtime: "native", state }); + const second = yield* secondNetwork.register({ + id: "db", + endpoints: { + sql: { + ...endpoint(target, Ref.get(secondEnabled)), + protocol: "tcp", + port: firstPort, + }, + }, + }); + const failure = yield* second.bind.pipe(Effect.flip); + expect(failure.operation).toBe("bind"); + expect((yield* state.read("second"))?.ports).toEqual([ + { key: "api", host: "127.0.0.1", port: 20001 }, + ]); + yield* Ref.set(firstEnabled, false); + yield* first.release; + yield* Ref.set(secondEnabled, false); + yield* second.release; + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("can bind a shared listener after saving its first assignment failed", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-save-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const failing = yield* Ref.make(true); + const network = yield* makeTestNetwork({ + stackId: "stack", + runtime: "native", + state: { + ...state, + save: (value) => + Ref.get(failing).pipe( + Effect.flatMap((fail) => + fail + ? Effect.fail( + new State.StateError({ operation: "save", message: "write failed" }), + ) + : state.save(value), + ), + ), + }, + }); + const target = yield* backend; + const namespace = yield* network.register({ + id: "rest", + endpoints: { + api: { ...endpoint(target, Effect.succeed(false)), shared: [{ prefix: "/rest" }] }, + }, + }); + yield* namespace.bind.pipe(Effect.flip); + yield* Ref.set(failing, false); + yield* namespace.bind; + const address = yield* namespace.address("api", "host"); + expect(yield* request(address.host, address.port, "/rest")).toBe("backend:/rest"); + yield* namespace.release; + yield* network.release; + expect((yield* state.read("stack"))?.ports).toHaveLength(0); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("releases dedicated HTTP activity after the response while keep-alive stays open", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "network-http-activity-" }); + const state = yield* makeTestState(root); + yield* state.save(stack("stack", "auto")); + const target = yield* backend; + const released = yield* Deferred.make(); + const network = yield* makeTestNetwork({ stackId: "stack", runtime: "native", state }); + const service = yield* network.register({ + id: "studio", + endpoints: { + http: { + ...endpoint(target, Effect.succeed(true)), + backend: Effect.acquireRelease(Effect.succeed(target), () => + Deferred.succeed(released, undefined), + ), + }, + }, + }); + yield* service.bind; + const address = yield* service.address("http", "host"); + const socket = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const connection = Net.createConnection({ host: address.host, port: address.port }); + connection.once("connect", () => resume(Effect.succeed(connection))); + connection.on("error", (cause) => + resume(Effect.fail(new FixtureError({ message: cause.message }))), + ); + return Effect.sync(() => connection.destroy()); + }), + (connection) => + Effect.sync(() => { + connection.destroy(); + }), + ); + yield* Effect.callback((resume) => { + let text = ""; + const onData = (bytes: Buffer) => { + text += bytes.toString(); + if (text.includes("backend:/hello")) resume(Effect.void); + }; + socket.on("data", onData); + socket.write("GET /hello HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); + return Effect.sync(() => socket.off("data", onData)); + }); + expect(socket.destroyed).toBe(false); + yield* Deferred.await(released).pipe(Effect.timeout("2 seconds")); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts new file mode 100644 index 0000000000..43be4c1ec7 --- /dev/null +++ b/packages/stack/src/Network.ts @@ -0,0 +1,318 @@ +import { Context, Data, Effect, Exit, Layer, Ref, Scope, Semaphore } from "effect"; +import * as State from "./State.ts"; +import { makePorts, PortError } from "./Ports.ts"; +import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; +import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; + +export type NetworkRuntime = "native" | "docker" | "podman"; + +export interface NetworkEndpoint { + readonly protocol: "tcp" | "http"; + readonly port: number | "auto"; + readonly backend: Effect.Effect; + readonly shared?: ReadonlyArray<{ + readonly prefix: string; + readonly upstreamPrefix?: string; + readonly upstreamHost?: string; + }>; + readonly enabled: Effect.Effect; +} + +class NetworkError extends Data.TaggedError("NetworkError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface NetworkBinding { + readonly name: string; + readonly protocol: "tcp" | "http"; + readonly host: string; + readonly port: number; +} + +export interface NetworkNamespace { + readonly bindings: Effect.Effect>; + readonly bind: Effect.Effect, NetworkError>; + readonly close: Effect.Effect; + readonly release: Effect.Effect; + readonly address: ( + name: string, + from: "host" | "runtime", + ) => Effect.Effect; +} + +export interface Interface { + readonly release: Effect.Effect; + readonly register: (options: { + readonly id: string; + readonly endpoints: Readonly>; + }) => Effect.Effect; +} + +export class Service extends Context.Service()("@supabase/stack/Network") {} + +const errorFor = (operation: string, cause: unknown) => + new NetworkError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const makeNetwork = (options: { + readonly stackId: string; + readonly runtime: NetworkRuntime; + readonly state: State.Interface; +}) => + Effect.gen(function* () { + const ports = yield* makePorts(options.state).pipe( + Effect.mapError((cause) => errorFor("ports", cause)), + ); + const owner = yield* Scope.Scope; + const gate = yield* Semaphore.make(1); + const shared = yield* Ref.make< + | { + readonly claim: number; + readonly proxy: HttpProxy; + readonly routes: ReadonlyArray; + readonly scope: Scope.Closeable; + } + | undefined + >(undefined); + const namespaces = yield* Ref.make(new Map>()); + + const listenHost = options.runtime === "native" ? "127.0.0.1" : "0.0.0.0"; + const hostAddress = "127.0.0.1"; + const runtimeAddress = + options.runtime === "native" + ? "127.0.0.1" + : options.runtime === "docker" + ? "host.docker.internal" + : "host.containers.internal"; + + const register = Effect.fn("Network.register")(function* ({ + id, + endpoints, + }: { + readonly id: string; + readonly endpoints: Readonly>; + }) { + const names = Object.keys(endpoints); + const duplicate = yield* Ref.modify(namespaces, (current) => [ + current.has(id), + current.has(id) ? current : new Map(current).set(id, names), + ]); + if (duplicate) return yield* errorFor("register", `Duplicate instance ${id}`); + const bound = yield* Ref.make>(new Map()); + const scopes = yield* Ref.make>(new Map()); + const closed = yield* Ref.make(false); + + const bind = Effect.fn("Network.bind")(() => + gate.withPermits(1)( + Effect.gen(function* () { + if (yield* Ref.get(closed)) + return yield* errorFor("bind", "Network namespace is closed"); + for (const [name, endpoint] of Object.entries(endpoints)) { + if ((yield* Ref.get(bound)).has(name)) continue; + yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const endpointScope = yield* Scope.fork(owner, "sequential"); + const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; + const result = yield* restore( + ports + .acquire<{ readonly proxy?: HttpProxy }, Scope.Scope>( + { stackId: options.stackId, key, host: listenHost, port: endpoint.port }, + (host, port) => + Effect.gen(function* () { + if (endpoint.shared !== undefined) { + const current = yield* Ref.get(shared); + if (current !== undefined) { + if (current.claim !== port) + return yield* new PortError({ + key: "api", + message: "Shared API port differs from existing claim", + }); + return { proxy: current.proxy }; + } + return { proxy: yield* makeHttpProxy({ host, port }) }; + } + if (endpoint.protocol === "http") { + const proxy = yield* makeHttpProxy({ host, port }); + yield* proxy.setRoutes([ + { id, prefix: "/", target: endpoint.backend }, + ]); + return { proxy }; + } + const listener = yield* bindTcp(host, port); + yield* Effect.forkIn( + serveTcp(listener, endpoint.backend).pipe( + Effect.provideService(Scope.Scope, endpointScope), + Effect.catch((cause) => + Effect.logWarning("Public listener failed", cause), + ), + ), + endpointScope, + { startImmediately: true }, + ); + return {}; + }), + ) + .pipe( + Effect.provideService(Scope.Scope, endpointScope), + Effect.mapError((cause) => errorFor("bind", cause)), + ), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? Scope.close(endpointScope, exit) : Effect.void, + ), + ); + if (endpoint.shared !== undefined && result.listener.proxy !== undefined) { + const previous = yield* Ref.get(shared); + const current = previous ?? { + claim: result.port, + proxy: result.listener.proxy, + routes: [], + scope: endpointScope, + }; + if (previous !== undefined) yield* Scope.close(endpointScope, Exit.void); + const routes = [ + ...current.routes, + ...endpoint.shared.map((route) => ({ + id, + prefix: route.prefix, + upstreamPrefix: route.upstreamPrefix, + upstreamHost: route.upstreamHost, + target: endpoint.backend, + })), + ]; + yield* current.proxy.setRoutes(routes); + yield* Ref.set(shared, { ...current, routes }); + } else { + yield* Ref.update(scopes, (current) => + new Map(current).set(name, endpointScope), + ); + } + yield* Ref.update(bound, (current) => + new Map(current).set(name, { + name, + protocol: endpoint.protocol, + host: hostAddress, + port: result.port, + }), + ); + }), + ); + } + return [...(yield* Ref.get(bound)).values()]; + }), + ), + ); + + const close = Effect.fn("Network.close")(() => + gate.withPermits(1)( + Effect.gen(function* () { + if (yield* Ref.get(closed)) return; + for (const endpoint of Object.values(endpoints)) if (yield* endpoint.enabled) return; + + const current = yield* Ref.get(shared); + let remainingRoutes = current?.routes ?? []; + if (current !== undefined) { + remainingRoutes = current.routes.filter((route) => route.id !== id); + yield* Ref.set(shared, { ...current, routes: remainingRoutes }); + yield* current.proxy + .setRoutes(remainingRoutes) + .pipe(Effect.mapError((cause) => errorFor("close", cause))); + if (remainingRoutes.length === 0) { + yield* Scope.close(current.scope, Exit.void); + yield* Ref.set(shared, undefined); + } + } + for (const [name, scope] of yield* Ref.get(scopes)) { + const endpoint = endpoints[name]; + if (endpoint?.shared !== undefined) continue; + yield* Scope.close(scope, Exit.void).pipe( + Effect.mapError((cause) => errorFor("close", cause)), + ); + } + yield* Ref.set(bound, new Map()); + yield* Ref.set(scopes, new Map()); + }), + ), + ); + const release = Effect.fn("Network.release")(() => + close().pipe( + Effect.andThen( + Effect.gen(function* () { + for (const endpoint of Object.values(endpoints)) + if (yield* endpoint.enabled) + return yield* errorFor( + "release", + "Stop the instance before releasing its endpoints", + ); + yield* Ref.set(closed, true); + for (const name of names) { + const endpoint = endpoints[name]; + if (endpoint?.shared === undefined) + yield* ports + .release(options.stackId, `${id}:${name}`) + .pipe(Effect.mapError((cause) => errorFor("release", cause))); + } + yield* Ref.update(namespaces, (current) => { + const next = new Map(current); + next.delete(id); + return next; + }); + }), + ), + ), + ); + const address = Effect.fn("Network.address")((name: string, from: "host" | "runtime") => + Effect.gen(function* () { + const endpoint = endpoints[name]; + if (endpoint === undefined) return yield* errorFor("address", `Unknown endpoint ${name}`); + const saved = yield* options.state + .read(options.stackId) + .pipe(Effect.mapError((cause) => errorFor("address", cause))); + const key = endpoint.shared === undefined ? `${id}:${name}` : "api"; + const claim = saved?.ports.find((value) => value.key === key); + if (claim === undefined) + return yield* errorFor("address", `Endpoint ${name} is not assigned`); + return { + name, + protocol: endpoint.protocol, + host: from === "host" ? hostAddress : runtimeAddress, + port: claim.port, + }; + }), + ); + return { + bind: bind(), + close: close(), + release: release(), + address, + bindings: Ref.get(bound).pipe(Effect.map((values) => [...values.values()])), + } satisfies NetworkNamespace; + }); + + const release = Effect.fn("Network.releaseNamespace")(() => + gate.withPermits(1)( + Effect.gen(function* () { + if ((yield* Ref.get(namespaces)).size !== 0) + return yield* errorFor("release", "Destroy instances before releasing the namespace"); + yield* ports + .release(options.stackId, "api") + .pipe(Effect.mapError((cause) => errorFor("release", cause))); + }), + ), + ); + return { register, release: release() } satisfies Interface; + }); + +export const layer = (options: { readonly stackId: string; readonly runtime: NetworkRuntime }) => + Layer.effect( + Service, + Effect.gen(function* () { + const state = yield* State.Service; + return yield* makeNetwork({ ...options, state }); + }).pipe(Effect.map(Service.of)), + ); diff --git a/packages/stack/src/Orchestrator.integration.test.ts b/packages/stack/src/Orchestrator.integration.test.ts new file mode 100644 index 0000000000..8bca0be8d1 --- /dev/null +++ b/packages/stack/src/Orchestrator.integration.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Context, Deferred, Effect, Exit, Fiber, Layer, Ref, Schema, Scope, Stream } from "effect"; +import * as TestClock from "effect/testing/TestClock"; +import * as Orchestrator from "./Orchestrator.ts"; +import type { RegisteredInstance } from "./Orchestrator.ts"; +import { makeService, ServiceError } from "./Service.ts"; + +const failure = (message: string) => new ServiceError({ operation: "fixture", message }); +const makeTestOrchestrator = () => + Layer.build(Layer.fresh(Orchestrator.layer)).pipe( + Effect.map((context) => Context.get(context, Orchestrator.Service)), + ); +const makeInstance = ( + orchestrator: Orchestrator.Interface, + id: string, + options: { + readonly endpoint?: boolean; + readonly health?: Effect.Effect; + readonly prepare?: Effect.Effect; + readonly launch?: Effect.Effect; + readonly stop?: Effect.Effect; + readonly exit?: Deferred.Deferred>; + readonly events?: Ref.Ref>; + } = {}, +) => + Effect.gen(function* () { + const starts = yield* Ref.make>>>([]); + const bound = yield* Ref.make(false); + const event = (name: string) => + options.events === undefined + ? Effect.void + : Ref.update(options.events, (values) => [...values, `${name}:${id}`]); + const core = yield* makeService>( + { + prepare: () => options.prepare ?? Effect.void, + launch: ({ config }) => + Effect.gen(function* () { + yield* options.launch ?? Effect.void; + yield* Ref.update(starts, (values) => [...values, config]); + yield* event("start"); + const exited = yield* Deferred.make>(); + return { + health: options.health ?? Effect.void, + exit: Deferred.await(options.exit ?? exited), + stop: (options.stop ?? Effect.void).pipe( + Effect.andThen(event("stop")), + Effect.andThen(Deferred.succeed(exited, Exit.void)), + Effect.asVoid, + ), + remove: Effect.void, + }; + }), + removeData: () => Effect.void, + }, + { id, config: {}, coordinate: orchestrator.admissionFor(id) }, + ); + const instance: RegisteredInstance = { + id, + core, + startAt: (revision, inputs, wake, guard) => core.startAt(revision, inputs, wake, guard), + restart: (revision, inputs, candidate, guard) => + (candidate === undefined + ? Effect.succeed(inputs) + : Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.String))(candidate).pipe( + Effect.mapError(() => failure("Invalid configuration")), + ) + ).pipe(Effect.flatMap((config) => core.restart(config, revision, guard))), + bind: Ref.set(bound, true), + close: core.get.pipe( + Effect.flatMap((state) => + state.lifecycle === "stopped" && !state.wakeEnabled ? Ref.set(bound, false) : Effect.void, + ), + ), + hasEndpoint: options.endpoint ?? true, + inputs: ["databaseUrl"], + outputs: { url: Effect.succeed(`postgres://${id}`) }, + }; + yield* orchestrator.register(instance); + return { ...instance, starts, bound }; + }); + +const stopped = (instance: RegisteredInstance) => + instance.core.observation.pipe( + Stream.filter((state) => state.lifecycle === "stopped" && state.currentOperation === undefined), + Stream.take(1), + Stream.runDrain, + ); + +describe("service composition", () => { + it.live("admits an inspector connection while its service is still becoming healthy", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const health = yield* Deferred.make(); + const functions = yield* makeInstance(orchestrator, "functions", { + health: Deferred.await(health), + }); + yield* orchestrator.start("functions"); + expect((yield* functions.core.get).health).toBe("starting"); + yield* Effect.scoped(orchestrator.acquire("functions", false)); + expect((yield* functions.core.get).health).toBe("starting"); + yield* Deferred.succeed(health, undefined); + yield* Effect.scoped(orchestrator.acquire("functions")); + expect((yield* functions.core.get).health).toBe("healthy"); + }), + ), + ); + + it.live( + "launches an individual service before health and invalidates a dependent readiness wait on stop", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const health = yield* Deferred.make(); + const database = yield* makeInstance(orchestrator, "database", { + health: Deferred.await(health), + }); + const rest = yield* makeInstance(orchestrator, "rest"); + yield* orchestrator.configure({ + members: [{ id: "rest", activation: "eager" }], + dependencies: [ + { from: "database", to: "rest", bindings: [{ output: "url", input: "databaseUrl" }] }, + ], + }); + yield* orchestrator.start("database"); + expect((yield* database.core.get).health).toBe("starting"); + const dependent = yield* orchestrator + .start("rest") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* orchestrator.stop("database"); + expect(Exit.isFailure(yield* Fiber.await(dependent))).toBe(true); + expect(yield* Ref.get(rest.starts)).toEqual([]); + expect((yield* database.core.get).lifecycle).toBe("stopped"); + }), + ), + ); + + it.live( + "rejects a stale dependency plan after its prerequisite was stopped and replaced during preparation", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const preparing = yield* Deferred.make(); + const continuePreparation = yield* Deferred.make(); + const database = yield* makeInstance(orchestrator, "database"); + const rest = yield* makeInstance(orchestrator, "rest", { + prepare: Deferred.succeed(preparing, undefined).pipe( + Effect.andThen(Deferred.await(continuePreparation)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "rest", activation: "eager" }], + dependencies: [{ from: "database", to: "rest" }], + }); + const request = yield* orchestrator.start("rest").pipe(Effect.forkChild); + yield* Deferred.await(preparing); + yield* orchestrator.stop("database"); + yield* orchestrator.start("database"); + yield* database.core.ready; + yield* Deferred.succeed(continuePreparation, undefined); + expect(Exit.isFailure(yield* Fiber.await(request))).toBe(true); + expect(yield* Ref.get(rest.starts)).toEqual([]); + expect((yield* database.core.get).lifecycle).toBe("running"); + }), + ), + ); + + it.live( + "restarts selected members in dependency order and keeps standalone instances independent", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const events = yield* Ref.make>([]); + const database = yield* makeInstance(orchestrator, "database", { events }); + const rest = yield* makeInstance(orchestrator, "rest", { events }); + const standalone = yield* makeInstance(orchestrator, "shadow", { events }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "eager" }, + { id: "rest", activation: "eager" }, + ], + dependencies: [ + { from: "database", to: "rest", bindings: [{ output: "url", input: "databaseUrl" }] }, + ], + }); + yield* orchestrator.start("shadow"); + yield* orchestrator.startComposition; + expect(yield* Ref.get(rest.starts)).toEqual([{ databaseUrl: "postgres://database" }]); + yield* orchestrator.stop("database").pipe(Effect.flip); + yield* Ref.set(events, []); + yield* orchestrator.restartComposition; + expect(yield* Ref.get(events)).toEqual([ + "stop:rest", + "stop:database", + "start:database", + "start:rest", + ]); + expect(yield* Ref.get(standalone.starts)).toHaveLength(1); + yield* orchestrator.stopNamespace; + expect((yield* standalone.core.get).lifecycle).toBe("stopped"); + expect((yield* database.core.get).lifecycle).toBe("stopped"); + }), + ), + ); + + it.live( + "does not restart after a failed stop and still settles independent selected members", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const failStop = yield* Ref.make(true); + const bad = yield* makeInstance(orchestrator, "bad", { + stop: Ref.get(failStop).pipe( + Effect.flatMap((fail) => (fail ? Effect.fail(failure("cannot stop")) : Effect.void)), + ), + }); + const independent = yield* makeInstance(orchestrator, "independent"); + yield* orchestrator.configure({ + members: [ + { id: "bad", activation: "eager" }, + { id: "independent", activation: "eager" }, + ], + dependencies: [], + }); + yield* orchestrator.startComposition; + const result = yield* orchestrator.restartComposition.pipe(Effect.exit); + expect(Exit.isFailure(result)).toBe(true); + expect((yield* bad.core.get).lifecycle).toBe("stopping"); + expect((yield* independent.core.get).lifecycle).toBe("stopped"); + expect(yield* Ref.get(bad.starts)).toHaveLength(1); + expect(yield* Ref.get(independent.starts)).toHaveLength(1); + yield* Ref.set(failStop, false); + yield* orchestrator.stopNamespace; + }), + ), + ); + + it.live( + "binds lazy endpoints without launching, counts traffic, and sleeps idle prerequisites after dependents", + () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const database = yield* makeInstance(orchestrator, "database"); + const rest = yield* makeInstance(orchestrator, "rest"); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy", idleMillis: 100 }, + { id: "rest", activation: "lazy", idleMillis: 1000 }, + ], + dependencies: [{ from: "database", to: "rest" }], + }); + yield* orchestrator.startComposition; + expect(yield* Ref.get(database.bound)).toBe(true); + expect(yield* Ref.get(rest.bound)).toBe(true); + expect(yield* Ref.get(database.starts)).toEqual([]); + const requestScope = yield* Scope.make(); + yield* orchestrator.acquire("rest").pipe(Scope.provide(requestScope)); + expect((yield* rest.core.get).health).toBe("healthy"); + yield* TestClock.adjust("2 seconds"); + expect((yield* rest.core.get).lifecycle).toBe("running"); + expect((yield* database.core.get).lifecycle).toBe("running"); + yield* Scope.close(requestScope, Exit.void); + const restStopped = yield* stopped(rest).pipe(Effect.forkChild); + const databaseStopped = yield* stopped(database).pipe(Effect.forkChild); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(restStopped); + yield* Fiber.join(databaseStopped); + expect((yield* database.core.get).wakeEnabled).toBe(true); + expect(yield* Ref.get(rest.bound)).toBe(true); + yield* orchestrator.stopComposition; + yield* orchestrator.acquire("rest").pipe(Effect.flip); + }), + ).pipe(Effect.provide(TestClock.layer())), + ); + + it.live("rechecks an idle prerequisite after dependent cleanup completes", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const dependentExit = yield* Deferred.make>(); + const cleanupStarted = yield* Deferred.make(); + const cleanupGate = yield* Deferred.make(); + const database = yield* makeInstance(orchestrator, "database"); + const rest = yield* makeInstance(orchestrator, "rest", { + exit: dependentExit, + stop: Deferred.succeed(cleanupStarted, undefined).pipe( + Effect.andThen(Deferred.await(cleanupGate)), + ), + }); + yield* orchestrator.configure({ + members: [ + { id: "database", activation: "lazy", idleMillis: 100 }, + { id: "rest", activation: "lazy", idleMillis: 10000 }, + ], + dependencies: [{ from: "database", to: "rest" }], + }); + yield* orchestrator.startComposition; + const requestScope = yield* Scope.make(); + yield* orchestrator.acquire("rest").pipe(Scope.provide(requestScope)); + yield* Scope.close(requestScope, Exit.void); + yield* TestClock.adjust("200 millis"); + expect((yield* database.core.get).lifecycle).toBe("running"); + const restStopped = yield* stopped(rest).pipe(Effect.forkChild); + const databaseStopped = yield* stopped(database).pipe(Effect.forkChild); + yield* Deferred.succeed( + dependentExit, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* Deferred.await(cleanupStarted); + expect((yield* database.core.get).lifecycle).toBe("running"); + yield* Deferred.succeed(cleanupGate, undefined); + yield* Fiber.join(restStopped); + yield* TestClock.adjust("1 second"); + yield* Fiber.join(databaseStopped); + expect((yield* database.core.get).lifecycle).toBe("stopped"); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), + ); +}); + +it.live("joins an in-progress wake and wakes again after an admitted idle stop", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const launching = yield* Deferred.make(); + const launch = yield* Deferred.make(); + const stopping = yield* Deferred.make(); + const stop = yield* Deferred.make(); + const instance = yield* makeInstance(orchestrator, "api", { + launch: Deferred.succeed(launching, undefined).pipe(Effect.andThen(Deferred.await(launch))), + stop: Deferred.succeed(stopping, undefined).pipe(Effect.andThen(Deferred.await(stop))), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy", idleMillis: 1000 }], + dependencies: [], + }); + yield* orchestrator.startComposition; + const first = yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.forkChild); + yield* Deferred.await(launching); + const second = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(launch, undefined); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(yield* Ref.get(instance.starts)).toHaveLength(1); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(stopping); + const third = yield* Effect.scoped(orchestrator.acquire("api")).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(stop, undefined); + yield* Fiber.join(third); + expect(yield* Ref.get(instance.starts)).toHaveLength(2); + yield* orchestrator.stopNamespace; + }), + ).pipe(Effect.provide(TestClock.layer())), +); + +it.live("allows later traffic to retry an armed service after a failed wake launch", () => + Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* makeTestOrchestrator(); + const failing = yield* Ref.make(true); + const instance = yield* makeInstance(orchestrator, "api", { + launch: Ref.get(failing).pipe( + Effect.flatMap((value) => (value ? Effect.fail(failure("launch failed")) : Effect.void)), + ), + }); + yield* orchestrator.configure({ + members: [{ id: "api", activation: "lazy" }], + dependencies: [], + }); + yield* orchestrator.startComposition; + yield* Effect.scoped(orchestrator.acquire("api")).pipe(Effect.flip); + yield* Ref.set(failing, false); + yield* Effect.scoped(orchestrator.acquire("api")); + expect((yield* instance.core.get).health).toBe("healthy"); + yield* orchestrator.stopNamespace; + }), + ), +); diff --git a/packages/stack/src/Orchestrator.ts b/packages/stack/src/Orchestrator.ts new file mode 100644 index 0000000000..3768183a61 --- /dev/null +++ b/packages/stack/src/Orchestrator.ts @@ -0,0 +1,907 @@ +import { + Clock, + Context, + Data, + Effect, + Exit, + FiberMap, + Layer, + Match, + Ref, + Schema, + Scope, + Semaphore, + Stream, +} from "effect"; +import { + ServiceError, + ServiceDestroyed, + ServiceNotRunning, + ServiceStaleLaunch, + type ServiceObservation, +} from "./Service.ts"; +import type { ServiceAdmission } from "./Service.ts"; + +export type OrchestratorOperation = + | "start" + | "stop" + | "restart" + | "sleep" + | "destroy" + | "arm" + | "bind" + | "close" + | "configure" + | "register" + | "unregister" + | "proxy"; + +export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{ + readonly operation: OrchestratorOperation; + readonly message: string; + readonly cause?: unknown; + readonly outcomes?: ReadonlyArray<{ + readonly id: string; + readonly result: Exit.Exit; + }>; +}> {} + +type CoreObservation = ServiceObservation; + +interface RegisteredCore { + readonly get: Effect.Effect; + readonly ready: Effect.Effect; + readonly stop: Effect.Effect; + readonly destroy: Effect.Effect; + readonly arm: Effect.Effect; + readonly armAt: ( + revision: number, + guard?: Effect.Effect, + ) => Effect.Effect; + readonly sleep: Effect.Effect; + readonly observation: Stream.Stream; +} + +export interface RegisteredInstance { + readonly id: string; + readonly core: RegisteredCore; + readonly startAt: ( + revision: number, + inputs: Record, + wake: boolean, + guard?: Effect.Effect, + ) => Effect.Effect; + readonly restart: ( + revision: number, + inputs: Record, + config?: unknown, + guard?: Effect.Effect, + ) => Effect.Effect; + readonly bind: Effect.Effect; + readonly close: Effect.Effect; + readonly hasEndpoint: boolean; + readonly inputs: ReadonlyArray; + readonly outputs: Readonly>>; +} + +export type LifecycleError = + | ServiceError + | ServiceDestroyed + | ServiceNotRunning + | ServiceStaleLaunch; + +interface CompositionMember { + readonly id: string; + readonly activation: "eager" | "lazy"; + readonly idleMillis?: number; +} + +interface CompositionBinding { + readonly output: string; + readonly input: string; +} + +interface CompositionDependency { + readonly from: string; + readonly to: string; + readonly bindings?: ReadonlyArray; +} + +export interface CompositionConfig { + readonly members: ReadonlyArray; + readonly dependencies: ReadonlyArray; +} + +export const CompositionConfig = Schema.Struct({ + members: Schema.Array( + Schema.Struct({ + id: Schema.String, + activation: Schema.Literals(["eager", "lazy"]), + idleMillis: Schema.optionalKey(Schema.Finite), + }), + ), + dependencies: Schema.Array( + Schema.Struct({ + from: Schema.String, + to: Schema.String, + bindings: Schema.optionalKey( + Schema.Array(Schema.Struct({ output: Schema.String, input: Schema.String })), + ), + }), + ), +}); + +export interface Interface { + readonly admissionFor: ( + id: string, + ) => ( + operation: ServiceAdmission, + transition: Effect.Effect, + ) => Effect.Effect; + readonly register: ( + instance: RegisteredInstance, + ) => Effect.Effect; + readonly unregister: (id: string) => Effect.Effect; + readonly get: ( + id: string, + ) => Effect.Effect; + readonly list: Effect.Effect< + ReadonlyArray, + OrchestratorError | LifecycleError + >; + readonly configure: ( + configuration: unknown, + ) => Effect.Effect; + readonly start: (id: string) => Effect.Effect; + readonly stop: (id: string) => Effect.Effect; + readonly restart: ( + id: string, + config?: unknown, + ) => Effect.Effect; + readonly destroy: (id: string) => Effect.Effect; + readonly startComposition: Effect.Effect< + ReadonlyArray, + OrchestratorError | LifecycleError + >; + readonly stopComposition: Effect.Effect< + ReadonlyArray, + OrchestratorError | LifecycleError + >; + readonly restartComposition: Effect.Effect< + ReadonlyArray, + OrchestratorError | LifecycleError + >; + readonly stopNamespace: Effect.Effect; + readonly destroyNamespace: Effect.Effect; + readonly acquire: ( + id: string, + awaitReady?: boolean, + ) => Effect.Effect; +} + +export class Service extends Context.Service()( + "@supabase/stack/Orchestrator", +) {} + +interface ActivityState { + readonly active: number; + readonly lastActivity: number; +} + +interface Graph { + readonly members: ReadonlyMap; + readonly prerequisites: ReadonlyMap>; + readonly dependents: ReadonlyMap>; +} + +interface StartPlan { + readonly order: ReadonlyArray; + readonly revisions: ReadonlyMap; + readonly structure: Graph; + readonly configuration: CompositionConfig; +} + +const serviceFailure = (operation: string, message: string): ServiceError => + new ServiceError({ operation, message }); + +const graphError = (operation: OrchestratorOperation, message: string, cause?: unknown) => + new OrchestratorError({ operation, message, ...(cause === undefined ? {} : { cause }) }); + +const isActive = (observation: CoreObservation): boolean => + observation.lifecycle !== "stopped" || observation.wakeEnabled; + +const topo = ( + ids: ReadonlyArray, + prerequisites: ReadonlyMap>, +): ReadonlyArray => { + const result: Array = []; + const seen = new Set(); + const visit = (id: string): void => { + if (seen.has(id)) return; + seen.add(id); + for (const prerequisite of prerequisites.get(id) ?? []) visit(prerequisite); + result.push(id); + }; + for (const id of ids) visit(id); + return result; +}; + +const makeOrchestrator = Effect.gen(function* () { + const owner = yield* Scope.Scope; + const graphGate = yield* Semaphore.make(1); + const idleTimers = yield* FiberMap.make(); + const registry = yield* Ref.make>(new Map()); + const composition = yield* Ref.make({ members: [], dependencies: [] }); + const activity = yield* Ref.make>(new Map()); + + const withGraph = Effect.fn("Orchestrator.withGraph")((effect: Effect.Effect) => + Effect.uninterruptibleMask((restore) => + restore(graphGate.take(1)).pipe( + Effect.andThen(effect.pipe(Effect.ensuring(graphGate.release(1)))), + ), + ), + ); + + const node = Effect.fn("Orchestrator.node")(function* ( + id: string, + ): Effect.fn.Return { + const nodes = yield* Ref.get(registry); + const value = nodes.get(id); + if (value === undefined) return yield* graphError("register", `Unknown instance ${id}`); + return value; + }); + + const graph = ( + configuration: CompositionConfig, + values: ReadonlyMap, + ): Graph | OrchestratorError => { + const members = new Map(); + for (const member of configuration.members) { + if (members.has(member.id)) return graphError("configure", `Duplicate member ${member.id}`); + members.set(member.id, member); + } + const boundInputs = new Set(); + const prerequisites = new Map>(); + const dependents = new Map>(); + for (const dependency of configuration.dependencies) { + const list = prerequisites.get(dependency.to) ?? []; + if (list.includes(dependency.from)) + return graphError("configure", `Duplicate dependency ${dependency.from}->${dependency.to}`); + list.push(dependency.from); + prerequisites.set(dependency.to, list); + const reverse = dependents.get(dependency.from) ?? []; + reverse.push(dependency.to); + dependents.set(dependency.from, reverse); + const target = values.get(dependency.to); + const source = values.get(dependency.from); + if (source === undefined || target === undefined) + return graphError("configure", `Dependency references an unregistered instance`); + for (const binding of dependency.bindings ?? []) { + if (source.outputs[binding.output] === undefined) + return graphError("configure", `Unknown output ${dependency.from}.${binding.output}`); + if (!target.inputs.includes(binding.input)) + return graphError("configure", `Unknown input ${dependency.to}.${binding.input}`); + const inputKey = `${dependency.to}:${binding.input}`; + if (boundInputs.has(inputKey)) + return graphError("configure", `Duplicate binding for ${dependency.to}.${binding.input}`); + boundInputs.add(inputKey); + } + } + for (const member of configuration.members) { + if (!values.has(member.id)) + return graphError("configure", `Member references an unregistered instance ${member.id}`); + if (member.activation === "lazy" && !values.get(member.id)?.hasEndpoint) + return graphError("configure", `Lazy member ${member.id} has no public endpoint`); + if (member.idleMillis !== undefined && member.idleMillis < 0) + return graphError("configure", `Negative idle timeout for ${member.id}`); + } + const visiting = new Set(); + const visited = new Set(); + const visit = (id: string): OrchestratorError | undefined => { + if (visiting.has(id)) + return graphError("configure", "Composition dependencies contain a cycle"); + if (visited.has(id)) return; + visiting.add(id); + for (const prerequisite of prerequisites.get(id) ?? []) { + const failure = visit(prerequisite); + if (failure !== undefined) return failure; + } + visiting.delete(id); + visited.add(id); + }; + for (const id of values.keys()) { + const failure = visit(id); + if (failure !== undefined) return failure; + } + return { members, prerequisites, dependents }; + }; + + const configurationGraph = Effect.fn("Orchestrator.configurationGraph")(function* ( + configuration: CompositionConfig, + ): Effect.fn.Return { + const values = yield* Ref.get(registry); + const result = graph(configuration, values); + if (result instanceof OrchestratorError) return yield* result; + return result; + }); + + const operationBlocksDependents = (operation: ServiceAdmission) => + Match.value(operation).pipe( + Match.when("stop", () => true), + Match.when("restart", () => true), + Match.when("destroy", () => true), + Match.when("sleep", () => true), + Match.when("start", () => false), + Match.when("storage", () => false), + Match.when("arm", () => false), + Match.exhaustive, + ); + const dependentBlocks = (operation: ServiceAdmission, observation: CoreObservation) => + Match.value(operation).pipe( + Match.when("destroy", () => observation.registered), + Match.when("sleep", () => observation.lifecycle !== "stopped"), + Match.when("stop", () => isActive(observation)), + Match.when("restart", () => isActive(observation)), + Match.when("start", () => false), + Match.when("storage", () => false), + Match.when("arm", () => false), + Match.exhaustive, + ); + const admissionFor = + (id: string) => (operation: ServiceAdmission, transition: Effect.Effect) => + withGraph( + Effect.gen(function* () { + const configured = yield* Ref.get(composition); + const structure = yield* configurationGraph(configured); + const dependents = structure.dependents.get(id) ?? []; + if (operationBlocksDependents(operation)) { + for (const dependentId of dependents) { + const dependent = yield* node(dependentId); + const observation = yield* dependent.core.get; + const blocked = dependentBlocks(operation, observation); + if (blocked) + return yield* serviceFailure( + "graph", + `Dependent ${dependentId} blocks ${operation} of ${id}`, + ); + } + } + if (operation === "sleep") { + const timeout = structure.members.get(id)?.idleMillis; + const current = (yield* Ref.get(activity)).get(id); + const now = yield* Clock.currentTimeMillis; + if ( + timeout === undefined || + current === undefined || + current.active > 0 || + now - current.lastActivity < timeout + ) + return yield* serviceFailure("graph", `Instance ${id} is not idle`); + } + if (operation === "start") { + for (const prerequisiteId of structure.prerequisites.get(id) ?? []) { + const prerequisite = yield* node(prerequisiteId); + const observation = yield* prerequisite.core.get; + if (observation.lifecycle !== "running" || observation.health !== "healthy") + return yield* serviceFailure( + "graph", + `Prerequisite ${prerequisiteId} is not healthy`, + ); + } + } + yield* transition; + }), + ).pipe( + Effect.mapError((error) => + error instanceof OrchestratorError ? serviceFailure("graph", error.message) : error, + ), + ); + + const resolveInputs = Effect.fn("Orchestrator.resolveInputs")(function* ( + id: string, + configuration: CompositionConfig, + ): Effect.fn.Return, OrchestratorError | LifecycleError> { + const inputs: Record = {}; + const edges = configuration.dependencies.filter((dependency) => dependency.to === id); + for (const dependency of edges) { + const source = yield* node(dependency.from); + for (const binding of dependency.bindings ?? []) { + const output = source.outputs[binding.output]; + if (output === undefined) + return yield* graphError("start", `Unknown output ${dependency.from}.${binding.output}`); + inputs[binding.input] = yield* output; + } + } + return inputs; + }); + + const snapshotPlan = Effect.fn("Orchestrator.snapshotPlan")(function* ( + id: string, + ): Effect.fn.Return { + return yield* withGraph( + Effect.gen(function* () { + const configuration = yield* Ref.get(composition); + const configured = yield* configurationGraph(configuration); + const structure: Graph = configured.members.has(id) + ? configured + : { + members: new Map([...configured.members, [id, { id, activation: "eager" as const }]]), + prerequisites: configured.prerequisites, + dependents: configured.dependents, + }; + const closure = new Set(); + const visit = (member: string) => { + if (closure.has(member)) return; + closure.add(member); + for (const prerequisite of structure.prerequisites.get(member) ?? []) visit(prerequisite); + }; + visit(id); + const revisions = new Map(); + for (const member of closure) { + const observation = yield* (yield* node(member)).core.get; + revisions.set(member, observation.intentRevision); + } + return { + order: topo([...closure], structure.prerequisites), + revisions, + structure, + configuration, + }; + }), + ); + }); + + const prerequisiteGuard = Effect.fn("Orchestrator.prerequisiteGuard")(function* ( + plan: StartPlan, + id: string, + ): Effect.fn.Return { + if ((yield* Ref.get(composition)) !== plan.configuration) + return yield* serviceFailure("graph", "Composition changed before admission"); + const values = yield* Ref.get(registry); + for (const prerequisite of plan.structure.prerequisites.get(id) ?? []) { + const expected = plan.revisions.get(prerequisite); + const current = values.get(prerequisite); + if (expected === undefined || current === undefined) + return yield* serviceFailure("graph", `Missing prerequisite ${prerequisite}`); + const observation = yield* current.core.get; + if (observation.intentRevision !== expected) + return yield* serviceFailure( + "graph", + `Prerequisite ${prerequisite} changed before ${id} admission`, + ); + } + }); + + const startNode = Effect.fn("Orchestrator.startNode")(function* ( + id: string, + wake = false, + awaitTarget = false, + savedPlan?: StartPlan, + ): Effect.fn.Return { + yield* Effect.gen(function* () { + const plan = savedPlan ?? (yield* snapshotPlan(id)); + for (const member of plan.order) { + const instance = yield* node(member); + const inputs = yield* resolveInputs(member, plan.configuration); + const revision = plan.revisions.get(member); + if (revision === undefined) + return yield* graphError("start", `Missing revision for ${member}`); + yield* instance.startAt(revision, inputs, wake, prerequisiteGuard(plan, member)); + yield* instance.bind; + if (member !== id || awaitTarget) yield* instance.core.ready; + } + }); + }); + + const reschedulePrerequisites = Effect.fn("Orchestrator.reschedulePrerequisites")(function* ( + id: string, + ): Effect.fn.Return { + yield* Effect.gen(function* () { + const structure = yield* configurationGraph(yield* Ref.get(composition)); + for (const prerequisite of structure.prerequisites.get(id) ?? []) + yield* scheduleIdle(prerequisite); + }); + }); + + const scheduleIdle = Effect.fn("Orchestrator.scheduleIdle")(function* ( + id: string, + ): Effect.fn.Return { + yield* Effect.gen(function* () { + const configured = yield* Ref.get(composition); + const timeout = configured.members.find((member) => member.id === id)?.idleMillis; + if (timeout === undefined || timeout < 0) return; + const now = yield* Clock.currentTimeMillis; + const current = (yield* Ref.get(activity)).get(id); + if (current === undefined) return; + const delay = Math.max(0, timeout - (now - current.lastActivity)); + yield* FiberMap.run( + idleTimers, + id, + Effect.gen(function* () { + if (delay > 0) yield* Effect.sleep(`${delay} millis`); + const now = yield* Clock.currentTimeMillis; + const state = yield* withGraph( + Effect.gen(function* () { + const values = yield* Ref.get(activity); + const current = values.get(id); + if ( + current === undefined || + current.active > 0 || + now - current.lastActivity < timeout + ) + return false; + yield* Ref.set(activity, new Map(values).set(id, { ...current })); + return true; + }), + ); + if (state) { + const result = yield* (yield* node(id)).core.sleep.pipe(Effect.exit); + if (Exit.isSuccess(result)) yield* reschedulePrerequisites(id); + } + }), + ); + }); + }); + + const release = Effect.fn("Orchestrator.release")(function* ( + id: string, + ): Effect.fn.Return { + yield* Effect.gen(function* () { + const now = yield* Clock.currentTimeMillis; + yield* withGraph( + Effect.gen(function* () { + const values = yield* Ref.get(activity); + const current = values.get(id); + if (current === undefined) return; + yield* Ref.set( + activity, + new Map(values).set(id, { + active: Math.max(0, current.active - 1), + lastActivity: now, + }), + ); + }), + ); + yield* scheduleIdle(id); + }); + }); + + const settle = Effect.fn("Orchestrator.settle")(function* ( + operation: "start" | "stop" | "destroy", + ids: ReadonlyArray, + action: (id: string) => Effect.Effect, + ): Effect.fn.Return, OrchestratorError | LifecycleError> { + return yield* Effect.gen(function* () { + const outcomes = yield* Effect.forEach(ids, (id) => + action(id).pipe( + Effect.exit, + Effect.map((result) => ({ id, result })), + ), + ); + if (outcomes.some(({ result }) => Exit.isFailure(result))) + return yield* new OrchestratorError({ + operation, + message: `Composition ${operation} had failures`, + outcomes, + }); + return yield* Effect.forEach(ids, (id) => + node(id).pipe(Effect.flatMap((instance) => instance.core.get)), + ); + }); + }); + + const startComposition = Effect.fn("Orchestrator.startComposition")( + function* (): Effect.fn.Return< + ReadonlyArray, + OrchestratorError | LifecycleError + > { + const configured = yield* Ref.get(composition); + const structure = yield* configurationGraph(configured); + const order = topo( + configured.members.map((member) => member.id), + structure.prerequisites, + ); + const eager = new Set( + topo( + [ + ...configured.members + .filter((member) => member.activation === "eager") + .map((member) => member.id), + ...order.filter((id) => !structure.members.has(id)), + ], + structure.prerequisites, + ), + ); + const plans = new Map(); + const bound = new Map>(); + for (const id of order) plans.set(id, yield* snapshotPlan(id)); + for (const id of order) + bound.set( + id, + yield* node(id).pipe( + Effect.flatMap((instance) => instance.bind), + Effect.exit, + ), + ); + return yield* settle("start", order, (id) => + Effect.gen(function* () { + const binding = bound.get(id); + if (binding !== undefined) yield* binding; + const instance = yield* node(id); + const plan = plans.get(id); + const revision = plan?.revisions.get(id); + if (plan === undefined || revision === undefined) + return yield* graphError("start", `Missing plan for ${id}`); + const policy = structure.members.get(id); + if ( + policy?.activation === "lazy" || + (policy?.idleMillis !== undefined && instance.hasEndpoint) + ) + yield* instance.core.armAt(revision, prerequisiteGuard(plan, id)); + if (eager.has(id)) { + const inputs = yield* resolveInputs(id, configured); + yield* instance.startAt(revision, inputs, false, prerequisiteGuard(plan, id)); + yield* instance.bind; + yield* instance.core.ready; + } + }), + ); + }, + ); + + const stopIds = Effect.fn("Orchestrator.stopIds")(function* (ids: ReadonlyArray) { + const structure = yield* configurationGraph(yield* Ref.get(composition)); + const order = topo(ids, structure.prerequisites) + .filter((id) => ids.includes(id)) + .toReversed(); + return yield* settle("stop", order, (id) => + Effect.gen(function* () { + const instance = yield* node(id); + yield* instance.core.stop; + yield* instance.close; + yield* reschedulePrerequisites(id); + }), + ); + }); + + const destroyNode = Effect.fn("Orchestrator.destroyNode")(function* (id: string) { + const instance = yield* node(id); + yield* instance.core.destroy; + yield* instance.close; + yield* withGraph( + Effect.gen(function* () { + const values = yield* Ref.get(registry); + const next = new Map(values); + next.delete(id); + yield* Ref.set(registry, next); + const configured = yield* Ref.get(composition); + yield* Ref.set(composition, { + members: configured.members.filter((member) => member.id !== id), + dependencies: configured.dependencies.filter( + (dependency) => dependency.from !== id && dependency.to !== id, + ), + }); + }), + ); + }); + + const stopComposition = Effect.fn("Orchestrator.stopComposition")(function* () { + const configured = yield* Ref.get(composition); + return yield* stopIds(configured.members.map((member) => member.id)); + }); + const restartComposition = Effect.fn("Orchestrator.restartComposition")(function* () { + const configured = yield* Ref.get(composition); + yield* stopIds(configured.members.map((member) => member.id)); + return yield* startComposition(); + }); + const stopNamespace = Effect.fn("Orchestrator.stopNamespace")(function* () { + const values = yield* Ref.get(registry); + yield* stopIds([...values.keys()]); + }); + const destroyNamespace = Effect.fn("Orchestrator.destroyNamespace")(function* () { + const values = yield* Ref.get(registry); + yield* stopIds([...values.keys()]); + const configured = yield* configurationGraph(yield* Ref.get(composition)); + for (const id of topo([...values.keys()], configured.prerequisites).toReversed()) + yield* destroyNode(id); + }); + + const orchestrator: Interface = { + admissionFor, + register: Effect.fn("Orchestrator.register")((instance) => + withGraph( + Effect.gen(function* () { + const values = yield* Ref.get(registry); + if (values.has(instance.id)) + return yield* graphError("register", `Duplicate instance ${instance.id}`); + yield* Ref.set(registry, new Map(values).set(instance.id, instance)); + const seenExit = yield* Ref.make(undefined); + const seenHealthy = yield* Ref.make(undefined); + yield* Effect.forkIn( + instance.core.observation.pipe( + Stream.takeUntil((observation) => !observation.registered), + Stream.runForEach((observation) => + Effect.gen(function* () { + if ( + observation.lifecycle === "running" && + observation.health === "healthy" && + observation.wakeEnabled && + observation.launchId !== (yield* Ref.get(seenHealthy)) + ) { + yield* Ref.set(seenHealthy, observation.launchId); + const now = yield* Clock.currentTimeMillis; + yield* withGraph( + Ref.update(activity, (values) => + new Map(values).set(instance.id, { + active: values.get(instance.id)?.active ?? 0, + lastActivity: now, + }), + ), + ); + yield* scheduleIdle(instance.id); + } + if (observation.exit === undefined) return; + if (observation.lifecycle === "stopped") { + yield* reschedulePrerequisites(instance.id); + return; + } + if (observation.wakeEnabled) return; + if ((yield* Ref.get(seenExit)) === observation.exit) return; + yield* Ref.set(seenExit, observation.exit); + yield* instance.close.pipe( + Effect.tapError((cause) => Effect.logError("Instance close failed", cause)), + Effect.ignore, + ); + }), + ), + ), + owner, + ); + }), + ), + ), + unregister: Effect.fn("Orchestrator.unregister")((id) => + withGraph( + Effect.gen(function* () { + const instance = yield* node(id); + const observation = yield* instance.core.get; + if (observation.registered || observation.lifecycle !== "stopped") + return yield* graphError("unregister", `Instance ${id} must be destroyed first`); + const structure = yield* configurationGraph(yield* Ref.get(composition)); + if ((structure.dependents.get(id) ?? []).some((dependentId) => dependentId !== id)) + return yield* graphError( + "unregister", + `Instance ${id} is still referenced by a dependent`, + ); + const values = yield* Ref.get(registry); + const next = new Map(values); + next.delete(id); + yield* Ref.set(registry, next); + const configured = yield* Ref.get(composition); + yield* Ref.set(composition, { + members: configured.members.filter((member) => member.id !== id), + dependencies: configured.dependencies.filter( + (dependency) => dependency.from !== id && dependency.to !== id, + ), + }); + }), + ), + ), + get: Effect.fn("Orchestrator.get")((id) => node(id)), + list: Ref.get(registry).pipe(Effect.map((values) => [...values.values()])), + configure: Effect.fn("Orchestrator.configure")((configuration) => + Schema.decodeUnknownEffect(CompositionConfig)(configuration).pipe( + Effect.mapError((cause) => graphError("configure", "Invalid composition", cause)), + Effect.flatMap((decoded) => + withGraph( + Effect.gen(function* () { + const previous = yield* Ref.get(composition); + yield* configurationGraph(decoded); + const affected = new Set([ + ...previous.members.map((member) => member.id), + ...decoded.members.map((member) => member.id), + ]); + for (const dependency of [...previous.dependencies, ...decoded.dependencies]) { + affected.add(dependency.from); + affected.add(dependency.to); + } + for (const affectedId of affected) { + const instance = yield* node(affectedId); + const observation = yield* instance.core.get; + if (observation.lifecycle !== "stopped" || observation.wakeEnabled) + return yield* graphError( + "configure", + `Instance ${affectedId} must be stopped and wake-disabled`, + ); + } + yield* Ref.set(composition, decoded); + }), + ), + ), + ), + ), + start: Effect.fn("Orchestrator.start")((id) => + Effect.gen(function* () { + const plan = yield* snapshotPlan(id); + for (const member of plan.order) yield* (yield* node(member)).bind; + yield* startNode(id, false, false, plan); + }), + ), + stop: Effect.fn("Orchestrator.stop")((id) => + Effect.gen(function* () { + const instance = yield* node(id); + yield* instance.core.stop; + yield* instance.close; + yield* reschedulePrerequisites(id); + }), + ), + restart: Effect.fn("Orchestrator.restart")((id, config) => + Effect.gen(function* () { + const plan = yield* snapshotPlan(id); + for (const member of plan.order) yield* (yield* node(member)).bind; + for (const member of plan.order.slice(0, -1)) { + const instance = yield* node(member); + const inputs = yield* resolveInputs(member, plan.configuration); + const revision = plan.revisions.get(member); + if (revision === undefined) + return yield* graphError("restart", `Missing revision for ${member}`); + yield* instance.startAt(revision, inputs, false, prerequisiteGuard(plan, member)); + yield* instance.bind; + yield* instance.core.ready; + } + const instance = yield* node(id); + const inputs = yield* resolveInputs(id, plan.configuration); + const revision = plan.revisions.get(id); + if (revision === undefined) + return yield* graphError("restart", `Missing revision for ${id}`); + yield* instance.restart(revision, inputs, config, prerequisiteGuard(plan, id)); + yield* instance.bind; + }), + ), + destroy: Effect.fn("Orchestrator.destroy")((id) => destroyNode(id)), + startComposition: startComposition(), + stopComposition: stopComposition(), + restartComposition: restartComposition(), + stopNamespace: stopNamespace(), + destroyNamespace: destroyNamespace(), + acquire: Effect.fn("Orchestrator.acquire")((id, awaitReady = true) => + Effect.gen(function* () { + const instance = yield* node(id); + const scope = yield* Scope.Scope; + const now = yield* Clock.currentTimeMillis; + const wake = yield* withGraph( + Effect.gen(function* () { + const observation = yield* instance.core.get; + if (observation.lifecycle === "stopped" && !observation.wakeEnabled) + return yield* serviceFailure("proxy", `Instance ${id} is explicitly stopped`); + const values = yield* Ref.get(activity); + yield* Ref.set( + activity, + new Map(values).set(id, { + active: (values.get(id)?.active ?? 0) + 1, + lastActivity: now, + }), + ); + yield* Scope.addFinalizer( + scope, + release(id).pipe( + Effect.tapError((cause) => Effect.logError("Instance release failed", cause)), + Effect.ignore, + ), + ); + return observation.lifecycle !== "running"; + }), + ); + if (wake) { + const plan = yield* snapshotPlan(id); + for (const member of plan.order) yield* (yield* node(member)).bind; + yield* startNode(id, true, awaitReady, plan); + } + if (awaitReady) yield* instance.core.ready; + }), + ), + }; + return orchestrator; +}); + +export const layer = Layer.effect(Service, makeOrchestrator.pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/Owner.integration.test.ts b/packages/stack/src/Owner.integration.test.ts new file mode 100644 index 0000000000..c19c6398ec --- /dev/null +++ b/packages/stack/src/Owner.integration.test.ts @@ -0,0 +1,428 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { PgClient } from "@effect/sql-pg"; +import { expect, it } from "@effect/vitest"; +import { Context, Effect, FileSystem, Layer, Path, Redacted, Ref } from "effect"; +import { HttpClient } from "effect/unstable/http"; +import { tmpdir } from "node:os"; +import * as Owner from "./Owner.ts"; +import * as State from "./State.ts"; +import type { SavedStack } from "./State.ts"; + +const stateFor = (root: string) => + Effect.gen(function* () { + const context = yield* Layer.build(State.layer({ root })); + return Context.get(context, State.Service); + }); + +const ownerFor = (options: { + readonly saved: SavedStack; + readonly state: State.Interface; + readonly root: string; + readonly cacheRoot: string; +}) => { + const { state, ...layerOptions } = options; + return Effect.gen(function* () { + const context = yield* Layer.build( + Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(State.Service, state))), + ); + return Context.get(context, Owner.Service); + }); +}; + +const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + +const initial = (id: string): SavedStack => ({ + id, + identity: { projectRoot: "/tmp/project", branchContext: "owner-test", stackName: id }, + runtime: "native", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], +}); + +const query = (url: string, statement: string) => + Effect.scoped( + Effect.gen(function* () { + const parsed = new URL(url); + const services = yield* Layer.build( + PgClient.layer({ + host: parsed.hostname, + port: Number(parsed.port), + database: parsed.pathname.slice(1), + username: decodeURIComponent(parsed.username), + password: Redacted.make(decodeURIComponent(parsed.password)), + }), + ); + return yield* Context.get(services, PgClient.PgClient).unsafe(statement); + }), + ); + +it.effect("rejects a malformed persisted composition", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-malformed-" }); + const state = yield* stateFor(`${root}/state`); + const saved = initial("owner-malformed"); + yield* state.save(saved); + const malformed = + '{"id":"owner-malformed","identity":{"projectRoot":"/tmp/project","branchContext":"owner-test","stackName":"owner-malformed"},"runtime":"native","instances":[],"composition":{"members":"invalid","dependencies":[]},"ports":[]}'; + yield* fs.writeFileString(path.join(root, "state", saved.id, "state.json"), malformed); + const reopened = yield* state.read(saved.id); + if (reopened === undefined) return yield* Effect.die("saved state disappeared"); + const failure = yield* ownerFor({ + saved: reopened, + state, + root: `${root}/state/${saved.id}/data`, + cacheRoot, + }).pipe(Effect.flip); + expect(failure).toBeInstanceOf(Owner.OwnerError); + expect(failure.operation).toBe("configure"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.effect("publishes service removal and composition pruning together", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-remove-" }); + const stack = initial("owner-remove"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const removalWrite = yield* Ref.make(false); + const failingState: State.Interface = { + ...state, + save: (next) => + Ref.get(removalWrite).pipe( + Effect.flatMap((alreadyWritten) => + next.instances.length === 0 + ? alreadyWritten + ? Effect.fail( + new State.StateError({ operation: "write", message: "injected failure" }), + ) + : Ref.set(removalWrite, true).pipe(Effect.andThen(state.save(next))) + : state.save(next), + ), + ), + }; + const owner = yield* ownerFor({ + saved: stack, + state: failingState, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const service = yield* owner.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" }, smtp: { port: "auto" }, pop3: { port: "auto" } }, + }); + yield* owner.composition.configure({ + members: [{ id: service.id, activation: "eager" }], + dependencies: [], + }); + + yield* owner.core.destroy(service.id); + + const saved = yield* state.read(stack.id); + if (saved === undefined) return yield* Effect.die("saved state disappeared"); + expect(yield* Ref.get(removalWrite)).toBe(true); + expect(saved.instances).toEqual([]); + expect(saved.composition).toEqual({ members: [], dependencies: [] }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "assembles a database and lazy REST service, keeps standalone instances independent, and reopens ports", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-" }); + const stack = initial("owner-integration"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const database = yield* owner.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-password"), + jwtSecret: Redacted.make("owner-jwt"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + const rest = yield* owner.services.create({ + service: "rest", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: "auto" } }, + }); + const shadow = yield* owner.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("shadow-password"), + jwtSecret: Redacted.make("shadow-jwt"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.composition.configure({ + members: [ + { id: database.id, activation: "eager" }, + { id: rest.id, activation: "lazy", idleMillis: 30_000 }, + ], + dependencies: [ + { + from: database.id, + to: rest.id, + bindings: [{ output: "databaseUrl", input: "databaseUrl" }], + }, + ], + }); + yield* owner.core.start(shadow.id); + yield* owner.core.ready(shadow.id); + yield* owner.composition.start; + const dbCredentials = yield* owner.credentials(database.id, "host"); + const databaseUrl = dbCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("database credentials missing"); + yield* query(databaseUrl, "CREATE TABLE owner_rows(value text NOT NULL)"); + yield* query(databaseUrl, "INSERT INTO owner_rows(value) VALUES ('wake')"); + const restCredentials = yield* owner.credentials(rest.id, "host"); + const restUrl = restCredentials.url; + if (restUrl === undefined) return yield* Effect.die("REST credentials missing"); + const restObservation = yield* owner.core.get(rest.id); + const restEndpoint = restObservation.endpoints.find((endpoint) => endpoint.name === "http"); + expect(restEndpoint?.host).toBe("127.0.0.1"); + expect(restEndpoint?.port).toBe(Number(new URL(restUrl).port)); + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`${restUrl}/owner_rows`); + expect(response.status).toBe(200); + const body = yield* response.json; + expect(body).toEqual([{ value: "wake" }]); + expect((yield* owner.core.get(shadow.id)).lifecycle).toBe("running"); + yield* owner.composition.stop; + expect((yield* owner.core.get(shadow.id)).lifecycle).toBe("running"); + const firstPort = new URL(databaseUrl).port; + yield* owner.core.stop(database.id); + yield* owner.core.start(database.id); + yield* owner.core.ready(database.id); + const reopened = yield* owner.credentials(database.id, "host"); + if (reopened.databaseUrl === undefined) + return yield* Effect.die("reopened credentials missing"); + expect(new URL(reopened.databaseUrl).port).toBe(firstPort); + yield* owner.namespace.stop; + const reopenedOwner = yield* ownerFor({ + saved: (yield* state.read(stack.id)) ?? stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => reopenedOwner.namespace.destroy.pipe(Effect.ignore)); + expect((yield* reopenedOwner.services.list).length).toBe(3); + expect((yield* reopenedOwner.core.get(database.id)).lifecycle).toBe("stopped"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.effect("isolates owner graphs built in one scope", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-layer-" }); + const state = yield* stateFor(`${root}/state`); + const first = initial("owner-layer-first"); + const second = initial("owner-layer-second"); + yield* state.save(first); + yield* state.save(second); + const memoMap = yield* Layer.makeMemoMap; + const scope = yield* Effect.scope; + const buildOwner = (saved: SavedStack) => + Layer.buildWithMemoMap( + Owner.layer({ + saved, + root: `${root}/data`, + cacheRoot, + }).pipe(Layer.provide(Layer.succeed(State.Service, state))), + memoMap, + scope, + ).pipe(Effect.map((context) => Context.get(context, Owner.Service))); + const firstOwner = yield* buildOwner(first); + const secondOwner = yield* buildOwner(second); + yield* Effect.addFinalizer(() => firstOwner.namespace.destroy.pipe(Effect.ignore)); + yield* Effect.addFinalizer(() => secondOwner.namespace.destroy.pipe(Effect.ignore)); + const created = yield* firstOwner.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" }, smtp: { port: "auto" }, pop3: { port: "auto" } }, + }); + const isolated = yield* secondOwner.composition + .configure({ + members: [{ id: created.id, activation: "eager" }], + dependencies: [], + }) + .pipe(Effect.flip); + expect(isolated.operation).toBe("configure"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "builds the default database, REST, and Mail composition with rendered bindings", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-factory-" }); + const stack = initial("owner-factory"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const created = yield* owner.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-factory-password"), + jwtSecret: Redacted.make("owner-factory-jwt"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { + service: "rest", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: "auto" } }, + }, + { + service: "mail", + config: {}, + endpoints: { http: { port: "auto" }, smtp: { port: "auto" } }, + }, + ]); + const database = created.find((entry) => entry.creation.service === "database"); + const rest = created.find((entry) => entry.creation.service === "rest"); + const mail = created.find((entry) => entry.creation.service === "mail"); + if (database === undefined || rest === undefined || mail === undefined) + return yield* Effect.die("factory did not create all requested services"); + if (rest.creation.service !== "rest") return yield* Effect.die("REST service missing"); + expect(rest.creation.config.databaseUrl).toContain("authenticator:"); + yield* owner.composition.start; + const databaseCredentials = yield* owner.credentials(database.id, "host"); + const databaseUrl = databaseCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("database URL missing"); + yield* query(databaseUrl, "CREATE TABLE factory_rows(value text NOT NULL)"); + yield* query(databaseUrl, "INSERT INTO factory_rows VALUES ('factory-row')"); + const restCredentials = yield* owner.credentials(rest.id, "host"); + const restUrl = restCredentials.url; + if (restUrl === undefined) return yield* Effect.die("REST URL missing"); + const mailCredentials = yield* owner.credentials(mail.id, "host"); + expect(mailCredentials.smtpUrl).toMatch(/^smtp:\/\//u); + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`${restUrl}/factory_rows`); + expect(response.status).toBe(200); + expect(yield* response.json).toEqual([{ value: "factory-row" }]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.effect("validates Supabase composition recipes before creating instances", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-owner-factory-errors-" }); + const stack = initial("owner-factory-errors"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const database = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("owner-errors-password"), + jwtSecret: Redacted.make("owner-errors-jwt"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, + }; + const duplicate = yield* owner.composition.supabase([database, database]).pipe(Effect.flip); + expect(duplicate.operation).toBe("supabase"); + expect(yield* owner.services.list).toHaveLength(0); + const conflicting = yield* owner.composition + .supabase([ + { + service: "rest", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: 41_001 } }, + }, + { + service: "auth", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: 41_002 } }, + }, + ]) + .pipe(Effect.flip); + expect(conflicting.operation).toBe("supabase"); + expect(yield* owner.services.list).toHaveLength(0); + + const badDataRoot = `${root}/data-file`; + const badOwner = yield* ownerFor({ + saved: stack, + state, + root: badDataRoot, + cacheRoot, + }); + yield* Effect.addFinalizer(() => badOwner.namespace.destroy.pipe(Effect.ignore)); + yield* fs.makeDirectory(badDataRoot); + yield* fs.remove(badDataRoot, { recursive: true }); + yield* fs.writeFileString(badDataRoot, "occupied"); + const registrationFailure = yield* badOwner.services.create(database).pipe(Effect.flip); + expect(registrationFailure.operation).toBe("register"); + expect(yield* badOwner.services.list).toHaveLength(0); + expect((yield* state.read(stack.id))?.instances).toHaveLength(0); + + const databaseWithoutSql = yield* owner.services.create({ + service: "database", + config: database.config, + endpoints: {}, + }); + expect(yield* owner.credentials(databaseWithoutSql.id, "host")).toEqual({}); + + const missingSql = yield* owner.composition + .supabase([ + { ...database, endpoints: {} }, + { + service: "rest", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: "auto" } }, + }, + ]) + .pipe(Effect.flip); + expect(missingSql.message).toBe("database requires configured sql endpoint"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts new file mode 100644 index 0000000000..c459ec053d --- /dev/null +++ b/packages/stack/src/Owner.ts @@ -0,0 +1,1034 @@ +import { + Context, + Crypto, + Data, + Effect, + FileSystem, + Path, + Redacted, + Ref, + Scope, + Schema, + Semaphore, + Stream, + Layer, +} from "effect"; +import { HttpClient } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import * as Network from "./Network.ts"; +import type { NetworkBinding, NetworkNamespace } from "./Network.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import type { CompositionConfig, RegisteredInstance } from "./Orchestrator.ts"; +import { + makeService, + ServiceError, + ServiceLaunchError, + type ServiceInstance, + type ServiceObservation, +} from "./Service.ts"; +import { ProxyError, type BackendAddress } from "./Proxy.ts"; +import type { NetworkEndpoint } from "./Network.ts"; +import { + apiRoute, + credentialsFor, + endpointNames, + endpointPort, + EndpointError, + outputsFor, + publicUrl, +} from "./host/Endpoints.ts"; +import { + makeSupabaseComposition, + SupabaseCompositionError, + type SupabaseCompositionOptions, +} from "./composition/Supabase.ts"; +import { + makeServiceRecipe, + ServiceCreation, + type CatalogLog, + type CatalogRecipe, + serviceSchemas, +} from "./services/Catalog.ts"; +import { makeDatabaseSnapshots, type DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; +import * as State from "./State.ts"; +import type { SavedInstance, SavedStack } from "./State.ts"; + +export class OwnerError extends Data.TaggedError("OwnerError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface OwnerOptions { + readonly saved: SavedStack; + readonly state: State.Interface; + readonly root: string; + readonly cacheRoot: string; +} + +export class Service extends Context.Service()("@supabase/stack/Owner") {} + +type OwnerObservation = ServiceObservation & { + readonly endpoints: ReadonlyArray; +}; + +export interface Interface { + readonly services: { + readonly create: ( + creation: unknown, + ) => Effect.Effect<{ id: string; creation: ServiceCreation }, OwnerError>; + readonly get: ( + id: string, + ) => Effect.Effect<{ id: string; creation: ServiceCreation }, OwnerError>; + readonly list: Effect.Effect< + ReadonlyArray<{ id: string; creation: ServiceCreation }>, + OwnerError + >; + }; + readonly core: { + readonly get: (id: string) => Effect.Effect; + readonly status: (id: string) => Effect.Effect; + readonly followStatus: (id: string) => Stream.Stream; + readonly prepare: (id: string) => Effect.Effect; + readonly logs: (id: string) => Stream.Stream; + readonly start: (id: string) => Effect.Effect; + readonly ready: (id: string) => Effect.Effect; + readonly stop: (id: string) => Effect.Effect; + readonly restart: (id: string, creation?: unknown) => Effect.Effect; + readonly destroy: (id: string) => Effect.Effect; + }; + readonly composition: { + readonly supabase: ( + creations: ReadonlyArray, + options?: SupabaseCompositionOptions, + ) => Effect.Effect, OwnerError>; + readonly configure: (configuration: CompositionConfig) => Effect.Effect; + readonly get: Effect.Effect; + readonly start: Effect.Effect, OwnerError>; + readonly stop: Effect.Effect, OwnerError>; + readonly restart: Effect.Effect, OwnerError>; + }; + readonly credentials: ( + id: string, + from: "host" | "runtime", + ) => Effect.Effect>, OwnerError>; + readonly snapshots: { + readonly exportSnapshot: ( + id: string, + destination: string, + ) => Effect.Effect; + readonly restoreSnapshot: ( + id: string, + source: string, + ) => Effect.Effect; + readonly resetData: (id: string) => Effect.Effect; + }; + readonly namespace: { + readonly stop: Effect.Effect; + readonly destroy: Effect.Effect; + }; + readonly setDraining: (draining: boolean) => Effect.Effect; + readonly getServing: Effect.Effect; +} + +const errorFor = (operation: string, cause: unknown) => + new OwnerError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const supabaseError = (cause: unknown) => + cause instanceof SupabaseCompositionError + ? cause + : new SupabaseCompositionError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const creationJson = Schema.toCodecJson(ServiceCreation); +const encodeCreation = (creation: ServiceCreation) => Schema.encodeEffect(creationJson)(creation); + +const makeOwnerWithDependencies = ( + options: OwnerOptions, + orchestrator: Orchestrator.Interface, + network: Network.Interface, +) => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const http = yield* HttpClient.HttpClient; + const ownerScope = yield* Scope.Scope; + const recipes = yield* Ref.make(new Map()); + const instances = yield* Ref.make(new Map>()); + const namespaces = yield* Ref.make(new Map()); + const composition = yield* Ref.make({ members: [], dependencies: [] }); + const registryGate = yield* Semaphore.make(1); + const compositionGate = yield* Semaphore.make(1); + const draining = yield* Ref.make(false); + const runtime = options.saved.runtime; + + const updateState = Effect.fn("Owner.updateState")(function* ( + update: (current: SavedStack) => Effect.Effect, + ) { + return yield* options.state + .withLock( + Effect.gen(function* () { + const current = yield* options.state + .read(options.saved.id) + .pipe(Effect.mapError((cause) => errorFor("state", cause))); + if (current === undefined) return yield* errorFor("state", "Saved stack is missing"); + const next = yield* update(current); + yield* options.state + .save(next) + .pipe(Effect.mapError((cause) => errorFor("state", cause))); + return next; + }), + ) + .pipe( + Effect.mapError((cause) => + cause instanceof OwnerError ? cause : errorFor("state", cause), + ), + ); + }); + + const persistCreation = Effect.fn("Owner.persistCreation")(function* ( + id: string, + creation: ServiceCreation, + ) { + const encoded = yield* encodeCreation(creation).pipe( + Effect.mapError((cause) => errorFor("state", cause)), + ); + yield* updateState((current) => + Effect.succeed({ + ...current, + instances: current.instances.map((instance) => + instance.id === id ? { ...instance, creation: encoded } : instance, + ), + }), + ); + yield* Ref.update(recipes, (values) => { + const recipe = values.get(id); + if (recipe === undefined) return values; + return new Map(values).set(id, { ...recipe, creation }); + }); + }); + + const addCreation = Effect.fn("Owner.addCreation")(function* ( + id: string, + creation: ServiceCreation, + ) { + const encoded = yield* encodeCreation(creation).pipe( + Effect.mapError((cause) => errorFor("state", cause)), + ); + yield* updateState((current) => + Effect.succeed({ + ...current, + instances: [...current.instances, { id, creation: encoded } satisfies SavedInstance], + }), + ); + }); + + const removeCreation = (id: string) => + updateState((current) => + Effect.succeed({ + ...current, + instances: current.instances.filter((instance) => instance.id !== id), + }), + ).pipe(Effect.asVoid); + + const persistComposition = (value: CompositionConfig) => + updateState((current) => Effect.succeed({ ...current, composition: value })).pipe( + Effect.asVoid, + ); + const removeInstance = (id: string) => { + const prune = (current: CompositionConfig): CompositionConfig => ({ + members: current.members.filter((member) => member.id !== id), + dependencies: current.dependencies.filter( + (dependency) => dependency.from !== id && dependency.to !== id, + ), + }); + return updateState((current) => + Schema.decodeUnknownEffect(Orchestrator.CompositionConfig)(current.composition).pipe( + Effect.mapError((cause) => errorFor("state", cause)), + Effect.map((savedComposition) => ({ + ...current, + instances: current.instances.filter((instance) => instance.id !== id), + composition: prune(savedComposition), + })), + ), + ).pipe( + Effect.tap(() => Ref.update(composition, prune)), + Effect.asVoid, + ); + }; + + const recipeFor = (input: unknown, id: string) => + makeServiceRecipe(input, { + stackId: options.saved.id, + instanceId: id, + root: options.root, + cacheRoot: options.cacheRoot, + runtime, + }).pipe(Effect.mapError((cause) => errorFor("recipe", cause))); + const recipeReady = (input: unknown, id: string) => + recipeFor(input, id).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HttpClient.HttpClient, http), + Effect.provideService(Scope.Scope, ownerScope), + ); + + const getRecipe = (id: string) => + Ref.get(recipes).pipe( + Effect.flatMap((values) => { + const recipe = values.get(id); + return recipe === undefined + ? Effect.fail(errorFor("get", `Unknown service ${id}`)) + : Effect.succeed(recipe); + }), + ); + + const getCreation = (id: string) => getRecipe(id).pipe(Effect.map((recipe) => recipe.creation)); + + const mergeInputs = (creation: ServiceCreation, inputs: Record) => + Schema.decodeUnknownEffect(ServiceCreation)({ + ...creation, + config: Object.fromEntries( + Object.entries({ ...creation.config, ...inputs }).filter( + ([, value]) => value !== undefined, + ), + ), + }).pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "inputs", message: String(cause) }), + ), + ); + + const restartCreation = ( + creation: ServiceCreation, + candidate: unknown, + ): Effect.Effect => { + if (candidate === undefined) return Effect.succeed(creation); + const config = isRecord(candidate) && "config" in candidate ? candidate.config : candidate; + return Schema.decodeUnknownEffect(ServiceCreation)({ + ...creation, + config, + }).pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "restart", message: String(cause) }), + ), + ); + }; + + const register = Effect.fn("Owner.register")(function* (id: string, recipe: CatalogRecipe) { + const initial = recipe.creation; + const namespaceRef = yield* Ref.make(undefined); + const instance = yield* makeService( + { + ...recipe.definition, + launch: (context) => + persistCreation(id, context.config).pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "state", message: cause.message, cause }), + ), + Effect.andThen(recipe.definition.launch(context)), + Effect.map((session) => ({ + ...session, + remove: session.remove.pipe( + Effect.andThen( + Ref.get(namespaceRef).pipe( + Effect.flatMap((value) => value?.close ?? Effect.void), + Effect.mapError( + (cause) => + new ServiceError({ operation: "close", message: cause.message, cause }), + ), + ), + ), + ), + })), + Effect.catchTag("ServiceLaunchError", (failure) => + Effect.fail( + new ServiceLaunchError({ + failure: failure.failure, + runtime: { + ...failure.runtime, + remove: failure.runtime.remove.pipe( + Effect.andThen( + Ref.get(namespaceRef).pipe( + Effect.flatMap((value) => value?.close ?? Effect.void), + Effect.mapError( + (cause) => + new ServiceError({ + operation: "close", + message: cause.message, + cause, + }), + ), + ), + ), + ), + }, + }), + ), + ), + ), + removeData: (context) => + recipe.definition.removeData(context).pipe( + Effect.andThen( + Ref.get(namespaceRef).pipe( + Effect.flatMap((value) => value?.release ?? Effect.void), + Effect.mapError( + (cause) => + new ServiceError({ operation: "release", message: cause.message, cause }), + ), + ), + ), + Effect.andThen( + removeInstance(id).pipe( + Effect.mapError( + (cause) => + new ServiceError({ operation: "state", message: cause.message, cause }), + ), + ), + ), + ), + }, + { + id, + config: initial, + coordinate: (operation, transition) => + Ref.get(draining).pipe( + Effect.flatMap((isDraining) => + isDraining && ["start", "arm", "restart", "storage"].includes(operation) + ? Effect.fail( + new ServiceError({ operation: "draining", message: "Owner is draining" }), + ) + : orchestrator.admissionFor(id)(operation, transition), + ), + ), + }, + ); + const enabled = instance.get.pipe( + Effect.map( + (observation) => + observation.registered && + observation.lifecycle !== "stopped" && + !(observation.exit !== undefined && !observation.wakeEnabled), + ), + ); + const endpointEntries = Object.fromEntries( + endpointNames(initial).map((name) => { + const route = name === "http" ? apiRoute(initial.service) : undefined; + const endpoint: NetworkEndpoint = { + protocol: name === "http" ? ("http" as const) : ("tcp" as const), + port: endpointPort(initial, name), + backend: orchestrator.acquire(id, name !== "inspector").pipe( + Effect.flatMap(() => recipe.endpoint(name)), + Effect.flatMap((address): Effect.Effect => { + if (address.kind === "unix") { + return address.path === undefined + ? Effect.fail(new ProxyError({ message: "Unix endpoint has no path" })) + : Effect.succeed({ path: `${address.path}/.s.PGSQL.${address.port}` }); + } + return Effect.succeed({ host: address.host ?? "127.0.0.1", port: address.port }); + }), + Effect.mapError((cause) => + cause instanceof ProxyError + ? cause + : new ProxyError({ message: String(cause), cause }), + ), + ), + enabled, + ...(route === undefined + ? {} + : { + shared: + initial.service === "realtime" + ? [ + { + prefix: "/realtime/v1/api", + upstreamPrefix: "/api", + upstreamHost: "realtime-dev", + }, + { + prefix: route, + upstreamPrefix: "/socket", + upstreamHost: "realtime-dev", + }, + ] + : [{ prefix: route, upstreamPrefix: "/" }], + }), + }; + return [name, endpoint]; + }), + ); + const namespace = yield* network + .register({ id, endpoints: endpointEntries }) + .pipe(Effect.mapError((cause) => errorFor("network", cause))); + yield* Ref.set(namespaceRef, namespace); + const endpointAddress = (name: string, from: "host" | "runtime") => + namespace.address(name, from).pipe( + Effect.mapError( + (cause) => + new EndpointError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + ), + ); + const databasePassword = () => + getCreation(id).pipe( + Effect.flatMap((creation) => + creation.service === "database" + ? Effect.succeed(Redacted.value(creation.config.databasePassword)) + : Effect.fail(new EndpointError({ message: "Output requires a database" })), + ), + Effect.mapError((cause) => + cause instanceof EndpointError + ? cause + : new EndpointError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + ), + ); + const outputs = Object.fromEntries( + Object.entries(outputsFor(initial, endpointAddress, databasePassword)).map( + ([name, value]) => [ + name, + value.pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "output", message: cause.message, cause }), + ), + ), + ], + ), + ); + const registered: RegisteredInstance = { + id, + core: { + get: instance.get, + ready: instance.ready, + stop: instance.stop, + destroy: instance.destroy, + arm: instance.arm, + armAt: instance.armAt, + sleep: instance.sleep, + observation: instance.observation, + }, + startAt: (revision, inputs, wake, guard) => + getCreation(id).pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "get", message: cause.message, cause }), + ), + Effect.flatMap((creation) => mergeInputs(creation, inputs)), + Effect.flatMap((candidate) => instance.startAt(revision, candidate, wake, guard)), + ), + restart: (revision, inputs, candidate, guard) => + getCreation(id).pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "get", message: cause.message, cause }), + ), + Effect.flatMap((creation) => restartCreation(creation, candidate)), + Effect.flatMap((creation) => mergeInputs(creation, inputs)), + Effect.flatMap((next) => instance.restart(next, revision, guard)), + ), + bind: namespace.bind.pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "bind", message: cause.message, cause }), + ), + ), + close: namespace.close.pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "close", message: cause.message, cause }), + ), + Effect.tap(() => + instance.get.pipe( + Effect.flatMap((observation) => + observation.registered + ? Effect.void + : Effect.all( + [ + Ref.update(recipes, (values) => { + const next = new Map(values); + next.delete(id); + return next; + }), + Ref.update(instances, (values) => { + const next = new Map(values); + next.delete(id); + return next; + }), + Ref.update(namespaces, (values) => { + const next = new Map(values); + next.delete(id); + return next; + }), + ], + { discard: true }, + ), + ), + ), + ), + ), + hasEndpoint: endpointNames(initial).length > 0, + inputs: Object.keys(serviceSchemas[initial.service].fields), + outputs, + }; + yield* registryGate.withPermits(1)( + orchestrator.register(registered).pipe( + Effect.catch((cause) => namespace.release.pipe(Effect.andThen(Effect.fail(cause)))), + Effect.mapError( + (cause) => new ServiceError({ operation: "register", message: String(cause), cause }), + ), + ), + ); + yield* Ref.update(recipes, (values) => new Map(values).set(id, recipe)); + yield* Ref.update(instances, (values) => new Map(values).set(id, instance)); + yield* Ref.update(namespaces, (values) => new Map(values).set(id, namespace)); + }); + const registerReady = (id: string, recipe: CatalogRecipe) => + register(id, recipe).pipe(Effect.provideService(Scope.Scope, ownerScope)); + + for (const saved of options.saved.instances) { + const creation = yield* Schema.decodeUnknownEffect(creationJson)(saved.creation).pipe( + Effect.mapError((cause) => errorFor("state", cause)), + ); + yield* registerReady(saved.id, yield* recipeReady(creation, saved.id)); + } + const savedComposition = yield* Schema.decodeUnknownEffect(Orchestrator.CompositionConfig)( + options.saved.composition, + ).pipe(Effect.mapError((cause) => errorFor("configure", cause))); + yield* orchestrator + .configure(savedComposition) + .pipe(Effect.mapError((cause) => errorFor("configure", cause))); + yield* Ref.set(composition, savedComposition); + + const get = Effect.fn("Owner.get")(function* (id: string) { + return yield* getCreation(id).pipe( + Effect.map((creation) => ({ id, creation })), + Effect.mapError((cause) => errorFor("get", cause)), + ); + }); + const enrichObservation = (id: string, value: ServiceObservation) => + getCreation(id).pipe( + Effect.mapError((cause) => errorFor("status", cause)), + Effect.flatMap((creation) => + Ref.get(namespaces).pipe( + Effect.mapError((cause) => errorFor("status", cause)), + Effect.flatMap((values) => { + const namespace = values.get(id); + return namespace === undefined + ? Effect.fail(errorFor("status", "Service namespace is missing")) + : namespace.bindings.pipe( + Effect.map((endpoints) => ({ ...value, config: creation, endpoints })), + Effect.mapError((cause) => errorFor("status", cause)), + ); + }), + ), + ), + ); + const observation = Effect.fn("Owner.status")(function* (id: string) { + return yield* orchestrator.get(id).pipe( + Effect.flatMap((instance) => instance.core.get), + Effect.flatMap((value) => enrichObservation(id, value)), + Effect.mapError((cause) => + cause instanceof OwnerError ? cause : errorFor("status", cause), + ), + ); + }); + const operation = ( + name: string, + effect: Effect.Effect, + ): Effect.Effect => + effect.pipe(Effect.mapError((cause) => errorFor(name, cause))); + const storage = ( + id: string, + action: Effect.Effect, + ): Effect.Effect => + Ref.get(instances).pipe( + Effect.flatMap((values) => { + const instance = values.get(id); + return instance === undefined + ? Effect.fail(errorFor("storage", `Unknown service ${id}`)) + : instance + .storage( + action.pipe( + Effect.mapError( + (cause) => new ServiceError({ operation: "storage", message: String(cause) }), + ), + ), + ) + .pipe(Effect.mapError((cause) => errorFor("storage", cause))); + }), + ); + const snapshotStore = (id: string) => + get(id).pipe( + Effect.flatMap(({ creation }) => + creation.service === "database" + ? makeDatabaseSnapshots({ + instanceRoot: path.join(options.root, id), + runtime, + version: creation.config.version, + stackId: options.saved.id, + instanceId: id, + }).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.mapError((cause) => errorFor("snapshot", cause)), + ) + : Effect.fail(errorFor("snapshot", "Snapshots are only supported for databases")), + ), + ); + + const createService = Effect.fn("Owner.createService")(function* (input: unknown) { + yield* Ref.get(draining).pipe( + Effect.flatMap((isDraining) => + isDraining ? Effect.fail(errorFor("create", "Owner is draining")) : Effect.void, + ), + ); + const creation = yield* Schema.decodeUnknownEffect(ServiceCreation)(input).pipe( + Effect.mapError((cause) => errorFor("create", cause)), + ); + const id = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("identity", cause)), + ); + yield* addCreation(id, creation); + yield* Effect.gen(function* () { + const recipe = yield* recipeReady(creation, id); + yield* registerReady(id, recipe); + }).pipe( + Effect.catch((cause) => removeCreation(id).pipe(Effect.andThen(Effect.fail(cause)))), + Effect.mapError((cause) => errorFor("register", cause)), + ); + return { id, creation }; + }); + + const configureComposition = Effect.fn("Owner.configureComposition")(function* ( + input: CompositionConfig, + ) { + yield* Ref.get(draining).pipe( + Effect.flatMap((isDraining) => { + if (isDraining) return Effect.fail(errorFor("configure", "Owner is draining")); + return Schema.decodeEffect(Orchestrator.CompositionConfig)(input).pipe( + Effect.mapError((cause) => errorFor("configure", cause)), + ); + }), + Effect.tap((value) => + orchestrator + .configure(value) + .pipe(Effect.mapError((cause) => errorFor("configure", cause))), + ), + Effect.tap((value) => Ref.set(composition, value)), + Effect.tap((value) => persistComposition(value)), + Effect.asVoid, + ); + }); + + const updateCreation = Effect.fn("Owner.updateCreation")(function* ( + id: string, + inputs: Record, + ) { + return yield* getCreation(id).pipe( + Effect.flatMap((creation) => mergeInputs(creation, inputs)), + Effect.mapError((cause) => errorFor("supabase", cause)), + Effect.tap((creation) => persistCreation(id, creation)), + Effect.map((creation) => ({ id, creation })), + ); + }); + + const supabaseComposition = Effect.fn("Owner.supabaseComposition")( + (inputs: ReadonlyArray, options?: SupabaseCompositionOptions) => + makeSupabaseComposition( + { + currentComposition: Ref.get(composition), + get: (id) => get(id).pipe(Effect.mapError(supabaseError)), + status: (id) => + observation(id).pipe( + Effect.map(({ lifecycle, wakeEnabled }) => ({ lifecycle, wakeEnabled })), + Effect.mapError(supabaseError), + ), + create: (creation) => createService(creation).pipe(Effect.mapError(supabaseError)), + destroy: (id) => destroy(id).pipe(Effect.mapError(supabaseError)), + bind: (id) => + orchestrator.get(id).pipe( + Effect.flatMap((registered) => registered.bind), + Effect.mapError(supabaseError), + ), + address: (id, endpoint, from) => + Ref.get(namespaces).pipe( + Effect.flatMap((values) => { + const namespace = values.get(id); + return namespace === undefined + ? Effect.fail(supabaseError("Service namespace is missing")) + : namespace.address(endpoint, from).pipe( + Effect.map(({ host, port }) => publicUrl(host, port)), + Effect.mapError(supabaseError), + ); + }), + ), + output: (id, name) => + orchestrator.get(id).pipe( + Effect.flatMap((registered) => { + const output = registered.outputs[name]; + return output === undefined + ? Effect.fail(supabaseError(`Missing output ${name}`)) + : output.pipe(Effect.mapError(supabaseError)); + }), + Effect.mapError(supabaseError), + ), + updateCreation: (id, values) => + updateCreation(id, values).pipe(Effect.mapError(supabaseError)), + configure: (configuration) => + configureComposition(configuration).pipe(Effect.mapError(supabaseError)), + }, + inputs, + options, + ).pipe( + Effect.mapError((cause) => { + if (cause instanceof OwnerError) return cause; + if (cause instanceof SupabaseCompositionError && cause.cause instanceof OwnerError) + return cause.cause; + return errorFor("supabase", cause); + }), + ), + ); + + const prepare = Effect.fn("Owner.prepare")(function* (id: string) { + return yield* getRecipe(id).pipe( + Effect.flatMap((recipe) => recipe.definition.prepare?.(recipe.creation) ?? Effect.void), + Effect.mapError((cause) => errorFor("prepare", cause)), + ); + }); + const start = Effect.fn("Owner.start")((id: string) => + operation("start", orchestrator.start(id)), + ); + const ready = Effect.fn("Owner.ready")((id: string) => + operation("ready", orchestrator.get(id).pipe(Effect.flatMap((value) => value.core.ready))), + ); + const stop = Effect.fn("Owner.stop")((id: string) => operation("stop", orchestrator.stop(id))); + const restart = Effect.fn("Owner.restart")((id: string, input?: unknown) => + operation( + "restart", + input === undefined ? orchestrator.restart(id) : orchestrator.restart(id, input), + ), + ); + const destroy = Effect.fn("Owner.destroy")((id: string) => + operation("destroy", orchestrator.destroy(id)), + ); + const credentials = Effect.fn("Owner.credentials")((id: string, from: "host" | "runtime") => + get(id).pipe( + Effect.flatMap(({ creation }) => + Ref.get(namespaces).pipe( + Effect.flatMap((values) => { + const namespace = values.get(id); + if (namespace === undefined) + return Effect.fail(errorFor("credentials", "Service namespace is missing")); + const address = (endpoint: string, source: "host" | "runtime") => + namespace.address(endpoint, source).pipe( + Effect.mapError( + (cause) => + new EndpointError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + ), + ); + const password = () => + getCreation(id).pipe( + Effect.flatMap((value) => + value.service === "database" + ? Effect.succeed(Redacted.value(value.config.databasePassword)) + : Effect.fail( + new EndpointError({ message: "Credentials require a database" }), + ), + ), + Effect.mapError((cause) => + cause instanceof EndpointError + ? cause + : new EndpointError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }), + ), + ); + return credentialsFor(creation, address, password, from).pipe( + Effect.mapError((cause) => errorFor("credentials", cause)), + ); + }), + ), + ), + ), + ); + const compose = Effect.fn("Owner.compose")( + (creations: ReadonlyArray, options?: SupabaseCompositionOptions) => + compositionGate.withPermits(1)(supabaseComposition(creations, options)), + ); + const configure = Effect.fn("Owner.configure")((input: CompositionConfig) => + compositionGate.withPermits(1)(configureComposition(input)), + ); + const compositionStart = orchestrator.startComposition.pipe( + Effect.mapError((cause) => errorFor("start", cause)), + Effect.flatMap((values) => Effect.forEach(values, (value) => observation(value.id))), + Effect.withSpan("Owner.startComposition"), + ); + const compositionStop = orchestrator.stopComposition.pipe( + Effect.mapError((cause) => errorFor("stop", cause)), + Effect.flatMap((values) => Effect.forEach(values, (value) => observation(value.id))), + Effect.withSpan("Owner.stopComposition"), + ); + const compositionRestart = orchestrator.restartComposition.pipe( + Effect.mapError((cause) => errorFor("restart", cause)), + Effect.flatMap((values) => Effect.forEach(values, (value) => observation(value.id))), + Effect.withSpan("Owner.restartComposition"), + ); + const exportSnapshot = Effect.fn("Owner.exportSnapshot")((id: string, destination: string) => + snapshotStore(id).pipe( + Effect.flatMap((store) => + storage( + id, + store + .exportSnapshot({ destination }) + .pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), + ), + ), + ), + ); + const restoreSnapshot = Effect.fn("Owner.restoreSnapshot")((id: string, source: string) => + snapshotStore(id).pipe( + Effect.flatMap((store) => + storage( + id, + store + .restoreSnapshot({ source }) + .pipe(Effect.mapError((cause) => errorFor("snapshot", cause))), + ), + ), + ), + ); + const resetData = Effect.fn("Owner.resetData")((id: string) => + get(id).pipe( + Effect.flatMap(({ creation }) => + creation.service === "database" + ? storage( + id, + Effect.gen(function* () { + const current = yield* getRecipe(id); + const reset = current.resetDatabaseData; + if (reset === undefined) + return yield* errorFor("resetData", "Database reset is unavailable"); + return yield* Effect.acquireUseRelease( + Scope.make("parallel"), + (scope) => + reset({ id, config: current.creation, scope }).pipe( + Effect.mapError((cause) => errorFor("resetData", cause)), + ), + (scope, exit) => Scope.close(scope, exit), + ); + }), + ) + : Effect.fail(errorFor("resetData", "Reset is only supported for databases")), + ), + ), + ); + const stopNamespace = operation("stopNamespace", orchestrator.stopNamespace).pipe( + Effect.withSpan("Owner.stopNamespace"), + ); + const destroyNamespace = operation( + "destroyNamespace", + orchestrator.destroyNamespace.pipe( + Effect.andThen(network.release), + Effect.andThen(options.state.remove(options.saved.id)), + ), + ).pipe(Effect.withSpan("Owner.destroyNamespace")); + const setDraining = Effect.fn("Owner.setDraining")((value: boolean) => + Ref.set(draining, value).pipe(Effect.mapError((cause) => errorFor("draining", cause))), + ); + const getServing = Effect.gen(function* () { + return !(yield* Ref.get(draining)); + }).pipe(Effect.withSpan("Owner.getServing")); + + const owner: Interface = { + services: { + create: createService, + get, + list: Ref.get(recipes).pipe( + Effect.map((values) => + [...values].map(([id, recipe]) => ({ id, creation: recipe.creation })), + ), + Effect.mapError((cause) => errorFor("list", cause)), + ), + }, + core: { + get: observation, + status: observation, + followStatus: (id) => + Stream.unwrap( + orchestrator.get(id).pipe( + Effect.map((instance) => instance.core.observation), + Effect.mapError((cause) => errorFor("status", cause)), + ), + ).pipe(Stream.mapEffect((value) => enrichObservation(id, value))), + prepare, + logs: (id) => + Stream.unwrap( + getRecipe(id).pipe( + Effect.map((recipe) => recipe.logs), + Effect.mapError((cause) => errorFor("logs", cause)), + ), + ).pipe(Stream.mapError((cause) => errorFor("logs", cause))), + start, + ready, + stop, + restart, + destroy, + }, + composition: { + supabase: compose, + configure, + get: Ref.get(composition), + start: compositionStart, + stop: compositionStop, + restart: compositionRestart, + }, + credentials, + snapshots: { + exportSnapshot, + restoreSnapshot, + resetData, + }, + namespace: { + stop: stopNamespace, + destroy: destroyNamespace, + }, + setDraining, + getServing, + }; + return owner; + }); + +export const layer = (options: Omit) => + Layer.effect( + Service, + Effect.gen(function* () { + const state = yield* State.Service; + const orchestrator = yield* Orchestrator.Service; + const network = yield* Network.Service; + return Service.of( + yield* makeOwnerWithDependencies({ ...options, state }, orchestrator, network), + ); + }), + ).pipe( + Layer.provide(Layer.fresh(Orchestrator.layer)), + Layer.provide(Network.layer({ stackId: options.saved.id, runtime: options.saved.runtime })), + ); diff --git a/packages/stack/src/Ports.integration.test.ts b/packages/stack/src/Ports.integration.test.ts new file mode 100644 index 0000000000..4e2428373a --- /dev/null +++ b/packages/stack/src/Ports.integration.test.ts @@ -0,0 +1,106 @@ +import { NodeServices, NodeSocketServer } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Effect, Exit, FileSystem, Layer, Scope } from "effect"; +import { makePorts, PortError } from "./Ports.ts"; +import * as State from "./State.ts"; + +const makeTestState = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); + +const bind = (host: string, port: number) => + NodeSocketServer.make({ host, port }).pipe( + Effect.mapError((cause) => new PortError({ key: "sql", message: "Cannot bind", cause })), + ); + +it.live("retains distinct claims for stopped stacks and rebinds the original public port", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const state = yield* makeTestState(root); + for (const id of ["first", "second"]) + yield* state.save({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const ports = yield* makePorts(state); + const firstScope = yield* Scope.make(); + const request = { stackId: "first", key: "db/sql", host: "127.0.0.1", port: "auto" as const }; + const first = yield* ports + .acquire(request, bind) + .pipe(Effect.provideService(Scope.Scope, firstScope)); + yield* Scope.close(firstScope, Exit.void); + const second = yield* ports.acquire({ ...request, stackId: "second" }, bind); + expect(second.port).not.toBe(first.port); + const reopened = yield* makePorts(yield* makeTestState(root)); + const again = yield* reopened.acquire(request, bind); + expect(again.port).toBe(first.port); + expect((yield* state.read("first"))?.ports).toEqual([ + { key: "db/sql", host: "127.0.0.1", port: first.port }, + ]); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("reports an occupied saved port without moving its assignment", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const state = yield* makeTestState(root); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const ports = yield* makePorts(state); + const scope = yield* Scope.make(); + const request = { stackId: "stack", key: "api", host: "127.0.0.1", port: "auto" as const }; + const first = yield* ports + .acquire(request, bind) + .pipe(Effect.provideService(Scope.Scope, scope)); + yield* Scope.close(scope, Exit.void); + yield* bind("127.0.0.1", first.port); + const failure = yield* ports.acquire(request, bind).pipe(Effect.flip); + expect(failure).toBeInstanceOf(PortError); + expect(failure.message).toContain(`api at 127.0.0.1:${first.port}`); + expect((yield* state.read("stack"))?.ports[0]?.port).toBe(first.port); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("refuses allocation when another stack has unreadable claims", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped(); + const state = yield* makeTestState(root); + yield* state.save({ + id: "healthy", + runtime: "native", + identity: { projectRoot: root, branchContext: "test", stackName: "ports" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + yield* fs.makeDirectory(`${root}/broken`); + yield* fs.writeFileString(`${root}/broken/state.json`, "{broken"); + const ports = yield* makePorts(state); + const error = yield* ports + .acquire({ stackId: "healthy", key: "sql", host: "127.0.0.1", port: "auto" }, bind) + .pipe(Effect.flip); + expect(error).toBeInstanceOf(State.StateError); + expect((yield* state.read("healthy"))?.ports).toEqual([]); + expect(yield* fs.readFileString(`${root}/broken/state.json`)).toBe("{broken"); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/Ports.ts b/packages/stack/src/Ports.ts new file mode 100644 index 0000000000..821af3467d --- /dev/null +++ b/packages/stack/src/Ports.ts @@ -0,0 +1,131 @@ +import { Cause, Crypto, Data, Effect, Exit, Option, Scope } from "effect"; +import type * as State from "./State.ts"; + +export class PortError extends Data.TaggedError("PortError")<{ + readonly key: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface PortRequest { + readonly stackId: string; + readonly key: string; + readonly host: string; + readonly port: number | "auto"; +} + +/** Coordinates durable public claims while retaining each successfully bound listener. */ +export const makePorts = (state: State.Interface) => + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + + const acquire = Effect.fn("Ports.acquire")( + ( + request: PortRequest, + bind: (host: string, port: number) => Effect.Effect, + ) => + state.withLock( + Effect.gen(function* () { + const stack = yield* state.read(request.stackId); + if (stack === undefined) + return yield* new PortError({ key: request.key, message: "Stack is not registered" }); + const saved = stack.ports.find((entry) => entry.key === request.key); + if ( + saved !== undefined && + (saved.host !== request.host || + (request.port !== "auto" && saved.port !== request.port)) + ) + return yield* new PortError({ + key: request.key, + message: "The requested listener differs from its saved assignment", + }); + const requested = saved?.port ?? request.port; + if ( + requested !== "auto" && + (!Number.isInteger(requested) || requested < 1 || requested > 65535) + ) + return yield* new PortError({ key: request.key, message: "Invalid public port" }); + + const claimed = new Set(); + for (const other of yield* state.list) + for (const claim of other.ports) + if (other.id !== request.stackId || claim.key !== request.key) + claimed.add(claim.port); + if (requested !== "auto" && claimed.has(requested)) + return yield* new PortError({ + key: request.key, + message: `Public port ${requested} is claimed by another listener`, + }); + + const owner = yield* Scope.Scope; + const first = yield* crypto.randomIntBetween(0, 29999); + let failures = 0; + for (let attempt = 0; attempt < 30000 && failures < 64; attempt++) { + const port = requested === "auto" ? 20000 + ((first + attempt) % 30000) : requested; + if (claimed.has(port)) continue; + const result = yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const scope = yield* Scope.fork(owner, "sequential"); + return yield* restore( + Effect.gen(function* () { + const listener = yield* bind(request.host, port).pipe( + Effect.mapError( + (cause) => + new PortError({ + key: request.key, + message: `Cannot bind ${request.key} at ${request.host}:${port}: ${cause.message}`, + cause, + }), + ), + Effect.provideService(Scope.Scope, scope), + ); + if (saved === undefined) + yield* state.save({ + ...stack, + ports: [...stack.ports, { key: request.key, host: request.host, port }], + }); + return { port, listener }; + }), + ).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) ? Scope.close(scope, exit) : Effect.void, + ), + Effect.exit, + ); + }), + ); + if (Exit.isSuccess(result)) return result.value; + const error = Cause.findErrorOption(result.cause); + if ( + requested !== "auto" || + Exit.hasInterrupts(result) || + Exit.hasDies(result) || + Option.isNone(error) || + !(error.value instanceof PortError) + ) + return yield* Effect.failCause(result.cause); + failures++; + } + return yield* new PortError({ + key: request.key, + message: "No public port is available", + }); + }), + ), + ); + + const release = Effect.fn("Ports.release")(function* (stackId: string, key: string) { + yield* state.withLock( + Effect.gen(function* () { + const stack = yield* state.read(stackId); + if (stack !== undefined) + yield* state.save({ + ...stack, + ports: stack.ports.filter((entry) => entry.key !== key), + }); + }), + ); + }); + + return { acquire, release }; + }); diff --git a/packages/stack/src/Proxy.integration.test.ts b/packages/stack/src/Proxy.integration.test.ts new file mode 100644 index 0000000000..c01cc30a1d --- /dev/null +++ b/packages/stack/src/Proxy.integration.test.ts @@ -0,0 +1,64 @@ +import { NodeHttpClient, NodeHttpServer } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Deferred, Effect, Fiber, Predicate } from "effect"; +import { + HttpClient, + HttpClientRequest, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- NodeHttpServer.make requires a native server factory. +import * as Http from "node:http"; +import { bindTcp, serveTcp } from "./Proxy.ts"; + +it.live( + "waits for target readiness and forwards a streamed response through its retained TCP listener", + () => + Effect.scoped( + Effect.gen(function* () { + const backend = yield* NodeHttpServer.make(() => Http.createServer(), { + host: "127.0.0.1", + port: 0, + }); + yield* backend.serve( + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + return HttpServerResponse.stream(request.stream, { + contentType: "application/octet-stream", + }); + }), + ); + const listener = yield* bindTcp("127.0.0.1", 0); + if ( + !Predicate.isTagged(backend.address, "TcpAddress") || + !Predicate.isTagged(listener.address, "TcpAddress") + ) + return yield* Effect.die("Expected TCP listeners"); + const address = { host: "127.0.0.1", port: backend.address.port }; + const acquired = yield* Deferred.make(); + const ready = yield* Deferred.make(); + const released = yield* Deferred.make(); + const target = Effect.acquireRelease(Deferred.succeed(acquired, undefined), () => + Deferred.succeed(released, undefined), + ).pipe(Effect.andThen(Deferred.await(ready)), Effect.as(address)); + yield* serveTcp(listener, target).pipe(Effect.forkScoped); + const body = new Uint8Array(2 * 1024 * 1024).fill(71); + const client = yield* HttpClient.HttpClient; + const request = client + .execute( + HttpClientRequest.post(`http://127.0.0.1:${listener.address.port}/echo`).pipe( + HttpClientRequest.bodyUint8Array(body), + HttpClientRequest.setHeader("connection", "close"), + ), + ) + .pipe(Effect.flatMap((response) => response.arrayBuffer)); + const response = yield* request.pipe(Effect.forkScoped); + yield* Deferred.await(acquired); + expect(yield* Deferred.isDone(released)).toBe(false); + yield* Deferred.succeed(ready, undefined); + const bytes = new Uint8Array(yield* Fiber.join(response)); + expect(bytes).toEqual(body); + yield* Deferred.await(released); + }), + ).pipe(Effect.provide(NodeHttpClient.layerNodeHttp)), +); diff --git a/packages/stack/src/Proxy.ts b/packages/stack/src/Proxy.ts new file mode 100644 index 0000000000..23006fe87d --- /dev/null +++ b/packages/stack/src/Proxy.ts @@ -0,0 +1,99 @@ +import { NodeSink, NodeSocket, NodeSocketServer, NodeStream } from "@effect/platform-node"; +import { Data, Effect, Option, Stream } from "effect"; +import type { Scope } from "effect"; +import type { SocketServer } from "effect/unstable/socket"; +import * as Net from "node:net"; +import { PortError } from "./Ports.ts"; + +export type BackendAddress = + | { readonly host: string; readonly port: number } + | { readonly path: string }; + +export class ProxyError extends Data.TaggedError("ProxyError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const proxyError = (cause: unknown) => + new ProxyError({ message: cause instanceof Error ? cause.message : String(cause), cause }); + +/** Binds a dedicated public listener and retains the socket until its scope closes. */ +export const bindTcp = (host: string, port: number) => + NodeSocketServer.make({ host, port, allowHalfOpen: true }).pipe( + Effect.mapError( + (cause) => new PortError({ key: "tcp", message: "Cannot bind TCP listener", cause }), + ), + ); + +const connect = Effect.fn("Proxy.connect")((address: BackendAddress) => + Effect.gen(function* () { + const socket = yield* Effect.acquireRelease( + Effect.try({ try: () => new Net.Socket({ allowHalfOpen: true }), catch: proxyError }), + (socket) => + Effect.sync(() => { + socket.destroy(); + }), + ); + yield* Effect.callback((resume) => { + const connected = () => resume(Effect.void); + const failed = (cause: Error) => resume(Effect.fail(proxyError(cause))); + socket.once("connect", connected); + socket.once("error", failed); + socket.connect(address); + return Effect.sync(() => { + socket.off("connect", connected); + socket.off("error", failed); + }); + }).pipe(Effect.timeout("10 seconds"), Effect.mapError(proxyError)); + return socket; + }), +); + +const copy = (source: Net.Socket, destination: Net.Socket) => + NodeStream.fromReadable({ evaluate: () => source, onError: proxyError, closeOnDone: false }).pipe( + Stream.run(NodeSink.fromWritable({ evaluate: () => destination, onError: proxyError })), + ); + +/** The scoped target acquisition owns request activity and performs orchestrated wake/readiness. */ +export const serveTcp = Effect.fn("Proxy.serveTcp")( + ( + listener: SocketServer.SocketServer["Service"], + target: Effect.Effect, + ) => + listener.run(() => + Effect.scoped( + Effect.gen(function* () { + // rc.112 supplies this service to handlers but does not remove it from run's requirements. + const incoming = yield* Effect.serviceOption(NodeSocket.NetSocket).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.fail(proxyError("TCP listener supplied no connection")), + onSome: Effect.succeed, + }), + ), + ); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + incoming.destroy(); + }), + ); + const closed = Effect.callback((resume) => { + const onClose = () => resume(Effect.fail(proxyError("Public connection closed"))); + incoming.once("close", onClose); + if (incoming.destroyed) onClose(); + return Effect.sync(() => { + incoming.off("close", onClose); + }); + }); + yield* Effect.gen(function* () { + const address = yield* target; + const backend = yield* connect(address); + yield* Effect.all([copy(incoming, backend), copy(backend, incoming)], { + concurrency: "unbounded", + discard: true, + }); + }).pipe(Effect.raceFirst(closed)); + }), + ).pipe(Effect.catchTag("ProxyError", () => Effect.void)), + ), +); diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts new file mode 100644 index 0000000000..f8186ff8d8 --- /dev/null +++ b/packages/stack/src/Rpc.ts @@ -0,0 +1,145 @@ +import { Data, Schema } from "effect"; +import { Rpc, RpcGroup } from "effect/unstable/rpc"; +import { ServiceCreation } from "./services/Catalog.ts"; +import { CompositionConfig } from "./Orchestrator.ts"; +import { SnapshotDescriptor } from "./services/DatabaseSnapshot.ts"; +import { PgProveOptions, PostgresTool } from "./Tools.ts"; + +const Outcome = Schema.Struct({ + id: Schema.String, + succeeded: Schema.Boolean, + error: Schema.optionalKey(Schema.String), +}); + +export const StackErrorSchema = Schema.TaggedStruct("StackError", { + operation: Schema.String, + message: Schema.String, + outcomes: Schema.optionalKey(Schema.Array(Outcome)), +}); +type StackErrorPayload = Omit, "_tag">; + +/** A typed failure returned by the stack owner. */ +export class StackError extends Data.TaggedError("StackError") {} + +const ServiceErrorSchema = Schema.TaggedStruct("ServiceError", { + operation: Schema.String, + message: Schema.String, +}); + +export const Observation = Schema.Struct({ + id: Schema.String, + endpoints: Schema.Array( + Schema.Struct({ + name: Schema.String, + protocol: Schema.Literals(["tcp", "http"]), + host: Schema.String, + port: Schema.Int, + }), + ), + config: ServiceCreation, + lifecycle: Schema.Literals(["stopped", "starting", "running", "stopping"]), + health: Schema.UndefinedOr(Schema.Literals(["starting", "healthy", "unhealthy"])), + error: Schema.UndefinedOr(ServiceErrorSchema), + cleanupError: Schema.UndefinedOr(ServiceErrorSchema), + exit: Schema.UndefinedOr(Schema.Exit(Schema.Void, ServiceErrorSchema, Schema.Defect())), + currentOperation: Schema.UndefinedOr( + Schema.Literals(["start", "stop", "restart", "storage", "destroy", "sleep"]), + ), + launchId: Schema.UndefinedOr(Schema.Int), + intentRevision: Schema.Int, + wakeEnabled: Schema.Boolean, + registered: Schema.Boolean, +}); +export interface Observation extends Schema.Schema.Type {} + +export const Definition = Schema.Struct({ id: Schema.String, creation: ServiceCreation }); +export interface Definition extends Schema.Schema.Type {} + +const Instance = { id: Schema.String }; +const Snapshot = Schema.Struct({ descriptor: SnapshotDescriptor, destination: Schema.String }); +const Log = Schema.Struct({ + stream: Schema.Literals(["stdout", "stderr"]), + bytes: Schema.Uint8ArrayFromBase64, +}); + +export const ToolEvent = Schema.TaggedUnion({ + Attached: { attachmentId: Schema.String }, + Stdout: { bytes: Schema.Uint8ArrayFromBase64 }, + Stderr: { bytes: Schema.Uint8ArrayFromBase64 }, + Completed: { jobId: Schema.String, exitCode: Schema.Int }, +}); + +/** The private transport contract; lifecycle admission remains in the owner. */ +export const StackRpc = RpcGroup.make( + Rpc.make("createService", { + payload: ServiceCreation, + success: Definition, + error: StackErrorSchema, + }), + Rpc.make("getService", { payload: Instance, success: Definition, error: StackErrorSchema }), + Rpc.make("listServices", { success: Schema.Array(Definition), error: StackErrorSchema }), + Rpc.make("startService", { payload: Instance, error: StackErrorSchema }), + Rpc.make("readyService", { payload: Instance, error: StackErrorSchema }), + Rpc.make("stopService", { payload: Instance, error: StackErrorSchema }), + Rpc.make("restartService", { + payload: { ...Instance, config: Schema.optionalKey(ServiceCreation) }, + error: StackErrorSchema, + }), + Rpc.make("destroyService", { payload: Instance, error: StackErrorSchema }), + Rpc.make("prepareService", { payload: Instance, error: StackErrorSchema }), + Rpc.make("status", { payload: Instance, success: Observation, error: StackErrorSchema }), + Rpc.make("followStatus", { + payload: Instance, + success: Observation, + error: StackErrorSchema, + stream: true, + }), + Rpc.make("logs", { payload: Instance, success: Log, error: StackErrorSchema, stream: true }), + Rpc.make("credentials", { + payload: { ...Instance, from: Schema.Literals(["host", "runtime"]) }, + success: Schema.Record(Schema.String, Schema.String), + error: StackErrorSchema, + }), + Rpc.make("exportSnapshot", { + payload: { ...Instance, destination: Schema.String }, + success: Snapshot, + error: StackErrorSchema, + }), + Rpc.make("restoreSnapshot", { + payload: { ...Instance, source: Schema.String }, + success: Snapshot, + error: StackErrorSchema, + }), + Rpc.make("resetData", { payload: Instance, error: StackErrorSchema }), + Rpc.make("supabaseComposition", { + payload: { + services: Schema.Array(ServiceCreation), + reuseIds: Schema.optionalKey(Schema.Array(Schema.String)), + }, + success: Schema.Array(Definition), + error: StackErrorSchema, + }), + Rpc.make("configureComposition", { payload: CompositionConfig, error: StackErrorSchema }), + Rpc.make("getComposition", { success: CompositionConfig, error: StackErrorSchema }), + Rpc.make("startComposition", { success: Schema.Array(Observation), error: StackErrorSchema }), + Rpc.make("stopComposition", { success: Schema.Array(Observation), error: StackErrorSchema }), + Rpc.make("restartComposition", { success: Schema.Array(Observation), error: StackErrorSchema }), + Rpc.make("shutdown", { payload: { destroy: Schema.Boolean }, error: StackErrorSchema }), + Rpc.make("runTool", { + payload: { + attachmentId: Schema.String, + tool: PostgresTool, + args: Schema.Array(Schema.String), + env: Schema.Record(Schema.String, Schema.String), + pgProve: Schema.optionalKey(PgProveOptions), + stdin: Schema.Boolean, + }, + success: ToolEvent, + error: StackErrorSchema, + stream: true, + }), + Rpc.make("toolInput", { + payload: { attachmentId: Schema.String, bytes: Schema.NullOr(Schema.Uint8ArrayFromBase64) }, + error: StackErrorSchema, + }), +); diff --git a/packages/stack/src/Service.integration.test.ts b/packages/stack/src/Service.integration.test.ts new file mode 100644 index 0000000000..403695fbf9 --- /dev/null +++ b/packages/stack/src/Service.integration.test.ts @@ -0,0 +1,1039 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Deferred, Effect, Exit, Fiber, Queue, Ref, Stream } from "effect"; +import { + makeService, + ServiceDestroyed, + ServiceError, + ServiceLaunchError, + type RuntimeSession, + type ServiceDefinition, + type ServiceInstance, + type ServiceObservation, +} from "./Service.ts"; + +type Config = { readonly version: number }; + +interface ResourceState { + readonly phase: "created" | "launched" | "stopping" | "stopped"; + readonly health: "not-started" | "starting" | "healthy"; + readonly removed: boolean; +} + +interface RuntimePlan { + readonly launchGate: Deferred.Deferred; + readonly launchStarted: Deferred.Deferred; + readonly healthGate: Deferred.Deferred; + readonly healthStarted: Deferred.Deferred; + readonly exit: Deferred.Deferred>; + readonly stopGate: Deferred.Deferred; + readonly stopStarted: Deferred.Deferred; + readonly stopRetryStarted: Deferred.Deferred; + readonly stopFailure: Ref.Ref; + readonly removeGate: Deferred.Deferred; + readonly removeStarted: Deferred.Deferred; + readonly removeFailure: Ref.Ref; + readonly state: Ref.Ref; +} + +interface PreparationPlan { + readonly gate: Deferred.Deferred; + readonly started: Deferred.Deferred; + readonly completed: Deferred.Deferred; +} + +const makeRuntimePlan = Effect.gen(function* () { + return { + launchGate: yield* Deferred.make(), + launchStarted: yield* Deferred.make(), + healthGate: yield* Deferred.make(), + healthStarted: yield* Deferred.make(), + exit: yield* Deferred.make>(), + stopGate: yield* Deferred.make(), + stopStarted: yield* Deferred.make(), + stopRetryStarted: yield* Deferred.make(), + stopFailure: yield* Ref.make(false), + removeGate: yield* Deferred.make(), + removeStarted: yield* Deferred.make(), + removeFailure: yield* Ref.make(false), + state: yield* Ref.make({ + phase: "created", + health: "not-started", + removed: false, + }), + } satisfies RuntimePlan; +}); + +const makePreparationPlan = Effect.gen(function* () { + return { + gate: yield* Deferred.make(), + started: yield* Deferred.make(), + completed: yield* Deferred.make(), + } satisfies PreparationPlan; +}); + +const open = (deferred: Deferred.Deferred) => Deferred.succeed(deferred, undefined); + +const waitForObservation = ( + service: ServiceInstance, + predicate: (observation: ServiceObservation) => boolean, +) => + service.observation.pipe( + Stream.filter(predicate), + Stream.runHead, + Effect.asVoid, + Effect.forkScoped, + ); + +const makeDefinition = ( + plans: Queue.Queue, + preparations?: Queue.Queue, + prepared?: Ref.Ref>, + invalid?: Ref.Ref, + launchFailure?: Ref.Ref, +): ServiceDefinition => ({ + prepare: + preparations === undefined && prepared === undefined && invalid === undefined + ? undefined + : (config) => { + const queue = preparations; + return Effect.gen(function* () { + if (invalid !== undefined && (yield* Ref.get(invalid))) { + return yield* new ServiceError({ + operation: "prepare", + message: "invalid configuration", + }); + } + if (prepared !== undefined) + yield* Ref.update(prepared, (values) => [...values, config]); + if (queue !== undefined) { + const preparation = yield* Queue.take(queue); + yield* Deferred.succeed(preparation.started, undefined); + yield* Deferred.await(preparation.gate); + yield* Deferred.succeed(preparation.completed, undefined); + } + }); + }, + launch: (_context) => + Effect.gen(function* () { + const plan = yield* Queue.take(plans); + yield* Deferred.succeed(plan.launchStarted, undefined); + yield* Deferred.await(plan.launchGate); + if (launchFailure !== undefined && (yield* Ref.get(launchFailure))) { + yield* Ref.set(launchFailure, false); + return yield* new ServiceError({ operation: "launch", message: "launch failed" }); + } + yield* Ref.update(plan.state, (state): ResourceState => ({ ...state, phase: "launched" })); + const session: RuntimeSession = { + health: Effect.gen(function* () { + const state = yield* Ref.get(plan.state); + if (state.health !== "not-started") { + return yield* new ServiceError({ + operation: "health", + message: "health started twice", + }); + } + yield* Ref.update(plan.state, (value): ResourceState => ({ + ...value, + health: "starting", + })); + yield* Deferred.succeed(plan.healthStarted, undefined); + yield* Deferred.await(plan.healthGate); + yield* Ref.update(plan.state, (value): ResourceState => ({ + ...value, + health: "healthy", + })); + }), + exit: Deferred.await(plan.exit), + stop: Effect.gen(function* () { + yield* Ref.update(plan.state, (value): ResourceState => ({ + ...value, + phase: "stopping", + })); + yield* Deferred.succeed(plan.stopStarted, undefined); + if (yield* Ref.get(plan.stopFailure)) { + yield* Ref.set(plan.stopFailure, false); + return yield* new ServiceError({ operation: "stop", message: "stop failed" }); + } + yield* Deferred.succeed(plan.stopRetryStarted, undefined); + yield* Deferred.await(plan.stopGate); + yield* Ref.update(plan.state, (value): ResourceState => ({ ...value, phase: "stopped" })); + }), + remove: Effect.gen(function* () { + yield* Deferred.succeed(plan.removeStarted, undefined); + if (yield* Ref.get(plan.removeFailure)) { + yield* Ref.set(plan.removeFailure, false); + return yield* new ServiceError({ + operation: "remove", + message: "exact cleanup failed", + }); + } + yield* Deferred.await(plan.removeGate); + yield* Ref.update(plan.state, (value) => ({ ...value, removed: true })); + }), + }; + return session; + }), + removeData: () => Effect.void, +}); + +const makeFixture = Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const service = yield* makeService(makeDefinition(plans), { + id: "database-1", + config: { version: 17 }, + }); + return { plans, service }; +}); + +const stopFixture = (service: ServiceInstance, plan: RuntimePlan) => + Effect.gen(function* () { + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* service.stop; + }); + +describe("service kernel", () => { + it.live("submits stop while launch is delayed and completes it after launch", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + const starting = yield* fixture.service.start.pipe(Effect.forkScoped); + yield* Deferred.await(plan.launchStarted); + + const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); + expect((yield* fixture.service.get).lifecycle).toBe("starting"); + yield* open(plan.launchGate); + yield* Fiber.join(starting); + yield* Deferred.await(plan.stopStarted); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + yield* Fiber.join(stopping); + + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), + ); + + it.live("shares one health program and a canceled readiness waiter does not poison it", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* Deferred.await(plan.healthStarted); + + const canceled = yield* fixture.service.ready.pipe(Effect.forkScoped); + const ready = yield* fixture.service.ready.pipe(Effect.forkScoped); + yield* Fiber.interrupt(canceled); + yield* open(plan.healthGate); + expect(Exit.isSuccess(yield* Fiber.await(ready))).toBe(true); + expect((yield* Ref.get(plan.state)).health).toBe("healthy"); + expect((yield* fixture.service.get).health).toBe("healthy"); + yield* stopFixture(fixture.service, plan); + }), + ), + ); + + it.live("cancels after preparation while waiting for the lifecycle gate", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const preparation = yield* makePreparationPlan; + const retryPreparation = yield* makePreparationPlan; + yield* Queue.offer(preparations, preparation); + yield* Queue.offer(preparations, retryPreparation); + const service = yield* makeService(makeDefinition(plans, preparations), { + id: "database-gate-wait", + config: { version: 17 }, + }); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + + const storageStarted = yield* Deferred.make(); + const storageGate = yield* Deferred.make(); + const storage = yield* service + .storage( + Effect.gen(function* () { + yield* Deferred.succeed(storageStarted, undefined); + yield* Deferred.await(storageGate); + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(storageStarted); + + const start = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(preparation.started); + yield* open(preparation.gate); + yield* Deferred.await(preparation.completed); + yield* Fiber.interrupt(start); + yield* open(storageGate); + yield* Fiber.join(storage); + + expect(yield* Deferred.isDone(plan.launchStarted)).toBe(false); + const retry = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(retryPreparation.started); + yield* open(retryPreparation.gate); + yield* open(plan.launchGate); + yield* Fiber.join(retry); + expect((yield* service.get).lifecycle).toBe("running"); + yield* stopFixture(service, plan); + }), + ), + ); + + it.live("does not prepare concurrent starts more than once", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const preparation = yield* makePreparationPlan; + const duplicatePreparation = yield* makePreparationPlan; + yield* Queue.offer(preparations, preparation); + yield* Queue.offer(preparations, duplicatePreparation); + const service = yield* makeService(makeDefinition(plans, preparations), { + id: "database-concurrent-prepare", + config: { version: 17 }, + }); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + + const first = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(preparation.started); + yield* open(preparation.gate); + yield* Deferred.await(preparation.completed); + yield* Deferred.await(plan.launchStarted); + const second = yield* service.start.pipe(Effect.forkScoped({ startImmediately: true })); + yield* open(plan.launchGate); + yield* Fiber.join(first); + expect(yield* Deferred.isDone(duplicatePreparation.started)).toBe(false); + yield* Fiber.join(second); + expect((yield* service.get).lifecycle).toBe("running"); + yield* stopFixture(service, plan); + }), + ), + ); + + it.live("reprepares a queued candidate after the current launch fails", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const firstPreparation = yield* makePreparationPlan; + const secondPreparation = yield* makePreparationPlan; + yield* Queue.offer(preparations, firstPreparation); + yield* Queue.offer(preparations, secondPreparation); + const launchFailure = yield* Ref.make(true); + const service = yield* makeService( + makeDefinition(plans, preparations, undefined, undefined, launchFailure), + { id: "database-retry-prepare", config: { version: 17 } }, + ); + const firstPlan = yield* makeRuntimePlan; + const secondPlan = yield* makeRuntimePlan; + yield* Queue.offer(plans, firstPlan); + yield* Queue.offer(plans, secondPlan); + + const first = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(firstPreparation.started); + yield* open(firstPreparation.gate); + yield* Deferred.await(firstPlan.launchStarted); + const second = yield* service + .startAt(0, { version: 18 }, false, Effect.void) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* open(firstPlan.launchGate); + expect(Exit.isFailure(yield* Fiber.await(first))).toBe(true); + yield* Deferred.await(secondPreparation.started); + yield* open(secondPreparation.gate); + yield* open(secondPlan.launchGate); + yield* Fiber.join(second); + expect((yield* service.get).config.version).toBe(18); + yield* stopFixture(service, secondPlan); + }), + ), + ); + + it.live("invalidates a prepared start when stop is admitted before preparation finishes", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const preparation = yield* makePreparationPlan; + yield* Queue.offer(preparations, preparation); + const service = yield* makeService(makeDefinition(plans, preparations), { + id: "database-stale-preparation", + config: { version: 17 }, + }); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + const start = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(preparation.started); + + yield* service.stop; + yield* open(preparation.gate); + const result = yield* Fiber.await(start); + expect(Exit.isFailure(result)).toBe(true); + expect(yield* Deferred.isDone(plan.launchStarted)).toBe(false); + expect((yield* service.get).lifecycle).toBe("stopped"); + }), + ), + ); + + it.live("records a launch failure as stopped and can launch successfully on retry", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const failLaunch = yield* Ref.make(true); + const definition = makeDefinition(plans); + const service = yield* makeService( + { + launch: (context) => + Effect.gen(function* () { + if (yield* Ref.get(failLaunch)) { + yield* Ref.set(failLaunch, false); + return yield* new ServiceError({ + operation: "launch", + message: "binary unavailable", + }); + } + return yield* definition.launch(context); + }), + removeData: definition.removeData, + }, + { id: "database-launch-retry", config: { version: 17 } }, + ); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + + const failed = yield* service.start.pipe(Effect.exit); + expect(Exit.isFailure(failed)).toBe(true); + expect((yield* service.get).lifecycle).toBe("stopped"); + expect((yield* service.get).error?.operation).toBe("launch"); + expect((yield* Ref.get(plan.state)).phase).toBe("created"); + + yield* open(plan.launchGate); + yield* service.start; + expect((yield* service.get).lifecycle).toBe("running"); + yield* stopFixture(service, plan); + }), + ), + ); + + it.live("retains a runtime acquired before launch failure for exact cleanup", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + const base = makeDefinition(fixture.plans); + const failure = new ServiceError({ operation: "launch", message: "initialization failed" }); + const partial = yield* makeService( + { + launch: (context) => + base + .launch(context) + .pipe( + Effect.flatMap((runtime) => + Effect.fail(new ServiceLaunchError({ failure, runtime })), + ), + ), + removeData: base.removeData, + }, + { id: "database-partial", config: { version: 17 } }, + ); + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* Ref.set(plan.stopFailure, true); + + const failed = yield* partial.start.pipe(Effect.exit); + expect(Exit.isFailure(failed)).toBe(true); + expect((yield* partial.get).lifecycle).toBe("stopping"); + expect((yield* partial.get).error).toEqual(failure); + expect((yield* partial.get).cleanupError).toBeDefined(); + expect((yield* Ref.get(plan.state)).removed).toBe(false); + + const retry = yield* partial.stop.pipe(Effect.forkScoped); + yield* Deferred.await(plan.stopRetryStarted); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + yield* Fiber.join(retry); + expect((yield* partial.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + + const plans2 = yield* Queue.unbounded(); + const plan2 = yield* makeRuntimePlan; + const base2 = makeDefinition(plans2); + const cleanFailure = new ServiceError({ + operation: "launch", + message: "clean partial failure", + }); + const cleaned = yield* makeService( + { + launch: (context) => + base2 + .launch(context) + .pipe( + Effect.flatMap((runtime) => + Effect.fail(new ServiceLaunchError({ failure: cleanFailure, runtime })), + ), + ), + removeData: base2.removeData, + }, + { id: "database-partial-clean", config: { version: 17 } }, + ); + yield* Queue.offer(plans2, plan2); + yield* open(plan2.launchGate); + yield* open(plan2.stopGate); + yield* open(plan2.removeGate); + expect(Exit.isFailure(yield* cleaned.start.pipe(Effect.exit))).toBe(true); + expect((yield* cleaned.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan2.state)).removed).toBe(true); + }), + ), + ); + + it.live("admits a prepared start after restart and does not relaunch", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const queuedStartPreparation = yield* makePreparationPlan; + const restartPreparation = yield* makePreparationPlan; + yield* Queue.offer(preparations, queuedStartPreparation); + yield* Queue.offer(preparations, restartPreparation); + const service = yield* makeService(makeDefinition(plans, preparations), { + id: "database-start-after-restart", + config: { version: 17 }, + }); + const first = yield* makeRuntimePlan; + const unused = yield* makeRuntimePlan; + yield* Queue.offer(plans, first); + yield* Queue.offer(plans, unused); + + const queuedStart = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(queuedStartPreparation.started); + + const restart = yield* service.restart().pipe(Effect.forkScoped); + yield* Deferred.await(restartPreparation.started); + yield* open(restartPreparation.gate); + yield* open(first.launchGate); + yield* Fiber.join(restart); + expect((yield* service.get).lifecycle).toBe("running"); + + yield* open(queuedStartPreparation.gate); + expect(Exit.isSuccess(yield* Fiber.await(queuedStart))).toBe(true); + expect((yield* service.get).launchId).toBe(1); + expect(yield* Deferred.isDone(unused.launchStarted)).toBe(false); + yield* stopFixture(service, first); + }), + ), + ); + + it.live("accepts a prepared start queued during restart once restart is running", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const preparations = yield* Queue.unbounded(); + const initialPreparation = yield* makePreparationPlan; + const queuedStartPreparation = yield* makePreparationPlan; + const restartPreparation = yield* makePreparationPlan; + yield* open(initialPreparation.gate); + yield* Queue.offer(preparations, initialPreparation); + yield* Queue.offer(preparations, restartPreparation); + yield* Queue.offer(preparations, queuedStartPreparation); + const service = yield* makeService(makeDefinition(plans, preparations), { + id: "database-restart-queue", + config: { version: 17 }, + }); + const first = yield* makeRuntimePlan; + const second = yield* makeRuntimePlan; + yield* Queue.offer(plans, first); + yield* Queue.offer(plans, second); + yield* open(first.launchGate); + yield* service.start; + + const restartRunning = yield* waitForObservation( + service, + (value) => value.currentOperation === "restart", + ); + const restart = yield* service.restart().pipe(Effect.forkScoped); + yield* Deferred.await(restartPreparation.started); + yield* open(restartPreparation.gate); + yield* Fiber.join(restartRunning); + const queuedStart = yield* service.start.pipe(Effect.forkScoped); + yield* Deferred.await(queuedStartPreparation.started); + yield* Deferred.await(first.stopStarted); + yield* open(first.stopGate); + yield* Deferred.await(first.removeStarted); + yield* open(first.removeGate); + yield* Deferred.await(second.launchStarted); + yield* open(queuedStartPreparation.gate); + yield* open(second.launchGate); + yield* Fiber.join(restart); + expect(Exit.isSuccess(yield* Fiber.await(queuedStart))).toBe(true); + expect((yield* service.get).lifecycle).toBe("running"); + yield* stopFixture(service, second); + }), + ), + ); + + it.live("keeps an admitted launch alive after its caller is canceled", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + const running = yield* waitForObservation( + fixture.service, + (value) => value.lifecycle === "running", + ); + const caller = yield* fixture.service.start.pipe(Effect.forkScoped); + yield* Deferred.await(plan.launchStarted); + yield* Fiber.interrupt(caller); + yield* open(plan.launchGate); + yield* Fiber.join(running); + expect((yield* Ref.get(plan.state)).phase).toBe("launched"); + yield* stopFixture(fixture.service, plan); + }), + ), + ); + + it.live("lets independent instances progress concurrently", () => + Effect.scoped( + Effect.gen(function* () { + const leftFixture = yield* makeFixture; + const rightFixture = yield* makeFixture; + const leftPlan = yield* makeRuntimePlan; + const rightPlan = yield* makeRuntimePlan; + yield* Queue.offer(leftFixture.plans, leftPlan); + yield* Queue.offer(rightFixture.plans, rightPlan); + const leftRunning = yield* waitForObservation( + leftFixture.service, + (value) => value.lifecycle === "running", + ); + const rightRunning = yield* waitForObservation( + rightFixture.service, + (value) => value.lifecycle === "running", + ); + const leftStart = yield* leftFixture.service.start.pipe(Effect.forkScoped); + const rightStart = yield* rightFixture.service.start.pipe(Effect.forkScoped); + yield* Deferred.await(leftPlan.launchStarted); + yield* Deferred.await(rightPlan.launchStarted); + yield* open(leftPlan.launchGate); + yield* Fiber.join(leftRunning); + expect((yield* leftFixture.service.get).lifecycle).toBe("running"); + expect((yield* rightFixture.service.get).lifecycle).toBe("starting"); + yield* open(rightPlan.launchGate); + yield* Fiber.join(leftStart); + yield* Fiber.join(rightRunning); + yield* Fiber.join(rightStart); + yield* stopFixture(leftFixture.service, leftPlan); + yield* stopFixture(rightFixture.service, rightPlan); + }), + ), + ); + + it.live("keeps exact cleanup authority after failure and retries only remaining cleanup", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* open(plan.stopGate); + yield* Ref.set(plan.removeFailure, true); + const failed = yield* fixture.service.stop.pipe(Effect.exit); + expect(Exit.isFailure(failed)).toBe(true); + expect((yield* fixture.service.get).lifecycle).toBe("stopping"); + expect((yield* Ref.get(plan.state)).removed).toBe(false); + yield* open(plan.removeGate); + yield* fixture.service.stop; + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + }), + ), + ); + + it.live("keeps registration after data removal failure", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + const dataFailure = yield* Ref.make(true); + const service = yield* makeService( + { + launch: makeDefinition(fixture.plans).launch, + removeData: () => + Effect.gen(function* () { + if (yield* Ref.get(dataFailure)) { + yield* Ref.set(dataFailure, false); + return yield* new ServiceError({ operation: "data", message: "retry" }); + } + }), + }, + { id: "database-data", config: { version: 17 } }, + ); + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* service.start; + yield* stopFixture(service, plan); + expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); + expect((yield* service.get).registered).toBe(true); + yield* service.destroy; + expect((yield* service.get).registered).toBe(false); + }), + ), + ); + + it.live("destroys a running instance and retains registration after stop failure", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* open(plan.stopGate); + yield* open(plan.removeGate); + yield* fixture.service.destroy; + expect((yield* fixture.service.get).registered).toBe(false); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + + const plans = yield* Queue.unbounded(); + const dataRemoved = yield* Ref.make(false); + const service = yield* makeService( + { + launch: makeDefinition(plans).launch, + removeData: () => Ref.set(dataRemoved, true), + }, + { id: "database-destroy-stop-failure", config: { version: 17 } }, + ); + const failedPlan = yield* makeRuntimePlan; + yield* Queue.offer(plans, failedPlan); + yield* open(failedPlan.launchGate); + yield* service.start; + yield* Ref.set(failedPlan.stopFailure, true); + + expect(Exit.isFailure(yield* service.destroy.pipe(Effect.exit))).toBe(true); + expect((yield* service.get).lifecycle).toBe("stopping"); + expect((yield* service.get).registered).toBe(true); + expect(yield* Ref.get(dataRemoved)).toBe(false); + + yield* open(failedPlan.stopGate); + yield* open(failedPlan.removeGate); + yield* service.stop; + yield* service.destroy; + expect(yield* Ref.get(dataRemoved)).toBe(true); + expect((yield* service.get).registered).toBe(false); + }), + ), + ); + + it.live("reports a destroyed instance as ServiceDestroyed to readiness callers", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + yield* fixture.service.destroy; + const result = yield* fixture.service.ready.pipe(Effect.exit); + expect(Exit.isFailure(result)).toBe(true); + if (Exit.isFailure(result)) { + const failure = result.cause.reasons.find(Cause.isFailReason); + expect(failure !== undefined && failure.error instanceof ServiceDestroyed).toBe(true); + } + }), + ), + ); + + it.live("waits for a pending launch before checking readiness", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + const start = yield* fixture.service.start.pipe(Effect.forkScoped); + yield* Deferred.await(plan.launchStarted); + const ready = yield* fixture.service.ready.pipe( + Effect.forkScoped({ startImmediately: true }), + ); + yield* open(plan.launchGate); + yield* Deferred.await(plan.healthStarted); + yield* open(plan.healthGate); + expect(Exit.isSuccess(yield* Fiber.await(ready))).toBe(true); + yield* Fiber.join(start); + yield* stopFixture(fixture.service, plan); + }), + ), + ); + + it.live("serializes storage and destroy through the same gate", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const storageStarted = yield* Deferred.make(); + const storageGate = yield* Deferred.make(); + const service = yield* makeService( + { + launch: makeDefinition(plans).launch, + removeData: () => Effect.void, + }, + { id: "database-storage", config: { version: 17 } }, + ); + const storage = yield* service + .storage( + Effect.gen(function* () { + yield* Deferred.succeed(storageStarted, undefined); + yield* Deferred.await(storageGate); + }), + ) + .pipe(Effect.forkScoped); + yield* Deferred.await(storageStarted); + const destroying = yield* service.destroy.pipe(Effect.forkScoped); + expect((yield* service.get).currentOperation).toBe("storage"); + yield* open(storageGate); + yield* Fiber.join(storage); + yield* Fiber.join(destroying); + expect((yield* service.get).registered).toBe(false); + }), + ), + ); + + it.live("prepares invalid restart configuration before stopping current runtime", () => + Effect.scoped( + Effect.gen(function* () { + const plans = yield* Queue.unbounded(); + const prepared = yield* Ref.make>([]); + const invalid = yield* Ref.make(false); + const service = yield* makeService(makeDefinition(plans, undefined, prepared, invalid), { + id: "database-restart", + config: { version: 17 }, + }); + const plan = yield* makeRuntimePlan; + yield* Queue.offer(plans, plan); + yield* open(plan.launchGate); + yield* service.start; + yield* Ref.set(invalid, true); + expect(Exit.isFailure(yield* service.restart({ version: 18 }).pipe(Effect.exit))).toBe( + true, + ); + expect((yield* service.get).lifecycle).toBe("running"); + expect((yield* service.get).config.version).toBe(17); + expect((yield* Ref.get(plan.state)).removed).toBe(false); + yield* stopFixture(service, plan); + }), + ), + ); + + it.live("cleans an exact runtime after an unexpected exit during health", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* Deferred.await(plan.healthStarted); + const ready = yield* fixture.service.ready.pipe(Effect.forkScoped); + const stopping = yield* waitForObservation( + fixture.service, + (value) => value.lifecycle === "stopping", + ); + const stopped = yield* waitForObservation( + fixture.service, + (value) => value.lifecycle === "stopped", + ); + yield* Deferred.succeed( + plan.exit, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* Deferred.await(plan.stopStarted); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + yield* Fiber.join(stopping); + yield* Fiber.join(stopped); + expect(Exit.isFailure(yield* Fiber.await(ready))).toBe(true); + expect((yield* fixture.service.get).lifecycle).toBe("stopped"); + expect((yield* Ref.get(plan.state)).removed).toBe(true); + expect((yield* fixture.service.get).wakeEnabled).toBe(false); + }), + ), + ); + + it.live("reports stop as the current operation during exit cleanup", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* Deferred.succeed( + plan.exit, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* Deferred.await(plan.stopStarted); + expect((yield* fixture.service.get).currentOperation).toBe("stop"); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + }), + ), + ); + + it.live("preserves cleanupError while a retry is admitted and blocked", () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const plan = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, plan); + yield* open(plan.launchGate); + yield* fixture.service.start; + yield* Ref.set(plan.stopFailure, true); + expect(Exit.isFailure(yield* fixture.service.stop.pipe(Effect.exit))).toBe(true); + expect((yield* fixture.service.get).cleanupError).toBeDefined(); + + const retry = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Deferred.await(plan.stopRetryStarted); + const exited = yield* waitForObservation( + fixture.service, + (value) => value.exit !== undefined, + ); + yield* Deferred.succeed( + plan.exit, + Exit.fail(new ServiceError({ operation: "process", message: "crashed" })), + ); + yield* Fiber.join(exited); + expect((yield* fixture.service.get).cleanupError).toBeDefined(); + yield* open(plan.stopGate); + yield* Deferred.await(plan.removeStarted); + yield* open(plan.removeGate); + yield* Fiber.join(retry); + expect((yield* fixture.service.get).cleanupError).toBeUndefined(); + }), + ), + ); + + it.live( + "makes an old readiness wait stale across stop and lets the new launch become ready", + () => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* makeFixture; + const first = yield* makeRuntimePlan; + const second = yield* makeRuntimePlan; + yield* Queue.offer(fixture.plans, first); + yield* Queue.offer(fixture.plans, second); + yield* open(first.launchGate); + yield* fixture.service.start; + yield* Deferred.await(first.healthStarted); + + const oldReady = yield* fixture.service.ready.pipe(Effect.forkScoped); + const stopping = yield* fixture.service.stop.pipe(Effect.forkScoped); + yield* Deferred.await(first.stopStarted); + yield* open(first.stopGate); + yield* Deferred.await(first.removeStarted); + yield* open(first.removeGate); + yield* Fiber.join(stopping); + expect(Exit.isFailure(yield* Fiber.await(oldReady))).toBe(true); + + yield* open(second.launchGate); + yield* fixture.service.start; + yield* Deferred.await(second.healthStarted); + const newReady = yield* fixture.service.ready.pipe(Effect.forkScoped); + yield* open(second.healthGate); + expect(Exit.isSuccess(yield* Fiber.await(newReady))).toBe(true); + expect((yield* fixture.service.get).launchId).toBe(2); + yield* stopFixture(fixture.service, second); + }), + ), + ); +}); + +it.live("retains armed wake after a partially launched process exits during cleanup", () => + Effect.scoped( + Effect.gen(function* () { + const exited = yield* Deferred.make>(); + const stopping = yield* Deferred.make(); + const finishStop = yield* Deferred.make(); + const failure = new ServiceError({ operation: "launch", message: "initialization failed" }); + const service = yield* makeService( + { + launch: () => + Effect.fail( + new ServiceLaunchError({ + failure, + runtime: { + health: Effect.never, + exit: Deferred.await(exited), + stop: Deferred.succeed(stopping, undefined).pipe( + Effect.andThen(Deferred.await(finishStop)), + ), + remove: Effect.void, + }, + }), + ), + removeData: () => Effect.void, + }, + { id: "armed-partial", config: {} }, + ); + yield* service.arm; + const observedExit = yield* waitForObservation(service, (state) => state.exit !== undefined); + const starting = yield* service.start.pipe(Effect.forkChild); + yield* Deferred.await(stopping); + yield* Deferred.succeed(exited, Exit.void); + yield* Fiber.join(observedExit); + yield* Deferred.succeed(finishStop, undefined); + expect(Exit.isFailure(yield* Fiber.await(starting))).toBe(true); + expect((yield* service.get).lifecycle).toBe("stopped"); + expect((yield* service.get).wakeEnabled).toBe(true); + }), + ), +); + +it.live("keeps a sleeping instance armed when exit observation retries failed removal", () => + Effect.scoped( + Effect.gen(function* () { + const exited = yield* Deferred.make>(); + const failRemoval = yield* Ref.make(true); + const service = yield* makeService( + { + launch: () => + Effect.succeed({ + health: Effect.void, + exit: Deferred.await(exited), + stop: Effect.void, + remove: Ref.getAndSet(failRemoval, false).pipe( + Effect.flatMap((fail) => + fail + ? Effect.fail( + new ServiceError({ + operation: "remove", + message: "temporary removal failure", + }), + ) + : Effect.void, + ), + ), + }), + removeData: () => Effect.void, + }, + { id: "sleep-cleanup-retry", config: {} }, + ); + yield* service.arm; + yield* service.start; + yield* service.ready; + expect(Exit.isFailure(yield* Effect.exit(service.sleep))).toBe(true); + expect((yield* service.get).wakeEnabled).toBe(true); + const stopped = yield* waitForObservation(service, (state) => state.lifecycle === "stopped"); + yield* Deferred.succeed(exited, Exit.void); + yield* Fiber.join(stopped); + expect((yield* service.get).wakeEnabled).toBe(true); + }), + ), +); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts new file mode 100644 index 0000000000..5ee3f910e4 --- /dev/null +++ b/packages/stack/src/Service.ts @@ -0,0 +1,635 @@ +import { + Cause, + Data, + Deferred, + Effect, + Exit, + Fiber, + Option, + Ref, + Scope, + Semaphore, + SubscriptionRef, +} from "effect"; +import type { Stream } from "effect"; + +type ServiceLifecycle = "stopped" | "starting" | "running" | "stopping"; +type ServiceHealth = "starting" | "healthy" | "unhealthy"; +type ServiceOperation = "start" | "stop" | "restart" | "storage" | "destroy" | "sleep"; +export type ServiceAdmission = ServiceOperation | "arm"; + +export interface ServiceObservation { + readonly id: string; + readonly config: Config; + readonly lifecycle: ServiceLifecycle; + readonly health: ServiceHealth | undefined; + readonly error: ServiceError | undefined; + readonly cleanupError: ServiceError | undefined; + readonly exit: Exit.Exit | undefined; + readonly currentOperation: ServiceOperation | undefined; + readonly launchId: number | undefined; + readonly intentRevision: number; + readonly wakeEnabled: boolean; + readonly registered: boolean; +} + +export class ServiceError extends Data.TaggedError("ServiceError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export class ServiceDestroyed extends Data.TaggedError("ServiceDestroyed")<{ + readonly id: string; +}> {} + +export class ServiceNotRunning extends Data.TaggedError("ServiceNotRunning")<{ + readonly id: string; +}> {} + +class ServiceNotStopped extends Data.TaggedError("ServiceNotStopped")<{ + readonly id: string; + readonly lifecycle: ServiceLifecycle; + readonly message: string; +}> {} + +export class ServiceStaleLaunch extends Data.TaggedError("ServiceStaleLaunch")<{ + readonly id: string; + readonly launchId: number; +}> {} + +/** The exact runtime resources owned by one service launch. */ +export interface RuntimeSession { + /** The one readiness program for this session. */ + readonly health: Effect.Effect; + /** Resolves with the runtime's terminal result and remains attached to this session. */ + readonly exit: Effect.Effect>; + readonly stop: Effect.Effect; + readonly remove: Effect.Effect; +} + +/** Retains cleanup authority when launch fails after acquiring runtime resources. */ +export class ServiceLaunchError extends Data.TaggedError("ServiceLaunchError")<{ + readonly failure: ServiceError; + readonly runtime: RuntimeSession; +}> {} + +export interface ServiceDefinition { + /** Preparation is performed before lifecycle admission. */ + readonly prepare?: (config: Config) => Effect.Effect; + /** Failures after resource acquisition carry the session for ordinary cleanup. */ + readonly launch: ( + context: ServiceInstanceContext, + ) => Effect.Effect; + readonly removeData: ( + context: ServiceInstanceContext, + ) => Effect.Effect; +} + +export interface ServiceInstanceContext { + readonly id: string; + readonly config: Config; + /** Owns auxiliary session resources; runtime stop/remove retain cleanup authority. */ + readonly scope: Scope.Closeable; +} + +export interface ServiceInstance { + readonly id: string; + readonly observation: Stream.Stream>; + readonly get: Effect.Effect>; + readonly start: Effect.Effect; + readonly startAt: ( + revision: number, + config?: Config, + wake?: boolean, + guard?: Effect.Effect, + ) => Effect.Effect; + readonly arm: Effect.Effect; + readonly armAt: ( + revision: number, + guard?: Effect.Effect, + ) => Effect.Effect; + readonly sleep: Effect.Effect; + readonly stop: Effect.Effect; + readonly restart: ( + config?: Config, + revision?: number, + guard?: Effect.Effect, + ) => Effect.Effect; + readonly ready: Effect.Effect< + void, + ServiceError | ServiceDestroyed | ServiceNotRunning | ServiceStaleLaunch + >; + /** Runs instance-owned storage work only while the instance is stopped. */ + readonly storage: ( + operation: Effect.Effect, + ) => Effect.Effect; + readonly destroy: Effect.Effect; +} + +interface SessionRecord { + readonly launchId: number; + readonly runtime: RuntimeSession; + readonly scope: Scope.Closeable; + readonly healthScope: Scope.Closeable; + readonly health: Deferred.Deferred>; + readonly stopped: Ref.Ref; + readonly removed: Ref.Ref; +} + +const contextFor = ( + id: string, + config: Config, + scope: Scope.Closeable, +): ServiceInstanceContext => ({ + id, + config, + scope, +}); + +const exitError = ( + operation: string, + exit: Exit.Exit, +): ServiceError | undefined => + Exit.isSuccess(exit) + ? undefined + : Option.getOrElse( + Cause.findErrorOption(exit.cause), + () => + new ServiceError({ + operation, + message: "Runtime session failed", + cause: exit.cause, + }), + ); + +/** Creates one independently serialized service instance. */ +export const makeService = ( + definition: ServiceDefinition, + options: { + readonly id: string; + readonly config: Config; + readonly coordinate?: ( + operation: ServiceAdmission, + transition: Effect.Effect, + ) => Effect.Effect; + }, +): Effect.Effect, never, Scope.Scope> => + Effect.gen(function* () { + const owner = yield* Scope.Scope; + const gate = yield* Semaphore.make(1); + const revision = yield* Ref.make(0); + const launchCounter = yield* Ref.make(0); + const current = yield* Ref.make(undefined); + const config = yield* Ref.make(options.config); + const observations = yield* SubscriptionRef.make>({ + id: options.id, + config: options.config, + lifecycle: "stopped", + health: undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: false, + registered: true, + }); + + const update = (change: Partial>) => + SubscriptionRef.update(observations, (value) => ({ ...value, ...change })); + + const setOperation = (operation: ServiceOperation | undefined) => + update({ currentOperation: operation }); + const coordinate = + options.coordinate ?? + ((_operation: ServiceAdmission, transition: Effect.Effect) => transition); + + // Mask only the permit handoff; admitted work belongs to the host scope. + const run = Effect.fn("Service.run")(function* < + A, + E extends ServiceError | ServiceDestroyed | ServiceNotStopped, + >(operation: Effect.Effect) { + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* restore(gate.take(1)); + const fiber = yield* Effect.forkIn( + operation.pipe(Effect.ensuring(gate.release(1))), + owner, + { uninterruptible: false }, + ); + return yield* restore(Fiber.join(fiber)); + }), + ); + }); + + const stopNow = Effect.fn("Service.stopNow")(function* ( + expected?: SessionRecord, + retainWake = false, + ) { + yield* Effect.gen(function* () { + const record = yield* Ref.get(current); + if (expected !== undefined && record !== expected) return; + if (record === undefined) { + yield* update({ lifecycle: "stopped", health: undefined }); + return; + } + + yield* update({ lifecycle: "stopping", health: undefined, wakeEnabled: retainWake }); + yield* Deferred.succeed( + record.health, + Exit.fail(new ServiceError({ operation: "health", message: "Session stopped" })), + ); + yield* Scope.close(record.healthScope, Exit.void); + if (!(yield* Ref.get(record.stopped))) { + yield* record.runtime.stop.pipe( + Effect.tapError((error) => + SubscriptionRef.update(observations, (value) => ({ + ...value, + error: value.error ?? error, + cleanupError: error, + })), + ), + ); + yield* Ref.set(record.stopped, true); + } + if (!(yield* Ref.get(record.removed))) { + yield* record.runtime.remove.pipe( + Effect.tapError((error) => + SubscriptionRef.update(observations, (value) => ({ + ...value, + error: value.error ?? error, + cleanupError: error, + })), + ), + ); + yield* Ref.set(record.removed, true); + } + yield* Scope.close(record.scope, Exit.void); + + yield* Ref.set(current, undefined); + yield* update({ + lifecycle: "stopped", + health: undefined, + launchId: undefined, + cleanupError: undefined, + }); + }); + }); + + const launchNow = Effect.fn("Service.launchNow")(function* ( + guard: Effect.Effect, + ) { + yield* Effect.gen(function* () { + let observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; + if (observation.lifecycle === "stopping") { + yield* stopNow(undefined, observation.wakeEnabled); + observation = yield* SubscriptionRef.get(observations); + if (observation.lifecycle !== "stopped") return; + } + + const launchId = yield* Ref.updateAndGet(launchCounter, (value) => value + 1); + yield* coordinate( + "start", + guard.pipe( + Effect.andThen( + update({ + lifecycle: "starting", + health: "starting", + error: undefined, + cleanupError: undefined, + exit: undefined, + wakeEnabled: observation.wakeEnabled, + launchId, + }), + ), + ), + ); + const runtimeScope = yield* Scope.fork(owner, "parallel"); + const launchExit = yield* Effect.exit( + definition.launch(contextFor(options.id, yield* Ref.get(config), runtimeScope)).pipe( + Effect.map((runtime) => ({ runtime, failure: undefined })), + Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => + Effect.succeed({ runtime, failure }), + ), + ), + ); + if (Exit.isFailure(launchExit)) { + yield* Scope.close(runtimeScope, launchExit); + yield* update({ + lifecycle: "stopped", + health: undefined, + wakeEnabled: observation.wakeEnabled, + error: exitError("launch", launchExit), + launchId: undefined, + }); + return yield* Effect.failCause(launchExit.cause); + } + const { runtime, failure: launchFailure } = launchExit.value; + const health = yield* Deferred.make>(); + const record: SessionRecord = { + launchId, + runtime, + scope: runtimeScope, + healthScope: yield* Scope.fork(runtimeScope, "parallel"), + health, + stopped: yield* Ref.make(false), + removed: yield* Ref.make(false), + }; + yield* Ref.set(current, record); + yield* update({ + lifecycle: launchFailure === undefined ? "running" : "stopping", + health: launchFailure === undefined ? "starting" : undefined, + error: launchFailure, + launchId, + config: yield* Ref.get(config), + }); + + const observeHealth = runtime.health.pipe( + Effect.onExit((exit) => + Effect.gen(function* () { + yield* SubscriptionRef.update( + observations, + (observation): ServiceObservation => { + if (observation.launchId !== launchId || observation.lifecycle !== "running") + return observation; + const error = exitError("health", exit); + return { + ...observation, + health: Exit.isSuccess(exit) ? "healthy" : "unhealthy", + error, + }; + }, + ); + yield* Deferred.succeed(record.health, exit); + }), + ), + ); + const observeExit = record.runtime.exit.pipe( + Effect.flatMap((exit) => + Effect.gen(function* () { + if ((yield* Ref.get(current)) !== record) return; + const observation = yield* SubscriptionRef.get(observations); + const error = + observation.lifecycle === "stopping" + ? observation.error + : (exitError("exit", exit) ?? + new ServiceError({ + operation: "exit", + message: "Runtime exited unexpectedly", + })); + yield* update({ + lifecycle: "stopping", + health: undefined, + error, + exit, + wakeEnabled: observation.lifecycle === "stopping" && observation.wakeEnabled, + }); + yield* run( + Effect.gen(function* () { + if ((yield* Ref.get(current)) !== record) return; + yield* setOperation("stop"); + const live = yield* SubscriptionRef.get(observations); + yield* stopNow(record, live.wakeEnabled).pipe( + Effect.ensuring(setOperation(undefined)), + ); + }), + ); + }), + ), + ); + if (launchFailure === undefined) + yield* Effect.forkIn(observeHealth, record.healthScope, { uninterruptible: false }); + yield* Effect.forkIn(observeExit, runtimeScope, { uninterruptible: false }); + if (launchFailure !== undefined) { + yield* Deferred.succeed(record.health, Exit.fail(launchFailure)); + yield* stopNow(undefined, observation.wakeEnabled); + return yield* launchFailure; + } + }); + }); + + const startAt = Effect.fn("Service.startAt")(function* ( + expectedRevision: number, + candidate?: Config, + wake = false, + guard: Effect.Effect = Effect.void, + ) { + let existing = yield* SubscriptionRef.get(observations); + if (!existing.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (existing.lifecycle === "running") return; + if (existing.lifecycle === "starting") { + yield* run(Effect.void); + existing = yield* SubscriptionRef.get(observations); + if (!existing.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (existing.lifecycle === "running") return; + } + const nextConfig = candidate ?? (yield* Ref.get(config)); + if (definition.prepare !== undefined) yield* definition.prepare(nextConfig); + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (observation.lifecycle === "running" || observation.lifecycle === "starting") return; + if ((yield* Ref.get(revision)) !== expectedRevision || (wake && !observation.wakeEnabled)) + return yield* new ServiceError({ + operation: "admission", + message: "Service intent changed before admission", + }); + yield* setOperation("start"); + yield* launchNow( + guard.pipe( + Effect.andThen(Ref.set(config, nextConfig)), + Effect.andThen(update({ config: nextConfig })), + ), + ).pipe(Effect.ensuring(setOperation(undefined))); + }), + ); + }); + const start = Ref.get(revision).pipe(Effect.flatMap((revision) => startAt(revision))); + + const invalidate = Effect.fn("Service.invalidate")(function* () { + const next = yield* Ref.updateAndGet(revision, (value) => value + 1); + const record = yield* Ref.get(current); + yield* update({ + intentRevision: next, + wakeEnabled: false, + lifecycle: record === undefined ? "stopped" : "stopping", + health: undefined, + }); + }); + + const armAt = Effect.fn("Service.armAt")(function* ( + expectedRevision: number, + guard: Effect.Effect = Effect.void, + ) { + yield* run( + Effect.gen(function* () { + if (!(yield* SubscriptionRef.get(observations)).registered) + return yield* new ServiceDestroyed({ id: options.id }); + if ((yield* Ref.get(revision)) !== expectedRevision) + return yield* new ServiceError({ + operation: "admission", + message: "Service intent changed before admission", + }); + yield* coordinate("arm", guard.pipe(Effect.andThen(update({ wakeEnabled: true })))); + }), + ); + }); + const arm = Ref.get(revision).pipe(Effect.flatMap((revision) => armAt(revision))); + + const sleep = Effect.fn("Service.sleep")(function* () { + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (observation.lifecycle !== "running" || !observation.wakeEnabled) return; + yield* coordinate( + "sleep", + update({ lifecycle: "stopping", health: undefined, currentOperation: "sleep" }), + ); + yield* stopNow(undefined, true).pipe(Effect.ensuring(setOperation(undefined))); + }), + ); + }); + + const stop = Effect.fn("Service.stop")(function* () { + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + yield* coordinate("stop", invalidate()); + yield* setOperation("stop"); + yield* stopNow().pipe(Effect.ensuring(setOperation(undefined))); + }), + ); + }); + + const restart = Effect.fn("Service.restart")(function* ( + candidate?: Config, + requestedRevision?: number, + guard: Effect.Effect = Effect.void, + ) { + const expectedRevision = requestedRevision ?? (yield* Ref.get(revision)); + const nextConfig = candidate ?? (yield* Ref.get(config)); + if (definition.prepare !== undefined) yield* definition.prepare(nextConfig); + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if ((yield* Ref.get(revision)) !== expectedRevision) + return yield* new ServiceError({ + operation: "admission", + message: "Service intent changed before admission", + }); + yield* coordinate("restart", invalidate()); + yield* setOperation("restart"); + yield* Effect.gen(function* () { + yield* stopNow(); + yield* Ref.set(config, nextConfig); + yield* update({ config: nextConfig }); + yield* launchNow(guard); + }).pipe(Effect.ensuring(setOperation(undefined))); + }), + ); + }); + + const ready = Effect.fn("Service.ready")(function* () { + const initial = yield* SubscriptionRef.get(observations); + if (!initial.registered) return yield* new ServiceDestroyed({ id: options.id }); + const expectedRevision = yield* Ref.get(revision); + const expectedLaunchId = initial.lifecycle === "starting" ? initial.launchId : undefined; + let record = yield* Ref.get(current); + if (record === undefined && initial.lifecycle === "starting") { + yield* run(Effect.void); + record = yield* Ref.get(current); + } + if (record === undefined) { + if ((yield* Ref.get(revision)) !== expectedRevision) + return yield* new ServiceStaleLaunch({ + id: options.id, + launchId: yield* Ref.get(launchCounter), + }); + return yield* new ServiceNotRunning({ id: options.id }); + } + const launchId = record.launchId; + if ((yield* Ref.get(revision)) !== expectedRevision) + return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + if (expectedLaunchId !== undefined && record.launchId !== expectedLaunchId) + return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + if ((yield* SubscriptionRef.get(observations)).lifecycle === "stopping") { + return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + } + const healthResult = yield* Deferred.await(record.health); + if ( + (yield* Ref.get(current)) !== record || + (yield* SubscriptionRef.get(observations)).lifecycle !== "running" + ) { + return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + } + yield* healthResult; + }); + + const storage = Effect.fn("Service.storage")(function* ( + operation: Effect.Effect, + ) { + return yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return yield* new ServiceDestroyed({ id: options.id }); + if (observation.lifecycle !== "stopped" || observation.wakeEnabled) { + return yield* new ServiceNotStopped({ + id: options.id, + lifecycle: observation.lifecycle, + message: `Service ${options.id} must be stopped with wake disabled before modifying data`, + }); + } + yield* coordinate("storage", setOperation("storage")); + return yield* operation.pipe(Effect.ensuring(setOperation(undefined))); + }), + ); + }); + + const destroy = Effect.fn("Service.destroy")(function* () { + yield* run( + Effect.gen(function* () { + const observation = yield* SubscriptionRef.get(observations); + if (!observation.registered) return; + yield* coordinate("destroy", invalidate()); + yield* setOperation("destroy"); + yield* Effect.gen(function* () { + yield* stopNow(); + { + const dataScope = yield* Scope.fork(owner, "parallel"); + yield* definition + .removeData(contextFor(options.id, yield* Ref.get(config), dataScope)) + .pipe( + Effect.tapError((error) => update({ error })), + Effect.ensuring(Scope.close(dataScope, Exit.void)), + ); + } + yield* update({ registered: false, lifecycle: "stopped" }); + }).pipe(Effect.ensuring(setOperation(undefined))); + }), + ); + }); + + return { + id: options.id, + observation: SubscriptionRef.changes(observations), + get: SubscriptionRef.get(observations), + start, + startAt, + arm, + armAt, + sleep: sleep(), + stop: stop(), + restart, + ready: ready(), + storage, + destroy: destroy(), + }; + }); diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts new file mode 100644 index 0000000000..0fbb129bce --- /dev/null +++ b/packages/stack/src/StackHost.integration.test.ts @@ -0,0 +1,518 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + Cause, + Context, + Data, + Deferred, + Effect, + Exit, + FileSystem, + Fiber, + Layer, + Redacted, + Ref, + Stream, +} from "effect"; +import { RpcClient, RpcSerialization } from "effect/unstable/rpc"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- integration observes exact listener closure. +import * as Net from "node:net"; +import { acquireHost, launchHost } from "./HostProcess.ts"; +import * as Owner from "./Owner.ts"; +import { OwnerError } from "./Owner.ts"; +import { OrchestratorError } from "./Orchestrator.ts"; +import { StackRpc } from "./Rpc.ts"; +import * as State from "./State.ts"; +import { makeRuntime } from "./StackHost.ts"; +import { postgres } from "./Tools.ts"; +import * as ToolRunner from "./host/ToolRunner.ts"; + +class HostTestError extends Data.TaggedError("HostTestError")<{ readonly message: string }> {} + +const stateFor = (root: string) => + Effect.gen(function* () { + const context = yield* Layer.build(State.layer({ root })); + return Context.get(context, State.Service); + }); + +const hostTestError = (cause: unknown) => + new HostTestError({ message: cause instanceof Error ? cause.message : String(cause) }); + +const ownerFor = (options: { + readonly saved: Parameters[0]["saved"]; + readonly state: State.Interface; + readonly root: string; + readonly cacheRoot: string; +}) => { + const { state, ...layerOptions } = options; + return Effect.gen(function* () { + const context = yield* Layer.build( + Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(State.Service, state))), + ); + return Context.get(context, Owner.Service); + }); +}; + +const clientFor = (port: number) => + RpcClient.make(StackRpc).pipe( + Effect.provide( + RpcClient.layerProtocolHttp({ url: `http://127.0.0.1:${port}/rpc` }).pipe( + Layer.provide(RpcSerialization.layerNdjson), + ), + ), + ); + +const openIdleSocket = (port: number) => + Effect.callback((resume) => { + let socket: Net.Socket; + try { + socket = Net.connect(port, "127.0.0.1"); + } catch (cause) { + resume(Effect.fail(hostTestError(cause))); + return Effect.void; + } + // Bun registers an HTTP connection only after it receives a request. + let response = ""; + const onData = (bytes: Buffer) => { + response += bytes.toString(); + if (response.includes("\r\n\r\n")) { + socket.off("data", onData); + socket.resume(); + resume(Effect.succeed(socket)); + } + }; + const onConnect = () => + socket.write("GET /identity HTTP/1.1\r\nHost: localhost\r\nConnection: keep-alive\r\n\r\n"); + const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); + socket.on("data", onData); + socket.once("connect", onConnect); + socket.once("error", onError); + return Effect.sync(() => { + socket.off("data", onData); + socket.off("connect", onConnect); + socket.off("error", onError); + socket.destroy(); + }); + }); + +const awaitClosed = (socket: Net.Socket) => + Effect.callback((resume) => { + if (socket.destroyed) { + resume(Effect.void); + return Effect.void; + } + const onClose = () => resume(Effect.void); + socket.once("close", onClose); + return Effect.sync(() => socket.off("close", onClose)); + }); + +const inProcessRuntime = ( + owner: Parameters[0], + state: State.Interface, + root: string, +) => + Effect.gen(function* () { + const acquired = yield* acquireHost(state, "stack"); + const toolContext = yield* Layer.build( + ToolRunner.layer({ + stackId: "stack", + root, + cacheRoot: "/tmp/supabase-stack-artifacts", + runtime: "native", + }), + ); + const runtime = yield* makeRuntime( + owner, + { + stackId: "stack", + identity: { projectRoot: root, branchContext: "main", stackName: "host" }, + pid: process.pid, + port: acquired.port, + }, + acquired.server, + acquired.closeConnections, + ).pipe(Effect.provideService(ToolRunner.Service, Context.get(toolContext, ToolRunner.Service))); + yield* runtime.serve; + return { runtime, port: acquired.port }; + }); + +const abortBeforeRpcBody = (port: number) => + Effect.callback((resume) => { + let socket: Net.Socket; + try { + socket = Net.connect(port, "127.0.0.1"); + } catch (cause) { + resume(Effect.fail(hostTestError(cause))); + return Effect.void; + } + const onConnect = () => + socket.write( + "POST /rpc HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Type: application/ndjson\r\nContent-Length: 1048576\r\nConnection: close\r\n\r\n", + () => socket.destroy(), + ); + const onClose = () => resume(Effect.void); + const onError = (cause: Error) => resume(Effect.fail(hostTestError(cause))); + socket.once("connect", onConnect); + socket.once("close", onClose); + socket.once("error", onError); + return Effect.sync(() => { + socket.off("connect", onConnect); + socket.off("close", onClose); + socket.off("error", onError); + socket.destroy(); + }); + }); + +it.live("preserves composition outcomes over RPC", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-outcomes-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "host-outcomes" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const failed = new OrchestratorError({ operation: "start", message: "member failed" }); + const delayedOwner = { + ...owner, + composition: { + ...owner.composition, + start: Effect.fail( + new OwnerError({ + operation: "composition", + message: "Composition start had failures", + cause: new OrchestratorError({ + operation: "start", + message: "Composition start had failures", + outcomes: [ + { id: "healthy", result: Exit.succeed(undefined) }, + { id: "failed", result: Exit.fail(failed) }, + ], + }), + }), + ), + }, + }; + const { runtime } = yield* inProcessRuntime(delayedOwner, state, root); + const client = yield* clientFor(runtime.endpoint.port); + const error = yield* client.startComposition().pipe(Effect.flip); + expect("outcomes" in error).toBe(true); + if (!("outcomes" in error)) return yield* Effect.die("Missing composition outcomes"); + expect(error.outcomes).toEqual([ + { id: "healthy", succeeded: true }, + { id: "failed", succeeded: false, error: expect.stringContaining("member failed") }, + ]); + yield* client.shutdown({ destroy: false }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("keeps serving when namespace shutdown fails", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-drain-failure-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "host-drain-failure" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const failedOwner = { + ...owner, + namespace: { + ...owner.namespace, + stop: Effect.fail(new OwnerError({ operation: "stop", message: "cleanup failed" })), + }, + }; + const { runtime } = yield* inProcessRuntime(failedOwner, state, root); + const client = yield* clientFor(runtime.endpoint.port); + const failure = yield* client.shutdown({ destroy: false }).pipe(Effect.flip); + expect("operation" in failure).toBe(true); + expect((yield* client.listServices()).length).toBe(0); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live( + "serves one detached owner through Effect RPC and retires after shutdown", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-" }); + const state = yield* stateFor(`${root}/state`); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "host" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const endpoint = yield* launchHost(state, { + stateRoot: `${root}/state`, + cacheRoot: "/tmp/supabase-stack-artifacts", + stackId: "stack", + }); + const http = yield* HttpClient.HttpClient; + const identity = yield* http.get(`http://127.0.0.1:${endpoint.port}/identity`); + expect(identity.status).toBe(200); + const client = yield* clientFor(endpoint.port); + const stopped = yield* Ref.make(false); + yield* Effect.addFinalizer(() => + Ref.get(stopped).pipe( + Effect.flatMap((value) => + value ? Effect.void : client.shutdown({ destroy: true }).pipe(Effect.ignore), + ), + ), + ); + yield* abortBeforeRpcBody(endpoint.port); + expect((yield* client.listServices()).length).toBe(0); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" }, smtp: { port: "auto" }, pop3: { port: "auto" } }, + }); + expect(mail.creation.service).toBe("mail"); + expect((yield* client.listServices()).length).toBe(1); + const toolAttachment = "early-stdin"; + const toolEvents = yield* client + .runTool({ + attachmentId: toolAttachment, + tool: postgres.psql({ major: 17 }), + args: ["--version"], + env: {}, + stdin: true, + }) + .pipe(Stream.runCollect); + expect(Array.from(toolEvents).some((event) => event._tag === "Completed")).toBe(true); + const closedInput = yield* client + .toolInput({ attachmentId: toolAttachment, bytes: null }) + .pipe(Effect.flip); + expect("operation" in closedInput).toBe(true); + if (!("operation" in closedInput)) return yield* Effect.die("Unexpected RPC error"); + expect(closedInput.operation).toBe("tool-input-closed"); + yield* client.startService({ id: mail.id }); + yield* client.readyService({ id: mail.id }); + expect((yield* client.status({ id: mail.id })).lifecycle).toBe("running"); + const database = yield* client.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("host-test-password"), + jwtSecret: Redacted.make("host-test-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* client.startService({ id: database.id }); + yield* client.readyService({ id: database.id }); + const databaseUrl = (yield* client.credentials({ id: database.id, from: "host" })) + .databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("Database credentials missing"); + const databaseEndpoint = new URL(databaseUrl); + const databaseEnv = { + PGHOST: databaseEndpoint.hostname, + PGPORT: databaseEndpoint.port, + PGUSER: decodeURIComponent(databaseEndpoint.username), + PGPASSWORD: decodeURIComponent(databaseEndpoint.password), + PGDATABASE: databaseEndpoint.pathname.slice(1), + }; + const idleSocket = yield* Effect.acquireRelease(openIdleSocket(endpoint.port), (socket) => + Effect.sync(() => socket.destroy()), + ); + const logs = yield* Effect.forkScoped(client.logs({ id: mail.id }).pipe(Stream.runDrain)); + const ready = yield* Deferred.make(); + const drainingTool = yield* Effect.forkScoped( + client + .runTool({ + attachmentId: "draining-stdin", + tool: postgres.psql({ major: 17 }), + args: [ + "-X", + "-t", + "-A", + "-c", + "SELECT 'stack-host-tool-ready'", + "-c", + "SELECT pg_sleep(600)", + ], + env: databaseEnv, + stdin: true, + }) + .pipe( + Stream.filter((event) => event._tag === "Stdout"), + Stream.map((event) => event.bytes), + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => + line === "stack-host-tool-ready" ? Deferred.succeed(ready, undefined) : Effect.void, + ), + ), + ); + yield* Deferred.await(ready); + expect(idleSocket.destroyed).toBe(false); + yield* client.shutdown({ destroy: false }); + yield* awaitClosed(idleSocket).pipe( + Effect.timeoutOrElse({ + duration: "5 seconds", + orElse: () => + Effect.fail( + new HostTestError({ + message: "Idle control connection did not close after shutdown", + }), + ), + }), + ); + yield* Fiber.await(logs).pipe( + Effect.timeoutOrElse({ + duration: "5 seconds", + orElse: () => + Effect.fail( + new HostTestError({ message: "Log observation did not close after shutdown" }), + ), + }), + ); + const drainingToolExit = yield* Fiber.await(drainingTool); + expect(Exit.isFailure(drainingToolExit)).toBe(true); + if (Exit.isFailure(drainingToolExit)) + expect(Cause.pretty(drainingToolExit.cause)).toContain("Stack host is draining"); + yield* Ref.set(stopped, true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, +); + +it.live("finishes detached shutdown after the caller disconnects", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-abort-" }); + const state = yield* stateFor(`${root}/state`); + yield* state.save({ + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "host-abort" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const owner = yield* ownerFor({ + saved: { + id: "stack", + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "host" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const entered = yield* Deferred.make(); + const allow = yield* Deferred.make(); + const delayedOwner = { + ...owner, + namespace: { + ...owner.namespace, + stop: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(allow)), + Effect.andThen(owner.namespace.stop), + ), + }, + }; + const { runtime } = yield* inProcessRuntime(delayedOwner, state, root); + yield* Effect.addFinalizer(() => Deferred.succeed(allow, undefined)); + const client = yield* clientFor(runtime.endpoint.port); + const shutdown = yield* Effect.forkScoped(client.shutdown({ destroy: false })); + yield* Deferred.await(entered); + yield* Fiber.interrupt(shutdown); + yield* Deferred.succeed(allow, undefined); + yield* Deferred.await(runtime.exit).pipe(Effect.timeout("5 seconds")); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("withdraws a command waiting for its prerequisite", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-host-command-abort-" }); + const state = yield* stateFor(`${root}/state`); + const saved = { + id: "stack", + runtime: "native" as const, + identity: { projectRoot: root, branchContext: "main", stackName: "host-command-abort" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const owner = yield* ownerFor({ + saved, + state, + root: `${root}/data`, + cacheRoot: "/tmp/supabase-stack-artifacts", + }); + const entered = yield* Deferred.make(); + const cancelled = yield* Deferred.make(); + const allow = yield* Deferred.make(); + const delayedOwner = { + ...owner, + core: { + ...owner.core, + start: (id: string) => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(allow)), + Effect.andThen(owner.core.start(id)), + Effect.ensuring(Deferred.succeed(cancelled, undefined)), + ), + }, + }; + const { runtime } = yield* inProcessRuntime(delayedOwner, state, root); + yield* Effect.addFinalizer(() => + Deferred.succeed(allow, undefined).pipe( + Effect.andThen(runtime.shutdown(false).pipe(Effect.ignore)), + ), + ); + const client = yield* clientFor(runtime.endpoint.port); + const mail = yield* client.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + const start = yield* Effect.forkScoped(client.startService({ id: mail.id })); + yield* Deferred.await(entered); + yield* Fiber.interrupt(start); + yield* Deferred.await(cancelled).pipe(Effect.timeout("5 seconds")); + yield* Deferred.succeed(allow, undefined); + expect((yield* client.status({ id: mail.id })).lifecycle).toBe("stopped"); + yield* client.shutdown({ destroy: false }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts new file mode 100644 index 0000000000..4fc986e0d8 --- /dev/null +++ b/packages/stack/src/StackHost.ts @@ -0,0 +1,424 @@ +import { NodeHttpServerRequest, NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { + Cause, + Context, + Data, + Deferred, + Effect, + Exit, + Fiber, + Layer, + Option, + Queue, + Ref, + Scope, + Semaphore, + Stream, + Path, +} from "effect"; +import * as HttpServer from "effect/unstable/http/HttpServer"; +import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; +import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; +import * as RpcServer from "effect/unstable/rpc/RpcServer"; +import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"; +import { acquireHost, HostEndpoint } from "./HostProcess.ts"; +import * as Owner from "./Owner.ts"; +import { OwnerError } from "./Owner.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import { StackError, StackRpc } from "./Rpc.ts"; +import * as State from "./State.ts"; +import { makeToolAttachments, type ToolAttachmentPayload } from "./host/ToolAttachments.ts"; +import * as ToolRunner from "./host/ToolRunner.ts"; +import type { CatalogLog } from "./services/Catalog.ts"; + +export interface StackHostOptions { + readonly stateRoot: string; + readonly cacheRoot: string; + readonly stackId: string; + readonly onReady?: (endpoint: HostEndpoint) => Effect.Effect; +} + +export class StackHostError extends Data.TaggedError("StackHostError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +const hostError = (operation: string, cause: unknown) => + new StackHostError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const stackError = (operation: string, cause: unknown): StackError => { + if (cause instanceof StackError) return cause; + const orchestration = + cause instanceof Orchestrator.OrchestratorError + ? cause + : cause instanceof OwnerError && cause.cause instanceof Orchestrator.OrchestratorError + ? cause.cause + : undefined; + if (orchestration?.outcomes !== undefined) { + return new StackError({ + operation, + message: orchestration.message, + outcomes: orchestration.outcomes.map(({ id, result }) => ({ + id, + succeeded: Exit.isSuccess(result), + ...(Exit.isFailure(result) ? { error: Cause.pretty(result.cause) } : {}), + })), + }); + } + return new StackError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + }); +}; + +const log = (value: CatalogLog) => ({ stream: value.stream, bytes: value.bytes }); + +const isOpen = (value: boolean): Effect.Effect => + value + ? Effect.void + : Effect.fail(new StackError({ operation: "host", message: "Stack host is draining" })); + +const requestSignal = () => + Effect.callback<"SIGTERM" | "SIGINT", never>((resume) => { + const onTerm = () => resume(Effect.succeed("SIGTERM")); + const onInt = () => resume(Effect.succeed("SIGINT")); + process.once("SIGTERM", onTerm); + process.once("SIGINT", onInt); + return Effect.sync(() => { + process.off("SIGTERM", onTerm); + process.off("SIGINT", onInt); + }); + }); + +const responseClosed = (response: ReturnType) => + Effect.callback((resume) => { + if (response.writableEnded || response.destroyed) { + resume(Effect.void); + return Effect.void; + } + const done = () => resume(Effect.void); + response.once("finish", done); + response.once("close", done); + return Effect.sync(() => { + response.off("finish", done); + response.off("close", done); + }); + }); + +export interface StackHostRuntime { + readonly endpoint: HostEndpoint; + readonly serve: Effect.Effect; + readonly closeConnections: Effect.Effect; + readonly shutdown: ( + destroy: boolean, + response?: ReturnType, + ) => Effect.Effect; + readonly exit: Deferred.Deferred; +} + +export const makeRuntime = Effect.fn("StackHost.makeRuntime")( + ( + owner: Owner.Interface, + endpoint: HostEndpoint, + server: HttpServer.HttpServer["Service"], + closeConnections: Effect.Effect, + ): Effect.Effect => + Effect.gen(function* () { + const scope = yield* Scope.Scope; + const runner = yield* ToolRunner.Service; + const attachments = yield* makeToolAttachments({ + admit: owner.getServing.pipe( + Effect.flatMap(isOpen), + Effect.mapError((cause) => stackError("host", cause)), + ), + run: runner.run, + toError: stackError, + }); + const exit = yield* Deferred.make(); + const gate = yield* Semaphore.make(1); + const current = yield* Ref.make< + { destroy: boolean; fiber: Fiber.Fiber } | undefined + >(undefined); + const watchResponse = ( + response: ReturnType, + closed: Deferred.Deferred, + ) => + Effect.forkIn( + responseClosed(response).pipe(Effect.andThen(Deferred.succeed(closed, undefined))), + scope, + ); + + const shutdown = Effect.fn("StackHost.shutdown")( + (destroy: boolean, response?: ReturnType) => + Effect.gen(function* () { + const fiber = yield* gate.withPermits(1)( + Effect.uninterruptibleMask(() => + Effect.gen(function* () { + const existing = yield* Ref.get(current); + if (existing !== undefined) { + if (existing.destroy !== destroy) + return yield* new StackError({ + operation: "shutdown", + message: "Shutdown mode is already selected", + }); + return existing.fiber; + } + yield* owner.setDraining(true); + const responseClosedSignal = + response === undefined ? undefined : yield* Deferred.make(); + if (response !== undefined && responseClosedSignal !== undefined) + yield* watchResponse(response, responseClosedSignal); + const cleanup = Effect.gen(function* () { + yield* attachments.stopAll; + if (destroy) yield* runner.cleanup; + yield* destroy ? owner.namespace.destroy : owner.namespace.stop; + if (response !== undefined) { + if (responseClosedSignal === undefined) return; + yield* Effect.forkIn( + Deferred.await(responseClosedSignal).pipe( + Effect.andThen(closeConnections), + Effect.andThen(Deferred.succeed(exit, undefined)), + ), + scope, + ); + } else { + yield* closeConnections; + yield* Deferred.succeed(exit, undefined); + } + }).pipe( + Effect.mapError((cause) => stackError("shutdown", cause)), + Effect.catchCause((cause) => + owner.setDraining(false).pipe( + Effect.mapError((reset) => stackError("shutdown", reset)), + Effect.andThen(gate.withPermits(1)(Ref.set(current, undefined))), + Effect.andThen(Effect.failCause(cause)), + ), + ), + ); + const fiber = yield* Effect.forkIn(cleanup, scope); + yield* Ref.set(current, { destroy, fiber }); + return fiber; + }), + ), + ); + yield* Fiber.join(fiber); + }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))), + ); + const handlers = { + createService: (creation: unknown) => + owner.services + .create(creation) + .pipe(Effect.mapError((cause) => stackError("createService", cause))), + getService: ({ id }: { readonly id: string }) => + owner.services.get(id).pipe(Effect.mapError((cause) => stackError("getService", cause))), + listServices: () => + owner.services.list.pipe(Effect.mapError((cause) => stackError("listServices", cause))), + startService: ({ id }: { readonly id: string }) => + owner.core.start(id).pipe(Effect.mapError((cause) => stackError("startService", cause))), + readyService: ({ id }: { readonly id: string }) => + owner.core.ready(id).pipe(Effect.mapError((cause) => stackError("readyService", cause))), + stopService: ({ id }: { readonly id: string }) => + owner.core.stop(id).pipe(Effect.mapError((cause) => stackError("stopService", cause))), + restartService: ({ id, config }: { readonly id: string; readonly config?: unknown }) => + owner.core + .restart(id, config) + .pipe(Effect.mapError((cause) => stackError("restartService", cause))), + destroyService: ({ id }: { readonly id: string }) => + owner.core + .destroy(id) + .pipe(Effect.mapError((cause) => stackError("destroyService", cause))), + prepareService: ({ id }: { readonly id: string }) => + owner.core + .prepare(id) + .pipe(Effect.mapError((cause) => stackError("prepareService", cause))), + status: ({ id }: { readonly id: string }) => + owner.core.status(id).pipe(Effect.mapError((cause) => stackError("status", cause))), + followStatus: ({ id }: { readonly id: string }) => + owner.core + .followStatus(id) + .pipe(Stream.mapError((cause) => stackError("followStatus", cause))), + logs: ({ id }: { readonly id: string }) => + owner.core.logs(id).pipe( + Stream.map(log), + Stream.mapError((cause) => stackError("logs", cause)), + ), + credentials: ({ id, from }: { readonly id: string; readonly from: "host" | "runtime" }) => + owner + .credentials(id, from) + .pipe(Effect.mapError((cause) => stackError("credentials", cause))), + exportSnapshot: ({ + id, + destination, + }: { + readonly id: string; + readonly destination: string; + }) => + owner.snapshots + .exportSnapshot(id, destination) + .pipe(Effect.mapError((cause) => stackError("exportSnapshot", cause))), + restoreSnapshot: ({ id, source }: { readonly id: string; readonly source: string }) => + owner.snapshots + .restoreSnapshot(id, source) + .pipe(Effect.mapError((cause) => stackError("restoreSnapshot", cause))), + resetData: ({ id }: { readonly id: string }) => + owner.snapshots + .resetData(id) + .pipe(Effect.mapError((cause) => stackError("resetData", cause))), + supabaseComposition: ({ + services, + reuseIds, + }: { + readonly services: Parameters[0]; + readonly reuseIds?: NonNullable< + Parameters[1] + >["reuseIds"]; + }) => + owner.composition + .supabase(services, { reuseIds }) + .pipe(Effect.mapError((cause) => stackError("supabaseComposition", cause))), + configureComposition: ( + configuration: Parameters[0], + ) => + owner.composition + .configure(configuration) + .pipe(Effect.mapError((cause) => stackError("configureComposition", cause))), + getComposition: () => + owner.composition.get.pipe( + Effect.mapError((cause) => stackError("getComposition", cause)), + ), + startComposition: () => + owner.composition.start.pipe( + Effect.mapError((cause) => stackError("startComposition", cause)), + ), + stopComposition: () => + owner.composition.stop.pipe( + Effect.mapError((cause) => stackError("stopComposition", cause)), + ), + restartComposition: () => + owner.composition.restart.pipe( + Effect.mapError((cause) => stackError("restartComposition", cause)), + ), + shutdown: ({ destroy }: { readonly destroy: boolean }) => + Effect.gen(function* () { + const request = yield* Effect.serviceOption(HttpServerRequest.HttpServerRequest); + yield* Option.match(request, { + onNone: () => + Effect.fail( + new StackError({ operation: "shutdown", message: "Request context is missing" }), + ), + onSome: (value) => shutdown(destroy, NodeHttpServerRequest.toServerResponse(value)), + }); + }).pipe(Effect.mapError((cause) => stackError("shutdown", cause))), + runTool: (input: ToolAttachmentPayload) => attachments.run(input), + toolInput: ({ + attachmentId, + bytes, + }: { + readonly attachmentId: string; + readonly bytes: Uint8Array | null; + }) => attachments.input(attachmentId, true, bytes), + }; + const rpc = yield* RpcServer.toHttpEffect(StackRpc, { streamBufferSize: 16 }).pipe( + Effect.provide(Layer.merge(StackRpc.toLayer(handlers), RpcSerialization.layerNdjson)), + ); + const application: Effect.Effect< + HttpServerResponse.HttpServerResponse, + never, + HttpServerRequest.HttpServerRequest | Scope.Scope + > = Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + if (request.method === "GET" && request.url === "/identity") { + return HttpServerResponse.jsonUnsafe(endpoint); + } + if (request.method === "POST" && request.url.startsWith("/rpc")) + return yield* rpc.pipe(Effect.interruptible); + return HttpServerResponse.empty({ status: 404 }); + }); + const serve: Effect.Effect = server.serve(application); + return { endpoint, serve, shutdown, closeConnections, exit }; + }), +); + +export const runStackHost = Effect.fn("StackHost.run")( + (options: StackHostOptions): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const signals = yield* Queue.bounded<"SIGTERM" | "SIGINT">(8); + yield* Effect.forkScoped( + Effect.forever( + requestSignal().pipe( + Effect.flatMap((signal) => Queue.offer(signals, signal).pipe(Effect.asVoid)), + ), + ), + ); + const started = yield* Effect.gen(function* () { + const stateContext = yield* Layer.build(State.layer({ root: options.stateRoot })); + const state = Context.get(stateContext, State.Service); + const path = yield* Path.Path; + const saved = yield* state.read(options.stackId); + if (saved === undefined) return yield* hostError("startup", "Stack is not registered"); + const acquired = yield* acquireHost(state, options.stackId); + const services = yield* Layer.build( + Layer.merge( + Owner.layer({ + saved, + root: path.join(options.stateRoot, saved.id, "data"), + cacheRoot: options.cacheRoot, + }), + ToolRunner.layer({ + stackId: saved.id, + root: path.join(options.stateRoot, saved.id, "data"), + cacheRoot: options.cacheRoot, + runtime: saved.runtime, + }), + ).pipe(Layer.provide(Layer.succeed(State.Service, state))), + ); + const owner = Context.get(services, Owner.Service); + const endpoint: HostEndpoint = { + stackId: saved.id, + identity: saved.identity, + pid: process.pid, + port: acquired.port, + }; + const runtime = yield* makeRuntime( + owner, + endpoint, + acquired.server, + acquired.closeConnections, + ).pipe( + Effect.provideService(ToolRunner.Service, Context.get(services, ToolRunner.Service)), + ); + yield* runtime.serve; + yield* options.onReady?.(endpoint) ?? Effect.void; + return runtime; + }).pipe( + Effect.raceFirst( + Queue.take(signals).pipe( + Effect.flatMap((signal) => hostError("startup", `Received ${signal} during startup`)), + ), + ), + ); + while (true) { + const event = yield* Deferred.await(started.exit).pipe( + Effect.map(() => "done" as const), + Effect.raceFirst(Queue.take(signals).pipe(Effect.map(() => "signal" as const))), + ); + if (event === "done") break; + const shutdownSucceeded = yield* started + .shutdown(false) + .pipe(Effect.match({ onSuccess: () => true, onFailure: () => false })); + if (shutdownSucceeded) break; + } + }), + ).pipe( + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + Effect.mapError((cause) => + cause instanceof StackHostError ? cause : hostError("host", cause), + ), + ), +); diff --git a/packages/stack/src/State.integration.test.ts b/packages/stack/src/State.integration.test.ts new file mode 100644 index 0000000000..2fbde496ab --- /dev/null +++ b/packages/stack/src/State.integration.test.ts @@ -0,0 +1,207 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { TestClock } from "effect/testing"; +import { Context, Deferred, Effect, Exit, Fiber, FileSystem, Layer, Path, Schema } from "effect"; +import * as State from "./State.ts"; +import type { SavedStack } from "./State.ts"; + +const makeTestState = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); + +const initial: SavedStack = { + id: "stack-main", + identity: { + projectRoot: "/tmp/project", + branchContext: "main", + stackName: "local", + }, + runtime: "docker", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], +}; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); + +describe("durable stack state", () => { + it.live("reopens saved state and serializes concurrent read-modify-write operations", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + yield* first.save(initial); + + const add = (store: typeof first, id: string) => + store.withLock( + Effect.gen(function* () { + const current = yield* store.read(initial.id); + if (current === undefined) return yield* Effect.die("state disappeared"); + yield* store.save({ + ...current, + instances: [ + ...current.instances, + { id, creation: { service: "database", config: { version: "17" } } }, + ], + }); + }), + ); + yield* Effect.all([add(first, "db-one"), add(second, "db-two")], { + concurrency: "unbounded", + }); + + const reopened = yield* (yield* makeTestState(root)).read(initial.id); + expect(reopened?.instances.map((instance) => instance.id).sort()).toEqual([ + "db-one", + "db-two", + ]); + expect(yield* fs.stat(path.join(root, "stack-main", "state.json"))).toBeDefined(); + }), + ), + ); + + it.effect("rejects unsafe paths and malformed documents without changing valid state", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-invalid-" }); + const store = yield* makeTestState(root); + yield* store.save(initial); + const unsafe = yield* store.read("../outside").pipe(Effect.exit); + expect(Exit.isFailure(unsafe)).toBe(true); + + yield* fs.makeDirectory(`${root}/broken`); + yield* fs.writeFileString(`${root}/broken/state.json`, '{"id":42}'); + const malformed = yield* store.read("broken").pipe(Effect.exit); + expect(Exit.isFailure(malformed)).toBe(true); + if (Exit.isFailure(malformed)) { + expect(String(malformed.cause)).toContain("stack broken"); + expect(String(malformed.cause)).toContain("state.json"); + } + const valid = yield* store.read(initial.id); + expect(Schema.is(State.SavedStack)(valid)).toBe(true); + }), + ), + ); + + it.live("removes empty stack parents without deleting unowned data", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-" }); + const store = yield* makeTestState(root); + yield* store.save(initial); + yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); + yield* store.remove(initial.id); + expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + + yield* store.save(initial); + yield* fs.makeDirectory(path.join(root, initial.id, "data"), { recursive: true }); + yield* fs.writeFileString(path.join(root, initial.id, "data", "owned-by-caller"), "keep"); + yield* store.remove(initial.id); + expect(yield* fs.exists(path.join(root, initial.id, "data", "owned-by-caller"))).toBe(true); + }), + ), + ); + + it.effect("keeps a lock owner protected when a waiting fiber is cancelled", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-lock-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + const entered = yield* Deferred.make(); + const release = yield* Deferred.make(); + const owner = yield* Effect.forkScoped( + first.withLock( + Effect.gen(function* () { + yield* Deferred.succeed(entered, undefined); + yield* Deferred.await(release); + }), + ), + ); + yield* Deferred.await(entered); + const waiter = yield* Effect.forkScoped(second.withLock(second.save(initial))); + yield* TestClock.adjust("100 millis"); + yield* Fiber.interrupt(waiter); + expect(yield* second.read(initial.id)).toBeUndefined(); + const later = yield* Effect.forkScoped(second.withLock(second.save(initial))); + yield* TestClock.adjust("100 millis"); + expect(yield* second.read(initial.id)).toBeUndefined(); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(owner); + yield* TestClock.adjust("50 millis"); + yield* Fiber.join(later); + expect(yield* second.read(initial.id)).toEqual(initial); + }), + ), + ); + it.effect("times out contention without releasing the holder's lock", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-timeout-" }); + const first = yield* makeTestState(root); + const second = yield* makeTestState(root); + const entered = yield* Deferred.make(); + const owner = yield* first + .withLock(Deferred.succeed(entered, undefined).pipe(Effect.andThen(Effect.never))) + .pipe(Effect.forkScoped); + yield* Deferred.await(entered); + const waiter = yield* second + .withLock(second.save(initial)) + .pipe(Effect.flip, Effect.forkScoped); + yield* TestClock.adjust("6 seconds"); + const result = yield* Fiber.join(waiter); + expect(result.operation).toBe("lock"); + expect(result.message).toBe("Stack registry is locked by another operation; retry shortly"); + expect(yield* second.read(initial.id)).toBeUndefined(); + yield* Fiber.interrupt(owner); + yield* second.withLock(second.save(initial)); + expect(yield* second.read(initial.id)).toEqual(initial); + }), + ), + ); + + it.live("releases ownership after failure and defects without replacing the lock file", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-failure-" }); + const state = yield* makeTestState(root); + const failure = new State.StateError({ operation: "test", message: "body failed" }); + expect(yield* state.withLock(Effect.fail(failure)).pipe(Effect.flip)).toBe(failure); + const defect = yield* state.withLock(Effect.die("body defect")).pipe(Effect.exit); + expect(Exit.hasDies(defect)).toBe(true); + yield* state.withLock(state.save(initial)); + expect(yield* state.read(initial.id)).toEqual(initial); + expect((yield* fs.readDirectory(root)).sort()).toEqual([ + ".registry-lock.sqlite", + initial.id, + ]); + expect((yield* fs.stat(`${root}/.registry-lock.sqlite`)).size).toBe(0n); + }), + ), + ); + it.effect("fails invalid lock files without retrying or changing saved state", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-invalid-" }); + const state = yield* makeTestState(root); + yield* fs.writeFileString(`${root}/.registry-lock.sqlite`, "not a database"); + const error = yield* state.withLock(state.save(initial)).pipe(Effect.flip); + expect(error).toBeInstanceOf(State.StateError); + expect(error.operation).toBe("lock"); + expect(yield* state.read(initial.id)).toBeUndefined(); + }), + ), + ); +}); diff --git a/packages/stack/src/State.process.integration.test.ts b/packages/stack/src/State.process.integration.test.ts new file mode 100644 index 0000000000..458911e7ec --- /dev/null +++ b/packages/stack/src/State.process.integration.test.ts @@ -0,0 +1,204 @@ +import { NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + Context, + Data, + Deferred, + Effect, + Fiber, + FileSystem, + Layer, + Ref, + Schema, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; +import * as State from "./State.ts"; + +class FixtureError extends Data.TaggedError("StateLockFixtureError")<{ + readonly message: string; +}> {} +const fixture = fileURLToPath(new URL("../tests/state-lock-fixture.ts", import.meta.url)); +const readOutput = (child: ChildProcessSpawner.ChildProcessHandle) => + Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + +for (const compiled of [false, true]) { + it.live( + `serializes writers and releases a killed holder while its child survives (${compiled ? "compiled" : "source"})`, + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-lock-process-" }); + const executable = compiled + ? `${root}/fixture${process.platform === "win32" ? ".exe" : ""}` + : process.execPath; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const spawn = (command: string, args: ReadonlyArray) => + spawner.spawn( + ChildProcess.make(command, args, { + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + detached: true, + forceKillAfter: "1 second", + }), + ); + if (compiled) { + const build = yield* spawn("bun", [ + "build", + fixture, + "--compile", + `--outfile=${executable}`, + ]); + const [, stderr, code] = yield* readOutput(build); + expect(code, stderr).toBe(0); + } + const stateRoot = `${root}/state`; + const context = yield* Layer.build(State.layer({ root: stateRoot })); + const state = Context.get(context, State.Service); + const saved: State.SavedStack = { + id: "stack-main", + identity: { projectRoot: root, branchContext: "test", stackName: "lock" }, + runtime: "native", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.save(saved); + const args = (mode: string, id?: string) => [ + ...(compiled ? [] : [fixture]), + mode, + stateRoot, + ...(id === undefined ? [] : [id]), + ]; + yield* Effect.forEach( + ["first", "second", "third"], + (id) => + Effect.scoped( + Effect.gen(function* () { + const writer = yield* spawn(executable, args("write", id)); + const [, stderr, code] = yield* readOutput(writer); + expect(code, stderr).toBe(0); + }), + ), + { concurrency: "unbounded" }, + ); + expect((yield* state.read(saved.id))?.instances.map(({ id }) => id).sort()).toEqual([ + "first", + "second", + "third", + ]); + + const holder = yield* spawn(executable, args("hold")); + const locked = yield* Deferred.make(); + const childReady = yield* Deferred.make(); + const observed = yield* Ref.make>([]); + const output = yield* holder.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((line) => + Ref.update(observed, (lines) => [...lines, line]).pipe( + Effect.andThen( + line === "locked" + ? Deferred.succeed(locked, undefined).pipe(Effect.asVoid) + : line.startsWith("child:") + ? Deferred.succeed(childReady, line.slice(6)).pipe(Effect.asVoid) + : Effect.void, + ), + ), + ), + Effect.forkScoped, + ); + const stderr = yield* Ref.make(""); + const diagnostics = yield* holder.stderr.pipe( + Stream.decodeText, + Stream.runForEach((chunk) => Ref.update(stderr, (text) => text + chunk)), + Effect.forkScoped, + ); + yield* Effect.all([Deferred.await(locked), Deferred.await(childReady)]).pipe( + Effect.raceFirst( + holder.exitCode.pipe( + Effect.matchEffect({ + onFailure: (cause) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail( + new FixtureError({ + message: `Holder exited before readiness: ${String(cause)}\n${text}`, + }), + ), + ), + ), + onSuccess: (code) => + Ref.get(stderr).pipe( + Effect.flatMap((text) => + Effect.fail( + new FixtureError({ + message: `Holder exited before readiness (${code}): ${text}`, + }), + ), + ), + ), + }), + ), + ), + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => + Effect.all([Ref.get(stderr), Ref.get(observed)]).pipe( + Effect.flatMap(([text, lines]) => + Effect.fail( + new FixtureError({ + message: `Holder readiness timed out: ${lines.join("\n")}\n${text}`, + }), + ), + ), + ), + }), + ); + const childPid = yield* Schema.decodeEffect(Schema.FiniteFromString)( + yield* Deferred.await(childReady), + ); + const kill = (pid: number) => + Effect.try({ + try: () => process.kill(pid, "SIGKILL"), + catch: (cause) => new FixtureError({ message: String(cause) }), + }); + yield* Effect.addFinalizer(() => + kill(childPid).pipe( + Effect.ignore, + Effect.andThen(Fiber.join(output)), + Effect.andThen(Fiber.join(diagnostics)), + Effect.timeout("5 seconds"), + Effect.orDie, + ), + ); + yield* kill(Number(holder.pid)); + yield* holder.exitCode.pipe(Effect.exit); + yield* state.withLock( + state.save({ ...saved, identity: { ...saved.identity, stackName: "after-crash" } }), + ); + expect((yield* state.read(saved.id))?.identity.stackName).toBe("after-crash"); + yield* Effect.try({ + try: () => process.kill(childPid, 0), + catch: (cause) => + new FixtureError({ message: `Child did not survive holder: ${String(cause)}` }), + }); + expect((yield* fs.stat(`${stateRoot}/.registry-lock.sqlite`)).size).toBe(0n); + expect((yield* fs.readDirectory(stateRoot)).sort()).toEqual([ + ".registry-lock.sqlite", + saved.id, + ]); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +} diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts new file mode 100644 index 0000000000..5214eec845 --- /dev/null +++ b/packages/stack/src/State.ts @@ -0,0 +1,252 @@ +import { + Data, + Effect, + FileSystem, + Context, + Layer, + Path, + Predicate, + Schedule, + Schema, +} from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. +import { rmdir } from "node:fs/promises"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no OS-owned cross-process lock primitive. +import { DatabaseSync } from "node:sqlite"; + +const SafeId = Schema.String.pipe( + Schema.refine((value): value is string => /^[a-zA-Z0-9_-]+$/u.test(value), { + identifier: "SafeStateId", + message: "Expected a safe state id", + }), +); + +export const SavedInstance = Schema.Struct({ + id: SafeId, + creation: Schema.Unknown, +}); +export interface SavedInstance extends Schema.Schema.Type {} + +export const SavedStack = Schema.Struct({ + id: SafeId, + identity: Schema.Struct({ + projectRoot: Schema.String, + branchContext: Schema.String, + stackName: Schema.String, + }), + runtime: Schema.Literals(["native", "docker", "podman"]), + instances: Schema.Array(SavedInstance), + composition: Schema.Unknown, + ports: Schema.Array( + Schema.Struct({ + key: Schema.String, + host: Schema.String, + port: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 65535 })), + }), + ), +}); +export interface SavedStack extends Schema.Schema.Type {} + +export class StateError extends Data.TaggedError("StateError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface Interface { + readonly read: (id: string) => Effect.Effect; + readonly list: Effect.Effect, StateError>; + readonly save: (state: SavedStack) => Effect.Effect; + readonly remove: (id: string) => Effect.Effect; + /** Not reentrant; wrap metadata updates here, while Ports operations acquire this lock themselves. */ + readonly withLock: ( + effect: Effect.Effect, + ) => Effect.Effect; +} + +export class Service extends Context.Service()("@supabase/stack/State") {} + +const stateError = (operation: string, cause: unknown): StateError => + new StateError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const checkId = (id: string): Effect.Effect => + Schema.is(SafeId)(id) + ? Effect.void + : Effect.fail(stateError("identity", `Invalid state id: ${id}`)); + +const decodeState = ( + text: string, + id: string, + target: string, +): Effect.Effect => + Schema.decodeEffect(Schema.fromJsonString(SavedStack))(text).pipe( + Effect.mapError( + (cause) => + new StateError({ + operation: "decode", + message: `Unable to decode state ${target} for stack ${id}: ${cause.message}`, + cause, + }), + ), + ); + +interface Options { + readonly root: string; + readonly onInvalidState?: (id: string, error: StateError) => Effect.Effect; +} + +const makeState = ( + options: Options, +): Effect.Effect => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = path.normalize(options.root); + // SQLite alone opens this persistent file; other descriptors can invalidate its POSIX locks. + const lock = path.join(root, ".registry-lock.sqlite"); + yield* fs + .makeDirectory(root, { recursive: true }) + .pipe(Effect.mapError((cause) => stateError("root", cause))); + yield* fs.chmod(root, 0o700).pipe(Effect.mapError((cause) => stateError("root", cause))); + + const stackRoot = (id: string) => path.join(root, id); + const statePath = (id: string) => path.join(stackRoot(id), "state.json"); + const removeEmptyDirectory = (directory: string) => + Effect.tryPromise({ + try: () => rmdir(directory), + catch: (cause) => stateError("cleanup", cause), + }).pipe( + Effect.catch((cause) => { + const code = + typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause + ? cause.cause.code + : undefined; + return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" + ? Effect.void + : Effect.fail(cause); + }), + ); + const read = Effect.fn("State.read")(function* (id: string) { + yield* checkId(id); + const target = statePath(id); + const exists = yield* fs + .exists(target) + .pipe(Effect.mapError((cause) => stateError("read", cause))); + if (!exists) return undefined; + const text = yield* fs + .readFileString(target) + .pipe(Effect.mapError((cause) => stateError("read", cause))); + const state = yield* decodeState(text, id, target); + if (state.id !== id) { + return yield* stateError("identity", "State document identity does not match its path"); + } + return state; + }); + const list = Effect.fn("State.list")(function* () { + const entries = yield* fs + .readDirectory(root) + .pipe(Effect.mapError((cause) => stateError("list", cause))); + const states: Array = []; + for (const entry of entries) { + if (!Schema.is(SafeId)(entry)) continue; + const id = entry; + const value = yield* read(id).pipe( + Effect.catch((error) => + options.onInvalidState !== undefined && + (error.operation === "decode" || error.operation === "identity") + ? options.onInvalidState(id, error).pipe(Effect.as(undefined)) + : Effect.fail(error), + ), + ); + if (value !== undefined) states.push(value); + } + return states; + }); + const save = Effect.fn("State.save")(function* (state: SavedStack) { + yield* checkId(state.id); + const target = statePath(state.id); + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(SavedStack))(state).pipe( + Effect.mapError((cause) => stateError("encode", cause)), + ); + yield* fs + .makeDirectory(stackRoot(state.id), { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => stateError("write", cause))); + yield* Effect.acquireUseRelease( + fs + .makeTempDirectory({ directory: root, prefix: ".state-write-" }) + .pipe(Effect.mapError((cause) => stateError("write", cause))), + (directory) => + Effect.gen(function* () { + const temporary = path.join(directory, "state.json"); + yield* fs + .writeFileString(temporary, serialized, { mode: 0o600 }) + .pipe(Effect.mapError((cause) => stateError("write", cause))); + yield* fs + .rename(temporary, target) + .pipe(Effect.mapError((cause) => stateError("publish", cause))); + }), + (directory) => + fs + .remove(directory, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => stateError("cleanup", cause))), + ); + }); + const remove = Effect.fn("State.remove")(function* (id: string) { + yield* checkId(id); + yield* fs + .remove(statePath(id), { force: true }) + .pipe(Effect.mapError((cause) => stateError("remove", cause))); + yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); + yield* removeEmptyDirectory(stackRoot(id)); + }); + const withLock = Effect.fn("State.withLock")((effect: Effect.Effect) => + Effect.acquireUseRelease( + Effect.try({ + try: () => new DatabaseSync(lock), + catch: (cause) => stateError("lock", cause), + }), + (connection) => + Effect.gen(function* () { + yield* Effect.try({ + try: () => connection.exec("PRAGMA busy_timeout = 0"), + catch: (cause) => stateError("lock", cause), + }); + yield* Effect.try({ + try: () => connection.exec("BEGIN IMMEDIATE"), + catch: (cause) => stateError("lock", cause), + }).pipe( + Effect.retry({ + schedule: Schedule.spaced("50 millis").pipe( + Schedule.upTo({ duration: "5 seconds" }), + ), + while: (error) => + Predicate.hasProperty(error.cause, "errcode") && error.cause.errcode === 5, + }), + Effect.mapError((error) => + Predicate.hasProperty(error.cause, "errcode") && error.cause.errcode === 5 + ? new StateError({ + operation: "lock", + message: "Stack registry is locked by another operation; retry shortly", + cause: error.cause, + }) + : error, + ), + ); + return yield* effect; + }), + (connection) => + Effect.try({ + try: () => connection.close(), + catch: (cause) => stateError("unlock", cause), + }), + ), + ); + return { read, list: list(), save, remove, withLock }; + }); + +export const layer = (options: Options) => + Layer.effect(Service, makeState(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/Tools.integration.test.ts b/packages/stack/src/Tools.integration.test.ts new file mode 100644 index 0000000000..f03ca504d2 --- /dev/null +++ b/packages/stack/src/Tools.integration.test.ts @@ -0,0 +1,382 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { + Context, + Data, + Deferred, + Effect, + Fiber, + FileSystem, + Layer, + Predicate, + Redacted, + Ref, + Schedule, + Stream, +} from "effect"; +import { tmpdir } from "node:os"; +import { randomUUID } from "node:crypto"; +import { postgres } from "./Tools.ts"; +import * as ToolRunner from "./host/ToolRunner.ts"; +import { makeDatabase } from "./services/Database.ts"; +import { makeService } from "./Service.ts"; +import { bindTcp, serveTcp } from "./Proxy.ts"; + +const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + +class PgProveTestError extends Data.TaggedError("PgProveTestError")<{ + readonly message: string; +}> {} + +const makeTestToolRunner = (options: { + readonly stackId: string; + readonly root: string; + readonly cacheRoot: string; + readonly runtime: "native" | "docker" | "podman"; +}) => + Layer.build(ToolRunner.layer(options)).pipe( + Effect.map((context) => Context.get(context, ToolRunner.Service)), + ); + +describe("finite PostgreSQL tools", { timeout: 180_000 }, () => { + for (const runtime of ["native", "docker"] as const) { + it.live(`${runtime} runs SQL from stdin, streams a dump, and returns client failure`, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-tools-" }); + const stackId = "tools-integration"; + const database = yield* makeDatabase({ + root, + cacheRoot, + stackId, + instanceId: "database", + runtime, + }); + const service = yield* makeService(database.definition, { + id: "database", + config: { + version: "17", + databasePassword: Redacted.make("tool-password"), + jwtSecret: Redacted.make("tool-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + }); + yield* service.start; + yield* service.ready; + const endpoint = yield* database.endpoint; + const listener = yield* bindTcp(runtime === "native" ? "127.0.0.1" : "0.0.0.0", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + yield* serveTcp( + listener, + Effect.succeed( + endpoint.kind === "unix" ? { path: `${endpoint.path}/.s.PGSQL.5432` } : endpoint, + ), + ).pipe(Effect.forkScoped); + const runner = yield* makeTestToolRunner({ root, cacheRoot, stackId, runtime }); + const env = { + PGHOST: runtime === "native" ? "127.0.0.1" : "host.docker.internal", + PGPORT: String(listener.address.port), + PGUSER: "supabase_admin", + PGPASSWORD: "tool-password", + PGDATABASE: "postgres", + }; + const stdout = yield* Ref.make(""); + const stderr = yield* Ref.make(""); + const output = { + stdout: (bytes: Uint8Array) => + Ref.update(stdout, (text) => text + new TextDecoder().decode(bytes)), + stderr: (bytes: Uint8Array) => + Ref.update(stderr, (text) => text + new TextDecoder().decode(bytes)), + }; + const sql = yield* runner.run({ + tool: postgres.psql({ major: 17 }), + args: ["-X", "-v", "ON_ERROR_STOP=1"], + env, + stdin: Stream.make( + new TextEncoder().encode( + "CREATE TABLE tool_story(value text); INSERT INTO tool_story VALUES ('streamed-row'); SELECT value FROM tool_story;", + ), + ), + ...output, + }); + expect(sql.exitCode).toBe(0); + expect(yield* Ref.get(stdout)).toContain("streamed-row"); + yield* Ref.set(stdout, ""); + const dump = yield* runner.run({ + tool: postgres.pgDump({ major: 17 }), + args: ["--data-only", "--table=tool_story"], + env, + ...output, + }); + expect(dump.exitCode).toBe(0); + expect(dump.jobId).not.toBe(sql.jobId); + expect(yield* Ref.get(stdout)).toContain("COPY public.tool_story"); + expect(yield* Ref.get(stdout)).toContain("streamed-row"); + const failed = yield* runner.run({ + tool: postgres.psql({ major: 17 }), + args: ["-X", "-v", "ON_ERROR_STOP=1", "-c", "SELECT missing_column FROM tool_story"], + env, + ...output, + }); + expect(failed.exitCode).not.toBe(0); + expect(yield* Ref.get(stderr)).toContain("missing_column"); + const rejected = yield* Effect.flip( + runner.run({ + tool: postgres.psql({ major: 17 }), + args: [], + env, + pgProve: { mounts: [] }, + ...output, + }), + ); + expect(rejected.message).toContain("pgProve options require the pg_prove tool"); + yield* Ref.set(stderr, ""); + const early = yield* runner.run({ + tool: postgres.psql({ major: 17 }), + args: ["-X", "-v", "ON_ERROR_STOP=1"], + env, + stdin: Stream.make( + new TextEncoder().encode( + "SELECT missing_column FROM tool_story;\n" + "SELECT 1;\n".repeat(150_000), + ), + ), + ...output, + }); + expect(early.exitCode).not.toBe(0); + expect(yield* Ref.get(stderr)).toContain("missing_column"); + const attached = yield* Deferred.make(); + const job = yield* runner + .run({ + tool: postgres.psql({ major: 17 }), + args: ["-X", "-q", "-t", "-A"], + env: { ...env, PGAPPNAME: "attached-tool-story" }, + stdin: Stream.make(new TextEncoder().encode("SELECT 'attached-ready';\n")).pipe( + Stream.concat(Stream.never), + ), + stdout: (bytes) => + new TextDecoder().decode(bytes).includes("attached-ready") + ? Deferred.succeed(attached, undefined).pipe(Effect.asVoid) + : Effect.void, + stderr: output.stderr, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(attached); + yield* Fiber.interrupt(job); + yield* Ref.set(stdout, ""); + const remaining = yield* runner.run({ + tool: postgres.psql({ major: 17 }), + args: [ + "-X", + "-t", + "-A", + "-c", + "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'attached-tool-story'", + ], + env, + ...output, + }); + expect(remaining.exitCode).toBe(0); + expect((yield* Ref.get(stdout)).trim()).toBe("0"); + expect(yield* fs.readDirectory(`${root}/jobs`)).toEqual([]); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + } + + for (const major of [15, 17] as const) { + for (const runtime of ["native", "docker"] as const) { + it.live(`${runtime} runs bundled pg_prove for PostgreSQL ${major}`, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-pgprove-${major}-` }); + const stackId = `tools-pgprove-${runtime}-${major}`; + const database = yield* makeDatabase({ + root, + cacheRoot, + stackId, + instanceId: "database", + runtime, + }); + const service = yield* makeService(database.definition, { + id: "database", + config: { + version: String(major), + databasePassword: Redacted.make(`pgprove-${runtime}-${major}-password`), + jwtSecret: Redacted.make("tool-jwt-secret-at-least-thirty-two-characters"), + jwtExpiry: 3600, + }, + }); + yield* service.start; + yield* service.ready; + const endpoint = yield* database.endpoint; + const listener = yield* bindTcp(runtime === "native" ? "127.0.0.1" : "0.0.0.0", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP listener"); + yield* serveTcp( + listener, + Effect.succeed( + endpoint.kind === "unix" ? { path: `${endpoint.path}/.s.PGSQL.5432` } : endpoint, + ), + ).pipe(Effect.forkScoped); + const runner = yield* makeTestToolRunner({ root, cacheRoot, stackId, runtime }); + const env = { + PGHOST: runtime === "native" ? "127.0.0.1" : "host.docker.internal", + PGPORT: String(listener.address.port), + PGUSER: "supabase_admin", + PGPASSWORD: `pgprove-${runtime}-${major}-password`, + PGDATABASE: "postgres", + }; + const tests = `${root}/tests`; + yield* fs.makeDirectory(tests, { recursive: true }); + yield* fs.writeFileString(`${tests}/main.sql`, "\\ir included.sql\n"); + yield* fs.writeFileString(`${tests}/included.sql`, "\\i nested.sql\n"); + yield* fs.writeFileString( + `${tests}/nested.sql`, + "SELECT plan(1);\nSELECT pass('relative include');\nSELECT * FROM finish();\n", + ); + const stdout = yield* Ref.make(""); + const stderr = yield* Ref.make(""); + const output = { + stdout: (bytes: Uint8Array) => + Ref.update(stdout, (text) => text + new TextDecoder().decode(bytes)), + stderr: (bytes: Uint8Array) => + Ref.update(stderr, (text) => text + new TextDecoder().decode(bytes)), + }; + const extension = yield* runner.run({ + tool: postgres.psql({ major }), + args: ["-X", "-v", "ON_ERROR_STOP=1", "-c", "CREATE EXTENSION IF NOT EXISTS pgtap"], + env, + ...output, + }); + expect(extension.exitCode).toBe(0); + const pgProve = (file: string) => + runner.run({ + tool: postgres.pgProve({ major }), + args: ["--ext", ".sql", file, "--verbose"], + env, + pgProve: { + mounts: [{ source: tests, target: "/tests" }], + cwd: tests, + workingDir: "/tests", + }, + ...output, + }); + yield* Ref.set(stdout, ""); + const passed = yield* pgProve("main.sql"); + expect(passed.exitCode).toBe(0); + expect(yield* Ref.get(stdout)).toMatch(/^ok 1\b/m); + expect(yield* Ref.get(stdout)).toContain("relative include"); + expect(yield* fs.readFileString(`${tests}/included.sql`)).toContain("\\i nested.sql"); + expect(yield* fs.readFileString(`${tests}/nested.sql`)).toContain("relative include"); + + yield* fs.writeFileString( + `${tests}/failed.sql`, + "SELECT plan(1);\nSELECT fail('expected failure');\nSELECT * FROM finish();\n", + ); + yield* Ref.set(stdout, ""); + const failed = yield* pgProve("failed.sql"); + expect(failed.exitCode).not.toBe(0); + expect(yield* Ref.get(stdout)).toContain("not ok"); + expect(yield* fs.readFileString(`${tests}/failed.sql`)).toContain("expected failure"); + if (runtime === "native" && major === 17) { + yield* Effect.gen(function* () { + const ready = yield* Deferred.make(); + const applicationName = `pgprove-cancel-${randomUUID()}`; + yield* fs.writeFileString( + `${tests}/cancel-ready.sql`, + "SELECT plan(1);\nSELECT pass('pgprove-ready');\nSELECT * FROM finish();\n", + ); + yield* fs.writeFileString( + `${tests}/cancel.sql`, + "SELECT plan(1);\nSELECT pass('pgprove-idle');\n\\watch 60\n", + ); + const cancelOutput = yield* Ref.make(""); + const running = yield* runner + .run({ + tool: postgres.pgProve({ major }), + args: ["--ext", ".sql", "cancel-ready.sql", "cancel.sql", "--verbose"], + env: { ...env, PGAPPNAME: applicationName }, + pgProve: { + mounts: [{ source: tests, target: "/tests" }], + cwd: tests, + workingDir: "/tests", + }, + stdout: (bytes) => + Effect.gen(function* () { + yield* Ref.update( + cancelOutput, + (text) => text + new TextDecoder().decode(bytes), + ); + if ((yield* Ref.get(cancelOutput)).includes("pgprove-ready")) + yield* Deferred.succeed(ready, undefined); + }), + stderr: output.stderr, + }) + .pipe(Effect.forkChild); + yield* Deferred.await(ready); + const idleOutput = yield* Ref.make(""); + const idle = Effect.gen(function* () { + yield* Ref.set(idleOutput, ""); + const probe = yield* runner.run({ + tool: postgres.psql({ major }), + args: [ + "-X", + "-t", + "-A", + "-c", + `SELECT count(*) FROM pg_stat_activity WHERE application_name = '${applicationName}' AND state = 'idle' AND query LIKE '%pgprove-idle%'`, + ], + env, + stdout: (bytes) => + Ref.update(idleOutput, (text) => text + new TextDecoder().decode(bytes)), + stderr: output.stderr, + }); + const count = (yield* Ref.get(idleOutput)).trim(); + if (probe.exitCode !== 0 || count !== "1") + return yield* new PgProveTestError({ + message: `Expected one idle pg_prove session, got ${count}`, + }); + return count; + }).pipe(Effect.retry({ schedule: Schedule.spaced("100 millis"), times: 50 })); + expect(yield* idle).toBe("1"); + yield* Fiber.interrupt(running); + const remainingOutput = yield* Ref.make(""); + const remaining = Effect.gen(function* () { + yield* Ref.set(remainingOutput, ""); + const probe = yield* runner.run({ + tool: postgres.psql({ major }), + args: [ + "-X", + "-t", + "-A", + "-c", + `SELECT count(*)::text || ':' || coalesce(string_agg(pid::text || ':' || state || ':' || coalesce(wait_event, '') || ':' || query, '|'), '') FROM pg_stat_activity WHERE application_name = '${applicationName}'`, + ], + env, + stdout: (bytes) => + Ref.update(remainingOutput, (text) => text + new TextDecoder().decode(bytes)), + stderr: output.stderr, + }); + const details = (yield* Ref.get(remainingOutput)).trim(); + const count = details.split(":", 1)[0]; + if (probe.exitCode !== 0 || count !== "0") + return yield* new PgProveTestError({ + message: `Expected no pg_prove session, got ${details}`, + }); + return details; + }).pipe(Effect.retry({ schedule: Schedule.spaced("100 millis"), times: 50 })); + expect((yield* remaining).split(":", 1)[0]).toBe("0"); + expect(yield* fs.readDirectory(`${root}/jobs`)).toEqual([]); + }); + } + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + } + } +}); diff --git a/packages/stack/src/Tools.ts b/packages/stack/src/Tools.ts new file mode 100644 index 0000000000..3df6c0d646 --- /dev/null +++ b/packages/stack/src/Tools.ts @@ -0,0 +1,33 @@ +import { Schema } from "effect"; + +const PgProveMount = Schema.Struct({ + source: Schema.String, + target: Schema.String, +}); + +export const PgProveOptions = Schema.Struct({ + mounts: Schema.Array(PgProveMount), + cwd: Schema.optional(Schema.String), + workingDir: Schema.optional(Schema.String), +}); +export interface PgProveOptions extends Schema.Schema.Type {} + +export const PostgresTool = Schema.Struct({ + command: Schema.Literals(["pg_dump", "pg_dumpall", "pg_prove", "psql"]), + major: Schema.Literals([15, 17]), +}); +export interface PostgresTool extends Schema.Schema.Type {} + +/** Describes finite PostgreSQL clients without managing a database service. */ +export const postgres = { + pgDump: ({ major }: { readonly major: 15 | 17 }): PostgresTool => ({ command: "pg_dump", major }), + pgDumpAll: ({ major }: { readonly major: 15 | 17 }): PostgresTool => ({ + command: "pg_dumpall", + major, + }), + pgProve: ({ major }: { readonly major: 15 | 17 }): PostgresTool => ({ + command: "pg_prove", + major, + }), + psql: ({ major }: { readonly major: 15 | 17 }): PostgresTool => ({ command: "psql", major }), +}; diff --git a/packages/stack/src/composition/Supabase.native.integration.test.ts b/packages/stack/src/composition/Supabase.native.integration.test.ts new file mode 100644 index 0000000000..a7a13afc19 --- /dev/null +++ b/packages/stack/src/composition/Supabase.native.integration.test.ts @@ -0,0 +1,604 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + Context, + Deferred, + Effect, + Exit, + FileSystem, + Fiber, + Layer, + Redacted, + Ref, + Schema, +} from "effect"; +import { PgClient } from "@effect/sql-pg"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { SignJWT } from "jose"; +import { tmpdir } from "node:os"; +import * as Owner from "../Owner.ts"; +import * as State from "../State.ts"; +import type { SavedStack } from "../State.ts"; +import { + makeSupabaseComposition, + SupabaseCompositionError, + type SupabaseCompositionOperations, +} from "./Supabase.ts"; + +const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + +const stateFor = (root: string) => + Effect.gen(function* () { + const context = yield* Layer.build(State.layer({ root })); + return Context.get(context, State.Service); + }); + +const ownerFor = (options: { + readonly saved: SavedStack; + readonly state: State.Interface; + readonly root: string; + readonly cacheRoot: string; +}) => { + const { state, ...layerOptions } = options; + return Effect.gen(function* () { + const context = yield* Layer.build( + Owner.layer(layerOptions).pipe(Layer.provide(Layer.succeed(State.Service, state))), + ); + return Context.get(context, Owner.Service); + }); +}; + +const query = (url: string, statement: string) => + Effect.scoped( + Effect.gen(function* () { + const parsed = new URL(url); + const services = yield* Layer.build( + PgClient.layer({ + host: parsed.hostname, + port: Number(parsed.port), + database: parsed.pathname.slice(1), + username: decodeURIComponent(parsed.username), + password: Redacted.make(decodeURIComponent(parsed.password)), + }), + ); + return yield* Context.get(services, PgClient.PgClient).unsafe(statement); + }), + ); + +const initial = (id: string): SavedStack => ({ + id, + identity: { projectRoot: "/tmp/project", branchContext: "catalog-native", stackName: id }, + runtime: "native", + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], +}); + +it.live("reports the failed binding and cleanup reason with the retained instance ID", () => + Effect.gen(function* () { + const unused = () => Effect.die("unused operation"); + const operations: SupabaseCompositionOperations = { + currentComposition: Effect.succeed({ members: [], dependencies: [] }), + get: unused, + status: unused, + create: (creation) => Effect.succeed({ id: "retained-mail", creation }), + destroy: () => + Effect.fail(new SupabaseCompositionError({ message: "cannot remove owned data" })), + bind: () => Effect.fail(new SupabaseCompositionError({ message: "port is occupied" })), + address: unused, + output: unused, + updateCreation: unused, + configure: unused, + }; + const error = yield* makeSupabaseComposition(operations, [ + { service: "mail", config: {}, endpoints: {} }, + ]).pipe(Effect.flip); + expect(error.message).toContain("mail retained-mail: port is occupied"); + expect(error.message).toContain("retained-mail: cannot remove owned data"); + }), +); + +it.live("removes a managed SMTP binding when Mail is excluded", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-exclude-mail-" }); + const saved = initial("catalog-exclude-mail"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(saved); + const owner = yield* ownerFor({ saved, state, root: `${root}/data`, cacheRoot }); + yield* Effect.acquireUseRelease( + Effect.succeed(owner), + () => + Effect.gen(function* () { + const auth = { + service: "auth" as const, + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: "exclude-mail-jwt-secret-with-32-chars", + jwtExpiry: 3600, + }, + endpoints: { http: { port: "auto" as const } }, + }; + const members = yield* owner.composition.supabase([ + auth, + { service: "mail", config: {}, endpoints: { smtp: { port: "auto" } } }, + ]); + const instance = members.find(({ creation }) => creation.service === "auth"); + if (instance?.creation.service !== "auth") return yield* Effect.die("Auth missing"); + expect(instance.creation.config.smtpUrl).toBeDefined(); + const selected = yield* owner.composition.supabase([auth], { reuseIds: [instance.id] }); + const reused = selected.find(({ id }) => id === instance.id); + if (reused?.creation.service !== "auth") return yield* Effect.die("Reused Auth missing"); + expect(reused.creation.config.smtpUrl).toBeUndefined(); + const external = yield* owner.composition.supabase( + [{ ...auth, config: { ...auth.config, smtpUrl: "smtp://mail.example:2525" } }], + { reuseIds: [instance.id] }, + ); + const configured = external.find(({ id }) => id === instance.id); + if (configured?.creation.service !== "auth") return yield* Effect.die("Auth missing"); + expect(configured.creation.config.smtpUrl).toBe("smtp://mail.example:2525"); + }), + () => owner.namespace.destroy, + ); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); + +it.live("cleans created registrations when composition is interrupted", () => + Effect.gen(function* () { + const entered = yield* Deferred.make(); + const registered = yield* Deferred.make(); + const destroyed = yield* Ref.make>([]); + const database = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("interrupt-password"), + jwtSecret: Redacted.make("interrupt-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, + }; + const unused = (operation: string) => + Effect.fail(new SupabaseCompositionError({ message: `${operation} is unused` })); + const operations: SupabaseCompositionOperations = { + currentComposition: Effect.succeed({ members: [], dependencies: [] }), + get: () => unused("get"), + status: () => unused("status"), + create: (creation) => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(registered)), + Effect.as({ id: "created-database", creation }), + ), + destroy: (id) => Ref.update(destroyed, (ids) => [...ids, id]), + bind: () => Effect.never, + address: () => Effect.die("address is unused"), + output: () => Effect.die("output is unused"), + updateCreation: () => Effect.die("updateCreation is unused"), + configure: () => Effect.die("configure is unused"), + }; + const running = yield* makeSupabaseComposition(operations, [database]).pipe(Effect.forkChild); + yield* Deferred.await(entered); + const interrupting = yield* Fiber.interrupt(running).pipe(Effect.forkChild); + yield* Deferred.succeed(registered, undefined); + yield* Fiber.join(interrupting); + expect(Exit.isFailure(yield* Fiber.await(running))).toBe(true); + expect(yield* Ref.get(destroyed)).toEqual(["created-database"]); + }), +); + +it.live( + "serves native Auth and Storage through an owned public database endpoint", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-native-auth-storage-" }); + const stack = initial("catalog-native-auth-storage"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const storageRoot = `${root}/storage`; + yield* fs.makeDirectory(storageRoot, { recursive: true }); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const jwtSecret = "catalog-native-auth-storage-secret-with-at-least-32-chars"; + const created = yield* owner.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(jwtSecret), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { + service: "auth", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret, + jwtExpiry: 3600, + }, + endpoints: { http: { port: "auto" } }, + }, + { + service: "storage", + config: { + databaseUrl: "postgresql://placeholder", + filePath: storageRoot, + jwtSecret, + }, + endpoints: { http: { port: "auto" } }, + }, + ]); + const database = created.find((entry) => entry.creation.service === "database"); + const auth = created.find((entry) => entry.creation.service === "auth"); + const storage = created.find((entry) => entry.creation.service === "storage"); + if (database === undefined || auth === undefined || storage === undefined) + return yield* Effect.die("Native composition members missing"); + + yield* owner.composition.start; + const databaseCredentials = yield* owner.credentials(database.id, "host"); + const databaseUrl = databaseCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("Native database URL missing"); + expect(databaseUrl).toMatch(/^postgresql:\/\/supabase_admin:/u); + const parsedDatabaseUrl = new URL(databaseUrl); + const databaseServices = yield* Layer.build( + PgClient.layer({ + host: parsedDatabaseUrl.hostname, + port: Number(parsedDatabaseUrl.port), + database: parsedDatabaseUrl.pathname.slice(1), + username: decodeURIComponent(parsedDatabaseUrl.username), + password: Redacted.make(decodeURIComponent(parsedDatabaseUrl.password)), + }), + ); + yield* Context.get(databaseServices, PgClient.PgClient).unsafe("SELECT 1"); + const authCredentials = yield* owner.credentials(auth.id, "host"); + const storageCredentials = yield* owner.credentials(storage.id, "host"); + if (authCredentials.url === undefined || storageCredentials.url === undefined) + return yield* Effect.die("Native public service URL missing"); + + const email = "catalog-native@example.test"; + const password = "catalog-native-password-123"; + const signup = yield* client.execute( + yield* HttpClientRequest.bodyJson({ email, password })( + HttpClientRequest.post(`${authCredentials.url}/signup`), + ), + ); + expect(signup.status).toBe(200); + const signupPayload = yield* signup.json; + const signupToken = yield* Schema.decodeUnknownEffect( + Schema.Struct({ access_token: Schema.String }), + )(signupPayload); + expect(signupToken.access_token.length).toBeGreaterThan(0); + + const login = yield* client.execute( + yield* HttpClientRequest.bodyJson({ email, password })( + HttpClientRequest.post(`${authCredentials.url}/token?grant_type=password`), + ), + ); + const loginBody = yield* login.text; + expect(login.status, loginBody).toBe(200); + const loginToken = yield* Schema.decodeEffect( + Schema.fromJsonString(Schema.Struct({ access_token: Schema.String })), + )(loginBody); + expect(loginToken.access_token.length).toBeGreaterThan(0); + + const serviceToken = yield* Effect.tryPromise(() => + new SignJWT({ role: "service_role" }) + .setProtectedHeader({ alg: "HS256", typ: "JWT" }) + .setSubject("catalog-native-service") + .setIssuedAt() + .setExpirationTime("1h") + .sign(new TextEncoder().encode(jwtSecret)), + ); + const headers = (request: HttpClientRequest.HttpClientRequest) => + request.pipe( + HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), + HttpClientRequest.setHeader("apikey", serviceToken), + ); + const bucket = yield* client.execute( + yield* HttpClientRequest.bodyJson({ name: "catalog" })( + headers(HttpClientRequest.post(`${storageCredentials.url}/bucket`)), + ), + ); + expect(bucket.status).toBe(200); + const upload = yield* client.execute( + headers( + HttpClientRequest.bodyText( + HttpClientRequest.post(`${storageCredentials.url}/object/catalog/hello.txt`), + "catalog-native-storage", + "text/plain", + ), + ), + ); + expect(upload.status).toBe(200); + const remove = yield* client.execute( + headers( + HttpClientRequest.make("DELETE")(`${storageCredentials.url}/object/catalog/hello.txt`), + ), + ); + expect(remove.status).toBe(200); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live( + "promotes a stopped database composition without replacing its identity or data", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-native-reuse-" }); + const stack = initial("catalog-native-reuse"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const databaseCreation = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("reuse-database-password"), + jwtSecret: Redacted.make("reuse-database-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, + }; + const databaseOnly = yield* owner.composition.supabase([databaseCreation]); + const database = databaseOnly[0]; + if (database === undefined) return yield* Effect.die("Database member missing"); + yield* owner.composition.start; + const credentials = yield* owner.credentials(database.id, "host"); + if (credentials.databaseUrl === undefined) + return yield* Effect.die("Database credentials missing"); + const port = new URL(credentials.databaseUrl).port; + yield* query(credentials.databaseUrl, "CREATE TABLE reuse_rows(value text NOT NULL)"); + yield* query(credentials.databaseUrl, "INSERT INTO reuse_rows VALUES ('preserved')"); + yield* owner.composition.stop; + + const expanded = yield* owner.composition.supabase( + [ + databaseCreation, + { + service: "rest", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: "auto" } }, + }, + ], + { reuseIds: [database.id] }, + ); + const reused = expanded.find((entry) => entry.creation.service === "database"); + if (reused === undefined) return yield* Effect.die("Reused database missing"); + expect(reused.id).toBe(database.id); + yield* owner.composition.start; + const reopened = yield* owner.credentials(database.id, "host"); + if (reopened.databaseUrl === undefined) + return yield* Effect.die("Reopened database credentials missing"); + expect(new URL(reopened.databaseUrl).port).toBe(port); + const rows = yield* query(reopened.databaseUrl, "SELECT value FROM reuse_rows"); + expect(rows).toEqual([{ value: "preserved" }]); + const rest = expanded.find((entry) => entry.creation.service === "rest"); + if (rest === undefined) return yield* Effect.die("REST missing"); + const before = yield* owner.credentials(rest.id, "host"); + yield* owner.composition.stop; + const withAuth = yield* owner.composition.supabase( + [ + databaseCreation, + rest.creation, + { + service: "auth", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: Redacted.value(databaseCreation.config.jwtSecret), + jwtExpiry: 3600, + }, + endpoints: { http: { port: "auto" } }, + }, + ], + { reuseIds: [database.id, rest.id] }, + ); + yield* owner.composition.start; + const auth = withAuth.find((entry) => entry.creation.service === "auth"); + if (auth === undefined) return yield* Effect.die("Auth missing"); + const restAddress = yield* owner.credentials(rest.id, "host"); + const authAddress = yield* owner.credentials(auth.id, "host"); + expect(restAddress.url).toBe(before.url); + if (restAddress.url === undefined || authAddress.url === undefined) + return yield* Effect.die("Shared API URLs missing"); + expect(new URL(authAddress.url).port).toBe(new URL(restAddress.url).port); + const client = yield* HttpClient.HttpClient; + expect((yield* client.get(restAddress.url)).status).toBe(200); + expect((yield* client.get(`${authAddress.url}/health`)).status).toBe(200); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live( + "reuses an excluded Studio instance and its fixed port", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-native-studio-reuse-" }); + const stack = initial("catalog-native-studio-reuse"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const database = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("studio-reuse-password"), + jwtSecret: Redacted.make("studio-reuse-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, + }; + const rest = { + service: "rest" as const, + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: "auto" as const } }, + }; + const pgmeta = { + service: "pgmeta" as const, + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: { port: "auto" as const } }, + }; + const studio = { + service: "studio" as const, + config: { + pgmetaUrl: "http://placeholder", + analyticsApiKey: "studio-reuse-analytics-key", + apiUrl: "http://placeholder", + publicApiUrl: "http://placeholder", + jwtSecret: "studio-reuse-jwt-secret-with-32-chars", + }, + endpoints: { http: { port: 54_391 } }, + }; + const initialMembers = yield* owner.composition.supabase([database, rest, pgmeta, studio]); + const databaseId = initialMembers.find( + (entry) => entry.creation.service === "database", + )?.id; + const restId = initialMembers.find((entry) => entry.creation.service === "rest")?.id; + const pgmetaId = initialMembers.find((entry) => entry.creation.service === "pgmeta")?.id; + const studioId = initialMembers.find((entry) => entry.creation.service === "studio")?.id; + if ( + databaseId === undefined || + restId === undefined || + pgmetaId === undefined || + studioId === undefined + ) + return yield* Effect.die("Studio composition members missing"); + yield* owner.composition.start; + yield* owner.composition.stop; + const reduced = yield* owner.composition.supabase([database, rest, pgmeta], { + reuseIds: [databaseId, restId, pgmetaId], + }); + expect(reduced.map((entry) => entry.id).sort()).toEqual( + [databaseId, restId, pgmetaId].sort(), + ); + yield* owner.composition.start; + yield* owner.composition.stop; + const restored = yield* owner.composition.supabase([database, rest, pgmeta, studio], { + reuseIds: [databaseId, restId, pgmetaId, studioId], + }); + expect(restored.find((entry) => entry.creation.service === "studio")?.id).toBe(studioId); + const studioCredentials = yield* owner.credentials(studioId, "host"); + if (studioCredentials.url === undefined) return yield* Effect.die("Studio URL missing"); + expect(new URL(studioCredentials.url).port).toBe("54391"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live( + "rejects reuse while a composition member is running before mutation", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-native-reuse-running-" }); + const stack = initial("catalog-native-reuse-running"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const databaseCreation = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("reuse-running-password"), + jwtSecret: Redacted.make("reuse-running-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, + }; + const created = yield* owner.composition.supabase([databaseCreation]); + const database = created[0]; + if (database === undefined) return yield* Effect.die("Database member missing"); + yield* owner.composition.start; + const failure = yield* owner.composition + .supabase([databaseCreation], { reuseIds: [database.id] }) + .pipe(Effect.flip); + expect(failure.message).toContain("stopped with wake disabled"); + expect((yield* owner.services.list).map((entry) => entry.id)).toEqual([database.id]); + yield* owner.composition.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); + +it.live( + "cleans newly created registrations when a composition bind fails", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-native-reuse-cleanup-" }); + const stack = initial("catalog-native-reuse-cleanup"); + const state = yield* stateFor(`${root}/state`); + yield* state.save(stack); + const owner = yield* ownerFor({ + saved: stack, + state, + root: `${root}/data`, + cacheRoot, + }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + const fixedPort = 54_392; + const standaloneMail = yield* owner.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: fixedPort } }, + }); + yield* owner.core.start(standaloneMail.id); + yield* owner.core.ready(standaloneMail.id); + const database = { + service: "database" as const, + config: { + version: "17", + databasePassword: Redacted.make("reuse-cleanup-password"), + jwtSecret: Redacted.make("reuse-cleanup-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" as const } }, + }; + const failure = yield* owner.composition + .supabase([ + database, + { service: "mail", config: {}, endpoints: { http: { port: fixedPort } } }, + ]) + .pipe(Effect.flip); + expect(failure.operation).toBe("supabase"); + expect((yield* owner.services.list).map((entry) => entry.id)).toEqual([standaloneMail.id]); + yield* owner.core.stop(standaloneMail.id); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, +); diff --git a/packages/stack/src/composition/Supabase.ts b/packages/stack/src/composition/Supabase.ts new file mode 100644 index 0000000000..6e60b98987 --- /dev/null +++ b/packages/stack/src/composition/Supabase.ts @@ -0,0 +1,467 @@ +import { Cause, Data, Effect, Exit, Schema } from "effect"; +import type { CompositionConfig } from "../Orchestrator.ts"; +import type { Observation } from "../Rpc.ts"; +import { ServiceCreation } from "../services/Catalog.ts"; +import { apiRoute, endpointNames, endpointPort } from "../host/Endpoints.ts"; + +export class SupabaseCompositionError extends Data.TaggedError("SupabaseCompositionError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +interface SupabaseCompositionEntry { + readonly id: string; + readonly creation: ServiceCreation; +} + +export interface SupabaseCompositionOperations { + readonly currentComposition: Effect.Effect; + readonly get: (id: string) => Effect.Effect; + readonly status: ( + id: string, + ) => Effect.Effect, SupabaseCompositionError>; + readonly create: ( + creation: ServiceCreation, + ) => Effect.Effect; + readonly destroy: (id: string) => Effect.Effect; + readonly bind: (id: string) => Effect.Effect; + readonly address: ( + id: string, + endpoint: string, + from: "host" | "runtime", + ) => Effect.Effect; + readonly output: (id: string, name: string) => Effect.Effect; + readonly updateCreation: ( + id: string, + inputs: Record, + ) => Effect.Effect; + readonly configure: ( + configuration: CompositionConfig, + ) => Effect.Effect; +} + +export interface SupabaseCompositionOptions { + /** Reuses stopped instances; inputs declare desired bindings, not previous resolved creations. */ + readonly reuseIds?: ReadonlyArray; +} + +const compositionError = (message: string, cause?: unknown) => + new SupabaseCompositionError({ message, cause }); + +const causeMessage = (cause: Cause.Cause) => + Cause.prettyErrors(cause) + .map((error) => error.message) + .join("; ") || "interrupted"; + +const compositionErrorFrom = (cause: unknown) => + cause instanceof SupabaseCompositionError + ? cause + : compositionError(cause instanceof Error ? cause.message : String(cause), cause); + +export const makeSupabaseComposition = Effect.fn("Supabase.compose")( + ( + operations: SupabaseCompositionOperations, + inputs: ReadonlyArray, + options: SupabaseCompositionOptions = {}, + ) => + Effect.gen(function* () { + const current = yield* operations.currentComposition; + const reuseIds = options.reuseIds; + if (reuseIds === undefined && (current.members.length > 0 || current.dependencies.length > 0)) + return yield* compositionError("Composition is already configured"); + + const decoded = yield* Effect.forEach(inputs, (input) => + Schema.decodeEffect(ServiceCreation)(input).pipe(Effect.mapError(compositionErrorFrom)), + ); + const seen = new Set(); + for (const creation of decoded) { + if (seen.has(creation.service)) + return yield* compositionError(`Duplicate service kind ${creation.service}`); + seen.add(creation.service); + } + + const reusedEntries: ReadonlyArray = + reuseIds === undefined + ? [] + : yield* Effect.gen(function* () { + const unique = new Set(); + for (const id of reuseIds) { + if (unique.has(id)) return yield* compositionError(`Duplicate reuse ID ${id}`); + unique.add(id); + } + const entries = yield* Effect.forEach(reuseIds, (id) => + operations.get(id).pipe(Effect.mapError(compositionErrorFrom)), + ); + const kinds = new Set(); + for (const entry of entries) { + if (!seen.has(entry.creation.service)) + return yield* compositionError( + `Reused service ${entry.id} (${entry.creation.service}) is not requested`, + ); + if (kinds.has(entry.creation.service)) + return yield* compositionError( + `Multiple reuse IDs request service kind ${entry.creation.service}`, + ); + kinds.add(entry.creation.service); + } + + const currentIds = new Set([ + ...current.members.map(({ id }) => id), + ...current.dependencies.flatMap(({ from, to }) => [from, to]), + ]); + const currentEntries = yield* Effect.forEach([...currentIds], (id) => + operations.get(id).pipe(Effect.mapError(compositionErrorFrom)), + ); + const reused = new Set(reuseIds); + for (const entry of currentEntries) { + if (seen.has(entry.creation.service) && !reused.has(entry.id)) + return yield* compositionError( + `Existing ${entry.creation.service} member ${entry.id} must be reused or removed from the requested composition`, + ); + } + const affectedIds = new Set([...currentIds, ...reuseIds]); + yield* Effect.forEach([...affectedIds], (id) => + operations.status(id).pipe( + Effect.mapError(compositionErrorFrom), + Effect.flatMap((status) => + status.lifecycle === "stopped" && !status.wakeEnabled + ? Effect.void + : Effect.fail( + compositionError( + `Service ${id} must be stopped with wake disabled before composition changes`, + ), + ), + ), + ), + ); + return entries; + }); + + const fixedPorts = new Set( + [...decoded, ...reusedEntries.map(({ creation }) => creation)] + .filter( + (creation) => + apiRoute(creation.service) !== undefined && endpointNames(creation).includes("http"), + ) + .map((creation) => endpointPort(creation, "http")) + .filter((port): port is number => port !== "auto"), + ); + if (fixedPorts.size > 1) + return yield* compositionError("Shared HTTP endpoints must use one port"); + const sharedPort = [...fixedPorts][0]; + const normalized = yield* Effect.forEach(decoded, (creation) => { + if ( + sharedPort === undefined || + apiRoute(creation.service) === undefined || + !endpointNames(creation).includes("http") || + endpointPort(creation, "http") !== "auto" + ) + return Effect.succeed(creation); + const endpoints = isRecord(creation.endpoints) ? creation.endpoints : {}; + return Schema.decodeUnknownEffect(ServiceCreation)({ + ...creation, + endpoints: { ...endpoints, http: { port: sharedPort } }, + }).pipe(Effect.mapError(compositionErrorFrom)); + }); + + const reusedByKind = new Map(reusedEntries.map((entry) => [entry.creation.service, entry])); + const targetCreations = normalized.map( + (creation) => reusedByKind.get(creation.service)?.creation ?? creation, + ); + const byKind = new Map(targetCreations.map((creation) => [creation.service, creation])); + const apiSource = targetCreations.find( + (creation) => + apiRoute(creation.service) !== undefined && endpointNames(creation).includes("http"), + ); + if ( + apiSource === undefined && + targetCreations.some((creation) => + ["auth", "functions", "studio"].includes(creation.service), + ) + ) + return yield* compositionError("API URL bindings require a configured HTTP endpoint"); + + const managedBindings: ReadonlyArray<{ + readonly sourceKind: ServiceCreation["service"]; + readonly sourceEndpoint: string; + readonly output: string; + readonly targetKind: ServiceCreation["service"]; + readonly input: string; + }> = [ + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "authenticatorUrl", + targetKind: "rest", + input: "databaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "authDatabaseUrl", + targetKind: "auth", + input: "databaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "storageDatabaseUrl", + targetKind: "storage", + input: "databaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "databaseUrl", + targetKind: "storage", + input: "vectorDatabaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "databaseUrl", + targetKind: "realtime", + input: "databaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "databaseUrl", + targetKind: "pgmeta", + input: "databaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "internalDatabaseUrl", + targetKind: "analytics", + input: "databaseUrl", + }, + { + sourceKind: "database", + sourceEndpoint: "sql", + output: "internalDatabaseUrl", + targetKind: "pooler", + input: "databaseUrl", + }, + { + sourceKind: "imgproxy", + sourceEndpoint: "http", + output: "url", + targetKind: "storage", + input: "imgproxyUrl", + }, + { + sourceKind: "pgmeta", + sourceEndpoint: "http", + output: "url", + targetKind: "studio", + input: "pgmetaUrl", + }, + { + sourceKind: "analytics", + sourceEndpoint: "http", + output: "url", + targetKind: "studio", + input: "analyticsUrl", + }, + { + sourceKind: "analytics", + sourceEndpoint: "http", + output: "url", + targetKind: "vector", + input: "analyticsUrl", + }, + { + sourceKind: "functions", + sourceEndpoint: "http", + output: "url", + targetKind: "studio", + input: "functionsUrl", + }, + { + sourceKind: "mail", + sourceEndpoint: "smtp", + output: "smtpUrl", + targetKind: "auth", + input: "smtpUrl", + }, + ]; + for (const { sourceKind, sourceEndpoint, targetKind } of managedBindings) { + if (byKind.has(sourceKind) && byKind.has(targetKind)) { + const source = byKind.get(sourceKind); + if (source === undefined || !endpointNames(source).includes(sourceEndpoint)) + return yield* compositionError( + `${sourceKind} requires configured ${sourceEndpoint} endpoint`, + ); + } + } + + const createdIds: Array = []; + const compose = Effect.gen(function* () { + const created = yield* Effect.forEach( + targetCreations.filter((creation) => !reusedByKind.has(creation.service)), + (creation) => + Effect.uninterruptible( + operations.create(creation).pipe( + Effect.mapError(compositionErrorFrom), + Effect.tap((entry) => Effect.sync(() => createdIds.push(entry.id))), + ), + ), + ); + const createdByKind = new Map(created.map((entry) => [entry.creation.service, entry])); + const entries = targetCreations.map((creation) => { + const reused = reusedByKind.get(creation.service); + return reused ?? createdByKind.get(creation.service); + }); + const configuredEntries = entries.filter( + (entry): entry is SupabaseCompositionEntry => entry !== undefined, + ); + if (configuredEntries.length !== entries.length) + return yield* compositionError("Composition instance is missing"); + for (const entry of configuredEntries) { + yield* operations + .bind(entry.id) + .pipe( + Effect.mapError((cause) => + compositionError(`${entry.creation.service} ${entry.id}: ${cause.message}`, cause), + ), + ); + } + + const addressUrl = ( + entry: { readonly id: string }, + endpoint: string, + from: "host" | "runtime", + ) => operations.address(entry.id, endpoint, from); + const apiEntry = + apiSource === undefined + ? undefined + : configuredEntries.find((entry) => entry.creation.service === apiSource.service); + if (apiSource !== undefined && apiEntry === undefined) + return yield* compositionError("API source instance is missing"); + const apiHostUrl = + apiEntry === undefined ? undefined : yield* addressUrl(apiEntry, "http", "host"); + const apiRuntimeUrl = + apiEntry === undefined ? undefined : yield* addressUrl(apiEntry, "http", "runtime"); + const entriesByKind = new Map( + configuredEntries.map((entry) => [entry.creation.service, entry]), + ); + const dependencyMap = new Map< + string, + { from: string; to: string; bindings: Array<{ output: string; input: string }> } + >(); + const configInputs = new Map>(); + for (const { + sourceKind, + output: outputName, + targetKind, + input: inputName, + } of managedBindings) { + const source = entriesByKind.get(sourceKind); + const target = entriesByKind.get(targetKind); + if (target === undefined) continue; + const values = configInputs.get(target.id) ?? {}; + if (source === undefined) { + const requested = normalized.find((creation) => creation.service === targetKind); + const fallback = + requested === undefined ? undefined : Reflect.get(requested.config, inputName); + values[inputName] = typeof fallback === "string" ? fallback : undefined; + configInputs.set(target.id, values); + continue; + } + values[inputName] = yield* operations.output(source.id, outputName); + configInputs.set(target.id, values); + const key = `${source.id}->${target.id}`; + const dependency = dependencyMap.get(key) ?? { + from: source.id, + to: target.id, + bindings: [], + }; + dependency.bindings.push({ output: outputName, input: inputName }); + dependencyMap.set(key, dependency); + } + const configured = yield* Effect.forEach(configuredEntries, (entry) => + Effect.gen(function* () { + const extra: Record = + entry.creation.service === "auth" && apiHostUrl !== undefined + ? { externalApiUrl: `${apiHostUrl}/auth/v1` } + : entry.creation.service === "studio" && + apiHostUrl !== undefined && + apiRuntimeUrl !== undefined + ? { + apiUrl: apiRuntimeUrl, + publicApiUrl: entry.creation.config.publicApiUrl ?? apiHostUrl, + } + : entry.creation.service === "functions" && apiRuntimeUrl !== undefined + ? { apiUrl: apiRuntimeUrl } + : {}; + const values = { ...configInputs.get(entry.id), ...extra }; + if (entry.creation.service === "studio") { + const analytics = entriesByKind.get("analytics"); + if ( + analytics?.creation.service === "analytics" && + analytics.creation.config.apiKey !== undefined + ) + values.analyticsApiKey = analytics.creation.config.apiKey; + const functions = entriesByKind.get("functions"); + if (functions?.creation.service === "functions") + values.functionsRoot = functions.creation.config.functionsRoot; + } + const database = entriesByKind.get("database"); + if ( + entry.creation.service === "functions" && + database !== undefined && + endpointNames(database.creation).includes("sql") + ) + values.databaseUrl = yield* operations.output(database.id, "databaseUrl"); + if (Object.keys(values).length > 0) { + return yield* operations.updateCreation(entry.id, values); + } + return entry; + }), + ); + const members = configured.map(({ id, creation }) => { + const lazy = creation.service !== "database" && endpointNames(creation).length > 0; + return lazy + ? creation.service === "functions" + ? { id, activation: "lazy" as const } + : { id, activation: "lazy" as const, idleMillis: 60_000 } + : { id, activation: "eager" as const }; + }); + yield* operations.configure({ + members, + dependencies: [...dependencyMap.values()], + }); + return configured; + }); + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const exit = yield* Effect.exit(restore(compose)); + if (Exit.isSuccess(exit)) return exit.value; + const cleanup = yield* Effect.forEach(createdIds, (id) => + operations.destroy(id).pipe( + Effect.exit, + Effect.map((result) => ({ id, result })), + ), + ); + const failures = cleanup.flatMap(({ id, result }) => + Exit.isFailure(result) ? [`${id}: ${causeMessage(result.cause)}`] : [], + ); + if (failures.length > 0) + return yield* compositionError( + `Composition failed: ${causeMessage(exit.cause)}. Cleanup failed for created services: ${failures.join("; ")}`, + { original: exit.cause, cleanup }, + ); + return yield* Effect.failCause(exit.cause); + }), + ); + }).pipe( + Effect.mapError((cause) => + cause instanceof SupabaseCompositionError ? cause : compositionErrorFrom(cause), + ), + ), +); diff --git a/packages/stack/src/control/ControlServer.ts b/packages/stack/src/control/ControlServer.ts deleted file mode 100644 index 97e19d60e2..0000000000 --- a/packages/stack/src/control/ControlServer.ts +++ /dev/null @@ -1,796 +0,0 @@ -import { - Cause, - Deferred, - Effect, - Exit, - Fiber, - FiberSet, - FileSystem, - Option, - Predicate, - Schema, - Scope, - Semaphore, -} from "effect"; -import { NodeSocket, NodeSocketServer } from "@effect/platform-node"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- Effect FileSystem exposes stat but no no-follow lstat; this security check must reject symlinked control directories. -import { lstat } from "node:fs/promises"; -import * as RpcClient from "effect/unstable/rpc/RpcClient"; -import { RpcClientError } from "effect/unstable/rpc/RpcClientError"; -import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"; -import * as RpcServer from "effect/unstable/rpc/RpcServer"; -import * as Socket from "effect/unstable/socket/Socket"; -import * as SocketServer from "effect/unstable/socket/SocketServer"; -import type { ControlEndpoint } from "../state/Ownership.ts"; -import { - decodeFrame, - encodeFrame, - encodeRawFrame, - FrameDecoder, - MAINTENANCE_MAX_FRAME_BYTES, - RPC_MAX_FRAME_BYTES, - type JsonValue, -} from "./FrameCodec.ts"; -import { - CONTROL_PREFACE_MAX_BYTES, - decodePreface, - encodePreface, - MAINTENANCE_MAX_CONCURRENT_REQUESTS, - MAINTENANCE_REQUEST_DEADLINE_MS, - MaintenanceRequestSchema, - MaintenanceProtocolError, - MaintenanceResponseSchema, - type MaintenanceRequest, - type MaintenanceResponse, -} from "./MaintenanceProtocol.ts"; -import { - releaseMismatch, - STACK_RPC_RELEASE, - StackRpcGroup, - type StackRpcClient, - type StackRpcHandlers, -} from "./StackRpc.ts"; - -interface ControlIdentity { - readonly stackId: string; - readonly ownerSessionId: string; -} - -export interface MaintenanceHandlers { - readonly probe: Effect.Effect; - readonly stop: Effect.Effect; -} - -export interface ControlServerOptions extends ControlIdentity { - readonly endpoint: ControlEndpoint; - readonly rpcRelease?: string; - readonly maintenanceHandlers: MaintenanceHandlers; - /** Re-evaluates owner shutdown after a lifecycle response or disconnect. */ - readonly onShutdownReady?: Effect.Effect; - readonly rpcHandlers: StackRpcHandlers; -} - -export interface ControlServer { - readonly endpoint: ControlEndpoint; -} - -const endpointPath = (endpoint: ControlEndpoint): string => - endpoint.kind === "unix" ? endpoint.path : endpoint.name; - -const controlDirectory = (endpoint: ControlEndpoint): string => { - const path = endpointPath(endpoint); - const separator = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")); - return separator < 0 ? path : path.slice(0, separator); -}; - -const controlServerError = (cause: unknown): SocketServer.SocketServerError => - new SocketServer.SocketServerError({ - reason: new SocketServer.SocketServerOpenError({ cause }), - }); - -/** - * Ensure the Unix endpoint is inside an owner-private directory. FileSystem.stat follows - * symlinks, so the no-follow lstat is required at this security boundary. - */ -const ensurePrivateControlDirectory = ( - fs: FileSystem.FileSystem, - directory: string, -): Effect.Effect => - Effect.gen(function* () { - const exists = yield* fs.exists(directory).pipe(Effect.mapError(controlServerError)); - let created = false; - if (!exists) { - yield* fs.makeDirectory(directory, { mode: 0o700 }).pipe(Effect.mapError(controlServerError)); - created = true; - } - const info = yield* Effect.tryPromise({ - try: () => lstat(directory), - catch: (cause) => controlServerError(cause), - }); - const uid = typeof process.getuid === "function" ? process.getuid() : undefined; - if (!info.isDirectory() || (uid !== undefined && info.uid !== uid) || (info.mode & 0o077) !== 0) - return yield* controlServerError( - new Error("Control endpoint directory is not owner-private"), - ); - yield* fs.chmod(directory, 0o700).pipe(Effect.mapError(controlServerError)); - return created; - }); - -const toBytes = (chunk: Uint8Array | string): Uint8Array => - typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk; - -const protocolFailure = ( - tag: - | "invalid-request" - | "stale-session" - | "timeout" - | "operation-failed" - | "unsupported-release" = "invalid-request", -): MaintenanceResponse => ({ - ok: false, - error: { tag, message: "Control request rejected" }, -}); - -const isResponseConnectionFailure = (cause: Cause.Cause): boolean => - Option.match(Cause.findErrorOption(cause), { - onNone: () => false, - onSome: (error) => - Predicate.isTagged(error.reason, "SocketWriteError") || - Predicate.isTagged(error.reason, "SocketCloseError"), - }); - -/** Wrap one accepted socket. This is the only reader for the connection. */ -const demuxSocket = ( - socket: Socket.Socket, - options: ControlServerOptions, - maintenanceSemaphore: Semaphore.Semaphore, - completionFibers: FiberSet.FiberSet, -): Socket.Socket => { - const expectedRelease = options.rpcRelease ?? STACK_RPC_RELEASE; - type Writer = ( - chunk: Uint8Array | string | Socket.CloseEvent, - ) => Effect.Effect; - let connectionWriter: Writer | undefined; - - const runRaw = ( - handler: (_: Uint8Array) => Effect.Effect | void, - runOptions?: { readonly onOpen?: Effect.Effect }, - ): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const writerReady = yield* Deferred.make(); - const underlyingWrite = yield* socket.writer; - const decoder = new FrameDecoder(); - const prefaceReady = yield* Deferred.make(); - let preface = new Uint8Array(0); - let phase: "preface" | "maintenance" | "rpc" = "preface"; - let closed = false; - - const markPrefaceReady = Deferred.succeed(prefaceReady, undefined).pipe(Effect.asVoid); - const close = Effect.suspend(() => { - if (closed) return Effect.void; - closed = true; - return markPrefaceReady.pipe( - Effect.andThen( - Deferred.await(writerReady).pipe( - Effect.flatMap((write) => write(new Socket.CloseEvent(1000))), - ), - ), - ); - }); - - const socketWriteError = (message: string) => - new Socket.SocketError({ - reason: new Socket.SocketWriteError({ cause: new Error(message) }), - }); - const sendJson = (value: JsonValue): Effect.Effect => - Deferred.await(writerReady).pipe( - Effect.flatMap((write) => - encodeFrame(value).pipe( - Effect.mapError((error) => socketWriteError(error.message)), - Effect.flatMap(write), - ), - ), - ); - - type MaintenanceValidation = - | { readonly _tag: "invalid-request" } - | { readonly _tag: "stale-session"; readonly request: MaintenanceRequest } - | { readonly _tag: "valid"; readonly request: MaintenanceRequest }; - - const dispatchMaintenance = ( - validation: MaintenanceValidation, - ): Effect.Effect => { - let responseValue: MaintenanceResponse | undefined; - let operationName: MaintenanceRequest["op"] | undefined; - let completionStarted = false; - const startCompletion = (completion: Effect.Effect) => - Effect.uninterruptible( - FiberSet.run(completionFibers, completion, { startImmediately: true }).pipe( - // Set only after the completion fiber is handed to the owner-scoped - // FiberSet; the uninterruptible region keeps fork and witness atomic - // with `onExit` below. - Effect.tap(() => - Effect.sync(() => { - completionStarted = true; - }), - ), - ), - ); - const dispatch = Effect.gen(function* () { - if (validation._tag === "invalid-request") { - yield* sendJson(protocolFailure("invalid-request")); - yield* close; - return; - } - if (validation._tag === "stale-session") { - yield* sendJson(protocolFailure("stale-session")); - yield* close; - return; - } - const request = validation.request; - operationName = request.op; - const operation = - request.op === "probe" - ? options.maintenanceHandlers.probe - : options.maintenanceHandlers.stop; - const result = yield* Effect.exit(operation); - const response = Exit.isSuccess(result) - ? result.value - : protocolFailure("operation-failed"); - responseValue = response; - yield* sendJson(response); - yield* close; - if (request.op === "stop" && options.onShutdownReady !== undefined) { - // The connection scope closes as soon as the close frame is sent; - // completion belongs to the owner session and must outlive it. - yield* startCompletion(options.onShutdownReady); - } - }); - return dispatch.pipe( - Effect.onExit((exit) => { - const response = responseValue; - if (response === undefined || operationName !== "stop") return Effect.void; - if (Exit.isSuccess(exit)) return Effect.void; - const connectionFailure = - Cause.hasInterruptsOnly(exit.cause) || isResponseConnectionFailure(exit.cause); - if (!connectionFailure || completionStarted || options.onShutdownReady === undefined) - return Effect.void; - return startCompletion(options.onShutdownReady).pipe(Effect.asVoid); - }), - Effect.catchReasons("SocketError", { - SocketWriteError: () => Effect.void, - SocketCloseError: () => Effect.void, - }), - ); - }; - - const processFrames = (input: Uint8Array): Effect.Effect => - Effect.gen(function* () { - const result = yield* Effect.exit( - decoder.push( - input, - phase === "rpc" ? RPC_MAX_FRAME_BYTES : MAINTENANCE_MAX_FRAME_BYTES, - ), - ); - if (Exit.isFailure(result)) { - yield* close; - return; - } - for (const frame of result.value) { - if (closed) break; - if (phase === "maintenance") { - // Probe/validation are bounded by the maintenance admission deadline; a - // validated stop owns its own cleanup and may outlive it. The preface deadline - // governs only admission until the first frame arrives — dispatch below owns - // policy after that. - yield* markPrefaceReady; - const operation = yield* Effect.exit( - decodeFrame(frame).pipe( - Effect.flatMap((decoded) => - Schema.decodeUnknownEffect(MaintenanceRequestSchema)(decoded, { - onExcessProperty: "error", - }), - ), - ), - ); - const validation: MaintenanceValidation = Exit.isFailure(operation) - ? { _tag: "invalid-request" } - : operation.value.stackId !== options.stackId - ? { _tag: "invalid-request" } - : operation.value.ownerSessionId !== options.ownerSessionId - ? { _tag: "stale-session", request: operation.value } - : { _tag: "valid", request: operation.value }; - const dispatch = maintenanceSemaphore.withPermit(dispatchMaintenance(validation)); - if (validation._tag === "valid" && validation.request.op !== "probe") { - yield* dispatch; - } else { - yield* dispatch.pipe( - Effect.timeoutOrElse({ - duration: MAINTENANCE_REQUEST_DEADLINE_MS, - orElse: () => - sendJson(protocolFailure("timeout")).pipe(Effect.andThen(close)), - }), - ); - } - } else { - const returned = handler(frame.slice(4)); - if (Effect.isEffect(returned)) yield* returned; - } - } - }); - - const processChunk = ( - chunk: Uint8Array | string, - ): Effect.Effect => - Effect.gen(function* () { - if (closed) return; - const input = toBytes(chunk); - if (phase === "preface") { - const combined = new Uint8Array(preface.byteLength + input.byteLength); - combined.set(preface); - combined.set(input, preface.byteLength); - const newline = combined.indexOf(10); - if ( - (newline < 0 && combined.byteLength > CONTROL_PREFACE_MAX_BYTES) || - (newline >= 0 && newline + 1 > CONTROL_PREFACE_MAX_BYTES) - ) { - yield* sendJson(protocolFailure("invalid-request")); - yield* close; - return; - } - preface = combined; - const decoded = yield* Effect.exit(decodePreface(combined)); - if (Exit.isFailure(decoded)) { - if (preface.includes(10)) { - yield* sendJson(protocolFailure()); - yield* close; - } - return; - } - phase = decoded.value.protocol.kind; - preface = new Uint8Array(0); - if (decoded.value.protocol.stackId !== options.stackId) { - yield* sendJson(protocolFailure("invalid-request")); - yield* close; - return; - } - if (decoded.value.protocol.ownerSessionId !== options.ownerSessionId) { - yield* sendJson(protocolFailure("stale-session")); - yield* close; - return; - } - if (phase === "rpc" && decoded.value.protocol.release !== expectedRelease) { - yield* sendJson({ - ok: false, - error: { - tag: "unsupported-release", - message: releaseMismatch(decoded.value.protocol.release), - }, - }); - yield* close; - return; - } - if (phase === "rpc") yield* markPrefaceReady; - const remainder = combined.slice(decoded.value.consumed); - if (remainder.byteLength > 0) yield* processFrames(remainder); - return; - } - yield* processFrames(input); - }); - - const onOpen = Effect.andThen( - Effect.sync(() => { - connectionWriter = underlyingWrite; - }).pipe( - Effect.andThen(Deferred.succeed(writerReady, underlyingWrite).pipe(Effect.asVoid)), - ), - runOptions?.onOpen ?? Effect.void, - ); - const prefaceDeadline = yield* Effect.forkChild( - Deferred.await(prefaceReady).pipe( - Effect.timeoutOrElse({ - duration: MAINTENANCE_REQUEST_DEADLINE_MS, - orElse: () => sendJson(protocolFailure("timeout")).pipe(Effect.andThen(close)), - }), - ), - ); - yield* socket.runRaw(processChunk, { onOpen }); - yield* Fiber.interrupt(prefaceDeadline); - }), - ).pipe( - Effect.catchReasons("SocketError", { - SocketReadError: () => Effect.void, - SocketCloseError: () => Effect.void, - }), - ); - - return Socket.make({ - runRaw, - writer: Effect.succeed((chunk: Uint8Array | string | Socket.CloseEvent) => { - const write = connectionWriter; - if (write === undefined) { - return Effect.fail( - new Socket.SocketError({ - reason: new Socket.SocketWriteError({ cause: new Error("Control socket is not open") }), - }), - ); - } - return Socket.isCloseEvent(chunk) - ? write(chunk) - : encodeRawFrame(chunk).pipe( - Effect.mapError( - (error) => - new Socket.SocketError({ - reason: new Socket.SocketWriteError({ cause: new Error(error.message) }), - }), - ), - Effect.flatMap(write), - ); - }), - }); -}; - -const wrappedServer = ( - base: SocketServer.SocketServer["Service"], - options: ControlServerOptions, - maintenanceSemaphore: Semaphore.Semaphore, - completionFibers: FiberSet.FiberSet, -): SocketServer.SocketServer["Service"] => - SocketServer.SocketServer.of({ - address: base.address, - run: (handler) => { - return base.run((socket) => { - return handler(demuxSocket(socket, options, maintenanceSemaphore, completionFibers)); - }); - }, - }); - -/** Bind the one local endpoint and serve maintenance plus exact-release RPC. */ -export const startControlServer = ( - options: ControlServerOptions, -): Effect.Effect< - ControlServer, - SocketServer.SocketServerError, - Scope.Scope | FileSystem.FileSystem -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - let bound = false; - if (options.endpoint.kind === "unix") { - const endpoint = options.endpoint; - const directory = controlDirectory(endpoint); - yield* Effect.acquireRelease( - ensurePrivateControlDirectory(fs, directory), - (directoryCreated) => - (bound ? fs.remove(endpoint.path, { force: true }) : Effect.void).pipe( - Effect.catchTag("PlatformError", () => Effect.void), - Effect.andThen( - directoryCreated - ? fs.remove(directory, { force: true, recursive: true }) - : Effect.void, - ), - Effect.catchTag("PlatformError", () => Effect.void), - ), - ); - } - const base = yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const base = yield* restore( - NodeSocketServer.make({ path: endpointPath(options.endpoint) }), - ); - bound = true; - return base; - }), - ); - if (options.endpoint.kind === "unix") { - yield* fs.chmod(options.endpoint.path, 0o600).pipe( - Effect.mapError( - (error) => - new SocketServer.SocketServerError({ - reason: new SocketServer.SocketServerOpenError({ cause: error }), - }), - ), - ); - } - const maintenanceSemaphore = yield* Semaphore.make(MAINTENANCE_MAX_CONCURRENT_REQUESTS); - const completionFibers = yield* FiberSet.make(); - const server = wrappedServer(base, options, maintenanceSemaphore, completionFibers); - const protocol = yield* RpcServer.makeProtocolSocketServer.pipe( - Effect.provideService(SocketServer.SocketServer, server), - Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.json), - ); - const isCompletionRequest = (tag: string): boolean => tag === "start" || tag === "destroy"; - const completionRequests = new Set(); - const startShutdown = (completion: Effect.Effect) => - Effect.uninterruptible( - FiberSet.run(completionFibers, completion, { startImmediately: true }).pipe(Effect.asVoid), - ); - const patchedProtocol = RpcServer.Protocol.of({ - ...protocol, - run: (handler) => - protocol.run((clientId, request) => { - if (Predicate.isTagged(request, "Request") && isCompletionRequest(request.tag)) - completionRequests.add(`${clientId}:${String(request.id)}`); - return handler(clientId, request); - }), - send: (clientId, response, transferables) => - Predicate.isTagged(response, "Exit") && - completionRequests.has(`${clientId}:${String(response.requestId)}`) - ? Effect.gen(function* () { - const key = `${clientId}:${String(response.requestId)}`; - const connected = (yield* protocol.clientIds).has(clientId); - if (connected) yield* Effect.exit(protocol.send(clientId, response, transferables)); - const completed = completionRequests.delete(key); - if (completed && options.onShutdownReady !== undefined) - yield* startShutdown(options.onShutdownReady); - }) - : protocol.send(clientId, response, transferables), - }); - // Keeps handler defects as keyed Exit responses so lifecycle-completion state can be - // cleaned by the same send path as typed failures; otherwise RpcServer emits a Defect - // frame with no requestId for the terminal handoff. - const rpcProgram: Effect.Effect = RpcServer.make(StackRpcGroup, { - disableTracing: true, - disableFatalDefects: true, - concurrency: MAINTENANCE_MAX_CONCURRENT_REQUESTS, - }).pipe( - Effect.provideService(RpcServer.Protocol, patchedProtocol), - Effect.provide(StackRpcGroup.toLayer(options.rpcHandlers)), - ); - yield* Effect.forkScoped(rpcProgram); - return { - endpoint: options.endpoint, - } satisfies ControlServer; - }); - -/** Client-side framed socket; writer emits the RPC preface exactly once. */ -const makeControlRpcSocket = ( - socket: Socket.Socket, - options: { - readonly rpcRelease?: string; - readonly stackId: string; - readonly ownerSessionId: string; - }, -): Socket.Socket => { - let prefaced = false; - return Socket.make({ - runRaw: (handler, options) => - Effect.scoped( - Effect.gen(function* () { - const decoder = new FrameDecoder(); - yield* socket.runRaw( - (chunk) => - decoder.push(toBytes(chunk), RPC_MAX_FRAME_BYTES).pipe( - Effect.mapError( - (error) => - new Socket.SocketError({ - reason: new Socket.SocketReadError({ cause: new Error(error.message) }), - }), - ), - Effect.flatMap((frames) => - Effect.forEach(frames, (frame) => - Effect.suspend(() => { - const effect = handler(frame.slice(4)); - return Effect.isEffect(effect) ? effect : Effect.void; - }), - ).pipe(Effect.asVoid), - ), - ), - options, - ); - }), - ), - writer: Effect.map( - socket.writer, - (write) => (chunk: Uint8Array | string | Socket.CloseEvent) => - Effect.gen(function* () { - if (Socket.isCloseEvent(chunk)) { - yield* write(chunk); - return; - } - const encodedFrame = yield* encodeRawFrame(chunk).pipe( - Effect.mapError( - (error) => - new Socket.SocketError({ - reason: new Socket.SocketWriteError({ cause: new Error(error.message) }), - }), - ), - ); - if (!prefaced) { - prefaced = true; - const preface = encodePreface({ - kind: "rpc", - release: options.rpcRelease ?? STACK_RPC_RELEASE, - stackId: options.stackId, - ownerSessionId: options.ownerSessionId, - }); - const combined = new Uint8Array(preface.byteLength + encodedFrame.byteLength); - combined.set(preface); - combined.set(encodedFrame, preface.byteLength); - yield* write(combined); - return; - } - yield* write(encodedFrame); - }), - ), - }); -}; - -export interface ControlClientOptions extends ControlIdentity { - readonly rpcRelease?: string; -} - -export interface ControlClient { - readonly probe: Effect.Effect; - readonly stop: Effect.Effect; - /** Connects with an RPC preface and completes when the owner closes the socket. */ - readonly awaitClose: (onOpen?: Effect.Effect) => Effect.Effect; - readonly rpc: Effect.Effect; -} - -/** A scoped client seam used by the Supervisor entrypoint and public Stack handles. */ -export const makeControlClient = ( - endpoint: ControlEndpoint, - options: ControlClientOptions, -): ControlClient => { - const maintenance = ( - op: MaintenanceRequest["op"], - ): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const socket = yield* NodeSocket.makeNet({ - path: endpointPath(endpoint), - openTimeout: MAINTENANCE_REQUEST_DEADLINE_MS, - }); - const underlyingWrite = yield* socket.writer; - const decoder = new FrameDecoder(); - const response = yield* Deferred.make(); - const writerReady = - yield* Deferred.make< - ( - chunk: Uint8Array | string | Socket.CloseEvent, - ) => Effect.Effect - >(); - const read = socket.runRaw( - (chunk) => - decoder.push(toBytes(chunk)).pipe( - Effect.flatMap((frames) => - Effect.forEach(frames, (frame) => - Effect.gen(function* () { - const decoded = yield* Effect.exit(decodeFrame(frame)); - if (Exit.isFailure(decoded)) return; - const parsed = yield* Effect.exit( - Schema.decodeUnknownEffect(MaintenanceResponseSchema)(decoded.value, { - onExcessProperty: "error", - }), - ); - if (Exit.isSuccess(parsed)) yield* Deferred.succeed(response, parsed.value); - }), - ).pipe(Effect.asVoid), - ), - ), - { - onOpen: Deferred.succeed(writerReady, underlyingWrite).pipe(Effect.asVoid), - }, - ); - const fiber = yield* Effect.forkChild(read); - // A connection can fail before NodeSocket runs the onOpen hook; join the reader - // alongside writer readiness so failure can't strand this handshake on an unresolved - // Deferred. - const readerReady = Fiber.join(fiber).pipe( - Effect.andThen( - Effect.fail( - new MaintenanceProtocolError({ - message: "Control connection closed", - reason: "transport", - }), - ), - ), - ); - // NodeSocket opens its writer as part of runRaw's onOpen hook. - const write = yield* Effect.raceFirst(Deferred.await(writerReady), readerReady); - yield* write( - encodePreface({ - kind: "maintenance", - release: "maintenance-v1", - stackId: options.stackId, - ownerSessionId: options.ownerSessionId, - }), - ); - yield* encodeFrame({ - op, - stackId: options.stackId, - ownerSessionId: options.ownerSessionId, - }).pipe(Effect.flatMap(write)); - // The server closes a successful connection immediately after flushing its response; - // check the response witness after the reader exits so the close event can't win - // that race. - const readerDone = Effect.exit(Fiber.join(fiber)).pipe( - Effect.flatMap(() => Deferred.poll(response)), - Effect.flatMap((completed) => - Option.isSome(completed) - ? completed.value - : Effect.fail( - new MaintenanceProtocolError({ - message: "Control connection closed", - reason: "transport", - }), - ), - ), - ); - const wait = Effect.raceFirst(Deferred.await(response), readerDone).pipe( - Effect.ensuring(Fiber.interrupt(fiber)), - ); - return yield* wait; - }), - ); - const probe = maintenance("probe").pipe( - Effect.timeoutOrElse({ - duration: MAINTENANCE_REQUEST_DEADLINE_MS, - orElse: () => - Effect.fail( - new MaintenanceProtocolError({ - message: "Control request timed out", - reason: "transport", - }), - ), - }), - ); - const stop = maintenance("stop"); - - return { - probe, - stop, - awaitClose: (onOpen = Effect.void) => - Effect.scoped( - Effect.gen(function* () { - const socket = yield* NodeSocket.makeNet({ - path: endpointPath(endpoint), - openTimeout: MAINTENANCE_REQUEST_DEADLINE_MS, - }); - const write = yield* socket.writer; - const prefaceFailure = yield* Deferred.make(); - const read = socket.runRaw(() => Effect.void, { - onOpen: write( - encodePreface({ - kind: "rpc", - release: options.rpcRelease ?? STACK_RPC_RELEASE, - stackId: options.stackId, - ownerSessionId: options.ownerSessionId, - }), - ).pipe( - Effect.matchEffect({ - onFailure: (error) => Deferred.fail(prefaceFailure, error).pipe(Effect.asVoid), - onSuccess: () => onOpen, - }), - ), - }); - return yield* Effect.raceFirst(read, Deferred.await(prefaceFailure)).pipe( - Effect.catchFilter( - Socket.SocketCloseError.filterClean((code) => code === 1000), - () => Effect.void, - ), - ); - }), - ), - rpc: Effect.gen(function* () { - const socket = yield* NodeSocket.makeNet({ - path: endpointPath(endpoint), - openTimeout: MAINTENANCE_REQUEST_DEADLINE_MS, - }); - const controlSocket = makeControlRpcSocket(socket, { - rpcRelease: options.rpcRelease, - stackId: options.stackId, - ownerSessionId: options.ownerSessionId, - }); - const protocol = yield* RpcClient.makeProtocolSocket().pipe( - Effect.provideService(Socket.Socket, controlSocket), - Effect.provideService(RpcSerialization.RpcSerialization, RpcSerialization.json), - ); - return yield* RpcClient.make(StackRpcGroup).pipe( - Effect.provideService(RpcClient.Protocol, protocol), - ); - }), - }; -}; diff --git a/packages/stack/src/control/FrameCodec.ts b/packages/stack/src/control/FrameCodec.ts deleted file mode 100644 index 418bf46631..0000000000 --- a/packages/stack/src/control/FrameCodec.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { Effect, Schema } from "effect"; -import { MaintenanceProtocolError } from "./MaintenanceProtocol.ts"; - -export const MAINTENANCE_MAX_FRAME_BYTES = 64 * 1024; -/** RPC payloads may include the complete retained log window plus serialization overhead. */ -export const RPC_MAX_FRAME_BYTES = 2 * 1024 * 1024; - -const textEncoder = new TextEncoder(); -const textDecoder = new TextDecoder("utf-8", { fatal: true }); - -/** JSON values accepted by the maintenance framing layer. RPC payload schemas remain in RpcGroup. */ -export type JsonValue = - | null - | boolean - | number - | string - | ReadonlyArray - | { readonly [key: string]: JsonValue }; - -const encodeJson = (value: JsonValue): Effect.Effect => - Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))(value).pipe( - Effect.map((text) => textEncoder.encode(text)), - Effect.mapError( - () => new MaintenanceProtocolError({ message: "Unable to encode control JSON" }), - ), - ); - -const encodeBoundedFrame = ( - body: Uint8Array, - maxBytes: number, -): Effect.Effect => { - if (body.byteLength > maxBytes) { - return Effect.fail( - new MaintenanceProtocolError({ message: "Control frame exceeds size limit" }), - ); - } - const frame = new Uint8Array(body.byteLength + 4); - new DataView(frame.buffer).setUint32(0, body.byteLength, false); - frame.set(body, 4); - return Effect.succeed(frame); -}; - -/** Encode one bounded uint32-big-endian frame containing JSON. */ -export const encodeFrame = ( - value: JsonValue, -): Effect.Effect => - encodeJson(value).pipe( - Effect.flatMap((body) => encodeBoundedFrame(body, MAINTENANCE_MAX_FRAME_BYTES)), - ); - -export const decodeFrame = ( - frame: Uint8Array, -): Effect.Effect => - Effect.gen(function* () { - if (frame.byteLength < 4) { - return yield* new MaintenanceProtocolError({ message: "Control frame is truncated" }); - } - const length = new DataView(frame.buffer, frame.byteOffset, 4).getUint32(0, false); - if (length > MAINTENANCE_MAX_FRAME_BYTES) { - return yield* new MaintenanceProtocolError({ message: "Control frame exceeds size limit" }); - } - if (frame.byteLength !== length + 4) { - return yield* new MaintenanceProtocolError({ message: "Control frame length mismatch" }); - } - const text = yield* Effect.try({ - try: () => textDecoder.decode(frame.slice(4)), - catch: () => new MaintenanceProtocolError({ message: "Invalid control JSON" }), - }); - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text).pipe( - Effect.mapError(() => new MaintenanceProtocolError({ message: "Invalid control JSON" })), - ); - if (!isJsonValue(parsed)) { - return yield* new MaintenanceProtocolError({ message: "Control JSON value is invalid" }); - } - return parsed; - }); - -const isJsonValue = (value: unknown): value is JsonValue => { - if (value === null || typeof value === "string" || typeof value === "boolean") return true; - if (typeof value === "number") return Number.isFinite(value); - if (Array.isArray(value)) return value.every(isJsonValue); - if (typeof value !== "object") return false; - return Object.values(value).every(isJsonValue); -}; - -/** Incremental bounded frame decoder. Instantiate one per connection execution. */ -export class FrameDecoder { - private buffer = new Uint8Array(0); - - push( - chunk: Uint8Array, - maxFrameBytes = MAINTENANCE_MAX_FRAME_BYTES, - ): Effect.Effect, MaintenanceProtocolError> { - if (chunk.byteLength === 0) return Effect.succeed([]); - const combined = new Uint8Array(this.buffer.byteLength + chunk.byteLength); - combined.set(this.buffer); - combined.set(chunk, this.buffer.byteLength); - this.buffer = combined; - const frames: Uint8Array[] = []; - while (this.buffer.byteLength >= 4) { - const length = new DataView(this.buffer.buffer, this.buffer.byteOffset, 4).getUint32( - 0, - false, - ); - if (length > maxFrameBytes) { - return Effect.fail( - new MaintenanceProtocolError({ message: "Control frame exceeds size limit" }), - ); - } - if (this.buffer.byteLength < length + 4) break; - frames.push(this.buffer.slice(0, length + 4)); - this.buffer = this.buffer.slice(length + 4); - } - return Effect.succeed(frames); - } -} - -/** Encode a raw RPC serialization payload in the same bounded frame format. */ -export const encodeRawFrame = ( - value: Uint8Array | string, -): Effect.Effect => - Effect.sync(() => (typeof value === "string" ? textEncoder.encode(value) : value)).pipe( - Effect.flatMap((body) => encodeBoundedFrame(body, RPC_MAX_FRAME_BYTES)), - ); diff --git a/packages/stack/src/control/MaintenanceProtocol.ts b/packages/stack/src/control/MaintenanceProtocol.ts deleted file mode 100644 index 795d120de5..0000000000 --- a/packages/stack/src/control/MaintenanceProtocol.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { Data, Effect, Predicate, Schema } from "effect"; -import { isStackId, StackIdSchema } from "../public/StackId.ts"; - -/** The stable protocol is intentionally frozen independently of Stack RPC releases. */ -const MAINTENANCE_PROTOCOL = "maintenance-v1" as const; -export const MAINTENANCE_MAX_CONCURRENT_REQUESTS = 16; -export const MAINTENANCE_REQUEST_DEADLINE_MS = 5_000; -export const CONTROL_PREFACE_MAX_BYTES = 256; - -export class MaintenanceProtocolError extends Data.TaggedError("MaintenanceProtocolError")<{ - readonly message: string; - readonly reason?: string; -}> {} - -/** Errors that indicate an owner endpoint is unavailable rather than rejecting a request. */ -export const isMaintenanceTransportFailure = (error: unknown): boolean => { - if (Predicate.isTagged(error, "SocketError")) return true; - if (!Predicate.isTagged(error, "MaintenanceProtocolError")) return false; - return Predicate.hasProperty(error, "reason") && error.reason === "transport"; -}; - -const ownerSessionIdIsValid = (value: string): boolean => - value.length > 0 && value.length <= 128 && /^[A-Za-z0-9_-]+$/.test(value); - -export const OwnerSessionIdSchema = Schema.String.pipe( - Schema.refine((value): value is string => ownerSessionIdIsValid(value), { - identifier: "OwnerSessionId", - message: "Expected a bounded owner session identifier", - }), -); - -const ReleaseTokenSchema = Schema.String.pipe( - Schema.refine( - (value): value is string => value.length > 0 && value.length <= 128 && /^[^\s]+$/.test(value), - { - identifier: "ControlRelease", - message: "Expected a bounded release token without whitespace", - }, - ), -); - -export const MaintenanceRequestSchema = Schema.Union([ - Schema.Struct({ - op: Schema.Literal("probe"), - stackId: StackIdSchema, - ownerSessionId: OwnerSessionIdSchema, - }), - Schema.Struct({ - op: Schema.Literal("stop"), - stackId: StackIdSchema, - ownerSessionId: OwnerSessionIdSchema, - }), -]); -export type MaintenanceRequest = Schema.Schema.Type; - -const MaintenanceErrorCodeSchema = Schema.Literals([ - "invalid-request", - "stale-session", - "timeout", - "operation-failed", - "unsupported-release", -] as const); -const MaintenanceStackErrorTagSchema = Schema.String; - -export const MaintenanceResponseSchema = Schema.Union([ - Schema.Struct({ - ok: Schema.Literal(true), - op: Schema.Literal("probe"), - ownerSessionId: OwnerSessionIdSchema, - stackId: StackIdSchema, - rpcRelease: ReleaseTokenSchema, - }), - Schema.Struct({ ok: Schema.Literal(true), op: Schema.Literal("stop") }), - Schema.Struct({ - ok: Schema.Literal(false), - error: Schema.Struct({ - tag: MaintenanceErrorCodeSchema, - message: Schema.String, - stackErrorTag: Schema.optionalKey(MaintenanceStackErrorTagSchema), - }), - }), -]); -export type MaintenanceResponse = Schema.Schema.Type; - -export type ControlProtocol = - | { - readonly kind: "maintenance"; - readonly release: typeof MAINTENANCE_PROTOCOL; - readonly stackId: string; - readonly ownerSessionId: string; - } - | { - readonly kind: "rpc"; - readonly release: string; - readonly stackId: string; - readonly ownerSessionId: string; - }; - -const textEncoder = new TextEncoder(); -const textDecoder = new TextDecoder("utf-8", { fatal: true }); - -export const encodePreface = (protocol: ControlProtocol): Uint8Array => - textEncoder.encode( - `supabase-stack/1 ${protocol.kind}:${protocol.release} stackId:${protocol.stackId} ownerSessionId:${protocol.ownerSessionId}\n`, - ); - -export const decodePreface = ( - bytes: Uint8Array, -): Effect.Effect< - { readonly protocol: ControlProtocol; readonly consumed: number }, - MaintenanceProtocolError -> => - Effect.gen(function* () { - const newline = bytes.indexOf(10); - if (newline < 0) { - return yield* new MaintenanceProtocolError({ message: "Control preface is incomplete" }); - } - if (newline + 1 > CONTROL_PREFACE_MAX_BYTES) { - return yield* new MaintenanceProtocolError({ message: "Control preface exceeds size limit" }); - } - const value = yield* Effect.try({ - try: () => textDecoder.decode(bytes.slice(0, newline)), - catch: () => new MaintenanceProtocolError({ message: "Control preface is invalid" }), - }); - const match = - /^supabase-stack\/1 (maintenance|rpc):([^ ]+) stackId:([^ ]+) ownerSessionId:([^ ]+)$/.exec( - value, - ); - if (match === null) { - return yield* new MaintenanceProtocolError({ message: "Control preface is invalid" }); - } - const kind = match[1]; - const release = match[2]; - const stackId = match[3]; - const ownerSessionId = match[4]; - if ( - kind === undefined || - release === undefined || - stackId === undefined || - ownerSessionId === undefined - ) { - return yield* new MaintenanceProtocolError({ message: "Control preface is invalid" }); - } - if (!isStackId(stackId) || !ownerSessionIdIsValid(ownerSessionId)) { - return yield* new MaintenanceProtocolError({ - message: "Control preface identity is invalid", - }); - } - if (kind === "maintenance" && release !== MAINTENANCE_PROTOCOL) { - return yield* new MaintenanceProtocolError({ - message: "Unsupported maintenance protocol release", - }); - } - return { - protocol: - kind === "maintenance" - ? { - kind: "maintenance", - release: MAINTENANCE_PROTOCOL, - stackId, - ownerSessionId, - } - : { - kind: "rpc", - release, - stackId, - ownerSessionId, - }, - consumed: newline + 1, - }; - }); diff --git a/packages/stack/src/control/StackRpc.ts b/packages/stack/src/control/StackRpc.ts deleted file mode 100644 index 97ea96a9d0..0000000000 --- a/packages/stack/src/control/StackRpc.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Schema } from "effect"; -import { Rpc, RpcGroup } from "effect/unstable/rpc"; -import * as RpcClient from "effect/unstable/rpc/RpcClient"; -import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"; -import { EffectStackCredentialsSchema } from "../public/Credentials.ts"; -import { StackConfigSchema } from "../public/Config.ts"; -import { LogQuerySchema, StackLogBatchSchema } from "../public/Logs.ts"; -import { StackStatusSchema } from "../public/Status.ts"; -import { STACK_ERROR_TAGS } from "../public/Errors.ts"; - -/** Pinned release identifier used to detect incompatible live owners. */ -export const STACK_RPC_RELEASE = "stack-rpc-v1@0.2.0" as const; - -const StackRpcErrorTagSchema = Schema.Literals([...STACK_ERROR_TAGS] as const); - -const StackRpcErrorSchema = Schema.Struct({ - tag: StackRpcErrorTagSchema, - message: Schema.String, -}); -export type StackRpcError = Schema.Schema.Type; - -const StackRpc = { - status: Rpc.make("status", { success: StackStatusSchema, error: StackRpcErrorSchema }), - credentials: Rpc.make("credentials", { - success: EffectStackCredentialsSchema, - error: StackRpcErrorSchema, - }), - start: Rpc.make("start", { - payload: Schema.Struct({ config: Schema.optionalKey(StackConfigSchema) }), - success: StackStatusSchema, - error: StackRpcErrorSchema, - }), - destroy: Rpc.make("destroy", { success: Schema.Void, error: StackRpcErrorSchema }), - resetDatabase: Rpc.make("resetDatabase", { - success: StackStatusSchema, - error: StackRpcErrorSchema, - }), - logs: Rpc.make("logs", { - payload: LogQuerySchema, - success: StackLogBatchSchema, - error: StackRpcErrorSchema, - }), -} as const; - -export const StackRpcGroup = RpcGroup.make( - StackRpc.status, - StackRpc.credentials, - StackRpc.start, - StackRpc.destroy, - StackRpc.resetDatabase, - StackRpc.logs, -); -type StackRpcDefinitions = RpcGroup.Rpcs; -export type StackRpcHandlers = { - readonly [Current in StackRpcDefinitions as Current["_tag"]]: Rpc.ToHandlerFn; -}; -export type StackRpcClient = RpcClient.FromGroup; - -export const releaseMismatch = (actualRelease: string): string => - `Incompatible Stack RPC release; expected ${STACK_RPC_RELEASE}, received ${actualRelease}`; diff --git a/packages/stack/src/control/control-transport.integration.test.ts b/packages/stack/src/control/control-transport.integration.test.ts deleted file mode 100644 index fe59588d88..0000000000 --- a/packages/stack/src/control/control-transport.integration.test.ts +++ /dev/null @@ -1,820 +0,0 @@ -import { NodeServices, NodeSocket } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Crypto, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Path, - PlatformError, - Redacted, - Ref, - Scope, -} from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization"; -import * as Socket from "effect/unstable/socket/Socket"; -import { deriveStackId, type StackIdentity } from "../identity/Identity.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import type { StackStatus } from "../public/Status.ts"; -import type { ControlEndpoint } from "../state/Ownership.ts"; -import { - makeControlClient, - startControlServer, - type MaintenanceHandlers, - type ControlServerOptions, -} from "./ControlServer.ts"; -import { - decodeFrame, - encodeFrame, - encodeRawFrame, - FrameDecoder, - MAINTENANCE_MAX_FRAME_BYTES, - type JsonValue, -} from "./FrameCodec.ts"; -import { - CONTROL_PREFACE_MAX_BYTES, - decodePreface, - encodePreface, - MaintenanceProtocolError, -} from "./MaintenanceProtocol.ts"; -import { STACK_RPC_RELEASE, type StackRpcError, type StackRpcHandlers } from "./StackRpc.ts"; - -interface ServerOverrides { - readonly rpcHandlers?: Partial; - readonly maintenanceHandlers?: Partial; - readonly onShutdownReady?: Effect.Effect; -} - -interface ServerSetup { - readonly endpoint?: (root: string, path: Path.Path) => string; - readonly prepare?: (context: { - readonly root: string; - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - }) => Effect.Effect; -} - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const testIdentity: StackIdentity = { - projectRoot: "/tmp/project", - branchContext: "ordinary-workspace", - stackName: "default", -}; - -const withServer = ( - f: (context: { - readonly endpoint: ControlEndpoint; - readonly stackId: string; - readonly ownerSessionId: string; - readonly completion: Deferred.Deferred; - readonly abandoned: Deferred.Deferred; - readonly completionStarted: Deferred.Deferred; - readonly completionRelease: Deferred.Deferred; - readonly stopCalls: Ref.Ref; - readonly maintenanceStarted: Deferred.Deferred; - readonly maintenanceRelease: Deferred.Deferred; - readonly rebind: Effect.Effect; - }) => Effect.Effect, - overrides?: (context: { - readonly stackId: string; - readonly ownerSessionId: string; - readonly status: StackStatus; - readonly completion: Deferred.Deferred; - readonly abandoned: Deferred.Deferred; - readonly completionStarted: Deferred.Deferred; - readonly completionRelease: Deferred.Deferred; - readonly stopCalls: Ref.Ref; - readonly maintenanceStarted: Deferred.Deferred; - readonly maintenanceRelease: Deferred.Deferred; - }) => ServerOverrides, - setup?: ServerSetup, -) => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-control-" }); - const stackId = yield* deriveStackId(testIdentity); - const ownerSessionId = yield* crypto.randomUUIDv4; - const completion = yield* Deferred.make(); - const abandoned = yield* Deferred.make(); - const completionStarted = yield* Deferred.make(); - const completionRelease = yield* Deferred.make(); - const stopCalls = yield* Ref.make(0); - const maintenanceStarted = yield* Deferred.make(); - const maintenanceRelease = yield* Deferred.make(); - const endpoint: ControlEndpoint = { - kind: "unix", - path: setup?.endpoint?.(root, path) ?? path.join(root, "control.sock"), - }; - if (setup?.prepare !== undefined) yield* setup.prepare({ root, fs, path }); - const status: StackStatus = { - id: stackId, - lifecycle: "stopped", - desiredLifecycle: "stopped", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: "eager", - state: "stopped", - })), - artifacts: [], - }; - const defaultRpcHandlers: StackRpcHandlers = { - status: () => Effect.succeed(status), - credentials: () => - Effect.succeed({ - database: { - url: Redacted.make("postgres://localhost"), - password: Redacted.make("secret"), - }, - api: { - publishableKey: "publishable", - secretKey: Redacted.make("secret"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service"), - }, - }), - start: () => Effect.succeed(status), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(status), - logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), - }; - const defaultMaintenanceHandlers: MaintenanceHandlers = { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }; - const custom = - overrides?.({ - stackId, - ownerSessionId, - status, - completion, - abandoned, - completionStarted, - completionRelease, - stopCalls, - maintenanceStarted, - maintenanceRelease, - }) ?? {}; - const options: ControlServerOptions = { - endpoint, - stackId, - ownerSessionId, - maintenanceHandlers: { ...defaultMaintenanceHandlers, ...custom.maintenanceHandlers }, - rpcHandlers: { ...defaultRpcHandlers, ...custom.rpcHandlers }, - onShutdownReady: custom.onShutdownReady, - }; - yield* startControlServer(options); - const rebind = Effect.scoped(startControlServer(options)).pipe( - Effect.exit, - Effect.map(Exit.isFailure), - ); - const info = yield* fs.stat(endpoint.path); - expect(info.mode & 0o777).toBe(0o600); - expect((yield* fs.stat(root)).mode & 0o777).toBe(0o700); - return yield* f({ - endpoint, - stackId, - ownerSessionId, - completion, - abandoned, - completionStarted, - completionRelease, - stopCalls, - maintenanceStarted, - maintenanceRelease, - rebind, - }); - }), - ); - -const concatBytes = (...chunks: ReadonlyArray): Uint8Array => { - const total = chunks.reduce((size, chunk) => size + chunk.byteLength, 0); - const result = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - result.set(chunk, offset); - offset += chunk.byteLength; - } - return result; -}; - -const sendRawAndReadFrame = ( - endpoint: ControlEndpoint, - bytes: Uint8Array, -): Effect.Effect => - sendRawSequenceAndReadFrame(endpoint, [bytes]); - -const sendRawSequenceAndReadFrame = ( - endpoint: ControlEndpoint, - chunks: ReadonlyArray, -): Effect.Effect => - Effect.gen(function* () { - const socket = yield* NodeSocket.makeNet({ - path: endpoint.kind === "unix" ? endpoint.path : endpoint.name, - }); - const write = yield* socket.writer; - const decoder = new FrameDecoder(); - const response = yield* Deferred.make(); - const read = socket - .runRaw((chunk) => - decoder.push(typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk).pipe( - Effect.flatMap((frames) => - Effect.forEach(frames, (frame) => - Deferred.succeed(response, frame).pipe(Effect.asVoid), - ), - ), - Effect.asVoid, - ), - ) - .pipe(Effect.catchTag("SocketError", () => Effect.void)); - const fiber = yield* Effect.forkChild(read); - for (const chunk of chunks) { - yield* write(chunk).pipe( - Effect.mapError((error) => new MaintenanceProtocolError({ message: error.message })), - ); - } - const frame = yield* Deferred.await(response).pipe( - Effect.timeoutOrElse({ - duration: 5_500, - orElse: () => - Effect.fail(new MaintenanceProtocolError({ message: "Timed out waiting for response" })), - }), - Effect.ensuring(Fiber.interrupt(fiber)), - ); - return yield* decodeFrame(frame); - }); - -const makeDestroyRequestFrame = (): Effect.Effect => - Effect.gen(function* () { - const parser = RpcSerialization.json.makeUnsafe(); - const encoded = parser.encode({ - _tag: "Request", - id: "1", - tag: "destroy", - payload: null, - headers: [], - }); - if (encoded === undefined) { - return yield* new MaintenanceProtocolError({ message: "RPC request could not be encoded" }); - } - return yield* encodeRawFrame(encoded); - }); - -describe("control transport", () => { - it.live("rejects a symlinked control directory", () => - Effect.gen(function* () { - const result = yield* withServer(() => Effect.void, undefined, { - endpoint: (root, path) => path.join(root, "control", "control.sock"), - prepare: ({ root, fs, path }) => - Effect.gen(function* () { - const target = path.join(root, "target"); - yield* fs.makeDirectory(target); - yield* fs.symlink(target, path.join(root, "control")); - }), - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - }), - ); - - it.live("creates and cleans a private nested control directory", () => - withPlatform( - Effect.gen(function* () { - const directoryPath = yield* Ref.make(undefined); - yield* withServer( - ({ endpoint }) => - Effect.gen(function* () { - if (endpoint.kind !== "unix") - return yield* Effect.die("control fixture did not select a Unix endpoint"); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const info = yield* fs.stat(path.dirname(endpoint.path)); - expect(info.mode & 0o777).toBe(0o700); - }), - undefined, - { - endpoint: (root, path) => { - const shortRoot = root.slice(root.lastIndexOf("/") + 1); - return path.join("/tmp", `${shortRoot}-nested`, "control.sock"); - }, - prepare: ({ root, path }) => { - const shortRoot = root.slice(root.lastIndexOf("/") + 1); - return Ref.set(directoryPath, path.join("/tmp", `${shortRoot}-nested`)); - }, - }, - ); - const fs = yield* FileSystem.FileSystem; - const directory = yield* Ref.get(directoryPath); - if (directory === undefined) return yield* Effect.die("control directory was not recorded"); - expect(yield* fs.exists(directory)).toBe(false); - }), - ), - ); - - it.live("rejects a control directory with public permissions", () => - Effect.gen(function* () { - const result = yield* withServer(() => Effect.void, undefined, { - prepare: ({ root, fs }) => fs.chmod(root, 0o755), - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - }), - ); - - it.live("keeps the first control server reachable after a competing bind fails", () => - withServer(({ endpoint, stackId, ownerSessionId, rebind }) => - Effect.gen(function* () { - expect(yield* rebind).toBe(true); - const probe = yield* makeControlClient(endpoint, { stackId, ownerSessionId }).probe; - expect(probe.ok).toBe(true); - }), - ), - ); - - it.live("fails maintenance requests when the control endpoint is unavailable", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-control-dead-" }); - const endpoint: ControlEndpoint = { kind: "unix", path: path.join(root, "missing.sock") }; - const client = makeControlClient(endpoint, { - stackId: "a".repeat(64), - ownerSessionId: "dead-owner", - }); - const result = yield* client.stop.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) expect(failure.value).toBeInstanceOf(Socket.SocketError); - } - }), - ), - ); - - it.live("accepts unknown maintenance stack error tags for forward compatibility", () => - withServer( - ({ endpoint, stackId, ownerSessionId }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const response = yield* client.stop; - expect(response).toMatchObject({ - ok: false, - error: { tag: "operation-failed", stackErrorTag: "FutureStackError" }, - }); - }), - () => ({ - maintenanceHandlers: { - stop: Effect.succeed({ - ok: false, - error: { - tag: "operation-failed", - message: "future owner failure", - stackErrorTag: "FutureStackError", - }, - }), - }, - }), - ), - ); - - it.live("reevaluates owner shutdown after a failed start response", () => - withServer( - ({ endpoint, stackId, ownerSessionId, completion }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const rpc = yield* client.rpc; - const result = yield* rpc.start({}).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - yield* Deferred.await(completion).pipe( - Effect.timeoutOrElse({ - duration: 5_500, - orElse: () => - Effect.fail( - new MaintenanceProtocolError({ - message: "failed start did not trigger shutdown reevaluation", - }), - ), - }), - ); - }), - ({ completion }) => ({ - rpcHandlers: { - start: () => - Effect.fail({ - tag: "StackPreparationError", - message: "injected start failure", - } satisfies StackRpcError), - }, - onShutdownReady: Deferred.succeed(completion, undefined).pipe(Effect.asVoid), - }), - ), - ); - - it.live("signals destroy shutdown only after writing the typed response", () => - withServer( - ({ endpoint, stackId, ownerSessionId, completion }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const rpc = yield* client.rpc; - yield* rpc.destroy(undefined); - yield* Deferred.await(completion).pipe( - Effect.timeoutOrElse({ - duration: 5_500, - orElse: () => - Effect.fail( - new MaintenanceProtocolError({ message: "destroy completion was not signaled" }), - ), - }), - ); - }), - ({ completion }) => ({ - onShutdownReady: Deferred.succeed(completion, undefined).pipe(Effect.asVoid), - }), - ), - ); - - it.live("counts a successful destroy send before an immediate disconnect", () => - withServer( - ({ endpoint, stackId, ownerSessionId, completionStarted, completionRelease, completion }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "rpc", - release: STACK_RPC_RELEASE, - stackId, - ownerSessionId, - }); - const frame = yield* makeDestroyRequestFrame(); - const response = yield* sendRawSequenceAndReadFrame(endpoint, [ - concatBytes(preface, frame), - ]); - expect(response).toMatchObject({ _tag: "Exit", requestId: "1" }); - yield* Deferred.await(completionStarted).pipe( - Effect.timeoutOrElse({ - duration: 1_000, - orElse: () => - Effect.fail( - new MaintenanceProtocolError({ - message: "destroy completion did not survive the response disconnect", - }), - ), - }), - ); - yield* Deferred.succeed(completionRelease, undefined); - yield* Deferred.await(completion).pipe( - Effect.timeoutOrElse({ - duration: 1_000, - orElse: () => - Effect.fail( - new MaintenanceProtocolError({ message: "destroy completion was not signaled" }), - ), - }), - ); - }), - ({ completionStarted, completionRelease, completion }) => ({ - onShutdownReady: Deferred.succeed(completionStarted, undefined).pipe( - Effect.andThen(Deferred.await(completionRelease)), - Effect.andThen(Deferred.succeed(completion, undefined)), - Effect.asVoid, - ), - }), - ), - ); - - it.live("reevaluates shutdown after a request-scoped defect and keeps the endpoint usable", () => - withServer( - ({ endpoint, stackId, ownerSessionId, completion }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "rpc", - release: STACK_RPC_RELEASE, - stackId, - ownerSessionId, - }); - const frame = yield* makeDestroyRequestFrame(); - const response = yield* sendRawSequenceAndReadFrame(endpoint, [ - concatBytes(preface, frame), - ]); - expect(response).toMatchObject({ - _tag: "Exit", - requestId: "1", - exit: { _tag: "Failure" }, - }); - yield* Deferred.await(completion); - - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const rpc = yield* client.rpc; - const status = yield* rpc.status(undefined); - expect(status.id).toBe(stackId); - }), - ({ completion }) => ({ - rpcHandlers: { - destroy: () => Effect.die(new Error("injected destroy defect")), - }, - onShutdownReady: Deferred.succeed(completion, undefined).pipe(Effect.asVoid), - }), - ), - ); - - it.live("round-trips typed RPC and stable maintenance on one endpoint", () => - withServer(({ endpoint, stackId, ownerSessionId }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const probe = yield* client.probe; - expect(probe).toMatchObject({ ok: true, op: "probe", stackId, ownerSessionId }); - const rpc = yield* client.rpc; - const observed = yield* rpc.status(undefined); - expect(observed.id).toBe(stackId); - expect(yield* client.stop).toEqual({ ok: true, op: "stop" }); - }), - ), - ); - - it.live("round-trips artifact preparation state alongside dormant capabilities", () => - withServer( - ({ endpoint, stackId, ownerSessionId }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const rpc = yield* client.rpc; - const observed = yield* rpc.status(undefined); - expect(observed.capabilities.find(({ name }) => name === "rest")?.state).toBe("dormant"); - expect(observed.artifacts).toEqual([ - { workloadId: "rest:rest", capability: "rest", state: "downloading" }, - ]); - expect(observed.recovery).toEqual({ - operation: "stop", - message: "injected cleanup diagnostic", - }); - }), - ({ status }) => ({ - rpcHandlers: { - status: () => - Effect.succeed({ - ...status, - lifecycle: "running", - desiredLifecycle: "running", - capabilities: status.capabilities.map((capability) => - capability.name === "rest" ? { ...capability, state: "dormant" } : capability, - ), - artifacts: [{ workloadId: "rest:rest", capability: "rest", state: "downloading" }], - recovery: { operation: "stop", message: "injected cleanup diagnostic" }, - }), - }, - }), - ), - ); - - it.live("round-trips an RPC log batch larger than the maintenance frame cap", () => - withServer( - ({ endpoint, stackId, ownerSessionId }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { stackId, ownerSessionId }); - const rpc = yield* client.rpc; - const result = yield* rpc.logs({}); - expect(result.entries).toHaveLength(1); - expect(result.entries[0]?.message).toHaveLength(MAINTENANCE_MAX_FRAME_BYTES + 16_384); - }), - () => ({ - rpcHandlers: { - logs: () => - Effect.succeed({ - entries: [ - { - cursor: { opaque: "v1_1" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "database" as const, - stream: "stdout" as const, - message: "x".repeat(MAINTENANCE_MAX_FRAME_BYTES + 16_384), - }, - ], - cursor: { opaque: "v1_1" }, - running: false, - }), - }, - }), - ), - ); - - it.live("holds a fragmented preface until the newline and reports the consumed boundary", () => - withPlatform( - Effect.gen(function* () { - const stackId = "a".repeat(64); - const ownerSessionId = "session"; - const preface = encodePreface({ - kind: "rpc", - release: STACK_RPC_RELEASE, - stackId, - ownerSessionId, - }); - const incomplete = yield* decodePreface(preface.slice(0, 3)).pipe(Effect.exit); - expect(Exit.isFailure(incomplete)).toBe(true); - const complete = yield* decodePreface(preface); - expect(complete.consumed).toBe(preface.byteLength); - }), - ), - ); - - it.live("rejects a mismatched stack identity before decoding an oversized RPC frame", () => - withServer(({ endpoint, ownerSessionId }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "rpc", - release: STACK_RPC_RELEASE, - stackId: "b".repeat(64), - ownerSessionId, - }); - const oversized = new Uint8Array(4); - new DataView(oversized.buffer).setUint32(0, MAINTENANCE_MAX_FRAME_BYTES + 1, false); - const response = yield* sendRawAndReadFrame(endpoint, concatBytes(preface, oversized)); - expect(response).toMatchObject({ ok: false, error: { tag: "invalid-request" } }); - }), - ), - ); - - it.live("rejects a stale owner session before decoding an invalid RPC frame", () => - withServer(({ endpoint, stackId }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "rpc", - release: STACK_RPC_RELEASE, - stackId, - ownerSessionId: "stale-session", - }); - const invalid = concatBytes(new Uint8Array([0, 0, 0, 3]), new Uint8Array([123, 125, 0])); - const response = yield* sendRawAndReadFrame(endpoint, concatBytes(preface, invalid)); - expect(response).toMatchObject({ ok: false, error: { tag: "stale-session" } }); - }), - ), - ); - - it.live("rejects an unsupported RPC release before payload parsing", () => - withServer(({ endpoint, stackId, ownerSessionId }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "rpc", - release: "stack-rpc-v9@0.0.0", - stackId, - ownerSessionId, - }); - const invalid = concatBytes(new Uint8Array([0, 0, 0, 1]), new Uint8Array([0xff])); - const response = yield* sendRawAndReadFrame(endpoint, concatBytes(preface, invalid)); - expect(response).toMatchObject({ ok: false, error: { tag: "unsupported-release" } }); - }), - ), - ); - - it.live("rejects an older stack RPC release with an upgrade error", () => - withServer(({ endpoint, stackId, ownerSessionId }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "rpc", - release: "stack-rpc-v1@0.1.0", - stackId, - ownerSessionId, - }); - const invalid = concatBytes(new Uint8Array([0, 0, 0, 1]), new Uint8Array([0xff])); - const response = yield* sendRawAndReadFrame(endpoint, concatBytes(preface, invalid)); - expect(response).toMatchObject({ - ok: false, - error: { - tag: "unsupported-release", - message: `Incompatible Stack RPC release; expected ${STACK_RPC_RELEASE}, received stack-rpc-v1@0.1.0`, - }, - }); - }), - ), - ); - - it.live("dispatches exactly one maintenance request on a connection", () => { - return withServer( - ({ endpoint, stackId, ownerSessionId, stopCalls }) => - Effect.gen(function* () { - const preface = encodePreface({ - kind: "maintenance", - release: "maintenance-v1", - stackId, - ownerSessionId, - }); - const first = yield* encodeFrame({ op: "stop", stackId, ownerSessionId }); - const second = yield* encodeFrame({ op: "stop", stackId, ownerSessionId }); - const response = yield* sendRawAndReadFrame( - endpoint, - concatBytes(preface, first, second), - ); - expect(response).toMatchObject({ ok: true, op: "stop" }); - expect(yield* Ref.get(stopCalls)).toBe(1); - }), - ({ stopCalls }) => ({ - maintenanceHandlers: { - stop: Ref.update(stopCalls, (count) => count + 1).pipe( - Effect.as({ ok: true, op: "stop" }), - ), - }, - }), - ); - }); - - it.effect("allows validated stop to outlive the admission deadline", () => { - return withServer( - ({ endpoint, stackId, ownerSessionId, maintenanceStarted, maintenanceRelease }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { - stackId, - ownerSessionId, - }); - const stopCompleted = yield* Deferred.make(); - const stopFiber = yield* Effect.forkChild( - client.stop.pipe(Effect.ensuring(Deferred.succeed(stopCompleted, undefined))), - ); - yield* Deferred.await(maintenanceStarted); - yield* TestClock.adjust("6 seconds"); - yield* Effect.yieldNow; - const pending = yield* Deferred.poll(stopCompleted); - expect(Option.isNone(pending)).toBe(true); - yield* Deferred.succeed(maintenanceRelease, undefined); - expect(yield* Fiber.join(stopFiber)).toMatchObject({ ok: true, op: "stop" }); - }), - ({ maintenanceStarted, maintenanceRelease }) => { - return { - maintenanceHandlers: { - stop: Effect.gen(function* () { - yield* Deferred.succeed(maintenanceStarted, undefined); - yield* Deferred.await(maintenanceRelease); - return { ok: true, op: "stop" }; - }), - }, - }; - }, - ); - }); - - it.live("runs maintenance completion only after the response is flushed", () => { - return withServer( - ({ endpoint, stackId, ownerSessionId, completionStarted, completionRelease, completion }) => - Effect.gen(function* () { - const client = makeControlClient(endpoint, { - stackId, - ownerSessionId, - }); - const stopFiber = yield* Effect.forkChild(client.stop, { startImmediately: true }); - yield* Deferred.await(completionStarted); - const response = yield* Fiber.join(stopFiber); - expect(response).toMatchObject({ ok: true, op: "stop" }); - yield* Deferred.succeed(completionRelease, undefined); - yield* Deferred.await(completion); - }), - ({ completionStarted, completionRelease, completion }) => ({ - onShutdownReady: Deferred.succeed(completionStarted, undefined).pipe( - Effect.andThen(Deferred.await(completionRelease)), - Effect.andThen(Deferred.succeed(completion, undefined)), - Effect.asVoid, - ), - }), - ); - }); - - it.live("fails closed on bounded frame and preface violations", () => - withPlatform( - Effect.gen(function* () { - const decoder = new FrameDecoder(); - const oversized = new Uint8Array(4); - new DataView(oversized.buffer).setUint32(0, MAINTENANCE_MAX_FRAME_BYTES + 1, false); - const frameResult = yield* decoder.push(oversized).pipe(Effect.exit); - expect(Exit.isFailure(frameResult)).toBe(true); - const preface = new Uint8Array(CONTROL_PREFACE_MAX_BYTES + 1); - preface.fill(120); - const prefaceResult = yield* decodePreface(preface).pipe(Effect.exit); - expect(Exit.isFailure(prefaceResult)).toBe(true); - }), - ), - ); - - it.live( - "closes a connection that never sends a preface by the maintenance deadline", - () => - withServer(({ endpoint }) => - Effect.gen(function* () { - const response = yield* sendRawSequenceAndReadFrame(endpoint, []); - expect(response).toMatchObject({ ok: false, error: { tag: "timeout" } }); - }), - ), - 7_000, - ); -}); diff --git a/packages/stack/src/effect.integration.test.ts b/packages/stack/src/effect.integration.test.ts new file mode 100644 index 0000000000..eb488207a7 --- /dev/null +++ b/packages/stack/src/effect.integration.test.ts @@ -0,0 +1,237 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, expectTypeOf, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; +import { tmpdir } from "node:os"; +import { + create, + discover, + open, + postgres, + type DatabaseInstance, + type ServiceInstance, +} from "./effect.ts"; + +const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +it.live("registers and discovers saved definitions without inventing live observations", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-api-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${root}/cache`, + runtime: "native", + } satisfies Parameters[0]; + const stack = yield* create(options); + const entries = yield* discover(options); + expect(entries).toHaveLength(1); + expect(entries[0]?.definition.id).toBe(stack.id); + expect(entries[0]?.host).toBeUndefined(); + expect(entries[0]?.definition.instances).toEqual([]); + expect(yield* stack.services.list).toEqual([]); + expect(yield* stack.composition.describe).toEqual({ members: [], dependencies: [] }); + expect((yield* discover(options))[0]?.host).toBeUndefined(); + const duplicate = yield* Effect.flip(create(options)); + expect(duplicate.message).toContain("already exists"); + const reopened = yield* open({ ...options, id: stack.id }); + expect(reopened.id).toBe(stack.id); + const missing = yield* Effect.flip(open({ ...options, id: "missing" })); + expect(missing.message).toContain("does not exist"); + + const database = stack.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("test"), + jwtSecret: Redacted.make("test"), + jwtExpiry: 3600, + }, + }); + const rest = stack.services.create({ + service: "rest", + config: { databaseUrl: "postgres://external" }, + }); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + }).pipe(Effect.scoped, Effect.provide(layer)), +); + +const resetDataStory = (runtime: "native" | "docker") => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-reset-data-${runtime}-` }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + runtime, + } satisfies Parameters[0]; + const stack = yield* create(options); + yield* Effect.acquireUseRelease( + Effect.succeed(stack), + (current) => + Effect.gen(function* () { + const target = yield* current.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("reset-target-password"), + jwtSecret: Redacted.make("reset-target-jwt-secret"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + const sibling = yield* current.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("reset-sibling-password"), + jwtSecret: Redacted.make("reset-sibling-jwt-secret"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + const composition = { + members: [ + { id: target.id, activation: "lazy" as const }, + { id: sibling.id, activation: "eager" as const }, + ], + dependencies: [], + }; + yield* current.composition.configure(composition); + + yield* current.composition.start; + const armed = yield* target.status; + expect(armed.lifecycle).toBe("stopped"); + expect(armed.wakeEnabled).toBe(true); + const armedReset = yield* Effect.flip(target.resetData); + expect(armedReset.message).toContain("must be stopped with wake disabled"); + yield* current.composition.stop; + expect((yield* target.status).wakeEnabled).toBe(false); + + yield* target.start; + yield* target.ready; + yield* sibling.start; + yield* sibling.ready; + const targetCredentials = yield* target.credentials({ from: "runtime" }); + const siblingCredentials = yield* sibling.credentials({ from: "runtime" }); + const targetUrl = targetCredentials.databaseUrl; + const siblingUrl = siblingCredentials.databaseUrl; + if (targetUrl === undefined || siblingUrl === undefined) + return yield* Effect.die("database credentials missing"); + + const runSql = Effect.fn("ResetData.runSql")(( + client: typeof current, + url: string, + sql: string, + ) => { + const stdout: Array = []; + const stderr: Array = []; + return client.tools + .run(postgres.psql({ major: 17 }), { + args: ["--dbname", url, "-Atc", sql], + stdout: (bytes) => Effect.sync(() => stdout.push(new TextDecoder().decode(bytes))), + stderr: (bytes) => Effect.sync(() => stderr.push(new TextDecoder().decode(bytes))), + }) + .pipe( + Effect.map((result) => ({ + ...result, + stdout: stdout.join(""), + stderr: stderr.join(""), + })), + ); + }); + const assertSql = Effect.fn("ResetData.assertSql")( + (client: typeof current, url: string, sql: string) => + runSql(client, url, sql).pipe( + Effect.tap((result) => + Effect.sync(() => { + expect(result.exitCode, result.stderr).toBe(0); + }), + ), + ), + ); + + const runningReset = yield* Effect.flip(target.resetData); + expect(runningReset.message).toContain("must be stopped with wake disabled"); + yield* assertSql( + current, + siblingUrl, + "CREATE TABLE reset_sibling_sentinel(value text NOT NULL); INSERT INTO reset_sibling_sentinel VALUES ('survives');", + ); + yield* assertSql( + current, + targetUrl, + "CREATE TABLE reset_rows(value text NOT NULL); INSERT INTO reset_rows VALUES ('owned'); CREATE ROLE reset_role", + ); + yield* assertSql(current, targetUrl, "CREATE DATABASE reset_extra"); + const beforeStatus = yield* target.status; + const beforeComposition = yield* current.composition.describe; + const targetPort = new URL(targetUrl).port; + + yield* target.stop; + yield* target.resetData; + const resetStatus = yield* target.status; + expect(resetStatus.lifecycle).toBe("stopped"); + expect(resetStatus.wakeEnabled).toBe(false); + expect(resetStatus.config).toEqual(beforeStatus.config); + expect(resetStatus.id).toBe(target.id); + expect(yield* current.composition.describe).toEqual(beforeComposition); + const resetCredentials = yield* target.credentials({ from: "runtime" }); + if (resetCredentials.databaseUrl === undefined) + return yield* Effect.die("reset database credentials missing"); + expect(new URL(resetCredentials.databaseUrl).port).toBe(targetPort); + expect((yield* sibling.status).lifecycle).toBe("running"); + const siblingMarker = yield* assertSql( + current, + siblingUrl, + "SELECT value FROM reset_sibling_sentinel", + ); + expect(siblingMarker.stdout.trim()).toBe("survives"); + + yield* target.start; + yield* target.ready; + const cleared = yield* assertSql( + current, + targetUrl, + "SELECT to_regclass('public.reset_rows') IS NULL AND NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'reset_role') AND NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'reset_extra')", + ); + expect(cleared.stdout.trim()).toBe("t"); + + yield* target.stop; + yield* target.resetData; + yield* current.stop; + const reopened = yield* open({ ...options, id: current.id }); + const reopenedTarget = yield* reopened.services.get(target.id); + if (reopenedTarget.service !== "database") + return yield* Effect.die("reopened target is not a database"); + yield* reopenedTarget.start; + yield* reopenedTarget.ready; + const reopenedCredentials = yield* reopenedTarget.credentials({ from: "runtime" }); + if (reopenedCredentials.databaseUrl === undefined) + return yield* Effect.die("reopened database credentials missing"); + expect(new URL(reopenedCredentials.databaseUrl).port).toBe(targetPort); + const reopenedCleared = yield* assertSql( + reopened, + reopenedCredentials.databaseUrl, + "SELECT to_regclass('public.reset_rows') IS NULL AND NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'reset_role') AND NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'reset_extra')", + ); + expect(reopenedCleared.stdout.trim()).toBe("t"); + }), + (owned) => + Effect.exit(owned.destroy).pipe( + Effect.map((exit) => { + expect(Exit.isSuccess(exit)).toBe(true); + }), + ), + ); + }).pipe(Effect.scoped, Effect.provide(layer)); + +it.live("resets native database data through the public RPC", () => resetDataStory("native"), { + timeout: 10 * 60_000, +}); + +it.live("resets Docker database data through the public RPC", () => resetDataStory("docker"), { + timeout: 15 * 60_000, +}); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 3c8709143e..54a602ff25 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -1 +1,458 @@ -export * from "./public/index.ts"; +import { Crypto, Deferred, Effect, Fiber, Layer, Match, Ref, Scope, Schema, Stream } from "effect"; +import { HttpClient } from "effect/unstable/http"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { RpcClient, RpcSerialization } from "effect/unstable/rpc"; +import { RpcClientError } from "effect/unstable/rpc/RpcClientError"; +import { connectHost, launchHost, waitForOwnerExit } from "./HostProcess.ts"; +import type { SupabaseCompositionOptions } from "./composition/Supabase.ts"; +import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; +import * as State from "./State.ts"; +import type { SavedStack } from "./State.ts"; +import { + StackError, + StackErrorSchema, + StackRpc, + type Definition, + type Observation, +} from "./Rpc.ts"; +import { ServiceCreation } from "./services/Catalog.ts"; +import type { DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import type { PgProveOptions, PostgresTool } from "./Tools.ts"; + +export { postgres } from "./Tools.ts"; +export { StackError } from "./Rpc.ts"; +export type { ServiceCreation } from "./services/Catalog.ts"; +export type { CompositionConfig } from "./Orchestrator.ts"; +export type { SupabaseCompositionOptions } from "./composition/Supabase.ts"; +export type { Observation } from "./Rpc.ts"; +export type { DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; +export type { PgProveOptions } from "./Tools.ts"; + +const stateFor = (root: string) => State.Service.pipe(Effect.provide(State.layer({ root }))); + +/** Storage locations shared by clients and the detached stack owner. */ +export interface StackLocations { + readonly stateRoot: string; + readonly cacheRoot: string; +} +/** Registers an isolated stack identity without starting service processes. */ +export interface CreateOptions extends StackLocations { + readonly projectRoot: string; + readonly name?: string; + readonly runtime: "native" | "docker" | "podman"; +} +/** Opens a previously registered stack. */ +export interface OpenOptions extends StackLocations { + readonly id: string; +} + +const failure = (operation: string, cause: unknown): StackError => + Schema.is(StackErrorSchema)(cause) + ? new StackError(cause) + : new StackError({ + operation, + message: Schema.is(RpcClientError)(cause) + ? `Owner response unavailable; the request outcome is uncertain: ${cause.message}` + : cause instanceof Error + ? cause.message + : String(cause), + }); + +type Kind = ServiceCreation["service"]; +/** An individually identified service controlled through the owner. */ +export interface ServiceInstance { + readonly id: string; + readonly service: K; + readonly start: Effect.Effect; + readonly ready: Effect.Effect; + readonly stop: Effect.Effect; + readonly restart: ( + input?: Pick, "config">, + ) => Effect.Effect; + readonly destroy: Effect.Effect; + readonly prepare: Effect.Effect; + readonly status: Effect.Effect; + readonly followStatus: Stream.Stream; + readonly logs: Stream.Stream< + { readonly stream: "stdout" | "stderr"; readonly bytes: Uint8Array }, + StackError + >; + readonly credentials: (options?: { + readonly from?: "host" | "runtime"; + }) => Effect.Effect>, StackError>; +} +/** A database instance with stopped-data snapshot operations. */ +export interface DatabaseInstance extends ServiceInstance<"database"> { + readonly exportSnapshot: (destination: string) => Effect.Effect; + readonly restoreSnapshot: (source: string) => Effect.Effect; + /** Removes database-owned data while preserving the instance registration. */ + readonly resetData: Effect.Effect; +} +/** Maps creation discriminators to their supported instance operations. */ +export type ServiceInstances = { + [K in Kind]: K extends "database" ? DatabaseInstance : ServiceInstance; +}; +type AnyInstance = ServiceInstances[Kind]; +/** An attached finite command with backpressured byte streams. */ +export interface ToolOptions { + readonly args?: ReadonlyArray; + readonly env?: Readonly>; + readonly pgProve?: PgProveOptions; + readonly stdin?: Stream.Stream; + readonly stdout: (bytes: Uint8Array) => Effect.Effect; + readonly stderr: (bytes: Uint8Array) => Effect.Effect; +} +/** A client handle; its lifetime does not own service processes. */ +export interface Stack { + readonly id: string; + readonly services: { + readonly create: ( + creation: Input, + ) => Effect.Effect; + readonly get: (id: string) => Effect.Effect; + readonly list: Effect.Effect, StackError>; + }; + readonly composition: { + readonly supabase: ( + services: ReadonlyArray, + options?: SupabaseCompositionOptions, + ) => Effect.Effect, StackError>; + readonly configure: (config: Orchestrator.CompositionConfig) => Effect.Effect; + readonly describe: Effect.Effect; + readonly start: Effect.Effect, StackError>; + readonly stop: Effect.Effect, StackError>; + readonly restart: Effect.Effect, StackError>; + }; + readonly stop: Effect.Effect; + readonly destroy: Effect.Effect; + readonly tools: { + readonly run: ( + tool: PostgresTool, + options: ToolOptions, + ) => Effect.Effect<{ readonly jobId: string; readonly exitCode: number }, E | StackError, R>; + }; +} + +type Client = Effect.Success>; +const clientFor = (port: number) => + RpcClient.make(StackRpc).pipe( + Effect.provide( + RpcClient.layerProtocolHttp({ url: `http://127.0.0.1:${port}/rpc` }).pipe( + Layer.provide(RpcSerialization.layerNdjson), + ), + ), + ); + +const makeHandle = Effect.fn("Stack.makeHandle")(function* ( + state: State.Interface, + saved: SavedStack, + locations: StackLocations, +) { + const http = yield* HttpClient.HttpClient; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const endpointFor = (live: boolean) => + (live + ? launchHost(state, { ...locations, stackId: saved.id }) + : connectHost(state, saved.id) + ).pipe( + Effect.provideService(HttpClient.HttpClient, http), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + ); + const client = (live: boolean) => + Effect.gen(function* () { + const endpoint = yield* endpointFor(live); + return yield* clientFor(endpoint.port); + }).pipe(Effect.provideService(HttpClient.HttpClient, http)); + const call = ( + operation: string, + run: (rpc: Client) => Effect.Effect, + live = true, + ) => + Effect.scoped(Effect.flatMap(client(live), run)).pipe( + Effect.mapError((cause) => failure(operation, cause)), + ); + const stream = (operation: string, run: (rpc: Client) => Stream.Stream) => + Stream.unwrap(Effect.map(client(false), run)).pipe( + Stream.mapError((cause) => failure(operation, cause)), + ); + const shutdown = Effect.fn("Stack.shutdown")(function* (destroy: boolean) { + const operation = destroy ? "destroy" : "shutdown"; + const endpoint = yield* Effect.scoped( + Effect.gen(function* () { + const endpoint = yield* endpointFor(destroy); + yield* clientFor(endpoint.port).pipe( + Effect.provideService(HttpClient.HttpClient, http), + Effect.flatMap((rpc) => rpc.shutdown({ destroy })), + ); + return endpoint; + }), + ).pipe(Effect.mapError((cause) => failure(operation, cause))); + yield* waitForOwnerExit(endpoint.pid).pipe( + Effect.mapError((cause) => failure("shutdown-exit", cause)), + ); + }); + + const common = (id: string, service: K): ServiceInstance => ({ + id, + service, + start: call("start", (rpc) => rpc.startService({ id })), + ready: call("ready", (rpc) => rpc.readyService({ id }), false), + stop: call("stop", (rpc) => rpc.stopService({ id })), + restart: (input) => + input === undefined + ? call("restart", (rpc) => rpc.restartService({ id })) + : Schema.decodeUnknownEffect(ServiceCreation)({ service, config: input.config }).pipe( + Effect.mapError((cause) => failure("restart", cause)), + Effect.flatMap((creation) => + call("restart", (rpc) => rpc.restartService({ id, config: creation })), + ), + ), + destroy: call("destroy", (rpc) => rpc.destroyService({ id })), + prepare: call("prepare", (rpc) => rpc.prepareService({ id })), + status: call("status", (rpc) => rpc.status({ id }), false), + followStatus: stream("followStatus", (rpc) => rpc.followStatus({ id })), + logs: stream("logs", (rpc) => rpc.logs({ id })), + credentials: (options) => + call("credentials", (rpc) => rpc.credentials({ id, from: options?.from ?? "host" })), + }); + function instance(definition: { + id: string; + creation: Input; + }): ServiceInstances[Input["service"]]; + function instance(definition: Definition): AnyInstance { + const { id, creation } = definition; + switch (creation.service) { + case "database": + return { + ...common(id, "database"), + exportSnapshot: (destination) => + call("exportSnapshot", (rpc) => rpc.exportSnapshot({ id, destination })), + restoreSnapshot: (source) => + call("restoreSnapshot", (rpc) => rpc.restoreSnapshot({ id, source })), + resetData: call("resetData", (rpc) => rpc.resetData({ id })), + }; + case "rest": + return common(id, "rest"); + case "auth": + return common(id, "auth"); + case "realtime": + return common(id, "realtime"); + case "storage": + return common(id, "storage"); + case "imgproxy": + return common(id, "imgproxy"); + case "functions": + return common(id, "functions"); + case "studio": + return common(id, "studio"); + case "pgmeta": + return common(id, "pgmeta"); + case "mail": + return common(id, "mail"); + case "analytics": + return common(id, "analytics"); + case "vector": + return common(id, "vector"); + case "pooler": + return common(id, "pooler"); + } + } + function create( + creation: Input, + ): Effect.Effect; + function create(creation: ServiceCreation): Effect.Effect { + return call("createService", (rpc) => rpc.createService(creation)).pipe(Effect.map(instance)); + } + const run = Effect.fn("Stack.runTool")(function* ( + tool: PostgresTool, + options: ToolOptions, + ) { + return yield* Effect.scoped( + Effect.gen(function* () { + const rpc = yield* client(true).pipe(Effect.mapError((cause) => failure("tool", cause))); + const attachmentId = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => failure("tool", cause)), + ); + const scope = yield* Scope.Scope; + const inputFailure = yield* Deferred.make(); + const result = yield* Ref.make<{ jobId: string; exitCode: number } | undefined>(undefined); + const sender = yield* Ref.make | undefined>(undefined); + yield* rpc + .runTool({ + attachmentId, + tool, + args: options.args ?? [], + env: options.env ?? {}, + ...(options.pgProve === undefined ? {} : { pgProve: options.pgProve }), + stdin: options.stdin !== undefined, + }) + .pipe( + Stream.mapError((cause) => failure("tool", cause)), + Stream.runForEach((event): Effect.Effect => + Match.valueTags(event, { + Attached: () => + options.stdin === undefined + ? Effect.void + : options.stdin.pipe( + Stream.runForEach((bytes) => + Effect.forEach( + Array.from({ length: Math.ceil(bytes.length / 65536) }, (_, index) => + bytes.subarray(index * 65536, (index + 1) * 65536), + ), + (chunk) => + rpc + .toolInput({ attachmentId, bytes: chunk }) + .pipe(Effect.mapError((cause) => failure("stdin", cause))), + { discard: true }, + ), + ), + Effect.andThen( + rpc + .toolInput({ attachmentId, bytes: null }) + .pipe(Effect.mapError((cause) => failure("stdin", cause))), + ), + Effect.catchIf( + (cause) => + Schema.is(StackErrorSchema)(cause) && + cause.operation === "tool-input-closed", + () => Effect.void, + ), + Effect.tapCause((cause) => Deferred.failCause(inputFailure, cause)), + Effect.forkIn(scope), + Effect.flatMap((fiber) => Ref.set(sender, fiber)), + ), + Stdout: (output) => options.stdout(output.bytes), + Stderr: (output) => options.stderr(output.bytes), + Completed: (completed) => + Ref.set(result, { jobId: completed.jobId, exitCode: completed.exitCode }), + }), + ), + Effect.raceFirst(Deferred.await(inputFailure)), + ); + const input = yield* Ref.get(sender); + if (input !== undefined) yield* Fiber.interrupt(input); + const completed = yield* Ref.get(result); + if (completed === undefined) + return yield* failure("tool", "Tool attachment ended without an exit result"); + return completed; + }), + ); + }); + const savedDefinition = Effect.gen(function* () { + const current = yield* state.read(saved.id); + if (current === undefined) return yield* failure("definition", "Stack does not exist"); + return current; + }).pipe(Effect.mapError((cause) => failure("definition", cause))); + const definitions = savedDefinition.pipe( + Effect.flatMap((current) => + Effect.forEach(current.instances, (entry) => + Schema.decodeUnknownEffect(Schema.toCodecJson(ServiceCreation))(entry.creation).pipe( + Effect.map((creation) => ({ id: entry.id, creation })), + Effect.mapError((cause) => failure("definition", cause)), + ), + ), + ), + ); + return { + id: saved.id, + services: { + create, + get: (id: string) => + definitions.pipe( + Effect.flatMap((entries) => { + const definition = entries.find((entry) => entry.id === id); + return definition === undefined + ? Effect.fail(failure("getService", `Unknown service ${id}`)) + : Effect.succeed(instance(definition)); + }), + ), + list: definitions.pipe(Effect.map((entries) => entries.map(instance))), + }, + composition: { + supabase: (services: ReadonlyArray, options?: SupabaseCompositionOptions) => + call("supabaseComposition", (rpc) => + rpc.supabaseComposition({ + services, + ...(options?.reuseIds === undefined ? {} : { reuseIds: options.reuseIds }), + }), + ).pipe(Effect.map((definitions) => definitions.map(instance))), + configure: (config: Orchestrator.CompositionConfig) => + call("configureComposition", (rpc) => rpc.configureComposition(config)), + describe: savedDefinition.pipe( + Effect.flatMap((current) => + Schema.decodeUnknownEffect(Orchestrator.CompositionConfig)(current.composition), + ), + Effect.mapError((cause) => failure("getComposition", cause)), + ), + start: call("startComposition", (rpc) => rpc.startComposition()), + stop: call("stopComposition", (rpc) => rpc.stopComposition()), + restart: call("restartComposition", (rpc) => rpc.restartComposition()), + }, + stop: shutdown(false), + destroy: shutdown(true), + tools: { run }, + } satisfies Stack; +}); + +/** Registers a stack; a matching existing identity must be opened explicitly. */ +export const create = Effect.fn("Stack.create")( + function* (options: CreateOptions) { + const state = yield* stateFor(options.stateRoot); + const identity = yield* resolveStackIdentity(options); + const id = yield* deriveStackId(identity); + const saved: SavedStack = { + id, + identity, + runtime: options.runtime, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }; + yield* state.withLock( + Effect.gen(function* () { + if ((yield* state.read(id)) !== undefined) + return yield* failure("create", "Stack already exists; use open"); + yield* state.save(saved); + }), + ); + return yield* makeHandle(state, saved, options); + }, + Effect.mapError((cause) => failure("create", cause)), +); + +/** Opens saved definitions without starting or reconstructing live services. */ +export const open = Effect.fn("Stack.open")( + function* (options: OpenOptions) { + const state = yield* stateFor(options.stateRoot); + const saved = yield* state.read(options.id); + if (saved === undefined) return yield* failure("open", "Stack does not exist"); + return yield* makeHandle(state, saved, options); + }, + Effect.mapError((cause) => failure("open", cause)), +); + +/** Lists saved resources separately from the availability of their live owners. */ +export const discover = Effect.fn("Stack.discover")( + function* ( + options: Pick & { + readonly onInvalidState?: (id: string, error: State.StateError) => Effect.Effect; + }, + ) { + const state = yield* State.Service.pipe( + Effect.provide( + State.layer({ root: options.stateRoot, onInvalidState: options.onInvalidState }), + ), + ); + const saved = yield* state.list; + return yield* Effect.forEach(saved, (definition) => + connectHost(state, definition.id).pipe( + Effect.map((host) => ({ definition, host })), + Effect.catchTag("HostProcessError", () => Effect.succeed({ definition, host: undefined })), + ), + ); + }, + Effect.mapError((cause) => failure("discover", cause)), +); diff --git a/packages/stack/src/entrypoints/supervisor-node.ts b/packages/stack/src/entrypoints/supervisor-node.ts deleted file mode 100644 index d3c1b77e35..0000000000 --- a/packages/stack/src/entrypoints/supervisor-node.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { Crypto, Data, Effect, FileSystem, Path, Schema } from "effect"; -// Node's fd3 readiness channel has no FileSystem abstraction, so it's used directly here. -// oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness uses inherited fd3 directly, which has no FileSystem service abstraction. -import * as NodeFs from "node:fs"; -import { - acquireOwnership, - publishOwnership, - type StackRuntimeEnvironmentValue, -} from "../state/Ownership.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { makeSupervisor } from "../supervisor/Supervisor.ts"; -import { makeProductionRuntime } from "../runtime/ProductionRuntime.ts"; -import { startControlServer } from "../control/ControlServer.ts"; -import { STACK_RPC_RELEASE } from "../control/StackRpc.ts"; -import { StackOwnershipConflictError } from "../public/Errors.ts"; -import { - SupervisorArgsSchema, - SupervisorReadySchema, - type SupervisorArgs, -} from "../supervisor/LaunchProtocol.ts"; -import { openSupervisorBootstrapLog } from "../supervisor/BootstrapLog.ts"; - -class SupervisorReadinessError extends Data.TaggedError("SupervisorReadinessError")<{ - readonly message: string; - readonly cause?: unknown; -}> {} - -interface ReadinessState { - written: boolean; -} - -const reportSupervisorFailure = (error: unknown, readiness: ReadinessState): void => { - if (!readiness.written) - try { - const conflict = error instanceof StackOwnershipConflictError; - // The descriptor is owned by the parent launcher, written directly at this process - // boundary. - NodeFs.writeSync( - 3, - `${Schema.encodeSync(Schema.fromJsonString(SupervisorReadySchema))({ ok: false, code: conflict ? "ownership-conflict" : "failed", message: error instanceof Error ? error.message : "Supervisor failed" })}\n`, - ); - NodeFs.closeSync(3); - } catch { - // The parent may have already closed the readiness descriptor. - } - process.exitCode = 1; -}; - -const writeReadiness = ( - value: Schema.Schema.Type, - readiness: ReadinessState, -): Effect.Effect => - Effect.gen(function* () { - const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(SupervisorReadySchema))( - value, - ).pipe( - Effect.mapError( - (cause) => - new SupervisorReadinessError({ - message: "Unable to encode supervisor readiness", - cause, - }), - ), - ); - yield* Effect.try({ - try: () => { - NodeFs.writeSync(3, `${encoded}\n`, undefined, "utf8"); - NodeFs.closeSync(3); - readiness.written = true; - }, - catch: (cause) => - new SupervisorReadinessError({ - message: "Unable to write supervisor readiness", - cause, - }), - }); - }); - -const runSupervisor = (args: SupervisorArgs, readiness: ReadinessState) => - Effect.scoped( - Effect.gen(function* () { - const environment: StackRuntimeEnvironmentValue = { - stateRoot: args.stateRoot, - ...(args.artifactCacheRoot === undefined - ? {} - : { artifactCacheRoot: args.artifactCacheRoot }), - tempRoot: args.tempRoot, - platform: args.platform, - }; - const store = yield* makeStackStateStore({ stateRoot: args.stateRoot }); - const ownerSessionId = args.ownerSessionId; - const lease = yield* acquireOwnership({ - stateRoot: args.stateRoot, - stackId: args.stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - environment, - }); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: args.stateRoot, - ...(args.artifactCacheRoot === undefined - ? {} - : { artifactCacheRoot: args.artifactCacheRoot }), - stackId: args.stackId, - ownerSessionId, - stateStore: store, - context, - }); - const supervisor = yield* makeSupervisor({ - stackId: args.stackId, - ownerSessionId, - stateStore: store, - context, - runtime, - }); - yield* startControlServer({ - endpoint: lease.metadata.endpoint, - stackId: args.stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - maintenanceHandlers: supervisor.maintenanceHandlers, - rpcHandlers: supervisor.rpcHandlers, - onShutdownReady: supervisor.shutdownIfIdle, - }); - yield* publishOwnership(lease); - yield* writeReadiness({ ok: true, stackId: args.stackId, ownerSessionId }, readiness); - yield* supervisor.shutdown; - }), - ).pipe(Effect.provide(NodeServices.layer)); - -const parseSupervisorArgs = (argv: ReadonlyArray) => - Schema.decodeEffect(Schema.fromJsonString(SupervisorArgsSchema))(argv[0] ?? "{}"); - -export const runSupervisorProcess = (argv: ReadonlyArray): Promise => { - const readiness = { written: false } satisfies ReadinessState; - return Effect.runPromise( - parseSupervisorArgs(argv).pipe( - Effect.tap((args) => - Effect.sync(() => { - const log = openSupervisorBootstrapLog(args.stateRoot, args.stackId); - if (log === undefined) return; - const writeChunk = (chunk: string | Uint8Array): void => { - if (typeof chunk === "string") NodeFs.writeSync(log.fd, chunk); - else NodeFs.writeSync(log.fd, chunk); - }; - const hijack = (stream: NodeJS.WriteStream): void => { - stream.write = ((chunk: unknown, encoding?: unknown, callback?: unknown) => { - if (typeof chunk === "string" || chunk instanceof Uint8Array) writeChunk(chunk); - if (typeof encoding === "function") encoding(); - else if (typeof callback === "function") (callback as () => void)(); - return true; - }) as typeof stream.write; - }; - hijack(process.stdout); - hijack(process.stderr); - }), - ), - Effect.flatMap((args) => runSupervisor(args, readiness)), - ), - ).catch((error) => reportSupervisorFailure(error, readiness)); -}; - -if (import.meta.main) { - await runSupervisorProcess(process.argv.slice(2)); -} diff --git a/packages/stack/src/functions/FunctionsBootstrap.ts b/packages/stack/src/functions/FunctionsBootstrap.ts index b8da240a65..a6fad9206c 100644 --- a/packages/stack/src/functions/FunctionsBootstrap.ts +++ b/packages/stack/src/functions/FunctionsBootstrap.ts @@ -1,109 +1,129 @@ -import { Crypto, Effect, FileSystem, Path, PlatformError } from "effect"; -import { StackPreparationError } from "../public/Errors.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import type { StackId } from "../public/StackId.ts"; +import { Crypto, Data, Effect, FileSystem, Path, PlatformError, Schema } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. +import { rmdir } from "node:fs/promises"; +import { StackIdSchema, type StackId } from "../identity/StackId.ts"; + +export class FunctionsBootstrapError extends Data.TaggedError("FunctionsBootstrapError")<{ + readonly message: string; + readonly cause?: unknown; + readonly path?: string; +}> {} export interface FunctionsBootstrapOwner { /** Publishes the stack-owned Edge Runtime main service for the current session. */ readonly write: (input: { readonly content: string; - }) => Effect.Effect; + }) => Effect.Effect; /** Removes only this stack's functions bootstrap root. */ - readonly cleanupAll: Effect.Effect; + readonly cleanupAll: Effect.Effect; } export interface FunctionsBootstrapOwnerOptions { - readonly stateRoot: string; + readonly root: string; readonly stackId: StackId; + readonly instanceId: string; } const failure = (message: string, fields: Readonly> = {}) => - new StackPreparationError({ message, ...fields }); + new FunctionsBootstrapError({ message, ...fields }); const mapFs = ( path: string, operation: string, effect: Effect.Effect, -): Effect.Effect => +): Effect.Effect => effect.pipe(Effect.mapError((cause) => failure(`Unable to ${operation}`, { path, cause }))); -export const makeFunctionsBootstrapOwner = ( +export const makeFunctionsBootstrapOwner = Effect.fn("FunctionsBootstrap.makeOwner")(function* ( options: FunctionsBootstrapOwnerOptions, -): Effect.Effect< - FunctionsBootstrapOwner, - StackPreparationError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const stackPaths = yield* resolveStackPaths(options).pipe( - Effect.mapError((cause) => failure("Unable to resolve functions bootstrap path", { cause })), +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( + Effect.mapError((cause) => failure("Invalid stack identity", { cause })), + ); + if (!/^[a-zA-Z0-9_-]+$/u.test(options.instanceId)) + return yield* failure("Invalid Functions instance identity"); + const root = path.join(options.root, options.instanceId, "runtime", "functions"); + const removeEmptyDirectory = (directory: string) => + Effect.tryPromise({ + try: () => rmdir(directory), + catch: (cause) => + failure("Unable to clean Functions parent directory", { path: directory, cause }), + }).pipe( + Effect.catch((cause) => { + const code = + typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause + ? cause.cause.code + : undefined; + return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" + ? Effect.void + : Effect.fail(cause); + }), ); - const root = path.join(stackPaths.runtime, "functions"); - const write = (input: { - readonly content: string; - }): Effect.Effect => { - if (input.content.includes("\u0000")) - return Effect.fail(failure("Functions bootstrap contains an invalid character")); - const target = path.join(root, "index.ts"); - return Effect.gen(function* () { - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((cause) => - failure("Unable to allocate functions bootstrap file", { cause }), - ), + const write = Effect.fn("FunctionsBootstrap.write")(function* (input: { + readonly content: string; + }) { + if (input.content.includes("\u0000")) + return yield* failure("Functions bootstrap contains an invalid character"); + const target = path.join(root, "index.ts"); + return yield* Effect.gen(function* () { + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => + failure("Unable to allocate functions bootstrap file", { cause }), + ), + ); + const temporary = path.join(root, `.index.ts.${token}.tmp`); + return yield* Effect.gen(function* () { + yield* mapFs( + root, + "create functions bootstrap directory", + fs.makeDirectory(root, { recursive: true, mode: 0o700 }), ); - const temporary = path.join(root, `.index.ts.${token}.tmp`); - return yield* Effect.gen(function* () { - yield* mapFs( - root, - "create functions bootstrap directory", - fs.makeDirectory(root, { recursive: true, mode: 0o700 }), - ); - yield* mapFs(root, "secure functions bootstrap directory", fs.chmod(root, 0o700)); - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* mapFs( - temporary, - "create functions bootstrap file", - fs.open(temporary, { flag: "w", mode: 0o600 }), - ); - yield* mapFs( - temporary, - "write functions bootstrap file", - file.writeAll(new TextEncoder().encode(input.content)), - ); - yield* mapFs(temporary, "sync functions bootstrap file", file.sync); - }), - ); - yield* mapFs(temporary, "secure functions bootstrap file", fs.chmod(temporary, 0o600)); - yield* mapFs(target, "publish functions bootstrap file", fs.rename(temporary, target)); - yield* mapFs( - target, - "secure published functions bootstrap file", - fs.chmod(target, 0o600), - ); - return yield* mapFs( - target, - "resolve published functions bootstrap file", - fs.realPath(target), - ); - }).pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), + yield* mapFs(root, "secure functions bootstrap directory", fs.chmod(root, 0o700)); + yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* mapFs( + temporary, + "create functions bootstrap file", + fs.open(temporary, { flag: "w", mode: 0o600 }), + ); + yield* mapFs( + temporary, + "write functions bootstrap file", + file.writeAll(new TextEncoder().encode(input.content)), + ); + yield* mapFs(temporary, "sync functions bootstrap file", file.sync); + }), ); - }); - }; - - const cleanupAll = mapFs( - root, - "clean functions bootstrap files", - fs.remove(root, { recursive: true, force: true }), - ); - return { write, cleanupAll }; + yield* mapFs(temporary, "secure functions bootstrap file", fs.chmod(temporary, 0o600)); + yield* mapFs(target, "publish functions bootstrap file", fs.rename(temporary, target)); + yield* mapFs(target, "secure published functions bootstrap file", fs.chmod(target, 0o600)); + return yield* mapFs( + target, + "resolve published functions bootstrap file", + fs.realPath(target), + ); + }).pipe( + Effect.ensuring( + fs + .remove(temporary, { force: true }) + .pipe(Effect.catchTag("PlatformError", () => Effect.void)), + ), + ); + }); }); + + const cleanupAll = mapFs( + root, + "clean functions bootstrap files", + fs.remove(root, { recursive: true, force: true }), + ).pipe( + Effect.andThen(removeEmptyDirectory(path.dirname(root))), + Effect.andThen(removeEmptyDirectory(path.dirname(path.dirname(root)))), + Effect.withSpan("FunctionsBootstrap.cleanupAll"), + ); + return { write, cleanupAll }; +}); diff --git a/packages/stack/src/functions/functions-bootstrap.integration.test.ts b/packages/stack/src/functions/functions-bootstrap.integration.test.ts index c1886a8d39..cb1e008534 100644 --- a/packages/stack/src/functions/functions-bootstrap.integration.test.ts +++ b/packages/stack/src/functions/functions-bootstrap.integration.test.ts @@ -1,7 +1,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Effect, FileSystem, Path } from "effect"; -import { StackIdSchema } from "../public/StackId.ts"; +import { StackIdSchema } from "../identity/StackId.ts"; import { makeFunctionsBootstrapOwner } from "./FunctionsBootstrap.ts"; const setupBootstrapOwner = (prefix: string) => @@ -9,7 +9,11 @@ const setupBootstrapOwner = (prefix: string) => const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix }); const stackId = StackIdSchema.make("a".repeat(64)); - const owner = yield* makeFunctionsBootstrapOwner({ stateRoot: root, stackId }); + const owner = yield* makeFunctionsBootstrapOwner({ + root, + stackId, + instanceId: "functions-one", + }); return { fs, root, stackId, owner }; }); @@ -17,29 +21,42 @@ describe("functions bootstrap owner", () => { it.live("publishes a private Functions bootstrap file with restrictive modes", () => Effect.gen(function* () { const path = yield* Path.Path; - const { fs, root, stackId, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-"); + const { fs, root, owner } = yield* setupBootstrapOwner("stack-functions-bootstrap-"); const target = yield* owner.write({ content: "export default 1" }); - expect(target).toContain(path.join(root, stackId, "runtime", "functions", "index.ts")); - expect((yield* fs.stat(path.dirname(target))).mode! & 0o777).toBe(0o700); - expect((yield* fs.stat(target)).mode! & 0o777).toBe(0o600); + expect(target).toContain( + path.join(root, "functions-one", "runtime", "functions", "index.ts"), + ); + expect(((yield* fs.stat(path.dirname(target))).mode ?? 0) & 0o777).toBe(0o700); + expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); expect(yield* fs.readFileString(target)).toBe("export default 1"); }).pipe(Effect.provide(NodeServices.layer)), ); - it.live("cleans the Functions bootstrap file and directory", () => + it.live("cleans one Functions bootstrap while preserving another instance", () => Effect.gen(function* () { const path = yield* Path.Path; const { fs, root, stackId, owner } = yield* setupBootstrapOwner( "stack-functions-bootstrap-cleanup-", ); const target = yield* owner.write({ content: "export default 1" }); + const other = yield* makeFunctionsBootstrapOwner({ + root, + stackId, + instanceId: "functions-two", + }); + const otherTarget = yield* other.write({ content: "export default 2" }); expect(yield* fs.exists(target)).toBe(true); yield* owner.cleanupAll; expect(yield* fs.exists(target)).toBe(false); - expect(yield* fs.exists(path.join(root, stackId, "runtime", "functions"))).toBe(false); + expect(yield* fs.readFileString(otherTarget)).toBe("export default 2"); + expect(yield* fs.exists(path.join(root, "functions-one", "runtime", "functions"))).toBe( + false, + ); + expect(yield* fs.exists(path.join(root, "functions-one"))).toBe(false); + expect(yield* fs.exists(path.join(root, "functions-two"))).toBe(true); }).pipe(Effect.provide(NodeServices.layer)), ); @@ -72,26 +89,27 @@ describe("functions bootstrap owner", () => { yield* fs.symlink(canonicalRoot, configuredRoot); const stackIdValue = StackIdSchema.make("b".repeat(64)); const owner = yield* makeFunctionsBootstrapOwner({ - stateRoot: configuredRoot, + root: configuredRoot, stackId: stackIdValue, + instanceId: "functions-one", }); const target = yield* owner.write({ content: "export default 2" }); const expected = path.join( yield* fs.realPath(canonicalRoot), - stackIdValue, + "functions-one", "runtime", "functions", "index.ts", ); expect(target).toBe(expected); expect(yield* fs.readFileString(target)).toBe("export default 2"); - expect((yield* fs.stat(target)).mode! & 0o777).toBe(0o600); + expect(((yield* fs.stat(target)).mode ?? 0) & 0o777).toBe(0o600); yield* owner.cleanupAll; - expect(yield* fs.exists(path.join(canonicalRoot, stackIdValue, "runtime", "functions"))).toBe( - false, - ); + expect( + yield* fs.exists(path.join(canonicalRoot, "functions-one", "runtime", "functions")), + ).toBe(false); }).pipe(Effect.provide(NodeServices.layer)), ); }); diff --git a/packages/stack/src/functions/serve-main-bundler.integration.test.ts b/packages/stack/src/functions/serve-main-bundler.integration.test.ts index 37e5eac87d..a0de0fa6c2 100644 --- a/packages/stack/src/functions/serve-main-bundler.integration.test.ts +++ b/packages/stack/src/functions/serve-main-bundler.integration.test.ts @@ -3,7 +3,7 @@ import { Data, Deferred, Effect, Exit, FileSystem, Path, Schema, Stream } from " import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { exportJWK, generateKeyPair, SignJWT } from "jose"; -import { bundleServeMainTemplate } from "./serve-main-bundler.ts"; +import { bundleServeMainTemplate } from "../../tests/serve-main-bundler.ts"; type ServeOptions = { readonly handler: (request: Request) => Promise; @@ -23,7 +23,7 @@ describe("stack-owned functions bootstrap", () => { yield* fs.makeDirectory(path.join(root, "hello")); yield* fs.writeFileString(path.join(root, "hello", "index.ts"), "export default 1"); const secret = "bootstrap-test-secret"; - const envRecord = { + const envRecord: Record = { SUPABASE_INTERNAL_FUNCTIONS_ROOT: root, SUPABASE_INTERNAL_JWT_SECRET: secret, KEEP: "yes", @@ -47,7 +47,7 @@ describe("stack-owned functions bootstrap", () => { const sandbox = { Deno: { env: { - get: (name: string) => envRecord[name as keyof typeof envRecord], + get: (name: string) => envRecord[name], toObject: () => envRecord, }, lstat: (filename: string) => @@ -202,12 +202,12 @@ describe("stack-owned functions bootstrap", () => { it.live("starts with malformed optional functions config", () => Effect.gen(function* () { const bundled = yield* bundleServeMainTemplate; - const envRecord = { SUPABASE_INTERNAL_FUNCTIONS_CONFIG: "{" }; + const envRecord: Record = { SUPABASE_INTERNAL_FUNCTIONS_CONFIG: "{" }; let serveOptions: ServeOptions | undefined; const sandbox = { Deno: { env: { - get: (name: string) => envRecord[name as keyof typeof envRecord], + get: (name: string) => envRecord[name], toObject: () => envRecord, }, errors: {}, @@ -278,7 +278,7 @@ describe("stack-owned functions bootstrap", () => { .sign(privateKey), ); const bundled = yield* bundleServeMainTemplate; - const envRecord = { + const envRecord: Record = { SUPABASE_INTERNAL_FUNCTIONS_ROOT: "/functions", SUPABASE_INTERNAL_JWT_SECRET: "secret", SUPABASE_INTERNAL_FUNCTIONS_CONFIG: '{"hello":{"verifyJWT":true}}', @@ -289,7 +289,7 @@ describe("stack-owned functions bootstrap", () => { const sandbox = { Deno: { env: { - get: (name: string) => envRecord[name as keyof typeof envRecord], + get: (name: string) => envRecord[name], toObject: () => envRecord, }, errors: {}, diff --git a/packages/stack/src/functions/serve-main-deps.ts b/packages/stack/src/functions/serve-main-deps.ts index 9d6f699a9f..650f97a13d 100644 --- a/packages/stack/src/functions/serve-main-deps.ts +++ b/packages/stack/src/functions/serve-main-deps.ts @@ -1,5 +1,4 @@ /** Minimal runtime dependencies bundled into the Edge Runtime bootstrap. */ -export const FUNCTIONS_CONTAINER_ROOT = "/__supabase_functions"; export const STATUS_CODE = { OK: 200, diff --git a/packages/stack/src/functions/serve-main-resolver.integration.test.ts b/packages/stack/src/functions/serve-main-resolver.integration.test.ts index fc574fdd3d..16f30cbc18 100644 --- a/packages/stack/src/functions/serve-main-resolver.integration.test.ts +++ b/packages/stack/src/functions/serve-main-resolver.integration.test.ts @@ -110,10 +110,11 @@ describe("Edge Runtime request-time function resolver", () => { fs: nodeFileSystem, }); expect(config).toBeDefined(); + if (config === undefined) return yield* Effect.die("function config was not resolved"); expect( yield* packageJsonContainedFor({ root, - config: { ...config!, importMapPath: "" }, + config: { ...config, importMapPath: "" }, fs: nodeFileSystem, }), ).toBe(true); @@ -310,10 +311,11 @@ describe("Edge Runtime request-time function resolver", () => { fs: nodeFileSystem, }); expect(config).toBeDefined(); + if (config === undefined) return yield* Effect.die("function config was not resolved"); expect( yield* packageJsonContainedFor({ root: configured, - config: { ...config!, importMapPath: "" }, + config: { ...config, importMapPath: "" }, fs: nodeFileSystem, }), ).toBe(true); diff --git a/packages/stack/src/functions/serve-main-resolver.ts b/packages/stack/src/functions/serve-main-resolver.ts index da1806c083..9c9f53e8bd 100644 --- a/packages/stack/src/functions/serve-main-resolver.ts +++ b/packages/stack/src/functions/serve-main-resolver.ts @@ -94,13 +94,14 @@ const relativePath = (base: string, value: string): string => value.length === 0 ? "" : value.startsWith("/") ? value : join(base, value); /** Resolves one request's persisted override/default against the live functions tree. */ -export const resolveFunctionConfig = (options: { - readonly root: string; - readonly slug: string; - readonly overrides: FunctionOverrides; - readonly fs: FunctionFileSystem; -}): Effect.Effect => - Effect.gen(function* () { +export const resolveFunctionConfig = Effect.fn("Functions.resolveFunctionConfig")( + function* (options: { + readonly root: string; + readonly filesRoot?: string; + readonly slug: string; + readonly overrides: FunctionOverrides; + readonly fs: FunctionFileSystem; + }) { const { root, slug, overrides, fs } = options; if (!root.startsWith("/") || !slugPattern.test(slug) || slug === "_shared") return undefined; const rootInfo = yield* optionalInfo(fs, root); @@ -111,6 +112,11 @@ export const resolveFunctionConfig = (options: { if (!canonicalRoot.startsWith("/")) return undefined; const canonicalInfo = yield* optionalInfo(fs, canonicalRoot); if (canonicalInfo === undefined || !canonicalInfo.isDirectory) return undefined; + const filesRoot = + options.filesRoot === undefined + ? canonicalRoot + : yield* fs.realPath(options.filesRoot).pipe(Effect.orElseSucceed(() => "")); + if (!filesRoot.startsWith("/") || !contained(filesRoot, canonicalRoot)) return undefined; const globalDefaults = overrides.$default; const functionOverride = overrides[slug]; // `$default` cannot be a function slug (the slug schema only accepts letters, digits, `_`, @@ -134,7 +140,7 @@ export const resolveFunctionConfig = (options: { if (!(yield* safeRealPath(fs, canonicalRoot, functionDirectory))) return undefined; } const entrypointPath = relativePath(functionDirectory, rawEntrypoint); - if (!(yield* safeRealPath(fs, canonicalRoot, entrypointPath))) return undefined; + if (!(yield* safeRealPath(fs, filesRoot, entrypointPath))) return undefined; const entrypointInfo = yield* optionalInfo(fs, entrypointPath); if (entrypointInfo === undefined || !entrypointInfo.isFile || entrypointInfo.isSymbolicLink) return undefined; @@ -150,7 +156,7 @@ export const resolveFunctionConfig = (options: { ? relativePath(canonicalRoot, globalImportMap) : relativePath(functionDirectory, ""); if (importMapPath.length > 0) { - if (!(yield* safeRealPath(fs, canonicalRoot, importMapPath))) return undefined; + if (!(yield* safeRealPath(fs, filesRoot, importMapPath))) return undefined; const info = yield* optionalInfo(fs, importMapPath); if (info === undefined || !info.isFile || info.isSymbolicLink) return undefined; } else { @@ -158,11 +164,7 @@ export const resolveFunctionConfig = (options: { const path = join(functionDirectory, candidate); const info = yield* optionalInfo(fs, path); if (info !== undefined) { - if ( - !info.isFile || - info.isSymbolicLink || - !(yield* safeRealPath(fs, canonicalRoot, path)) - ) + if (!info.isFile || info.isSymbolicLink || !(yield* safeRealPath(fs, filesRoot, path))) return undefined; importMapPath = path; break; @@ -174,19 +176,19 @@ export const resolveFunctionConfig = (options: { relativePath(functionDirectory, pattern), ); for (const pattern of staticFiles) { - if (!contained(canonicalRoot, pattern)) return undefined; + if (!contained(filesRoot, pattern)) return undefined; const wildcardIndex = pattern.search(globPattern); const prefix = wildcardIndex < 0 ? pattern : pattern.slice(0, wildcardIndex); const searchRoot = wildcardIndex < 0 ? dirname(pattern) : prefix.slice(0, Math.max(0, prefix.lastIndexOf("/"))) || canonicalRoot; - if (!(yield* rejectSymlinkDescendants(fs, canonicalRoot, searchRoot))) return undefined; + if (!(yield* rejectSymlinkDescendants(fs, filesRoot, searchRoot))) return undefined; if (!globPattern.test(pattern)) { const info = yield* optionalInfo(fs, pattern); if ( info !== undefined && - (!(yield* safeRealPath(fs, canonicalRoot, pattern)) || info.isSymbolicLink) + (!(yield* safeRealPath(fs, filesRoot, pattern)) || info.isSymbolicLink) ) return undefined; } @@ -199,7 +201,8 @@ export const resolveFunctionConfig = (options: { verifyJWT: override.verifyJWT ?? override.verify_jwt ?? true, env: override.env, }; - }); + }, +); const packageJsonPathFor = (config: FunctionConfig): string => join(dirname(config.entrypointPath), "package.json"); @@ -221,12 +224,12 @@ export const createWorkerServicePathResolver = (makeTempDirectory: () => string) }; /** Checks package discovery without allowing a package.json symlink to leave the root. */ -export const packageJsonContainedFor = (options: { - readonly root: string; - readonly config: FunctionConfig; - readonly fs: FunctionFileSystem; -}): Effect.Effect => - Effect.gen(function* () { +export const packageJsonContainedFor = Effect.fn("Functions.packageJsonContainedFor")( + function* (options: { + readonly root: string; + readonly config: FunctionConfig; + readonly fs: FunctionFileSystem; + }) { if (!options.root.startsWith("/")) return false; const rootInfo = yield* optionalInfo(options.fs, options.root); if (rootInfo === undefined || (!rootInfo.isDirectory && !rootInfo.isSymbolicLink)) return false; @@ -241,4 +244,5 @@ export const packageJsonContainedFor = (options: { if (packageInfo === undefined || !packageInfo.isFile || packageInfo.isSymbolicLink) return false; return yield* safeRealPath(options.fs, canonicalRoot, packagePath); - }); + }, +); diff --git a/packages/stack/src/functions/serve.main.ts b/packages/stack/src/functions/serve.main.ts index f305d98fe9..c1df67236c 100644 --- a/packages/stack/src/functions/serve.main.ts +++ b/packages/stack/src/functions/serve.main.ts @@ -43,7 +43,6 @@ interface WorkerCreateOptions { readonly workerTimeoutMs: number; readonly noModuleCache: boolean; readonly noNpm: boolean; - readonly importMapPath?: string; readonly envVars: ReadonlyArray; readonly forceCreate: boolean; readonly customModuleRoot: string; @@ -51,7 +50,7 @@ interface WorkerCreateOptions { readonly cpuTimeHardLimitMs: number; readonly decoratorType: string; readonly maybeEntrypoint: string; - readonly context: { readonly useReadSyncFileAPI: boolean }; + readonly context: { readonly useReadSyncFileAPI: boolean; readonly importMapPath?: string }; readonly staticPatterns: ReadonlyArray; } declare const Deno: DenoApi; @@ -76,6 +75,7 @@ const bootstrapConfig = Effect.runSync( return { hostPort: yield* read(Config.string("SUPABASE_INTERNAL_HOST_PORT")), functionsRoot: yield* read(Config.string("SUPABASE_INTERNAL_FUNCTIONS_ROOT")), + filesRoot: yield* read(Config.string("SUPABASE_INTERNAL_FUNCTIONS_FILES_ROOT")), jwtSecret: yield* read(Config.string("SUPABASE_INTERNAL_JWT_SECRET")), supabaseUrl: yield* read(Config.string("SUPABASE_URL")), wallclock: yield* read(Config.string("SUPABASE_INTERNAL_WALLCLOCK_LIMIT_SEC")), @@ -326,18 +326,25 @@ const denoFileSystem: FunctionFileSystem = { Effect.mapError((cause) => new FunctionFileSystemError({ cause })), ), readDirectory: (path) => - Stream.fromAsyncIterable( - Deno.readDir(path), - (cause) => new BootstrapOperationError({ cause }), + Stream.suspend(() => + Stream.fromAsyncIterable( + Deno.readDir(path), + (cause) => new BootstrapOperationError({ cause }), + ).pipe(Stream.map((entry) => entry.name)), ).pipe( - Stream.map((entry) => entry.name), Stream.runCollect, Effect.mapError((cause) => new FunctionFileSystemError({ cause })), ), }; const functionConfig = (slug: string): Effect.Effect => - resolveFunctionConfig({ root: FUNCTIONS_ROOT, slug, overrides: configured, fs: denoFileSystem }); + resolveFunctionConfig({ + root: FUNCTIONS_ROOT, + filesRoot: Option.getOrUndefined(bootstrapConfig.filesRoot), + slug, + overrides: configured, + fs: denoFileSystem, + }); const workerServicePath = createWorkerServicePathResolver(() => Deno.makeTempDirSync({ prefix: "supabase-worker-" }), ); @@ -345,7 +352,11 @@ const workerServicePath = createWorkerServicePathResolver(() => const shouldUsePackageJsonDiscovery = (config: FunctionConfig): Effect.Effect => config.importMapPath ? Effect.succeed(false) - : packageJsonContainedFor({ root: FUNCTIONS_ROOT, config, fs: denoFileSystem }); + : packageJsonContainedFor({ + root: valueOr(bootstrapConfig.filesRoot, FUNCTIONS_ROOT), + config, + fs: denoFileSystem, + }); export function prepareUserRequest(request: Request): Request { const url = new URL(request.url); @@ -405,7 +416,6 @@ Deno.serve({ : 400_000, noModuleCache: true, noNpm, - importMapPath: config.importMapPath, envVars, forceCreate: true, customModuleRoot: "", @@ -413,7 +423,10 @@ Deno.serve({ cpuTimeHardLimitMs: 2000, decoratorType: "tc39", maybeEntrypoint: toFileUrl(config.entrypointPath).href, - context: { useReadSyncFileAPI: true }, + context: { + useReadSyncFileAPI: true, + ...(config.importMapPath === "" ? {} : { importMapPath: config.importMapPath }), + }, staticPatterns: config.staticFiles, }), ); diff --git a/packages/stack/src/gateway/ActivityTracker.ts b/packages/stack/src/gateway/ActivityTracker.ts deleted file mode 100644 index b88524ab0a..0000000000 --- a/packages/stack/src/gateway/ActivityTracker.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Effect } from "effect"; -import type { CapabilityName } from "../public/Capability.ts"; - -/** Tracks gateway work that must keep a lazy capability running. */ -export interface GatewayActivity { - readonly track: ( - capability: CapabilityName, - effect: Effect.Effect, - ) => Effect.Effect; -} - -export interface GatewayActivityCallbacks { - readonly begin: (capability: CapabilityName) => Effect.Effect; - readonly end: (capability: CapabilityName, lease: Lease) => Effect.Effect; -} - -/** Adapts gateway lifetimes to the Supervisor-owned traffic controller. */ -export const makeGatewayActivity = ( - callbacks: GatewayActivityCallbacks, -): Effect.Effect => - Effect.succeed({ - track: (capability, effect) => - Effect.acquireUseRelease( - callbacks.begin(capability), - () => effect, - (lease) => callbacks.end(capability, lease), - ), - } satisfies GatewayActivity); diff --git a/packages/stack/src/gateway/Gateway.ts b/packages/stack/src/gateway/Gateway.ts deleted file mode 100644 index 3932d35db8..0000000000 --- a/packages/stack/src/gateway/Gateway.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { Data, Effect, Exit } from "effect"; -import { GatewayActivationError } from "../public/Errors.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import type { PortField } from "../public/Status.ts"; -import { makeHttpGateway, type HttpGateway, type HttpGatewayOptions } from "./HttpGateway.ts"; -import { makeTcpGateway, type TcpGateway, type TcpGatewayOptions } from "./TcpGateway.ts"; -import type { GatewayActivity } from "./ActivityTracker.ts"; - -/** A private backend endpoint returned only after activation has completed. */ -export interface BackendEndpoint { - readonly host: string; - readonly port: number; -} - -export interface ActivationResult { - readonly capability: CapabilityName; - readonly endpoint: BackendEndpoint; -} - -/** Internal route preflight failure used for request-time discovery. */ -export class GatewayRouteNotFoundError extends Data.TaggedError("GatewayRouteNotFoundError")<{ - readonly message: string; -}> {} - -export interface PreparedGatewayRoute { - readonly resolveBackend: ( - activation: ActivationResult, - ) => Effect.Effect; - /** Optional request path sent to the activated backend. */ - readonly upstreamPath?: (request: GatewayRouteRequest) => string; - /** Optional header transform applied after hop-by-hop filtering. */ - readonly upstreamHeaders?: GatewayHeaderTransform; -} - -export interface GatewayRouteRequest { - readonly path: string; - readonly method?: string; - readonly headers: Readonly>; -} - -/** A response produced locally without activating or resolving a backend. */ -export interface GatewayLocalResponse { - readonly status?: number; - readonly body: string | Uint8Array; - readonly contentType?: string; -} - -type GatewayHeaderValue = string | string[]; -export type GatewayHeaders = Readonly>; -export type GatewayHeaderTransform = ( - request: GatewayRouteRequest, - headers: GatewayHeaders, -) => GatewayHeaders; - -export interface GatewayProxyRoute { - readonly capability: CapabilityName; - /** Workload binding selected when a capability exposes multiple endpoints. */ - readonly binding?: string; - readonly match: (request: GatewayRouteRequest) => boolean; - /** Optional path transform applied before proxying when no prepared route overrides it. */ - readonly upstreamPath?: (request: GatewayRouteRequest) => string; - /** Optional header transform applied after hop-by-hop filtering. */ - readonly upstreamHeaders?: GatewayHeaderTransform; - /** Request-time validation and preparation performed before activation. */ - readonly prepare?: ( - request: GatewayRouteRequest, - ) => Effect.Effect; - /** Proxy routes never produce a local response. */ - readonly localResponse?: never; -} - -/** HTTP-only route that produces a response without activating a capability. */ -interface GatewayLocalRoute { - readonly capability?: never; - readonly binding?: never; - readonly match: (request: GatewayRouteRequest) => boolean; - readonly upstreamPath?: never; - readonly upstreamHeaders?: never; - readonly prepare?: never; - readonly localResponse: ( - request: GatewayRouteRequest, - ) => Effect.Effect; -} - -export type GatewayRoute = GatewayProxyRoute | GatewayLocalRoute; - -export const isGatewayProxyRoute = (route: GatewayRoute): route is GatewayProxyRoute => - route.capability !== undefined; - -type GatewayHttpKey = PortField | "api:internal"; - -export interface StackGateway { - readonly http: ReadonlyMap; - readonly tcp: ReadonlyMap; - readonly close: Effect.Effect; -} - -export interface HttpGatewayListenerOptions { - readonly field: PortField; - /** Optional internal map key when multiple listeners serve one logical field. */ - readonly key?: GatewayHttpKey; - readonly options: Omit; -} - -interface TcpGatewayListenerOptions { - readonly field: PortField; - readonly options: Omit; -} - -export interface StackGatewayOptions { - readonly http?: ReadonlyArray; - readonly tcp?: ReadonlyArray; - readonly activate: ( - capability: CapabilityName, - ) => Effect.Effect; - /** Optional traffic lease shared by the supervisor's idle-stop controller. */ - readonly activity?: GatewayActivity; -} - -/** Compose the protocol gateways under one Supervisor-owned lifecycle scope. */ -export const makeGateway = ( - options: StackGatewayOptions, -): Effect.Effect => - Effect.gen(function* () { - const http = new Map(); - const tcp = new Map(); - const closeValues = (values: Iterable<{ readonly close: Effect.Effect }>) => - Effect.forEach(values, (gateway) => gateway.close.pipe(Effect.exit), { - concurrency: "unbounded", - discard: true, - }); - for (const entry of options.http ?? []) { - const key = entry.key ?? entry.field; - if (http.has(key)) { - yield* closeValues(http.values()); - return yield* new GatewayActivationError({ - message: `Duplicate HTTP gateway listener ${key}`, - }); - } - const acquired = yield* Effect.exit( - makeHttpGateway({ - ...entry.options, - activate: options.activate, - activity: options.activity, - }), - ); - if (Exit.isFailure(acquired)) { - yield* closeValues(http.values()); - return yield* Effect.failCause(acquired.cause); - } - http.set(key, acquired.value); - } - for (const entry of options.tcp ?? []) { - if (tcp.has(entry.field) || http.has(entry.field)) { - yield* closeValues([...http.values(), ...tcp.values()]); - return yield* new GatewayActivationError({ - message: `Duplicate gateway listener ${entry.field}`, - }); - } - const acquired = yield* Effect.exit( - makeTcpGateway({ - ...entry.options, - activate: options.activate, - activity: options.activity, - }), - ); - if (Exit.isFailure(acquired)) { - yield* closeValues([...http.values(), ...tcp.values()]); - return yield* Effect.failCause(acquired.cause); - } - tcp.set(entry.field, acquired.value); - } - const closeOperation = closeValues([...http.values(), ...tcp.values()]); - const close = yield* Effect.cached(closeOperation); - return { http, tcp, close } satisfies StackGateway; - }); diff --git a/packages/stack/src/gateway/HttpGateway.ts b/packages/stack/src/gateway/HttpGateway.ts deleted file mode 100644 index 0ed7fe669d..0000000000 --- a/packages/stack/src/gateway/HttpGateway.ts +++ /dev/null @@ -1,705 +0,0 @@ -import { Cause, Data, Effect, Exit, FiberSet, Option, Queue, Scope } from "effect"; -import { - createServer, - request as proxyRequest, - type IncomingMessage, - type Server, - type ServerResponse, - // oxlint-disable-next-line effecttsgo/node-builtin-import -- transparent upgrade proxying needs raw handshake bytes; Effect upgrades terminate WebSockets. -} from "node:http"; -import { Socket } from "node:net"; -import type { Duplex } from "node:stream"; -import type { Fiber } from "effect/Fiber"; -import { GatewayActivationError } from "../public/Errors.ts"; -import type { - ActivationResult, - BackendEndpoint, - GatewayLocalResponse, - GatewayProxyRoute, - GatewayRoute, - GatewayRouteRequest, - GatewayHeaders, - PreparedGatewayRoute, -} from "./Gateway.ts"; -import { GatewayRouteNotFoundError, isGatewayProxyRoute } from "./Gateway.ts"; -import type { GatewayActivity } from "./ActivityTracker.ts"; -import type { - HostListener, - HostListenerHttpEvent, - HostListenerHttpEvents, -} from "../supervisor/HostListener.ts"; - -class GatewayBackendError extends Data.TaggedError("GatewayBackendError")<{ - readonly cause?: unknown; -}> {} - -export interface HttpGatewayOptions { - readonly address?: string; - readonly port?: number; - readonly listener?: HostListener; - readonly routes: ReadonlyArray; - readonly activate: ( - capability: import("../public/Capability.ts").CapabilityName, - ) => Effect.Effect; - readonly resolveBackend?: ( - route: GatewayProxyRoute, - request: GatewayRouteRequest, - activation: ActivationResult, - ) => Effect.Effect; - readonly cors?: Readonly>; - readonly healthPaths?: ReadonlyArray; - readonly activity?: GatewayActivity; -} - -export interface HttpGateway { - readonly address: string; - readonly port: number; - readonly close: Effect.Effect; - readonly server: Server; -} - -const hopByHop = new Set([ - "connection", - "keep-alive", - "proxy-authenticate", - "proxy-authorization", - "te", - "trailer", - "transfer-encoding", - "upgrade", -]); - -const gatewayCorsHeaders = new Set([ - "access-control-allow-origin", - "access-control-allow-methods", - "access-control-allow-headers", -]); - -const requestView = (request: IncomingMessage): GatewayRouteRequest => ({ - path: request.url ?? "/", - method: request.method, - headers: request.headers, -}); - -const setCors = ( - response: ServerResponse, - options: HttpGatewayOptions, - requestedHeaders?: string | string[], -) => { - response.setHeader( - "access-control-allow-origin", - options.cors?.["access-control-allow-origin"] ?? "*", - ); - response.setHeader( - "access-control-allow-methods", - options.cors?.["access-control-allow-methods"] ?? "GET,POST,PUT,PATCH,DELETE,OPTIONS", - ); - response.setHeader( - "access-control-allow-headers", - options.cors?.["access-control-allow-headers"] ?? - requestedHeaders ?? - "authorization,apikey,content-type,x-client-info", - ); - for (const [name, value] of Object.entries(options.cors ?? {})) response.setHeader(name, value); -}; - -const respond = ( - response: ServerResponse, - status: number, - body: string, - options: HttpGatewayOptions, -) => { - setCors(response, options); - response.statusCode = status; - response.setHeader("content-type", "application/json"); - response.end(body); -}; - -const respondLocal = ( - response: ServerResponse, - local: GatewayLocalResponse, - options: HttpGatewayOptions, -): void => { - setCors(response, options); - response.statusCode = local.status ?? 200; - response.setHeader("content-type", local.contentType ?? "application/octet-stream"); - response.end(local.body); -}; - -const routeFor = ( - request: GatewayRouteRequest, - routes: ReadonlyArray, -): GatewayRoute | undefined => routes.find((route) => route.match(request)); - -const preparedPath = ( - route: GatewayRoute, - prepared: PreparedGatewayRoute | undefined, - request: GatewayRouteRequest, -): string => prepared?.upstreamPath?.(request) ?? route.upstreamPath?.(request) ?? request.path; - -const filteredRequestHeaders = (request: IncomingMessage): Record => { - const headers: Record = {}; - for (const [name, value] of Object.entries(request.headers)) { - if (value !== undefined && !hopByHop.has(name.toLowerCase())) headers[name] = value; - } - return headers; -}; - -const withForwardedHeaders = ( - request: IncomingMessage, - headers: GatewayHeaders, -): Record => { - const forwarded = new Map(); - for (const [name, value] of Object.entries(headers)) { - if (!name.toLowerCase().startsWith("x-forwarded-")) forwarded.set(name, value); - } - forwarded.set("x-forwarded-host", request.headers.host ?? ""); - forwarded.set("x-forwarded-proto", "http"); - forwarded.set("x-forwarded-for", request.socket.remoteAddress ?? ""); - return Object.fromEntries(forwarded); -}; - -const preparedHeaders = ( - route: GatewayRoute, - prepared: PreparedGatewayRoute | undefined, - view: GatewayRouteRequest, - request: IncomingMessage, -): Record => - withForwardedHeaders( - request, - prepared?.upstreamHeaders?.(view, filteredRequestHeaders(request)) ?? - route.upstreamHeaders?.(view, filteredRequestHeaders(request)) ?? - filteredRequestHeaders(request), - ); - -interface ResolvedProxyRequest { - readonly backend: BackendEndpoint; - readonly path: string; - readonly headers: Record; -} - -const resolveProxyRequest = ( - request: IncomingMessage, - view: GatewayRouteRequest, - route: GatewayProxyRoute, - options: HttpGatewayOptions, -): Effect.Effect< - ResolvedProxyRequest, - GatewayRouteNotFoundError | GatewayActivationError | GatewayBackendError -> => { - const preparation: Effect.Effect< - PreparedGatewayRoute | undefined, - GatewayRouteNotFoundError | GatewayActivationError - > = route.prepare === undefined ? Effect.as(Effect.void, undefined) : route.prepare(view); - return preparation.pipe( - Effect.flatMap((prepared) => - Effect.suspend(() => options.activate(route.capability)).pipe( - Effect.flatMap((result) => { - const resolved = - prepared === undefined - ? options.resolveBackend === undefined - ? Effect.succeed(result.endpoint) - : options.resolveBackend(route, view, result) - : prepared.resolveBackend(result); - return resolved.pipe( - Effect.map((backend) => ({ - backend, - path: preparedPath(route, prepared, view), - headers: preparedHeaders(route, prepared, view, request), - })), - Effect.mapError((cause) => - cause instanceof GatewayActivationError ? cause : new GatewayBackendError({ cause }), - ), - ); - }), - ), - ), - ); -}; - -const proxy = ( - request: IncomingMessage, - response: ServerResponse, - backend: BackendEndpoint, - options: HttpGatewayOptions, - upstreamPath: string, - headers: Record, -): Effect.Effect => - Effect.callback((resume) => { - let settled = false; - let outgoing: ReturnType | undefined; - let incoming: IncomingMessage | undefined; - const onIncomingError = (cause: Error) => { - if (settled) return; - settled = true; - outgoing?.destroy(); - incoming?.destroy(); - cleanup(); - resume(Effect.fail(new GatewayBackendError({ cause }))); - }; - const onIncomingAborted = () => onIncomingError(new Error("backend response aborted")); - const onIncomingEnd = () => { - if (settled) return; - settled = true; - cleanup(); - resume(Effect.void); - }; - const onOutgoingError = (cause: Error) => onIncomingError(cause); - const onRequestAborted = () => onIncomingError(new Error("request aborted")); - const onResponseClose = () => { - if (settled || response.writableEnded) return; - onIncomingError(new Error("response closed")); - }; - const cleanup = () => { - request.off("aborted", onRequestAborted); - response.off("close", onResponseClose); - if (incoming !== undefined) { - incoming.off("error", onIncomingError); - incoming.off("aborted", onIncomingAborted); - incoming.off("end", onIncomingEnd); - } - }; - const onIncoming = (value: IncomingMessage) => { - incoming = value; - value.once("error", onIncomingError); - value.once("aborted", onIncomingAborted); - value.once("end", onIncomingEnd); - setCors(response, options); - response.statusCode = value.statusCode ?? 502; - for (const [name, headerValue] of Object.entries(value.headers)) { - if ( - headerValue !== undefined && - !hopByHop.has(name.toLowerCase()) && - !gatewayCorsHeaders.has(name.toLowerCase()) - ) - response.setHeader(name, headerValue); - } - // Keep the gateway's CORS policy authoritative while retaining backend exposure metadata - // such as Content-Range. Explicit options.cors values override both defaults and upstream. - setCors(response, options); - value.pipe(response); - }; - - outgoing = proxyRequest( - { - host: backend.host, - port: backend.port, - method: request.method, - path: upstreamPath, - headers, - }, - onIncoming, - ); - outgoing.once("error", onOutgoingError); - request.once("aborted", onRequestAborted); - response.once("close", onResponseClose); - request.pipe(outgoing); - return Effect.sync(() => { - settled = true; - outgoing.destroy(); - incoming?.destroy(); - cleanup(); - }); - }); - -const mapFailure = (cause: Cause.Cause): number => { - const error = Cause.findErrorOption(cause); - if (Option.isSome(error) && error.value instanceof GatewayRouteNotFoundError) return 404; - if (Option.isSome(error) && error.value instanceof GatewayActivationError) return 503; - if (Option.isSome(error) && error.value instanceof GatewayBackendError) return 502; - return 503; -}; - -const recoveryFrom = (cause: Cause.Cause) => { - const error = Cause.findErrorOption(cause); - return Option.isSome(error) && error.value instanceof GatewayActivationError - ? error.value.recovery - : undefined; -}; - -const recoveryResponse = (operation: "stop" | "destroy") => ({ - error: "STACK_RECOVERY_REQUIRED", - recovery: { - operation, - message: - operation === "stop" - ? "Retry stack stop before activating workloads" - : "Retry stack destroy before activating workloads", - }, -}); - -const cancelOnRequestClose = ( - request: IncomingMessage, - response: ServerResponse, - fiber: Fiber, -): void => { - let active = true; - const cleanup = () => { - if (!active) return; - active = false; - request.off("aborted", cancel); - response.off("close", cancel); - }; - const cancel = () => { - if (!active || response.writableEnded) return; - cleanup(); - fiber.interruptUnsafe(); - }; - request.once("aborted", cancel); - response.once("close", cancel); - fiber.addObserver(cleanup); -}; - -const handleRequest = ( - request: IncomingMessage, - response: ServerResponse, - options: HttpGatewayOptions, - runFork: (effect: Effect.Effect) => Fiber, -): void => { - const view = requestView(request); - setCors( - response, - options, - request.method === "OPTIONS" ? request.headers["access-control-request-headers"] : undefined, - ); - const route = routeFor(view, options.routes); - if (request.method === "OPTIONS") { - if (route !== undefined && !isGatewayProxyRoute(route)) { - respond(response, 404, JSON.stringify({ error: "Not found" }), options); - return; - } - response.statusCode = 204; - response.end(); - return; - } - const health = options.healthPaths; - const pathname = view.path.split("?", 1)[0] ?? view.path; - if (health?.some((path) => pathname === path)) { - response.statusCode = 200; - response.setHeader("content-type", "application/json"); - response.end(JSON.stringify({ ok: true })); - return; - } - if (route === undefined) { - respond(response, 404, JSON.stringify({ error: "Not found" }), options); - return; - } - if (!isGatewayProxyRoute(route)) { - const local = route.localResponse(view); - const fiber = runFork(local); - fiber.addObserver((exit) => { - if (Exit.isSuccess(exit) && !response.writableEnded && !response.destroyed) - respondLocal(response, exit.value, options); - else if (!response.writableEnded && !response.destroyed) - respond(response, 404, JSON.stringify({ error: "Not found" }), options); - }); - return; - } - const activation = resolveProxyRequest(request, view, route, options).pipe( - Effect.flatMap(({ backend, path, headers }) => - proxy(request, response, backend, options, path, headers), - ), - ); - const tracked = options.activity?.track(route.capability, activation) ?? activation; - // Node invokes this handler outside Effect; use the owner-scoped FiberSet - // runtime so cancellation of the gateway interrupts in-flight activation. - const fiber = runFork(tracked); - cancelOnRequestClose(request, response, fiber); - fiber.addObserver((exit) => { - if (!Exit.isFailure(exit) || response.writableEnded || response.destroyed) return; - if (response.headersSent) { - response.destroy(); - return; - } - const status = mapFailure(exit.cause); - const recovery = recoveryFrom(exit.cause); - respond( - response, - status, - JSON.stringify( - recovery === undefined - ? { - error: - status === 404 - ? "Not found" - : status === 503 - ? "Service unavailable" - : "Bad gateway", - } - : recoveryResponse(recovery.operation), - ), - options, - ); - }); -}; - -const writeUpgrade = ( - request: IncomingMessage, - path: string, - forwardedHeaders: Readonly>, -): string => { - const lines = [`${request.method ?? "GET"} ${path} HTTP/${request.httpVersion}`]; - for (const [name, value] of Object.entries(forwardedHeaders)) { - const values = Array.isArray(value) ? value : [value]; - for (const item of values) lines.push(`${name}: ${item}`); - } - lines.push("Connection: Upgrade", "Upgrade: websocket"); - return `${lines.join("\r\n")}\r\n\r\n`; -}; - -const trackSocket = (active: Set, socket: Duplex): void => { - if (active.has(socket)) return; - active.add(socket); - socket.once("close", () => active.delete(socket)); -}; - -const handleUpgrade = ( - request: IncomingMessage, - socket: Duplex, - head: Buffer, - options: HttpGatewayOptions, - runFork: (effect: Effect.Effect) => Fiber, - active: Set, -): void => { - trackSocket(active, socket); - const view = requestView(request); - const route = routeFor(view, options.routes); - if (route === undefined) { - socket.destroy(); - return; - } - // Local routes are HTTP-only and must never invoke their handler for upgrades. - if (!isGatewayProxyRoute(route)) { - socket.destroy(); - return; - } - // Observes EOF on a paused upgrade without consuming queued protocol bytes; removed before - // the backend tunnel starts so it can't interfere with flowing-mode piping. - const onSocketReadable = () => { - socket.read(0); - }; - socket.on("readable", onSocketReadable); - const activation = resolveProxyRequest(request, view, route, options).pipe( - Effect.flatMap(({ backend, path, headers }) => - Effect.callback((resume) => { - const target = new Socket({ allowHalfOpen: true }); - let settled = false; - const fail = (cause: unknown) => { - if (settled) return; - settled = true; - cleanup(); - target.destroy(); - socket.destroy(); - resume(Effect.fail(new GatewayBackendError({ cause }))); - }; - const onTargetError = (cause: Error) => fail(cause); - const onSocketClose = () => { - target.destroy(); - if (!settled) { - settled = true; - cleanup(); - resume(Effect.void); - } - }; - const onTargetClose = () => { - socket.destroy(); - if (!settled) { - settled = true; - cleanup(); - resume(Effect.void); - } - }; - const onSocketEnd = () => target.end(); - const onTargetEnd = () => socket.end(); - const onConnect = () => { - removePreConnectCancellation(); - socket.off("readable", onSocketReadable); - target.write(writeUpgrade(request, path, headers)); - if (head.byteLength > 0) target.write(head); - socket.pipe(target, { end: false }); - target.pipe(socket, { end: false }); - socket.once("end", onSocketEnd); - target.once("end", onTargetEnd); - }; - const cleanup = () => { - socket.off("readable", onSocketReadable); - target.off("error", onTargetError); - target.off("connect", onConnect); - target.off("close", onTargetClose); - socket.off("close", onSocketClose); - socket.off("end", onSocketEnd); - target.off("end", onTargetEnd); - }; - target.once("error", onTargetError); - target.once("connect", onConnect); - target.once("close", onTargetClose); - socket.once("close", onSocketClose); - target.connect(backend.port, backend.host); - return Effect.sync(() => { - settled = true; - cleanup(); - target.destroy(); - socket.destroy(); - }); - }), - ), - ); - const tracked = options.activity?.track(route.capability, activation) ?? activation; - const onSocketClose = () => fiber.interruptUnsafe(); - const onSocketEnd = () => fiber.interruptUnsafe(); - const onSocketError = () => fiber.interruptUnsafe(); - const onRequestAborted = () => fiber.interruptUnsafe(); - const removePreConnectCancellation = () => { - socket.off("end", onSocketEnd); - request.off("aborted", onRequestAborted); - }; - const fiber = runFork(tracked); - socket.once("close", onSocketClose); - socket.once("end", onSocketEnd); - socket.once("error", onSocketError); - request.once("aborted", onRequestAborted); - const removeSocketCancellation = () => { - socket.off("readable", onSocketReadable); - removePreConnectCancellation(); - socket.off("close", onSocketClose); - socket.off("error", onSocketError); - }; - fiber.addObserver(removeSocketCancellation); - fiber.addObserver((exit) => { - if (!Exit.isFailure(exit)) return; - const recovery = recoveryFrom(exit.cause); - if (recovery === undefined || socket.destroyed) { - socket.destroy(); - return; - } - const body = JSON.stringify(recoveryResponse(recovery.operation)); - const response = [ - "HTTP/1.1 503 Service Unavailable", - "Content-Type: application/json", - `Content-Length: ${Buffer.byteLength(body)}`, - "Connection: close", - "", - body, - ].join("\r\n"); - const onRecoveryError = () => socket.destroy(); - socket.once("error", onRecoveryError); - socket.once("close", () => socket.off("error", onRecoveryError)); - socket.end(response, () => socket.destroy()); - }); -}; - -const drainPendingEvents = ( - events: HostListenerHttpEvents, - onRequest: (request: IncomingMessage, response: ServerResponse) => void, - onUpgrade: (request: IncomingMessage, socket: Duplex, head: Buffer) => void, -): Effect.Effect => - Effect.suspend(() => - Queue.poll(events.queue).pipe( - Effect.flatMap((event) => { - if (Option.isNone(event)) return Effect.void; - return Effect.sync(() => { - const value: HostListenerHttpEvent = event.value; - if (value._tag === "request") { - if (!value.request.destroyed && !value.response.destroyed) - onRequest(value.request, value.response); - } else if (!value.socket.destroyed) { - onUpgrade(value.request, value.socket, value.head); - } - }).pipe(Effect.andThen(drainPendingEvents(events, onRequest, onUpgrade))); - }), - ), - ); - -/** Start an HTTP gateway, optionally adopting an already-bound native server. */ -export const makeHttpGateway = ( - options: HttpGatewayOptions, -): Effect.Effect => - Effect.gen(function* () { - const fibers = yield* FiberSet.make(); - const runFork = yield* FiberSet.runtime(fibers)(); - if (options.listener?.binding?.kind === "tcp") - return yield* new GatewayActivationError({ message: "Gateway listener protocol mismatch" }); - const server = - options.listener?.binding?.kind === "http" ? options.listener.binding.server : createServer(); - if (options.listener !== undefined && !server.listening) - return yield* new GatewayActivationError({ message: "Gateway listener is not bound" }); - // Cold workload activation may take longer than Node/Bun's default request and headers - // timeouts. The gateway owns cancellation through the request/socket lifecycle instead. - server.requestTimeout = 0; - server.headersTimeout = 0; - server.timeout = 0; - const active = - options.listener === undefined ? new Set() : options.listener.connections.sockets; - const connectionHandler = (socket: Duplex) => trackSocket(active, socket); - const requestHandler = (request: IncomingMessage, response: ServerResponse) => - handleRequest(request, response, options, runFork); - const upgradeHandler = (request: IncomingMessage, socket: Duplex, head: Buffer) => - handleUpgrade(request, socket, head, options, runFork, active); - server.on("connection", connectionHandler); - server.on("request", requestHandler); - server.on("upgrade", upgradeHandler); - if (!server.listening) { - yield* Effect.callback((resume) => { - let settled = false; - const cleanup = () => { - server.off("error", onError); - server.close(() => undefined); - }; - const onError = (cause: Error) => { - if (settled) return; - settled = true; - cleanup(); - resume( - Effect.fail( - new GatewayActivationError({ message: "Gateway listener unavailable", cause }), - ), - ); - }; - server.once("error", onError); - server.listen({ host: options.address ?? "127.0.0.1", port: options.port ?? 0 }, () => { - if (settled) return; - settled = true; - server.off("error", onError); - resume(Effect.void); - }); - return Effect.sync(() => { - if (settled) return; - settled = true; - cleanup(); - }); - }); - } - const address = server.address(); - if (typeof address !== "object" || address === null) - return yield* new GatewayActivationError({ - message: "Gateway listener did not expose an endpoint", - }); - const pendingEvents = - options.listener?.binding.kind === "http" - ? options.listener.binding.pendingEvents - : undefined; - if (pendingEvents !== undefined) { - pendingEvents.detach(); - yield* drainPendingEvents(pendingEvents, requestHandler, upgradeHandler); - } - if (options.listener !== undefined) options.listener.connections.release?.(); - const closeOperation = Effect.gen(function* () { - server.off("request", requestHandler); - server.off("upgrade", upgradeHandler); - server.off("connection", connectionHandler); - if (pendingEvents !== undefined) { - pendingEvents.detach(); - yield* Queue.shutdown(pendingEvents.queue); - } - yield* FiberSet.clear(fibers); - for (const socket of active) socket.destroy(); - if (options.listener !== undefined) yield* options.listener.close; - yield* Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }); - }); - const close = yield* Effect.cached(closeOperation); - yield* Effect.addFinalizer(() => close); - return { address: address.address, port: address.port, close, server } satisfies HttpGateway; - }); diff --git a/packages/stack/src/gateway/RouteCatalog.ts b/packages/stack/src/gateway/RouteCatalog.ts deleted file mode 100644 index 6d06609471..0000000000 --- a/packages/stack/src/gateway/RouteCatalog.ts +++ /dev/null @@ -1,311 +0,0 @@ -import type { ExecutionPlan } from "../model/ExecutionPlan.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import type { PortField } from "../public/Status.ts"; -import type { - GatewayHeaderTransform, - GatewayHeaders, - GatewayProxyRoute, - GatewayRoute, - GatewayRouteRequest, -} from "./Gateway.ts"; - -export interface GatewayRouteCatalog { - readonly http: ReadonlyMap>; - readonly tcp: ReadonlyMap>; -} - -/** Owner-resolved API material used only for forwarding transformations. */ -export interface GatewayApiMaterial { - readonly publishableKey: string; - readonly secretKey: string; - readonly anonJwt: string; - readonly serviceRoleJwt: string; -} - -const pathStartsWith = - (prefix: string) => - (path: string): boolean => - path === prefix || path.startsWith(`${prefix}/`); - -const pathAndQuery = (requestPath: string): readonly [string, string] => { - const queryIndex = requestPath.indexOf("?"); - return queryIndex < 0 - ? [requestPath, ""] - : [requestPath.slice(0, queryIndex), requestPath.slice(queryIndex)]; -}; - -const stripPrefix = - (prefix: string) => - (request: GatewayRouteRequest): string => { - const [pathname, query] = pathAndQuery(request.path); - const suffix = pathname.slice(prefix.length); - return `${suffix.length === 0 ? "/" : suffix}${query}`; - }; - -const appendSuffix = - (base: string, prefix: string) => - (request: GatewayRouteRequest): string => { - const [pathname, query] = pathAndQuery(request.path); - const suffix = pathname.slice(prefix.length); - return `${base}${suffix}${query}`; - }; - -const headerValue = (headers: GatewayHeaders, name: string): string | undefined => { - const value = Object.entries(headers).find( - ([key]) => key.toLowerCase() === name.toLowerCase(), - )?.[1]; - return Array.isArray(value) ? value[0] : value; -}; - -const removeHeader = (headers: Record, name: string): void => { - for (const key of Object.keys(headers)) - if (key.toLowerCase() === name.toLowerCase()) delete headers[key]; -}; - -const apiKeyJwt = (value: string | undefined, material: GatewayApiMaterial): string | undefined => { - if (value === undefined) return undefined; - const trimmed = value.trim(); - const candidate = /^Bearer\s+/iu.test(trimmed) ? trimmed.slice(7).trim() : trimmed; - if (candidate === material.publishableKey) return material.anonJwt; - if (candidate === material.secretKey) return material.serviceRoleJwt; - return undefined; -}; - -const authorizationTransform = - (material: GatewayApiMaterial, target: "authorization" | "sb-api-key"): GatewayHeaderTransform => - (_request, headers) => { - const output: Record = { ...headers }; - const authorization = headerValue(headers, "authorization"); - const apikey = headerValue(headers, "apikey"); - const jwt = - target === "sb-api-key" - ? (apiKeyJwt(apikey, material) ?? apiKeyJwt(authorization, material)) - : authorization === undefined - ? apiKeyJwt(apikey, material) - : apiKeyJwt(authorization, material); - if (jwt === undefined) { - if (target === "sb-api-key") removeHeader(output, target); - return output; - } - removeHeader(output, target); - output[target] = `Bearer ${jwt}`; - return output; - }; - -const composeHeaders = - (...transforms: ReadonlyArray): GatewayHeaderTransform => - (request, headers) => - transforms.reduce((current, transform) => transform(request, current), headers); - -const preserveHeaders: GatewayHeaderTransform = (_request, headers) => ({ ...headers }); - -const graphqlHeaders = (material: GatewayApiMaterial): GatewayHeaderTransform => - composeHeaders(authorizationTransform(material, "authorization"), (_request, headers) => { - const output: Record = { ...headers }; - removeHeader(output, "content-profile"); - output["content-profile"] = "graphql_public"; - return output; - }); - -const realtimeWebsocketPath = - (material: GatewayApiMaterial | undefined) => - (request: GatewayRouteRequest): string => { - const forwarded = appendSuffix("/socket", "/realtime/v1")(request); - if (material === undefined) return forwarded; - const [pathname, query] = pathAndQuery(forwarded); - if (query.length === 0) return forwarded; - const mapped = query - .slice(1) - .split("&") - .map((part) => { - const separator = part.indexOf("="); - if (separator < 0) return part; - const key = decodeURIComponent(part.slice(0, separator)); - if (key !== "apikey") return part; - const value = decodeURIComponent(part.slice(separator + 1)); - const jwt = apiKeyJwt(value, material); - return jwt === undefined - ? part - : `${part.slice(0, separator + 1)}${encodeURIComponent(jwt)}`; - }) - .join("&"); - return `${pathname}?${mapped}`; - }; - -const realtimeWebsocketHeaders = - (material: GatewayApiMaterial | undefined): GatewayHeaderTransform => - (request, headers) => { - const output: Record = { - ...(material === undefined - ? headers - : authorizationTransform(material, "authorization")(request, headers)), - }; - removeHeader(output, "host"); - output.host = "realtime-dev"; - return output; - }; - -const prefixRoute = ( - capability: CapabilityName, - prefix: string, - upstreamPath: (request: GatewayRouteRequest) => string = stripPrefix(prefix), - match: (request: GatewayRouteRequest) => boolean = (request) => - pathStartsWith(prefix)(pathAndQuery(request.path)[0]), - upstreamHeaders?: GatewayHeaderTransform, -): GatewayProxyRoute => ({ - capability, - match, - upstreamPath, - ...(upstreamHeaders === undefined ? {} : { upstreamHeaders }), -}); - -const directRoute = (capability: CapabilityName, binding?: string): GatewayProxyRoute => ({ - capability, - ...(binding === undefined ? {} : { binding }), - match: () => true, -}); - -const realtimeWebsocketRoute = (material?: GatewayApiMaterial): GatewayProxyRoute => - prefixRoute( - "realtime", - "/realtime/v1", - realtimeWebsocketPath(material), - (request) => { - const [pathname] = pathAndQuery(request.path); - const upgrade = request.headers["upgrade"]; - const isUpgrade = Array.isArray(upgrade) - ? upgrade.some((value) => value.toLowerCase() === "websocket") - : upgrade?.toLowerCase() === "websocket"; - return ( - pathStartsWith("/realtime/v1")(pathname) && - (isUpgrade || pathname === "/realtime/v1/websocket") - ); - }, - realtimeWebsocketHeaders(material), - ); - -const apiRoutes = ( - capability: CapabilityName, - material?: GatewayApiMaterial, -): ReadonlyArray => { - switch (capability) { - case "rest": - return [ - prefixRoute( - capability, - "/rest/v1", - stripPrefix("/rest/v1"), - undefined, - material === undefined ? undefined : authorizationTransform(material, "authorization"), - ), - prefixRoute( - capability, - "/graphql/v1", - appendSuffix("/rpc/graphql", "/graphql/v1"), - undefined, - material === undefined ? undefined : graphqlHeaders(material), - ), - ]; - case "auth": - return [ - prefixRoute( - capability, - "/auth/v1", - stripPrefix("/auth/v1"), - undefined, - material === undefined ? undefined : authorizationTransform(material, "authorization"), - ), - ]; - case "realtime": - return [ - realtimeWebsocketRoute(material), - prefixRoute( - capability, - "/realtime/v1/api", - stripPrefix("/realtime/v1"), - undefined, - material === undefined ? undefined : authorizationTransform(material, "authorization"), - ), - prefixRoute(capability, "/socket"), - ]; - case "storage": - return [ - prefixRoute( - capability, - "/storage/v1/s3", - appendSuffix("/s3", "/storage/v1/s3"), - undefined, - preserveHeaders, - ), - prefixRoute( - capability, - "/storage/v1", - stripPrefix("/storage/v1"), - undefined, - material === undefined ? undefined : authorizationTransform(material, "authorization"), - ), - ]; - case "functions": - return [ - prefixRoute( - capability, - "/functions/v1", - stripPrefix("/functions/v1"), - undefined, - material === undefined ? undefined : authorizationTransform(material, "sb-api-key"), - ), - ]; - case "analytics": - return [ - prefixRoute( - capability, - "/analytics/v1", - stripPrefix("/analytics/v1"), - undefined, - material === undefined ? undefined : authorizationTransform(material, "authorization"), - ), - ]; - case "database": - case "pooler": - case "studio": - case "mail": - return []; - } -}; - -/** Build the closed public route set from the enabled plan, in collision-safe order. */ -export const routeCatalogFor = ( - plan: ExecutionPlan, - material?: GatewayApiMaterial, -): GatewayRouteCatalog => { - const http = new Map(); - const tcp = new Map(); - const append = ( - target: Map, - field: PortField, - routes: ReadonlyArray, - ) => { - const current = target.get(field) ?? []; - current.push(...routes); - target.set(field, current); - }; - for (const route of plan.routes) { - if (route.listener === "api" && route.protocol === "http") - append(http, route.listener, apiRoutes(route.capability, material)); - else if (route.listener === "studio" && route.protocol === "http") - append(http, route.listener, [directRoute(route.capability)]); - else if (route.listener === "mailUi" && route.protocol === "http") - append(http, route.listener, [directRoute(route.capability, "ui")]); - else if (route.listener === "functionsInspector" && route.protocol === "http") - append(http, route.listener, [directRoute(route.capability, "inspector")]); - else if (route.listener === "database" && route.protocol === "tcp") - append(tcp, route.listener, [directRoute(route.capability, "primary")]); - else if (route.listener === "pooler" && route.protocol === "tcp") - append(tcp, route.listener, [directRoute(route.capability, "primary")]); - else if (route.listener === "smtp" && route.protocol === "tcp") - append(tcp, route.listener, [directRoute(route.capability, "smtp")]); - else if (route.listener === "pop3" && route.protocol === "tcp") - append(tcp, route.listener, [directRoute(route.capability, "pop3")]); - } - return { http, tcp }; -}; diff --git a/packages/stack/src/gateway/TcpGateway.ts b/packages/stack/src/gateway/TcpGateway.ts deleted file mode 100644 index 5e18721584..0000000000 --- a/packages/stack/src/gateway/TcpGateway.ts +++ /dev/null @@ -1,242 +0,0 @@ -import { Effect, Exit, FiberSet, Scope } from "effect"; -import { createServer, Socket, type Server } from "node:net"; -import type { Duplex } from "node:stream"; -import type { Fiber } from "effect/Fiber"; -import { GatewayActivationError } from "../public/Errors.ts"; -import type { - ActivationResult, - BackendEndpoint, - GatewayProxyRoute, - GatewayRouteRequest, -} from "./Gateway.ts"; -import type { HostListener } from "../supervisor/HostListener.ts"; -import type { GatewayActivity } from "./ActivityTracker.ts"; - -export interface TcpGatewayOptions { - readonly address?: string; - readonly port?: number; - readonly listener?: HostListener; - readonly routes: ReadonlyArray; - readonly activate: ( - capability: import("../public/Capability.ts").CapabilityName, - ) => Effect.Effect; - readonly resolveBackend?: ( - route: GatewayProxyRoute, - request: GatewayRouteRequest, - activation: ActivationResult, - ) => Effect.Effect; - readonly activity?: GatewayActivity; -} - -export interface TcpGateway { - readonly address: string; - readonly port: number; - readonly close: Effect.Effect; - readonly server: Server; -} - -const routeFor = (routes: ReadonlyArray): GatewayProxyRoute | undefined => - routes.find((route) => route.match({ path: "/", headers: {} })); - -const trackSocket = (active: Set, socket: Duplex): void => { - if (active.has(socket)) return; - active.add(socket); - socket.once("close", () => active.delete(socket)); -}; - -const tunnel = ( - source: Socket, - backend: BackendEndpoint, -): Effect.Effect => - Effect.callback((resume) => { - const target = new Socket({ allowHalfOpen: true }); - let settled = false; - let targetEnded = false; - const onSourceError = (cause: Error) => { - if (settled) return; - settled = true; - cleanup(); - source.destroy(); - target.destroy(); - resume(Effect.fail(new GatewayActivationError({ message: "Gateway backend failed", cause }))); - }; - const onTargetError = (cause: Error) => onSourceError(cause); - const onSourceClose = () => { - target.destroy(); - if (!settled) { - settled = true; - cleanup(); - resume(Effect.void); - } - }; - const onTargetClose = () => { - if (!settled) { - if (!targetEnded) { - settled = true; - cleanup(); - source.destroy(); - resume(Effect.void); - } - } - }; - const onSourceEnd = () => target.end(); - const onTargetEnd = () => { - targetEnded = true; - source.end(); - }; - const onConnect = () => { - // Node's pipe implementation naturally propagates backpressure. Keeping - // both streams open here preserves protocol half-close semantics. - source.pipe(target, { end: false }); - target.pipe(source, { end: false }); - }; - const cleanup = () => { - source.off("error", onSourceError); - target.off("error", onTargetError); - source.off("close", onSourceClose); - target.off("close", onTargetClose); - target.off("connect", onConnect); - source.off("end", onSourceEnd); - target.off("end", onTargetEnd); - }; - source.once("error", onSourceError); - target.once("error", onTargetError); - source.once("close", onSourceClose); - target.once("close", onTargetClose); - source.once("end", onSourceEnd); - target.once("end", onTargetEnd); - target.once("connect", onConnect); - target.connect(backend.port, backend.host); - return Effect.sync(() => { - if (!settled) settled = true; - cleanup(); - source.destroy(); - target.destroy(); - }); - }); - -const handleConnection = ( - source: Socket, - options: TcpGatewayOptions, - runFork: (effect: Effect.Effect) => Fiber, - active: Set, -): void => { - source.pause(); - trackSocket(active, source); - const route = routeFor(options.routes); - if (route === undefined) { - source.destroy(); - return; - } - const onPreActivationError = () => source.destroy(); - const view: GatewayRouteRequest = { path: "/", headers: {} }; - const operation = options.activate(route.capability).pipe( - Effect.flatMap((result) => - options.resolveBackend === undefined - ? Effect.succeed(result.endpoint) - : options.resolveBackend(route, view, result), - ), - Effect.flatMap((backend) => { - source.off("error", onPreActivationError); - return tunnel(source, backend); - }), - ); - const tracked = options.activity?.track(route.capability, operation) ?? operation; - source.once("error", onPreActivationError); - // Node invokes this handler outside Effect; use the owner-scoped FiberSet - // runtime for the exact accepted connection's lifecycle. - const fiber = runFork(tracked); - const onSourceClose = () => fiber.interruptUnsafe(); - source.once("close", onSourceClose); - fiber.addObserver(() => { - source.off("close", onSourceClose); - source.off("error", onPreActivationError); - }); - fiber.addObserver((exit) => { - if (Exit.isFailure(exit)) source.destroy(); - }); -}; - -/** Start a transparent TCP gateway, optionally adopting an already-bound server. */ -export const makeTcpGateway = ( - options: TcpGatewayOptions, -): Effect.Effect => - Effect.gen(function* () { - const fibers = yield* FiberSet.make(); - const runFork = yield* FiberSet.runtime(fibers)(); - if (options.listener?.binding?.kind === "http") - return yield* new GatewayActivationError({ message: "Gateway listener protocol mismatch" }); - if ( - options.listener?.binding?.kind === "tcp" && - options.listener.binding.allowHalfOpen !== true - ) - return yield* new GatewayActivationError({ - message: "Gateway listener must preserve half-close", - }); - const server = - options.listener?.binding?.kind === "tcp" - ? options.listener.binding.server - : createServer({ allowHalfOpen: true }); - if (options.listener !== undefined && !server.listening) - return yield* new GatewayActivationError({ message: "Gateway listener is not bound" }); - const active = - options.listener === undefined ? new Set() : options.listener.connections.sockets; - const connectionHandler = (socket: Socket) => - handleConnection(socket, options, runFork, active); - server.on("connection", connectionHandler); - // A listener may have accepted sockets while it was waiting for gateway adoption. Node emits - // `connection` only once, so explicitly hand those sockets to the gateway before resuming them. - for (const socket of active) { - if (socket instanceof Socket) connectionHandler(socket); - } - if (!server.listening) { - yield* Effect.callback((resume) => { - let settled = false; - const cleanup = () => { - server.off("error", onError); - server.close(() => undefined); - }; - const onError = (cause: Error) => { - if (settled) return; - settled = true; - cleanup(); - resume( - Effect.fail( - new GatewayActivationError({ message: "Gateway listener unavailable", cause }), - ), - ); - }; - server.once("error", onError); - server.listen({ host: options.address ?? "127.0.0.1", port: options.port ?? 0 }, () => { - if (settled) return; - settled = true; - server.off("error", onError); - resume(Effect.void); - }); - return Effect.sync(() => { - if (settled) return; - settled = true; - cleanup(); - }); - }); - } - const address = server.address(); - if (typeof address !== "object" || address === null) - return yield* new GatewayActivationError({ - message: "Gateway listener did not expose an endpoint", - }); - if (options.listener !== undefined) options.listener.connections.release?.(); - const closeOperation = Effect.gen(function* () { - server.off("connection", connectionHandler); - yield* FiberSet.clear(fibers); - for (const socket of active) socket.destroy(); - if (options.listener !== undefined) yield* options.listener.close; - yield* Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }); - }); - const close = yield* Effect.cached(closeOperation); - yield* Effect.addFinalizer(() => close); - return { address: address.address, port: address.port, close, server } satisfies TcpGateway; - }); diff --git a/packages/stack/src/gateway/activity.integration.test.ts b/packages/stack/src/gateway/activity.integration.test.ts deleted file mode 100644 index ac609f0514..0000000000 --- a/packages/stack/src/gateway/activity.integration.test.ts +++ /dev/null @@ -1,337 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Option, Queue, Ref } from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- The integration fixture needs a real Node HTTP client. -import { Agent, createServer as createHttpServer, request as httpRequest } from "node:http"; -import { connect as tcpConnect, createServer as createTcpServer, type Socket } from "node:net"; -import { makeGatewayActivity } from "./ActivityTracker.ts"; -import { makeHttpGateway } from "./HttpGateway.ts"; -import { makeTcpGateway } from "./TcpGateway.ts"; - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const route = { - capability: "rest" as const, - match: (request: { path: string }) => request.path === "/data", -}; - -const tcpRoute = { - capability: "rest" as const, - match: () => true, -}; - -const activityFixture = Effect.gen(function* () { - const events = yield* Queue.unbounded(); - const active = yield* Ref.make(0); - const activity = yield* makeGatewayActivity({ - begin: (capability) => - Ref.update(active, (value) => value + 1).pipe( - Effect.andThen(Queue.offer(events, `begin:${capability}`)), - ), - end: (capability) => - Ref.update(active, (value) => value - 1).pipe( - Effect.andThen(Queue.offer(events, `end:${capability}`)), - ), - }); - return { active, activity, events }; -}); - -const nextEvent = (events: Queue.Queue) => Queue.take(events); - -describe("gateway traffic activity", () => { - it.live("tracks HTTP requests through streamed response completion", () => - run( - Effect.gen(function* () { - const { activity, active, events } = yield* activityFixture; - let finishResponse: (() => void) | undefined; - const backend = createHttpServer((_request, response) => { - response.write("part"); - finishResponse = () => response.end("done"); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - return Effect.sync(() => backend.close()); - }); - yield* Effect.addFinalizer(() => - Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }), - ); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend unavailable"); - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [route], - activity, - activate: () => - Effect.succeed({ - capability: "rest" as const, - endpoint: { host: "127.0.0.1", port: address.port }, - }), - }); - const response = yield* Effect.callback< - { first: string; done: Effect.Effect }, - Error - >((resume) => { - const request = httpRequest( - { host: "127.0.0.1", port: gateway.port, path: "/data" }, - (incoming) => { - incoming.once("data", (chunk) => - resume( - Effect.succeed({ - first: String(chunk), - done: Effect.callback((done) => { - incoming.once("end", () => done(Effect.void)); - return Effect.sync(() => incoming.destroy()); - }), - }), - ), - ); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - expect(yield* nextEvent(events)).toBe("begin:rest"); - expect(yield* Ref.get(active)).toBe(1); - expect(response.first).toBe("part"); - finishResponse?.(); - yield* response.done; - expect(yield* nextEvent(events)).toBe("end:rest"); - expect(yield* Ref.get(active)).toBe(0); - yield* gateway.close; - yield* Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }); - }), - ), - ); - - it.live("does not track OPTIONS or local health responses", () => - run( - Effect.gen(function* () { - const { activity, active, events } = yield* activityFixture; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - healthPaths: ["/health"], - routes: [route], - activity, - activate: () => Effect.die("should not activate"), - }); - const request = (method: string, path: string) => - Effect.callback((resume) => { - const client = httpRequest( - { host: "127.0.0.1", port: gateway.port, path, method }, - (response) => { - response.resume(); - response.once("end", () => resume(Effect.succeed(response.statusCode ?? 0))); - }, - ); - client.once("error", (error) => resume(Effect.fail(error))); - client.end(); - return Effect.sync(() => client.destroy()); - }); - expect(yield* request("OPTIONS", "/data")).toBe(204); - expect(yield* request("GET", "/health")).toBe(200); - expect(Option.isNone(yield* Queue.poll(events))).toBe(true); - expect(yield* Ref.get(active)).toBe(0); - yield* gateway.close; - }), - ), - ); - - it.live("holds silent TCP activity until the client connection closes", () => - run( - Effect.gen(function* () { - const { activity, active, events } = yield* activityFixture; - const backend = createTcpServer(() => undefined); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - return Effect.sync(() => backend.close()); - }); - yield* Effect.addFinalizer(() => - Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }), - ); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend unavailable"); - const gateway = yield* makeTcpGateway({ - address: "127.0.0.1", - port: 0, - routes: [tcpRoute], - activity, - activate: () => - Effect.succeed({ - capability: "rest" as const, - endpoint: { host: "127.0.0.1", port: address.port }, - }), - }); - const client = yield* Effect.callback((resume) => { - const socket = tcpConnect(gateway.port, "127.0.0.1", () => - resume(Effect.succeed(socket)), - ); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - yield* Effect.addFinalizer(() => Effect.sync(() => client.destroy())); - expect(yield* nextEvent(events)).toBe("begin:rest"); - expect(yield* Ref.get(active)).toBe(1); - expect(Option.isNone(yield* Queue.poll(events))).toBe(true); - client.destroy(); - expect(yield* nextEvent(events)).toBe("end:rest"); - expect(yield* Ref.get(active)).toBe(0); - yield* gateway.close; - yield* Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }); - }), - ), - ); - - it.live("holds a silent WebSocket upgrade until its socket closes", () => - run( - Effect.gen(function* () { - const { activity, active, events } = yield* activityFixture; - const backend = createHttpServer(); - backend.on("upgrade", (_request, socket) => - socket.write( - "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ), - ); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - return Effect.sync(() => backend.close()); - }); - yield* Effect.addFinalizer(() => - Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }), - ); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend unavailable"); - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "rest" as const, match: () => true }], - activity, - activate: () => - Effect.succeed({ - capability: "rest" as const, - endpoint: { host: "127.0.0.1", port: address.port }, - }), - }); - const client = yield* Effect.callback((resume) => { - const socket = tcpConnect(gateway.port, "127.0.0.1", () => { - socket.write( - "GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ); - resume(Effect.succeed(socket)); - }); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - yield* Effect.addFinalizer(() => Effect.sync(() => client.destroy())); - expect(yield* nextEvent(events)).toBe("begin:rest"); - expect(yield* Ref.get(active)).toBe(1); - expect(Option.isNone(yield* Queue.poll(events))).toBe(true); - client.destroy(); - expect(yield* nextEvent(events)).toBe("end:rest"); - expect(yield* Ref.get(active)).toBe(0); - yield* gateway.close; - yield* Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }); - }), - ), - ); - - it.live("releases activity when an HTTP client aborts and ignores keep-alive sockets", () => - run( - Effect.gen(function* () { - const { activity, active, events } = yield* activityFixture; - let requestCount = 0; - let responseToFinish: (() => void) | undefined; - let requestObserved = false; - let requestObservedResume: (() => void) | undefined; - const backend = createHttpServer((_request, response) => { - requestCount += 1; - requestObserved = true; - requestObservedResume?.(); - if (requestCount > 1) response.end("done"); - else responseToFinish = () => response.end("done"); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - return Effect.sync(() => backend.close()); - }); - yield* Effect.addFinalizer(() => - Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }), - ); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend unavailable"); - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [route], - activity, - activate: () => - Effect.succeed({ - capability: "rest" as const, - endpoint: { host: "127.0.0.1", port: address.port }, - }), - }); - const client = httpRequest({ host: "127.0.0.1", port: gateway.port, path: "/data" }); - client.on("error", () => undefined); - yield* Effect.addFinalizer(() => Effect.sync(() => client.destroy())); - client.end(); - expect(yield* nextEvent(events)).toBe("begin:rest"); - if (!requestObserved) - yield* Effect.callback((resume) => { - requestObservedResume = () => resume(Effect.void); - return Effect.sync(() => (requestObservedResume = undefined)); - }); - client.destroy(); - expect(yield* nextEvent(events)).toBe("end:rest"); - expect(yield* Ref.get(active)).toBe(0); - responseToFinish?.(); - const agent = new Agent({ keepAlive: true }); - yield* Effect.addFinalizer(() => Effect.sync(() => agent.destroy())); - yield* Effect.callback((resume) => { - const keepAliveRequest = httpRequest( - { host: "127.0.0.1", port: gateway.port, path: "/data", agent }, - (response) => { - response.resume(); - response.once("end", () => resume(Effect.void)); - }, - ); - keepAliveRequest.on("error", (error) => resume(Effect.fail(error))); - keepAliveRequest.end(); - return Effect.sync(() => keepAliveRequest.destroy()); - }); - expect(yield* nextEvent(events)).toBe("begin:rest"); - expect(yield* nextEvent(events)).toBe("end:rest"); - expect(yield* Ref.get(active)).toBe(0); - agent.destroy(); - yield* gateway.close; - yield* Effect.callback((resume) => { - backend.close(() => resume(Effect.void)); - }); - }), - ), - ); -}); diff --git a/packages/stack/src/gateway/gateway.integration.test.ts b/packages/stack/src/gateway/gateway.integration.test.ts deleted file mode 100644 index c1e0ecaebf..0000000000 --- a/packages/stack/src/gateway/gateway.integration.test.ts +++ /dev/null @@ -1,1235 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Deferred, Effect, Exit, Fiber, Option, Queue, Schema } from "effect"; -import { GatewayActivationError } from "../public/Errors.ts"; -import { connect as connectNet, createServer, type Server, type Socket } from "node:net"; -import { - Agent, - type ClientRequest, - createServer as createHttpServer, - request as requestHttp, - type IncomingMessage, - type ServerResponse, - // oxlint-disable-next-line effecttsgo/node-builtin-import -- fixture controls raw HTTP proxy responses and connection behavior at the gateway boundary. -} from "node:http"; -import { - GatewayRouteNotFoundError, - makeGateway, - type BackendEndpoint, - type GatewayRouteRequest, -} from "./Gateway.ts"; -import { makeHttpGateway } from "./HttpGateway.ts"; -import { makeTcpGateway } from "./TcpGateway.ts"; -import type { HostListener } from "../supervisor/HostListener.ts"; -import { bindHostListener } from "../supervisor/HostListener.ts"; - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const closeServer = (server: Server): Effect.Effect => - Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }); - -const getStatus = (port: number, path: string) => - Effect.callback((resume) => { - const request = requestHttp({ host: "127.0.0.1", port, path }, (response) => { - response.resume(); - response.once("end", () => resume(Effect.succeed(response.statusCode ?? 0))); - }); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - -const recoveryResponseSchema = Schema.fromJsonString( - Schema.Struct({ - error: Schema.String, - recovery: Schema.Struct({ - operation: Schema.Literals(["stop", "destroy"] as const), - message: Schema.String, - }), - }), -); - -describe("stack gateway", () => { - it.live("serves local HTTP routes before activation and rejects upgrades", () => - withPlatform( - Effect.gen(function* () { - let localCalls = 0; - let activations = 0; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [ - { - match: (request) => request.path.startsWith("/email/"), - localResponse: (request) => - request.method === "GET" - ? Effect.sync(() => { - localCalls += 1; - return { body: "template", contentType: "text/plain; charset=utf-8" }; - }) - : Effect.fail(new GatewayRouteNotFoundError({ message: "Not found" })), - }, - ], - activate: () => - Effect.sync(() => { - activations += 1; - return { capability: "auth" as const, endpoint: { host: "127.0.0.1", port: 1 } }; - }), - }); - const response = yield* Effect.callback<{ status: number; body: string }, Error>( - (resume) => { - const request = requestHttp( - { host: "127.0.0.1", port: gateway.port, path: "/email/confirmation.html" }, - (result) => { - const chunks: Buffer[] = []; - result.on("data", (chunk: Buffer) => chunks.push(chunk)); - result.once("end", () => - resume( - Effect.succeed({ - status: result.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - }), - ), - ); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }, - ); - expect(response).toEqual({ status: 200, body: "template" }); - expect(localCalls).toBe(1); - expect(activations).toBe(0); - const failedMethod = yield* Effect.callback((resume) => { - const request = requestHttp( - { - host: "127.0.0.1", - port: gateway.port, - path: "/email/confirmation.html", - method: "POST", - }, - (result) => { - result.resume(); - result.once("end", () => resume(Effect.succeed(result.statusCode ?? 0))); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - expect(failedMethod).toBe(404); - expect(localCalls).toBe(1); - expect(activations).toBe(0); - yield* Effect.callback((resume) => { - const socket = connectNet(gateway.port, "127.0.0.1"); - socket.once("connect", () => - socket.write( - "GET /email/confirmation.html HTTP/1.1\r\nHost: gateway.test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ), - ); - socket.once("close", () => resume(Effect.void)); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(localCalls).toBe(1); - expect(activations).toBe(0); - yield* gateway.close; - }), - ), - ); - - for (const operation of ["stop", "destroy"] as const) - it.live(`returns recovery guidance for ${operation}-required HTTP activation`, () => - withPlatform( - Effect.gen(function* () { - const sensitive = "internal cleanup detail that must stay private"; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "rest", match: () => true }], - activate: () => - Effect.fail( - new GatewayActivationError({ - message: sensitive, - recovery: { operation, message: sensitive }, - }), - ), - }); - const response = yield* Effect.callback< - { readonly status: number; readonly body: string }, - Error - >((resume) => { - const request = requestHttp( - { host: "127.0.0.1", port: gateway.port, path: "/rest/items" }, - (result) => { - const chunks: Buffer[] = []; - result.on("data", (chunk: Buffer) => chunks.push(chunk)); - result.once("end", () => - resume( - Effect.succeed({ - status: result.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - }), - ), - ); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - expect(response.status).toBe(503); - const body = yield* Schema.decodeEffect(recoveryResponseSchema)(response.body); - expect(body).toEqual({ - error: "STACK_RECOVERY_REQUIRED", - recovery: { - operation, - message: - operation === "stop" - ? "Retry stack stop before activating workloads" - : "Retry stack destroy before activating workloads", - }, - }); - expect(response.body).not.toContain(sensitive); - yield* gateway.close; - }), - ), - ); - - for (const operation of ["stop", "destroy"] as const) - it.live(`returns recovery guidance before a ${operation}-required WebSocket upgrade`, () => - withPlatform( - Effect.gen(function* () { - const sensitive = "private activation failure detail"; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "realtime", match: () => true }], - activate: () => - Effect.fail( - new GatewayActivationError({ - message: sensitive, - recovery: { operation, message: sensitive }, - }), - ), - }); - const response = yield* Effect.callback((resume) => { - const socket = connectNet(gateway.port, "127.0.0.1"); - const chunks: Buffer[] = []; - socket.once("connect", () => - socket.write( - "GET /realtime/v1/websocket HTTP/1.1\r\nHost: gateway.test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ), - ); - socket.on("data", (chunk: Buffer) => chunks.push(chunk)); - socket.once("close", () => resume(Effect.succeed(Buffer.concat(chunks).toString()))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(response).toContain("HTTP/1.1 503 Service Unavailable"); - const separator = response.indexOf("\r\n\r\n"); - expect(separator).toBeGreaterThan(0); - const body = yield* Schema.decodeEffect(recoveryResponseSchema)( - response.slice(separator + 4), - ); - expect(body).toEqual({ - error: "STACK_RECOVERY_REQUIRED", - recovery: { - operation, - message: - operation === "stop" - ? "Retry stack stop before activating workloads" - : "Retry stack destroy before activating workloads", - }, - }); - expect(response).not.toContain(sensitive); - yield* gateway.close; - }), - ), - ); - - it.live("keeps local routing usable when a client disconnects mid-response", () => - withPlatform( - Effect.gen(function* () { - const started = yield* Deferred.make(); - const release = yield* Deferred.make(); - let localCalls = 0; - let activations = 0; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [ - { - match: (request) => request.path.startsWith("/email/"), - localResponse: () => - Effect.gen(function* () { - localCalls += 1; - yield* Deferred.succeed(started, undefined); - yield* Deferred.await(release); - return { body: "template", contentType: "text/plain; charset=utf-8" }; - }), - }, - ], - activate: () => - Effect.sync(() => { - activations += 1; - return { capability: "auth" as const, endpoint: { host: "127.0.0.1", port: 1 } }; - }), - }); - let request: ClientRequest | undefined; - const pending = yield* Effect.forkChild( - Effect.callback((resume) => { - request = requestHttp( - { host: "127.0.0.1", port: gateway.port, path: "/email/confirmation.html" }, - (response) => { - response.resume(); - response.once("end", () => resume(Effect.void)); - }, - ); - request.once("error", () => resume(Effect.void)); - request.end(); - return Effect.sync(() => request?.destroy()); - }), - { startImmediately: true }, - ); - yield* Deferred.await(started); - request?.destroy(); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(pending).pipe(Effect.exit); - expect(yield* getStatus(gateway.port, "/email/confirmation.html")).toBe(200); - expect(localCalls).toBe(2); - expect(activations).toBe(0); - yield* gateway.close; - }), - ), - ); - - it.live("interrupts lazy activation when the client disconnects before proxying", () => - withPlatform( - Effect.gen(function* () { - const started = yield* Deferred.make(); - const interrupted = yield* Deferred.make(); - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "rest", match: () => true }], - activate: () => - Effect.gen(function* () { - yield* Deferred.succeed(started, undefined); - return yield* Effect.never.pipe( - Effect.ensuring(Deferred.succeed(interrupted, undefined)), - ); - }), - }); - let request: ClientRequest | undefined; - const requestFiber = yield* Effect.forkChild( - Effect.callback((resume) => { - request = requestHttp( - { host: "127.0.0.1", port: gateway.port, path: "/rest/v1/items" }, - (response) => response.resume(), - ); - request.once("error", () => resume(Effect.void)); - request.end(); - return Effect.sync(() => request?.destroy()); - }), - { startImmediately: true }, - ); - yield* Deferred.await(started); - request?.destroy(); - yield* Deferred.await(interrupted).pipe(Effect.timeout("5 seconds")); - yield* Fiber.interrupt(requestFiber); - yield* gateway.close; - }), - ), - ); - - it.live("routes an HTTP request after lazy activation and forwards the response", () => - withPlatform( - Effect.gen(function* () { - const backend = createHttpServer((req: IncomingMessage, res: ServerResponse) => { - res.setHeader("x-backend", "yes"); - res.setHeader("x-seen-forwarded-host", req.headers["x-forwarded-host"] ?? ""); - res.setHeader("x-seen-forwarded-proto", req.headers["x-forwarded-proto"] ?? ""); - res.setHeader("x-seen-forwarded-for", req.headers["x-forwarded-for"] ?? ""); - res.end(`${req.method}:${req.url}`); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - const backendEndpoint: BackendEndpoint = { host: "127.0.0.1", port: address.port }; - const activated: string[] = []; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [ - { - capability: "rest", - match: (request) => request.path.startsWith("/rest"), - upstreamPath: (request) => `/internal${request.path}`, - }, - { capability: "database", match: (request) => request.path.startsWith("/db") }, - ], - activate: (capability) => - Effect.sync(() => { - activated.push(capability); - return { capability, endpoint: backendEndpoint }; - }), - resolveBackend: () => Effect.succeed(backendEndpoint), - cors: { "access-control-allow-origin": "https://example.test" }, - }); - const response = yield* Effect.callback< - { status: number; body: string; headers: Record }, - Error - >((resume) => { - const req = requestHttp( - { - host: "127.0.0.1", - port: gateway.port, - path: "/rest/hello?limit=1", - headers: { host: "api.example.test" }, - }, - (res) => { - const chunks: Buffer[] = []; - res.on("data", (chunk: Buffer) => chunks.push(chunk)); - res.on("end", () => - resume( - Effect.succeed({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - headers: res.headers, - }), - ), - ); - }, - ); - req.on("error", (error) => resume(Effect.fail(error))); - req.end(); - return Effect.sync(() => req.destroy()); - }); - expect(response.status).toBe(200); - expect(response.body).toBe("GET:/internal/rest/hello?limit=1"); - expect(response.headers["access-control-allow-origin"]).toBe("https://example.test"); - expect(response.headers["x-seen-forwarded-host"]).toBe("api.example.test"); - expect(response.headers["x-seen-forwarded-proto"]).toBe("http"); - expect(String(response.headers["x-seen-forwarded-for"])).toContain("127.0.0.1"); - expect(activated).toEqual(["rest"]); - yield* closeServer(backend); - }), - ), - ); - - it.live("adopts an already-bound native HTTP listener without rebinding", () => - withPlatform( - Effect.gen(function* () { - const prebound = createHttpServer(); - yield* Effect.callback((resume) => { - prebound.once("error", (error) => resume(Effect.fail(error))); - prebound.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = prebound.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("prebound listener did not expose an address"); - let released = false; - let releaseCalls = 0; - const listener: HostListener = { - field: "api", - address: "127.0.0.1", - port: address.port, - binding: { kind: "http", server: prebound }, - connections: { sockets: new Set() }, - close: Effect.sync(() => { - released = true; - releaseCalls += 1; - }), - }; - const gateway = yield* makeHttpGateway({ - listener, - routes: [], - activate: () => Effect.fail(new GatewayActivationError({ message: "not reached" })), - }); - expect(gateway.server).toBe(prebound); - expect(gateway.port).toBe(address.port); - yield* Effect.all([gateway.close, gateway.close], { concurrency: "unbounded" }); - expect(released).toBe(true); - expect(releaseCalls).toBe(1); - expect(prebound.listening).toBe(false); - }), - ), - ); - - it.live("holds an HTTP request until an adopted listener is ready", () => - withPlatform( - Effect.gen(function* () { - const backend = createHttpServer((request, response) => { - const chunks: Buffer[] = []; - request.on("data", (chunk: Buffer) => chunks.push(chunk)); - request.once("end", () => { - response.setHeader("access-control-allow-origin", "https://backend.example"); - response.setHeader("access-control-expose-headers", "content-range"); - response.setHeader("content-range", "0-0/1"); - response.end(Buffer.concat(chunks)); - }); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const backendAddress = backend.address(); - if (typeof backendAddress !== "object" || backendAddress === null) - return yield* Effect.die("backend did not expose an address"); - const listener = yield* bindHostListener("127.0.0.1", 0, "api"); - if (listener.binding.kind !== "http") - return yield* Effect.die("API listener did not expose HTTP binding"); - const listenerAddress = listener.binding.server.address(); - if (typeof listenerAddress !== "object" || listenerAddress === null) - return yield* Effect.die("API listener did not expose an address"); - const clientReady = yield* Deferred.make(); - const getClientReady = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const releaseActivation = yield* Deferred.make(); - const responseReady = yield* Deferred.make<{ - readonly status: number; - readonly body: Buffer; - readonly expose: string | undefined; - readonly range: string | undefined; - readonly origin: string | undefined; - }>(); - const getResponseReady = yield* Deferred.make(); - const requestFiber = yield* Effect.forkChild( - Effect.callback((resume) => { - const request = requestHttp( - { - host: "127.0.0.1", - port: listenerAddress.port, - path: "/rest/items", - method: "POST", - }, - (response) => { - const chunks: Buffer[] = []; - response.on("data", (chunk: Buffer) => chunks.push(chunk)); - response.once("end", () => { - Deferred.doneUnsafe( - responseReady, - Effect.succeed({ - status: response.statusCode ?? 0, - body: Buffer.concat(chunks), - expose: response.headers["access-control-expose-headers"]?.toString(), - range: response.headers["content-range"]?.toString(), - origin: response.headers["access-control-allow-origin"]?.toString(), - }), - ); - resume(Effect.void); - }); - }, - ); - request.once("socket", () => { - Deferred.doneUnsafe(clientReady, Effect.void); - }); - request.once("error", () => resume(Effect.void)); - request.end("request-body"); - return Effect.sync(() => request.destroy()); - }), - { startImmediately: true }, - ); - const getRequestFiber = yield* Effect.forkChild( - Effect.callback((resume) => { - const request = requestHttp( - { host: "127.0.0.1", port: listenerAddress.port, path: "/rest/items", method: "GET" }, - (response) => { - response.resume(); - response.once("end", () => { - Deferred.doneUnsafe(getResponseReady, Effect.succeed(response.statusCode ?? 0)); - resume(Effect.void); - }); - }, - ); - request.once("socket", () => { - Deferred.doneUnsafe(getClientReady, Effect.void); - }); - request.once("error", () => resume(Effect.void)); - request.end(); - return Effect.sync(() => request.destroy()); - }), - { startImmediately: true }, - ); - yield* Deferred.await(clientReady); - yield* Deferred.await(getClientReady); - const pendingEvents = listener.binding.pendingEvents; - if (pendingEvents === undefined) - return yield* Effect.die("API listener did not expose pending events"); - const queuedPost = yield* Queue.take(pendingEvents.queue); - const queuedGet = yield* Queue.take(pendingEvents.queue); - Queue.offerUnsafe(pendingEvents.queue, queuedPost); - Queue.offerUnsafe(pendingEvents.queue, queuedGet); - const gateway = yield* makeHttpGateway({ - listener, - routes: [{ capability: "rest", match: (request) => request.path === "/rest/items" }], - activate: () => - Effect.gen(function* () { - yield* Deferred.succeed(activationStarted, undefined); - yield* Deferred.await(releaseActivation); - return { - capability: "rest" as const, - endpoint: { host: "127.0.0.1", port: backendAddress.port }, - }; - }), - cors: { "access-control-allow-origin": "https://gateway.example" }, - }); - yield* Deferred.await(activationStarted); - expect(Option.isNone(yield* Deferred.poll(responseReady))).toBe(true); - yield* Deferred.succeed(releaseActivation, undefined); - const result = yield* Deferred.await(responseReady); - expect(yield* Deferred.await(getResponseReady)).toBe(200); - expect(result.status).toBe(200); - expect(result.body.toString()).toBe("request-body"); - expect(result.expose).toBe("content-range"); - expect(result.range).toBe("0-0/1"); - expect(result.origin).toBe("https://gateway.example"); - const preflight = yield* Effect.callback< - Record, - Error - >((resume) => { - const request = requestHttp( - { - host: "127.0.0.1", - port: listenerAddress.port, - path: "/rest/items", - method: "OPTIONS", - headers: { - origin: "https://example.test", - "access-control-request-method": "POST", - "access-control-request-headers": "apikey,x-client-info", - }, - }, - (response) => { - response.resume(); - response.once("end", () => resume(Effect.succeed(response.headers))); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - expect(preflight["access-control-allow-headers"]?.toString()).toBe("apikey,x-client-info"); - yield* Fiber.join(requestFiber); - yield* Fiber.join(getRequestFiber); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("closes a socket accepted before HTTP gateway adoption", () => - withPlatform( - Effect.gen(function* () { - const listener = yield* bindHostListener("127.0.0.1", 0, "api"); - if (listener.binding.kind !== "http") - return yield* Effect.die("API listener did not expose HTTP binding"); - const server = listener.binding.server; - const address = server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("API listener did not expose an address"); - const accepted = yield* Effect.forkChild( - Effect.callback((resume) => { - const onConnection = (socket: Socket) => resume(Effect.succeed(socket)); - server.once("connection", onConnection); - return Effect.sync(() => server.off("connection", onConnection)); - }), - { startImmediately: true }, - ); - yield* Effect.acquireRelease( - Effect.callback((resume) => { - const client = connectNet(address.port, "127.0.0.1"); - client.once("connect", () => resume(Effect.succeed(client))); - client.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => { - client.destroy(); - }); - }), - (client) => Effect.sync(() => client.destroy()), - ); - const socket = yield* Fiber.join(accepted).pipe(Effect.timeout("5 seconds")); - const gateway = yield* makeHttpGateway({ - listener, - routes: [], - activate: () => Effect.fail(new GatewayActivationError({ message: "not reached" })), - }); - yield* gateway.close; - expect(socket.destroyed).toBe(true); - }), - ), - ); - - it.live("applies static and prepared header transforms to HTTP and WebSocket upstreams", () => - withPlatform( - Effect.gen(function* () { - let httpHeaders: IncomingMessage["headers"] | undefined; - let websocketHeaders: IncomingMessage["headers"] | undefined; - const backend = createHttpServer((_request, response) => { - httpHeaders = _request.headers; - response.end("ok"); - }); - backend.on("upgrade", (request, socket) => { - websocketHeaders = request.headers; - socket.end( - "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - const endpoint: BackendEndpoint = { host: "127.0.0.1", port: address.port }; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [ - { - capability: "rest", - match: (request) => request.path === "/http", - upstreamHeaders: (_request, headers) => { - const { "x-remove": _removed, ...rest } = headers; - return { ...rest, "x-route": "static", "x-forwarded-host": "spoofed" }; - }, - }, - { - capability: "functions", - match: (request) => request.path === "/ws", - prepare: () => - Effect.succeed({ - resolveBackend: () => Effect.succeed(endpoint), - upstreamHeaders: (_request, headers) => { - const { "x-remove": _removed, ...rest } = headers; - return { ...rest, "x-route": "prepared", "x-forwarded-for": "spoofed" }; - }, - }), - }, - ], - activate: (capability) => Effect.succeed({ capability, endpoint }), - }); - const response = yield* Effect.callback((resume) => { - const request = requestHttp( - { - host: "127.0.0.1", - port: gateway.port, - path: "/http", - headers: { - host: "gateway.test", - "x-remove": "client-secret", - "x-forwarded-host": "client-spoof", - }, - }, - (result) => { - result.resume(); - result.once("end", () => resume(Effect.succeed(result.statusCode ?? 0))); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - expect(response).toBe(200); - expect(httpHeaders?.["x-remove"]).toBeUndefined(); - expect(httpHeaders?.["x-route"]).toBe("static"); - expect(httpHeaders?.["x-forwarded-host"]).toBe("gateway.test"); - const websocket = yield* Effect.callback((resume) => { - const socket = connectNet(gateway.port, "127.0.0.1"); - const chunks: Buffer[] = []; - socket.once("connect", () => - socket.write( - "GET /ws HTTP/1.1\r\nHost: gateway.test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nx-remove: client-secret\r\n\r\n", - ), - ); - socket.on("data", (chunk: Buffer) => chunks.push(chunk)); - socket.once("end", () => resume(Effect.succeed(Buffer.concat(chunks).toString()))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(websocket).toContain("101 Switching Protocols"); - expect(websocketHeaders?.["x-remove"]).toBeUndefined(); - expect(websocketHeaders?.["x-route"]).toBe("prepared"); - expect(websocketHeaders?.["x-forwarded-for"]).toContain("127.0.0.1"); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("closes an idle keep-alive connection it owns", () => - withPlatform( - Effect.gen(function* () { - const backend = createHttpServer((_request, response) => response.end("ok")); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - const endpoint: BackendEndpoint = { host: "127.0.0.1", port: address.port }; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "rest", match: () => true }], - activate: () => Effect.succeed({ capability: "rest", endpoint }), - }); - const agent = new Agent({ keepAlive: true }); - yield* Effect.callback((resume) => { - const request = requestHttp( - { host: "127.0.0.1", port: gateway.port, path: "/", agent }, - (response) => { - response.resume(); - response.once("end", () => resume(Effect.void)); - }, - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }); - yield* gateway.close; - expect(gateway.server.listening).toBe(false); - agent.destroy(); - yield* closeServer(backend); - }), - ), - ); - - it.live("adopts a pre-bound TCP listener while preserving half-close", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer({ allowHalfOpen: true }, (socket) => { - const chunks: Buffer[] = []; - socket.on("data", (chunk: Buffer) => chunks.push(chunk)); - socket.once("end", () => socket.end(Buffer.concat(chunks))); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const backendAddress = backend.address(); - if (typeof backendAddress !== "object" || backendAddress === null) - return yield* Effect.die("backend did not expose an address"); - const prebound = createServer({ allowHalfOpen: true }); - yield* Effect.callback((resume) => { - prebound.once("error", (error) => resume(Effect.fail(error))); - prebound.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const preboundAddress = prebound.address(); - if (typeof preboundAddress !== "object" || preboundAddress === null) - return yield* Effect.die("prebound listener did not expose an address"); - const endpoint: BackendEndpoint = { host: "127.0.0.1", port: backendAddress.port }; - const listener: HostListener = { - field: "database", - address: "127.0.0.1", - port: preboundAddress.port, - binding: { kind: "tcp", server: prebound, allowHalfOpen: true }, - connections: { sockets: new Set() }, - close: closeServer(prebound), - }; - const gateway = yield* makeTcpGateway({ - listener, - routes: [{ capability: "database", match: () => true }], - activate: () => Effect.succeed({ capability: "database", endpoint }), - }); - const fixture = Buffer.from("half-close-through-adopted-listener"); - const response = yield* Effect.callback((resume) => { - const socket = connectNet(gateway.port, "127.0.0.1"); - const chunks: Buffer[] = []; - socket.once("connect", () => { - socket.write(fixture); - socket.end(); - }); - socket.on("data", (chunk: Buffer) => chunks.push(chunk)); - socket.once("end", () => resume(Effect.succeed(Buffer.concat(chunks)))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(response.equals(fixture)).toBe(true); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("closes a socket accepted before TCP gateway adoption", () => - withPlatform( - Effect.gen(function* () { - const listener = yield* bindHostListener("127.0.0.1", 0, "database"); - if (listener.binding.kind !== "tcp") - return yield* Effect.die("database listener did not expose TCP binding"); - const server = listener.binding.server; - const address = server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("database listener did not expose an address"); - const accepted = yield* Effect.forkChild( - Effect.callback((resume) => { - const onConnection = (socket: Socket) => resume(Effect.succeed(socket)); - server.once("connection", onConnection); - return Effect.sync(() => server.off("connection", onConnection)); - }), - { startImmediately: true }, - ); - yield* Effect.acquireRelease( - Effect.callback((resume) => { - const client = connectNet(address.port, "127.0.0.1"); - client.once("connect", () => resume(Effect.succeed(client))); - client.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => { - client.destroy(); - }); - }), - (client) => Effect.sync(() => client.destroy()), - ); - const socket = yield* Fiber.join(accepted).pipe(Effect.timeout("5 seconds")); - const gateway = yield* makeTcpGateway({ - listener, - routes: [], - activate: () => Effect.fail(new GatewayActivationError({ message: "not reached" })), - }); - yield* gateway.close; - expect(socket.destroyed).toBe(true); - }), - ), - ); - - it.live("rolls back an HTTP listener when TCP acquisition fails", () => - withPlatform( - Effect.gen(function* () { - const httpServer = createHttpServer(); - const badTcpServer = createHttpServer(); - const listen = (server: Server) => - Effect.callback((resume) => { - server.once("error", (error) => resume(Effect.fail(error))); - server.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - yield* listen(httpServer); - yield* listen(badTcpServer); - const httpAddress = httpServer.address(); - const badTcpAddress = badTcpServer.address(); - if ( - typeof httpAddress !== "object" || - httpAddress === null || - typeof badTcpAddress !== "object" || - badTcpAddress === null - ) - return; - const closeNative = (server: Server) => closeServer(server); - const httpListener: HostListener = { - field: "api", - address: "127.0.0.1", - port: httpAddress.port, - binding: { kind: "http", server: httpServer }, - connections: { sockets: new Set() }, - close: closeNative(httpServer), - }; - const badTcpListener: HostListener = { - field: "database", - address: "127.0.0.1", - port: badTcpAddress.port, - binding: { kind: "http", server: badTcpServer }, - connections: { sockets: new Set() }, - close: closeNative(badTcpServer), - }; - const result = yield* makeGateway({ - http: [{ field: "api", options: { listener: httpListener, routes: [] } }], - tcp: [{ field: "database", options: { listener: badTcpListener, routes: [] } }], - activate: () => Effect.fail(new GatewayActivationError({ message: "not reached" })), - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(httpServer.listening).toBe(false); - expect(badTcpServer.listening).toBe(true); - yield* closeServer(badTcpServer); - }), - ), - ); - - it.live("owns multiple HTTP and TCP listeners keyed by public port field", () => - withPlatform( - Effect.gen(function* () { - const gateway = yield* makeGateway({ - http: [ - { - field: "api", - options: { - address: "127.0.0.1", - port: 0, - routes: [], - }, - }, - { - field: "studio", - options: { - address: "127.0.0.1", - port: 0, - routes: [], - }, - }, - ], - tcp: [ - { - field: "database", - options: { - address: "127.0.0.1", - port: 0, - routes: [], - }, - }, - { - field: "smtp", - options: { - address: "127.0.0.1", - port: 0, - routes: [], - }, - }, - ], - activate: () => Effect.fail(new GatewayActivationError({ message: "not reached" })), - }); - expect(gateway.http.get("api")?.port).toBeGreaterThan(0); - expect(gateway.http.get("studio")?.port).toBeGreaterThan(0); - expect(gateway.tcp.get("database")?.port).toBeGreaterThan(0); - expect(gateway.tcp.get("smtp")?.port).toBeGreaterThan(0); - yield* gateway.close; - }), - ), - ); - - it.live("preserves HTTP WebSocket upgrades and tunnel bytes", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer((socket) => { - let handshake = false; - let buffered = Buffer.alloc(0); - socket.on("data", (chunk: Buffer) => { - buffered = Buffer.concat([buffered, chunk]); - if (!handshake) { - const boundary = buffered.indexOf("\r\n\r\n"); - if (boundary < 0) return; - handshake = true; - socket.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n\r\n"); - const remainder = buffered.subarray(boundary + 4); - if (remainder.byteLength > 0) socket.write(remainder); - buffered = Buffer.alloc(0); - return; - } - socket.write(chunk); - }); - socket.once("end", () => socket.destroy()); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - const endpoint: BackendEndpoint = { host: "127.0.0.1", port: address.port }; - const gateway = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "rest", match: () => true }], - activate: () => Effect.succeed({ capability: "rest", endpoint }), - }); - const tunneled = yield* Effect.callback((resume) => { - const socket = connectNet(gateway.port, "127.0.0.1"); - let handshake = false; - let output = Buffer.alloc(0); - socket.on("data", (chunk: Buffer) => { - output = Buffer.concat([output, chunk]); - if (!handshake && output.includes("\r\n\r\n")) { - handshake = true; - socket.write("ws-ping"); - } - if (handshake && output.includes("ws-ping")) { - resume(Effect.succeed(output.toString())); - socket.destroy(); - } - }); - socket.once("connect", () => - socket.write( - "GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ), - ); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(tunneled).toContain("101 Switching Protocols"); - expect(tunneled).toContain("ws-ping"); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("copies TCP bytes transparently", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer((socket) => { - socket.on("data", (chunk) => socket.write(chunk)); - socket.once("end", () => socket.end()); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - const endpoint: BackendEndpoint = { host: "127.0.0.1", port: address.port }; - const gateway = yield* makeTcpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "database", match: () => true }], - activate: () => Effect.succeed({ capability: "database", endpoint }), - resolveBackend: () => Effect.succeed(endpoint), - }); - const largeFixture = Buffer.alloc(512 * 1024); - for (let index = 0; index < largeFixture.byteLength; index += 1) - largeFixture[index] = (index * 31 + 17) % 256; - const fixtures = [ - Buffer.from("postgres-bytes"), - Buffer.from([0x16, 0x03, 0x01, 0x00, 0x2a, 0xff, 0x00]), - Buffer.from("220 smtp.example ESMTP\r\nSTARTTLS\r\n"), - Buffer.from("+OK pop3.example\r\nSTLS\r\n"), - largeFixture, - ]; - const responses = yield* Effect.forEach(fixtures, (fixture) => - Effect.callback((resume) => { - const socket = connectNet(gateway.port, "127.0.0.1"); - const chunks: Buffer[] = []; - socket.once("connect", () => socket.end(fixture)); - socket.on("data", (chunk: Buffer) => chunks.push(chunk)); - socket.on("end", () => resume(Effect.succeed(Buffer.concat(chunks)))); - socket.on("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }), - ); - expect(responses).toEqual(fixtures); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("preserves a backend-initiated TCP half-close", () => - withPlatform( - Effect.gen(function* () { - let receivedAfterBackendFin = Buffer.alloc(0); - const backendReceived = yield* Deferred.make(); - const backend = createServer({ allowHalfOpen: true }, (socket) => { - socket.write("backend-fin"); - socket.end(); - socket.on("data", (chunk) => { - receivedAfterBackendFin = Buffer.concat([receivedAfterBackendFin, Buffer.from(chunk)]); - Deferred.doneUnsafe(backendReceived, Effect.void); - }); - }); - yield* Effect.callback((resume) => { - backend.once("error", (error) => resume(Effect.fail(error))); - backend.listen(0, "127.0.0.1", () => resume(Effect.void)); - }); - const address = backend.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - const endpoint: BackendEndpoint = { host: "127.0.0.1", port: address.port }; - const gateway = yield* makeTcpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "database", match: () => true }], - activate: () => Effect.succeed({ capability: "database", endpoint }), - resolveBackend: () => Effect.succeed(endpoint), - }); - const response = yield* Effect.callback((resume) => { - const socket = connectNet({ port: gateway.port, host: "127.0.0.1", allowHalfOpen: true }); - const chunks: Buffer[] = []; - socket.on("data", (chunk: Buffer) => { - chunks.push(chunk); - if (Buffer.concat(chunks).includes("backend-fin")) socket.end("client-after-fin"); - }); - socket.once("end", () => resume(Effect.succeed(Buffer.concat(chunks)))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(response.toString()).toContain("backend-fin"); - yield* Deferred.await(backendReceived); - expect(receivedAfterBackendFin.toString()).toContain("client-after-fin"); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("keeps probes dormant and maps activation/backend failures", () => - withPlatform( - Effect.gen(function* () { - let activations = 0; - const route = { - capability: "rest" as const, - match: (request: GatewayRouteRequest) => request.path === "/api", - }; - const dormant = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [route], - activate: () => { - activations += 1; - return Effect.fail(new GatewayActivationError({ message: "should not run" })); - }, - }); - expect(yield* getStatus(dormant.port, "/health")).toBe(404); - expect(activations).toBe(0); - yield* dormant.close; - - const activationFailure = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [route], - activate: () => Effect.fail(new GatewayActivationError({ message: "activation failed" })), - }); - expect(yield* getStatus(activationFailure.port, "/api")).toBe(503); - yield* activationFailure.close; - - const backendFailure = yield* makeHttpGateway({ - address: "127.0.0.1", - port: 0, - routes: [route], - activate: () => - Effect.succeed({ capability: "rest", endpoint: { host: "127.0.0.1", port: 1 } }), - resolveBackend: () => - Effect.fail(new GatewayActivationError({ message: "Gateway backend failed" })), - }); - expect(yield* getStatus(backendFailure.port, "/api")).toBe(503); - yield* backendFailure.close; - - const tcpFailure = yield* makeTcpGateway({ - address: "127.0.0.1", - port: 0, - routes: [{ capability: "database", match: () => true }], - activate: () => Effect.fail(new GatewayActivationError({ message: "activation failed" })), - }); - const bytesOnActivationFailure = yield* Effect.callback((resume) => { - const socket = connectNet(tcpFailure.port, "127.0.0.1"); - let bytes = 0; - socket.on("data", (chunk: Buffer) => (bytes += chunk.byteLength)); - socket.once("close", () => resume(Effect.succeed(bytes))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }); - expect(bytesOnActivationFailure).toBe(0); - yield* tcpFailure.close; - }), - ), - ); -}); diff --git a/packages/stack/src/gateway/route-catalog.integration.test.ts b/packages/stack/src/gateway/route-catalog.integration.test.ts deleted file mode 100644 index c223015dea..0000000000 --- a/packages/stack/src/gateway/route-catalog.integration.test.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; -import { compileStack } from "../model/Compiler.ts"; -import { routeCatalogFor, type GatewayRouteCatalog } from "./RouteCatalog.ts"; - -const material = { - publishableKey: "sb_publishable", - secretKey: "sb_secret", - anonJwt: "anon-jwt", - serviceRoleJwt: "service-role-jwt", -} as const; - -const catalogFixture = () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: "/tmp/route-catalog", - runtime: { kind: "native" }, - config: { capabilities: { pooler: { enabled: true } } }, - }); - return routeCatalogFor(compiled.executionPlan, material); - }); - -const routeFor = ( - catalog: GatewayRouteCatalog, - path: string, - headers: Readonly> = {}, -) => { - const route = (catalog.http.get("api") ?? []).find((candidate) => - candidate.match({ path, headers }), - ); - if (route === undefined) throw new Error(`No API route matched ${path}`); - return route; -}; - -const upstreamFor = ( - catalog: GatewayRouteCatalog, - path: string, - headers: Readonly> = {}, -) => { - const route = routeFor(catalog, path, headers); - if (route.upstreamPath === undefined) throw new Error(`Route has no upstream path: ${path}`); - return route.upstreamPath({ path, headers }); -}; - -const headersFor = ( - catalog: GatewayRouteCatalog, - path: string, - headers: Readonly>, -): Readonly> => { - const route = routeFor(catalog, path, headers); - if (route.upstreamHeaders === undefined) - throw new Error(`Route has no header transform: ${path}`); - return route.upstreamHeaders({ path, headers }, headers); -}; - -const headerCases = [ - { - name: "publishable REST key", - path: "/rest/v1/items", - headers: { apikey: material.publishableKey }, - expected: { apikey: material.publishableKey, authorization: "Bearer anon-jwt" }, - }, - { - name: "secret REST authorization", - path: "/rest/v1/items", - headers: { authorization: "Bearer sb_secret" }, - expected: { authorization: "Bearer service-role-jwt" }, - }, - { - name: "user authorization", - path: "/rest/v1/items", - headers: { authorization: "Bearer user-jwt", apikey: material.secretKey }, - expected: { authorization: "Bearer user-jwt", apikey: material.secretKey }, - }, - { - name: "unknown REST key", - path: "/rest/v1/items", - headers: { apikey: "legacy-unknown" }, - expected: { apikey: "legacy-unknown" }, - }, - { - name: "publishable Auth key", - path: "/auth/v1/token", - headers: { apikey: material.publishableKey }, - expected: { apikey: material.publishableKey, authorization: "Bearer anon-jwt" }, - }, - { - name: "secret Realtime API key", - path: "/realtime/v1/api/tenants", - headers: { apikey: material.secretKey }, - expected: { apikey: material.secretKey, authorization: "Bearer service-role-jwt" }, - }, - { - name: "publishable Storage key", - path: "/storage/v1/object/list", - headers: { apikey: material.publishableKey }, - expected: { apikey: material.publishableKey, authorization: "Bearer anon-jwt" }, - }, - { - name: "spoofed Functions key", - path: "/functions/v1/hello", - headers: { "sb-api-key": "spoofed", apikey: material.publishableKey }, - expected: { apikey: material.publishableKey, "sb-api-key": "Bearer anon-jwt" }, - }, - { - name: "secret Functions key", - path: "/functions/v1/hello", - headers: { apikey: material.secretKey }, - expected: { apikey: material.secretKey, "sb-api-key": "Bearer service-role-jwt" }, - }, - { - name: "unknown Functions key removes spoof", - path: "/functions/v1/hello", - headers: { "sb-api-key": "spoofed", apikey: "legacy-unknown" }, - expected: { apikey: "legacy-unknown" }, - }, - { - name: "GraphQL profile", - path: "/graphql/v1", - headers: { authorization: "Bearer sb_publishable", "content-profile": "spoofed" }, - expected: { authorization: "Bearer anon-jwt", "content-profile": "graphql_public" }, - }, - { - name: "S3 signature", - path: "/storage/v1/s3/bucket/object", - headers: { authorization: "AWS4-HMAC-SHA256 Credential=example" }, - expected: { authorization: "AWS4-HMAC-SHA256 Credential=example" }, - }, -] as const; - -describe("public gateway route catalog", () => { - it.live("rewrites each enabled API prefix while preserving query strings", () => - catalogFixture().pipe( - Effect.map((catalog) => { - const expected = { - "/rest/v1/items?limit=1": "/items?limit=1", - "/graphql/v1?query=items": "/rpc/graphql?query=items", - "/auth/v1/token?grant_type=password": "/token?grant_type=password", - "/realtime/v1/websocket?vsn=1": "/socket/websocket?vsn=1", - "/realtime/v1/api/tenants?limit=1": "/api/tenants?limit=1", - "/storage/v1/s3/bucket/object?X-Amz-Signature=abc": - "/s3/bucket/object?X-Amz-Signature=abc", - "/storage/v1/object/list?limit=1": "/object/list?limit=1", - "/functions/v1/hello?x=1": "/hello?x=1", - "/analytics/v1/logs?limit=1": "/logs?limit=1", - } as const; - for (const [path, upstream] of Object.entries(expected)) - expect(upstreamFor(catalog, path)).toBe(upstream); - }), - Effect.provide(NodeServices.layer), - ), - ); - - it.live.each(headerCases)("transforms $name headers", ({ path, headers, expected }) => - catalogFixture().pipe( - Effect.map((catalog) => expect(headersFor(catalog, path, headers)).toEqual(expected)), - Effect.provide(NodeServices.layer), - ), - ); - - it.live("rewrites WebSocket host and API-key query credentials", () => - catalogFixture().pipe( - Effect.map((catalog) => { - const headers = headersFor(catalog, "/realtime/v1/websocket", { - Host: "127.0.0.1:40000", - apikey: material.publishableKey, - }); - expect(headers).toMatchObject({ host: "realtime-dev", authorization: "Bearer anon-jwt" }); - expect(headers).not.toHaveProperty("Host"); - expect(upstreamFor(catalog, "/realtime/v1/websocket?apikey=sb_secret&vsn=1")).toBe( - "/socket/websocket?apikey=service-role-jwt&vsn=1", - ); - }), - Effect.provide(NodeServices.layer), - ), - ); - - it.live("registers direct HTTP and TCP listeners for every enabled public service", () => - catalogFixture().pipe( - Effect.map((catalog) => { - const studioRoutes = catalog.http.get("studio"); - if (studioRoutes === undefined || studioRoutes.length !== 1) - throw new Error("Expected one studio route"); - expect(studioRoutes[0]?.capability).toBe("studio"); - expect(studioRoutes[0]?.match({ path: "/", headers: {} })).toBe(true); - expect(catalog.http.get("mailUi")).toEqual([ - expect.objectContaining({ capability: "mail", binding: "ui" }), - ]); - expect(catalog.http.get("functionsInspector")).toEqual([ - expect.objectContaining({ capability: "functions", binding: "inspector" }), - ]); - expect(catalog.tcp.get("database")).toEqual([ - expect.objectContaining({ capability: "database" }), - ]); - expect(catalog.tcp.get("pooler")).toEqual([ - expect.objectContaining({ capability: "pooler" }), - ]); - expect(catalog.tcp.get("smtp")).toEqual([expect.objectContaining({ binding: "smtp" })]); - expect(catalog.tcp.get("pop3")).toEqual([expect.objectContaining({ binding: "pop3" })]); - }), - Effect.provide(NodeServices.layer), - ), - ); -}); diff --git a/packages/stack/src/gateway/startup-transports.integration.test.ts b/packages/stack/src/gateway/startup-transports.integration.test.ts deleted file mode 100644 index d6937bad45..0000000000 --- a/packages/stack/src/gateway/startup-transports.integration.test.ts +++ /dev/null @@ -1,382 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Data, Deferred, Effect, Option, Predicate, Queue, Scope } from "effect"; -import { connect as connectNet, createServer, type Server, type Socket } from "node:net"; -import { makeHttpGateway } from "./HttpGateway.ts"; -import { makeTcpGateway } from "./TcpGateway.ts"; -import type { BackendEndpoint } from "./Gateway.ts"; -import { bindHostListener } from "../supervisor/HostListener.ts"; - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const closeServer = (server: Server): Effect.Effect => - Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }); - -const listen = (server: Server): Effect.Effect => { - const connections = new Set(); - const onConnection = (socket: Socket) => { - connections.add(socket); - socket.once("close", () => connections.delete(socket)); - }; - server.on("connection", onConnection); - return Effect.acquireRelease( - Effect.callback((resume) => { - server.once("error", (error) => resume(Effect.fail(error))); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - if (typeof address === "object" && address !== null) resume(Effect.succeed(address.port)); - }); - return Effect.sync(() => server.close()); - }), - () => - Effect.sync(() => { - server.off("connection", onConnection); - for (const socket of connections) socket.destroy(); - }).pipe(Effect.andThen(closeServer(server))), - ); -}; - -const socketAt = (port: number, allowHalfOpen = false): Effect.Effect => - Effect.acquireRelease( - Effect.callback((resume) => { - const socket = connectNet({ port, host: "127.0.0.1", allowHalfOpen }); - socket.once("connect", () => resume(Effect.succeed(socket))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => socket.destroy()); - }), - (socket) => Effect.sync(() => socket.destroy()), - ); - -class HeldSocketError extends Data.TaggedError("HeldSocketError")<{ - readonly message: string; -}> {} - -describe("gateway startup transport adoption", () => { - it.live("preserves TCP bytes sent before adoption while activation is delayed", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer({ allowHalfOpen: true }, (socket) => { - socket.on("data", (chunk) => { - socket.write(chunk); - }); - socket.once("end", () => socket.end()); - }); - const backendPort = yield* listen(backend); - const listener = yield* bindHostListener("127.0.0.1", 0, "database"); - if (listener.binding.kind !== "tcp") - return yield* Effect.die("database listener did not expose TCP binding"); - const address = listener.binding.server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("TCP listener did not expose an address"); - const accepted = yield* Deferred.make(); - listener.binding.server.once("connection", (socket) => - Deferred.doneUnsafe(accepted, Effect.succeed(socket)), - ); - const activationStarted = yield* Deferred.make(); - const releaseActivation = yield* Deferred.make(); - const responseReady = yield* Deferred.make(); - const client = yield* socketAt(address.port); - const serverSocket = yield* Deferred.await(accepted); - const chunks: Buffer[] = []; - client.on("data", (chunk) => chunks.push(Buffer.from(chunk))); - client.once("end", () => - Deferred.doneUnsafe(responseReady, Effect.succeed(Buffer.concat(chunks))), - ); - const fixture = Buffer.from("tcp-before-adoption"); - const buffered = yield* Deferred.make(); - serverSocket.once("readable", () => Deferred.doneUnsafe(buffered, Effect.void)); - client.write(fixture); - yield* Deferred.await(buffered); - expect(serverSocket.readableLength).toBeGreaterThanOrEqual(fixture.byteLength); - const gateway = yield* makeTcpGateway({ - listener, - routes: [{ capability: "database", match: () => true }], - activate: () => - Effect.gen(function* () { - yield* Deferred.succeed(activationStarted, undefined); - yield* Deferred.await(releaseActivation); - return { capability: "database", endpoint: { host: "127.0.0.1", port: backendPort } }; - }), - }); - yield* Deferred.await(activationStarted); - expect(Option.isNone(yield* Deferred.poll(responseReady))).toBe(true); - yield* Deferred.succeed(releaseActivation, undefined); - client.end(); - expect((yield* Deferred.await(responseReady)).equals(fixture)).toBe(true); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("contains errors during TCP activation without affecting other connections", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer({ allowHalfOpen: true }, (socket) => { - socket.on("data", (chunk) => socket.write(chunk)); - }); - const backendPort = yield* listen(backend); - const listener = yield* bindHostListener("127.0.0.1", 0, "database"); - if (listener.binding.kind !== "tcp") - return yield* Effect.die("database listener did not expose TCP binding"); - const address = listener.binding.server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("TCP listener did not expose an address"); - const firstAccepted = yield* Deferred.make(); - const secondAccepted = yield* Deferred.make(); - listener.binding.server.once("connection", (socket) => - Deferred.doneUnsafe(firstAccepted, Effect.succeed(socket)), - ); - const firstClient = yield* socketAt(address.port); - const firstServer = yield* Deferred.await(firstAccepted); - listener.binding.server.once("connection", (socket) => - Deferred.doneUnsafe(secondAccepted, Effect.succeed(socket)), - ); - const secondClient = yield* socketAt(address.port); - yield* Deferred.await(secondAccepted); - const firstClosed = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const releaseActivation = yield* Deferred.make(); - firstClient.on("error", () => undefined); - firstClient.once("close", () => Deferred.doneUnsafe(firstClosed, Effect.void)); - const responseReady = yield* Deferred.make(); - secondClient.once("data", (chunk) => - Deferred.doneUnsafe(responseReady, Effect.succeed(Buffer.from(chunk))), - ); - const gateway = yield* makeTcpGateway({ - listener, - routes: [{ capability: "database", match: () => true }], - activate: () => - Effect.gen(function* () { - yield* Deferred.succeed(activationStarted, undefined); - yield* Deferred.await(releaseActivation); - return { - capability: "database", - endpoint: { host: "127.0.0.1", port: backendPort }, - }; - }), - }); - yield* Deferred.await(activationStarted); - firstServer.destroy(new HeldSocketError({ message: "held socket failed" })); - yield* Deferred.await(firstClosed).pipe(Effect.timeout("5 seconds")); - expect(Option.isNone(yield* Deferred.poll(responseReady))).toBe(true); - yield* Deferred.succeed(releaseActivation, undefined); - secondClient.write("unrelated-connection"); - expect((yield* Deferred.await(responseReady)).toString()).toBe("unrelated-connection"); - secondClient.destroy(); - yield* gateway.close; - }), - ), - ); - - it.live("preserves an early WebSocket handshake and payload across delayed adoption", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer((socket) => { - let upgraded = false; - let pending = Buffer.alloc(0); - const headerEnd = Buffer.from("\r\n\r\n"); - socket.on("data", (chunk) => { - const bytes = Buffer.from(chunk); - if (upgraded) { - socket.write(bytes); - return; - } - pending = Buffer.concat([pending, bytes]); - const end = pending.indexOf(headerEnd); - if (end < 0) return; - upgraded = true; - socket.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n\r\nws-ready"); - const payload = pending.subarray(end + headerEnd.byteLength); - if (payload.byteLength > 0) socket.write(payload); - pending = Buffer.alloc(0); - }); - }); - const backendPort = yield* listen(backend); - const listener = yield* bindHostListener("127.0.0.1", 0, "api"); - if (listener.binding.kind !== "http") - return yield* Effect.die("API listener did not expose HTTP binding"); - const address = listener.binding.server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("HTTP listener did not expose an address"); - const client = yield* socketAt(address.port); - const upgradeQueued = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const releaseActivation = yield* Deferred.make(); - const responseReady = yield* Deferred.make(); - const output: Buffer[] = []; - const preAdoptionPayload = Buffer.from("ws-before-adoption"); - let pingSent = false; - listener.binding.server.once("upgrade", () => - Deferred.doneUnsafe(upgradeQueued, Effect.void), - ); - client.on("data", (chunk) => { - output.push(Buffer.from(chunk)); - const received = Buffer.concat(output); - if (!pingSent && received.includes(Buffer.from("ws-ready"))) { - pingSent = true; - client.write("ws-ping"); - } - if (received.includes(Buffer.from("ws-ping"))) - Deferred.doneUnsafe(responseReady, Effect.succeed(received)); - }); - client.write( - Buffer.concat([ - Buffer.from( - "GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ), - preAdoptionPayload, - ]), - ); - yield* Deferred.await(upgradeQueued); - const pendingEvents = listener.binding.pendingEvents; - if (pendingEvents === undefined) - return yield* Effect.die("HTTP listener did not expose pending events"); - const captured = yield* Queue.take(pendingEvents.queue); - if (!Predicate.isTagged(captured, "upgrade")) - return yield* Effect.die("upgrade event was not captured"); - if (captured.head.byteLength === 0 && captured.socket.readableLength === 0) { - const bytesBuffered = yield* Deferred.make(); - captured.socket.once("readable", () => Deferred.doneUnsafe(bytesBuffered, Effect.void)); - yield* Deferred.await(bytesBuffered); - } - expect(captured.head.byteLength + captured.socket.readableLength).toBeGreaterThan(0); - Queue.offerUnsafe(pendingEvents.queue, captured); - const gateway = yield* makeHttpGateway({ - listener, - routes: [{ capability: "rest", match: (request) => request.path === "/socket" }], - activate: () => - Deferred.succeed(activationStarted, undefined).pipe( - Effect.andThen(Deferred.await(releaseActivation)), - Effect.as({ capability: "rest", endpoint: { host: "127.0.0.1", port: backendPort } }), - ), - }); - yield* Deferred.await(activationStarted); - expect(Option.isNone(yield* Deferred.poll(responseReady))).toBe(true); - yield* Deferred.succeed(releaseActivation, undefined); - const response = yield* Deferred.await(responseReady).pipe(Effect.timeout("5 seconds")); - expect(response.toString()).toContain("101 Switching Protocols"); - expect(response.toString()).toContain(preAdoptionPayload.toString()); - expect(response.toString()).toContain("ws-ping"); - client.destroy(); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live("forwards backend trailing bytes after a client WebSocket half-close", () => - withPlatform( - Effect.gen(function* () { - const backend = createServer({ allowHalfOpen: true }, (socket) => { - let upgraded = false; - socket.on("data", (chunk) => { - if (!upgraded && Buffer.from(chunk).includes(Buffer.from("\r\n\r\n"))) { - upgraded = true; - socket.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n\r\n"); - } - }); - socket.once("end", () => socket.end("trailingbytes")); - }); - const backendPort = yield* listen(backend); - const listener = yield* bindHostListener("127.0.0.1", 0, "api"); - if (listener.binding.kind !== "http") - return yield* Effect.die("API listener did not expose HTTP binding"); - const address = listener.binding.server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("HTTP listener did not expose an address"); - const client = yield* socketAt(address.port, true); - const handshakeReady = yield* Deferred.make(); - const responseReady = yield* Deferred.make(); - const output: Buffer[] = []; - client.on("data", (chunk) => { - output.push(Buffer.from(chunk)); - const response = Buffer.concat(output); - if (response.includes(Buffer.from("101 Switching Protocols"))) - Deferred.doneUnsafe(handshakeReady, Effect.void); - if (response.includes(Buffer.from("trailingbytes"))) - Deferred.doneUnsafe(responseReady, Effect.succeed(response)); - }); - client.once("end", () => - Deferred.doneUnsafe(responseReady, Effect.succeed(Buffer.concat(output))), - ); - client.write( - "GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ); - const gateway = yield* makeHttpGateway({ - listener, - routes: [{ capability: "rest", match: (request) => request.path === "/socket" }], - activate: () => - Effect.succeed({ - capability: "rest", - endpoint: { host: "127.0.0.1", port: backendPort }, - }), - }); - yield* Deferred.await(handshakeReady); - client.end(); - const response = yield* Deferred.await(responseReady).pipe(Effect.timeout("5 seconds")); - expect(response.toString()).toContain("trailingbytes"); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); - - it.live( - "preserves an early WebSocket upgrade and interrupts activation when the client closes", - () => - withPlatform( - Effect.gen(function* () { - const backend = createServer((socket) => { - socket.on("data", (chunk) => { - if (Buffer.from(chunk).includes(Buffer.from("\r\n\r\n"))) - socket.write("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n\r\n"); - }); - }); - const backendPort = yield* listen(backend); - const listener = yield* bindHostListener("127.0.0.1", 0, "api"); - if (listener.binding.kind !== "http") - return yield* Effect.die("API listener did not expose HTTP binding"); - const address = listener.binding.server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("HTTP listener did not expose an address"); - const client = yield* socketAt(address.port); - const upgradeQueued = yield* Deferred.make(); - const clientClosed = yield* Deferred.make(); - listener.binding.server.once("upgrade", () => - Deferred.doneUnsafe(upgradeQueued, Effect.void), - ); - client.once("close", () => Deferred.doneUnsafe(clientClosed, Effect.void)); - client.write( - "GET /socket HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n", - ); - yield* Deferred.await(upgradeQueued); - const activationStarted = yield* Deferred.make(); - const activationInterrupted = yield* Deferred.make(); - const gateway = yield* makeHttpGateway({ - listener, - routes: [{ capability: "rest", match: (request) => request.path === "/socket" }], - activate: () => - Effect.gen(function* () { - yield* Deferred.succeed(activationStarted, undefined); - return yield* Effect.never.pipe( - Effect.ensuring(Deferred.succeed(activationInterrupted, undefined)), - ); - }), - resolveBackend: () => - Effect.succeed({ host: "127.0.0.1", port: backendPort }), - }); - yield* Deferred.await(activationStarted); - client.destroy(); - yield* Deferred.await(clientClosed); - yield* Deferred.await(activationInterrupted).pipe(Effect.timeout("5 seconds")); - yield* gateway.close; - yield* closeServer(backend); - }), - ), - ); -}); diff --git a/packages/stack/src/host/Endpoints.ts b/packages/stack/src/host/Endpoints.ts new file mode 100644 index 0000000000..bd8515dde3 --- /dev/null +++ b/packages/stack/src/host/Endpoints.ts @@ -0,0 +1,146 @@ +import { allowedEndpointNames, type ServiceCreation } from "../services/Catalog.ts"; +import { Data, Effect } from "effect"; + +export class EndpointError extends Data.TaggedError("EndpointError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +export const endpointNames = (creation: ServiceCreation): ReadonlyArray => { + const configured: Readonly> = Object.fromEntries( + Object.entries(creation.endpoints ?? {}), + ); + return allowedEndpointNames(creation.service).filter((name) => isRecord(configured[name])); +}; + +export const endpointPort = (creation: ServiceCreation, name: string): number | "auto" => { + const endpoints: unknown = creation.endpoints; + if (!isRecord(endpoints)) return "auto"; + const endpoint = endpoints[name]; + if (!isRecord(endpoint)) return "auto"; + return endpoint.port === "auto" || typeof endpoint.port === "number" ? endpoint.port : "auto"; +}; + +export const apiRoute = (service: ServiceCreation["service"]): string | undefined => { + switch (service) { + case "rest": + return "/rest/v1"; + case "auth": + return "/auth/v1"; + case "storage": + return "/storage/v1"; + case "functions": + return "/functions/v1"; + case "realtime": + return "/realtime/v1"; + default: + return undefined; + } +}; + +export const publicUrl = (host: string, port: number) => `http://${host}:${port}`; + +const postgresUrl = ( + host: string, + port: number, + role: string, + password: string, + database: string, +) => + `postgresql://${encodeURIComponent(role)}:${encodeURIComponent(password)}@${host}:${port}/${database}`; + +interface EndpointAddress { + readonly host: string; + readonly port: number; +} + +export type EndpointAddressFor = ( + endpoint: string, + from: "host" | "runtime", +) => Effect.Effect; + +export type EndpointPassword = () => Effect.Effect; + +export const outputsFor = ( + creation: ServiceCreation, + address: EndpointAddressFor, + password: EndpointPassword, +): Readonly>> => { + const output = (name: string) => + address(name, "runtime").pipe(Effect.map(({ host, port }) => publicUrl(host, port))); + const outputs: Record> = {}; + if (endpointNames(creation).includes("http")) { + outputs.url = output("http"); + outputs.hostUrl = address("http", "host").pipe( + Effect.map(({ host, port }) => publicUrl(host, port)), + ); + outputs.runtimeUrl = output("http"); + } + if (creation.service === "mail" && endpointNames(creation).includes("smtp")) + outputs.smtpUrl = address("smtp", "runtime").pipe( + Effect.map(({ host, port }) => `smtp://${host}:${port}`), + ); + if (creation.service === "database" && endpointNames(creation).includes("sql")) { + for (const [name, role, database] of [ + ["databaseUrl", "supabase_admin", "postgres"], + ["authenticatorUrl", "authenticator", "postgres"], + ["authDatabaseUrl", "supabase_auth_admin", "postgres"], + ["storageDatabaseUrl", "supabase_storage_admin", "postgres"], + ["internalDatabaseUrl", "supabase_admin", "_supabase"], + ] as const) + outputs[name] = address("sql", "runtime").pipe( + Effect.flatMap(({ host, port }) => + password().pipe(Effect.map((value) => postgresUrl(host, port, role, value, database))), + ), + ); + } + return outputs; +}; + +export const credentialsFor = Effect.fn("Endpoints.credentialsFor")( + ( + creation: ServiceCreation, + address: EndpointAddressFor, + password: EndpointPassword, + from: "host" | "runtime", + ): Effect.Effect>, EndpointError> => + Effect.gen(function* () { + const credentials: Record = {}; + if (endpointNames(creation).includes("http")) { + const { host, port } = yield* address("http", from); + const origin = publicUrl(host, port); + credentials.url = `${origin}${apiRoute(creation.service) ?? ""}`; + if (apiRoute(creation.service) !== undefined) credentials.apiUrl = origin; + } + if (creation.service === "mail") { + if (endpointNames(creation).includes("smtp")) { + const { host, port } = yield* address("smtp", from); + credentials.smtpUrl = `smtp://${host}:${port}`; + } + if (endpointNames(creation).includes("pop3")) { + const { host, port } = yield* address("pop3", from); + credentials.pop3Url = `pop3://${host}:${port}`; + } + } + if (creation.service === "pooler" && endpointNames(creation).includes("sql")) { + const { host, port } = yield* address("sql", from); + credentials.sqlUrl = `postgresql://${host}:${port}`; + } + if (creation.service === "database" && endpointNames(creation).includes("sql")) { + const { host, port } = yield* address("sql", from); + const value = yield* password(); + for (const [name, role, database] of [ + ["databaseUrl", "supabase_admin", "postgres"], + ["authenticatorUrl", "authenticator", "postgres"], + ["authDatabaseUrl", "supabase_auth_admin", "postgres"], + ["storageDatabaseUrl", "supabase_storage_admin", "postgres"], + ["internalDatabaseUrl", "supabase_admin", "_supabase"], + ] as const) + credentials[name] = postgresUrl(host, port, role, value, database); + } + return credentials; + }), +); diff --git a/packages/stack/src/host/ToolAttachments.ts b/packages/stack/src/host/ToolAttachments.ts new file mode 100644 index 0000000000..c45a73233c --- /dev/null +++ b/packages/stack/src/host/ToolAttachments.ts @@ -0,0 +1,159 @@ +import { Cause, Effect, Fiber, Queue, Ref, Schema, Stream } from "effect"; +import { StackError, ToolEvent as ToolEventSchema } from "../Rpc.ts"; +import type { PgProveOptions, PostgresTool } from "../Tools.ts"; +import type * as ToolRunner from "./ToolRunner.ts"; +import type { ToolInput } from "./ToolRunner.ts"; + +type ToolEvent = Schema.Schema.Type; + +export interface ToolAttachmentPayload { + readonly attachmentId: string; + readonly tool: PostgresTool; + readonly args: ReadonlyArray; + readonly env: Readonly>; + readonly pgProve?: PgProveOptions; + readonly stdin: boolean; +} + +interface Attachment { + readonly attachmentId: string; + readonly stdin: boolean; + readonly input: Queue.Queue; + readonly output: Queue.Queue; + readonly fiber: Ref.Ref | undefined>; +} + +interface ToolAttachmentOperations { + readonly run: (input: ToolAttachmentPayload) => Stream.Stream; + readonly input: ( + attachmentId: string, + stdin: boolean, + bytes: Uint8Array | null, + ) => Effect.Effect; + readonly stopAll: Effect.Effect; +} + +export interface ToolAttachmentOptions { + readonly admit: Effect.Effect; + readonly run: ToolRunner.Interface["run"]; + readonly toError: (operation: string, cause: unknown) => StackError; +} + +export const makeToolAttachments = (options: ToolAttachmentOptions) => + Effect.gen(function* () { + const entries = yield* Ref.make(new Map()); + const stopAll = Effect.gen(function* () { + yield* Ref.get(entries).pipe( + Effect.flatMap((values) => + Effect.forEach( + [...values.values()], + (entry) => + Effect.gen(function* () { + yield* Queue.shutdown(entry.input); + yield* Queue.fail(entry.output, options.toError("tool", "Stack host is draining")); + const runner = yield* Ref.get(entry.fiber); + if (runner !== undefined) yield* Fiber.interrupt(runner); + }), + { discard: true }, + ), + ), + ); + yield* Ref.set(entries, new Map()); + }).pipe(Effect.withSpan("ToolAttachments.stopAll")); + + const run = (input: ToolAttachmentPayload): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + yield* options.admit; + const existing = yield* Ref.get(entries).pipe( + Effect.map((values) => values.get(input.attachmentId)), + ); + if (existing !== undefined) + return yield* options.toError("tool", "Attachment is already in use"); + const attachment: Attachment = { + attachmentId: input.attachmentId, + stdin: input.stdin, + input: yield* Queue.bounded(1), + output: yield* Queue.make({ capacity: 16 }), + fiber: yield* Ref.make | undefined>(undefined), + }; + yield* Ref.update(entries, (values) => + new Map(values).set(input.attachmentId, attachment), + ); + const stdin = input.stdin + ? Stream.fromQueue(attachment.input).pipe( + Stream.takeWhile((chunk): chunk is Uint8Array => chunk !== null), + ) + : Stream.empty; + const write = (event: ToolEvent) => + Queue.offer(attachment.output, event).pipe(Effect.asVoid); + const runner = options + .run({ + tool: input.tool, + args: input.args, + env: input.env, + pgProve: input.pgProve, + stdin, + stdout: (bytes) => write({ _tag: "Stdout", bytes }), + stderr: (bytes) => write({ _tag: "Stderr", bytes }), + } satisfies ToolInput) + .pipe( + Effect.flatMap((result) => + write({ _tag: "Completed", jobId: result.jobId, exitCode: result.exitCode }), + ), + Effect.mapError((cause) => options.toError("tool", cause)), + Effect.tapError((cause) => Queue.fail(attachment.output, cause)), + Effect.ensuring( + Effect.gen(function* () { + yield* Queue.shutdown(attachment.input); + yield* Queue.end(attachment.output); + yield* Ref.update(entries, (values) => { + const next = new Map(values); + next.delete(input.attachmentId); + return next; + }); + }), + ), + ); + const fiber = yield* Effect.forkScoped(runner); + yield* Ref.set(attachment.fiber, fiber); + const stream = Stream.concat( + Stream.succeed({ + _tag: "Attached", + attachmentId: input.attachmentId, + } satisfies ToolEvent), + Stream.fromQueue(attachment.output), + ); + return stream.pipe( + Stream.ensuring( + Effect.gen(function* () { + yield* Queue.shutdown(attachment.input); + yield* Queue.shutdown(attachment.output); + yield* Fiber.interrupt(fiber); + }), + ), + ); + }).pipe(Effect.withSpan("ToolAttachments.run")), + ); + + const input = Effect.fn("ToolAttachments.input")( + (attachmentId: string, stdin: boolean, bytes: Uint8Array | null) => + Ref.get(entries).pipe( + Effect.flatMap((values) => { + const attachment = values.get(attachmentId); + if (attachment === undefined) + return Effect.fail(options.toError("tool-input-closed", "Attachment is closed")); + if (!stdin || !attachment.stdin) + return Effect.fail(options.toError("toolInput", "Tool did not request stdin")); + return Queue.offer(attachment.input, bytes).pipe( + Effect.flatMap((accepted) => + accepted + ? Effect.void + : Effect.fail(options.toError("tool-input-closed", "Attachment is closed")), + ), + ); + }), + ), + ); + return { run, input, stopAll } satisfies ToolAttachmentOperations; + }); diff --git a/packages/stack/src/host/ToolRunner.ts b/packages/stack/src/host/ToolRunner.ts new file mode 100644 index 0000000000..017c93677f --- /dev/null +++ b/packages/stack/src/host/ToolRunner.ts @@ -0,0 +1,195 @@ +import { + Context, + Crypto, + Data, + Effect, + FileSystem, + Layer, + Path, + Schema, + Sink, + Stream, +} from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- FileSystem has no non-recursive directory removal operation. +import { rmdir } from "node:fs/promises"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { HttpClient } from "effect/unstable/http"; +import { prepareNativeArtifact, postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { makeContainerRuntime } from "../runtime/Container.ts"; +import { PostgresTool, type PgProveOptions } from "../Tools.ts"; + +class ToolError extends Data.TaggedError("ToolError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +class StdinWriteError extends Data.TaggedError("StdinWriteError")<{ readonly cause: ToolError }> {} + +export interface ToolInput { + readonly tool: PostgresTool; + readonly args: ReadonlyArray; + readonly env: Readonly>; + readonly pgProve?: PgProveOptions; + readonly stdin?: Stream.Stream; + readonly stdout: (bytes: Uint8Array) => Effect.Effect; + readonly stderr: (bytes: Uint8Array) => Effect.Effect; +} + +export interface Interface { + readonly run: ( + input: ToolInput, + ) => Effect.Effect<{ readonly jobId: string; readonly exitCode: number }, E | ToolError, R>; + readonly cleanup: Effect.Effect; +} + +export class Service extends Context.Service()("@supabase/stack/ToolRunner") {} + +const failure = (cause: unknown) => + new ToolError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +/** Creates an attached byte-stream runner whose invocation scope owns each finite process. */ +const makeToolRunner = (options: { + readonly stackId: string; + readonly root: string; + readonly cacheRoot: string; + readonly runtime: "native" | "docker" | "podman"; +}) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const http = yield* HttpClient.HttpClient; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + if (!/^[a-zA-Z0-9_-]+$/u.test(options.stackId)) return yield* failure("Invalid stack identity"); + const container = + options.runtime === "native" + ? undefined + : yield* makeContainerRuntime({ engine: options.runtime }); + const jobsRoot = path.join(options.root, "jobs"); + yield* fs + .makeDirectory(jobsRoot, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError(failure)); + + const cleanup = Effect.tryPromise({ try: () => rmdir(jobsRoot), catch: failure }).pipe( + Effect.catch((cause) => { + const code = + typeof cause.cause === "object" && cause.cause !== null && "code" in cause.cause + ? cause.cause.code + : undefined; + return code === "ENOENT" || code === "ENOTEMPTY" || code === "EEXIST" + ? Effect.void + : Effect.fail(cause); + }), + ); + + const run = Effect.fn("ToolRunner.run")(function* (input: ToolInput) { + return yield* Effect.scoped( + Effect.gen(function* () { + const tool = yield* Schema.decodeEffect(PostgresTool)(input.tool).pipe( + Effect.mapError(failure), + ); + if (tool.command !== "pg_prove" && input.pgProve !== undefined) + return yield* failure("pgProve options require the pg_prove tool"); + const version = postgresVersion(String(tool.major)); + const jobId = yield* crypto.randomUUIDv4.pipe(Effect.mapError(failure)); + const directory = yield* fs + .makeTempDirectoryScoped({ directory: jobsRoot, prefix: `${jobId}-` }) + .pipe(Effect.mapError(failure)); + const process = yield* Effect.gen(function* () { + if (container === undefined) { + const artifact = yield* prepareNativeArtifact( + { service: "database", version }, + options.cacheRoot, + ).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HttpClient.HttpClient, http), + ); + const child = yield* spawner.spawn( + ChildProcess.make(path.join(artifact.root, "bin", tool.command), input.args, { + env: input.env, + cwd: input.pgProve?.cwd ?? directory, + // pg_prove exits on SIGTERM before its psql child, leaving that descendant alive. + killSignal: tool.command === "pg_prove" ? "SIGKILL" : undefined, + stdin: "pipe", + forceKillAfter: "5 seconds", + }), + ); + return { + stdin: child.stdin.pipe(Sink.mapError(failure)), + stdout: child.stdout.pipe(Stream.mapError(failure)), + stderr: child.stderr.pipe(Stream.mapError(failure)), + exitCode: child.exitCode.pipe(Effect.map(Number), Effect.mapError(failure)), + cleanup: Effect.void, + }; + } + const artifact = yield* resolveArtifact({ service: "database", version }); + yield* container.prepare(artifact.image); + const child = yield* container + .launchTool({ + image: artifact.image, + stackId: options.stackId, + instanceId: jobId, + env: input.env, + args: input.args, + entrypoint: tool.command, + workingDir: input.pgProve?.workingDir, + mounts: input.pgProve?.mounts.map((mount) => ({ ...mount, readOnly: true })), + }) + .pipe( + Effect.catchTag("ContainerLaunchError", (error) => + error.process.stop.pipe( + Effect.andThen(error.process.remove), + Effect.andThen(Effect.fail(error.failure)), + ), + ), + ); + return { + stdin: child.stdin.pipe(Sink.mapError(failure)), + stdout: child.stdout.pipe(Stream.mapError(failure)), + stderr: child.stderr.pipe(Stream.mapError(failure)), + exitCode: child.exitCode.pipe(Effect.mapError(failure)), + cleanup: child.stop.pipe(Effect.andThen(child.remove), Effect.mapError(failure)), + }; + }).pipe(Effect.mapError(failure)); + const [, , , exitCode] = yield* Effect.acquireUseRelease( + Effect.succeed(process), + (process) => + Effect.all( + [ + (input.stdin ?? Stream.empty).pipe( + Stream.run( + process.stdin.pipe(Sink.mapError((cause) => new StdinWriteError({ cause }))), + ), + Effect.catchIf( + (error): error is StdinWriteError => error instanceof StdinWriteError, + () => process.exitCode.pipe(Effect.asVoid), + ), + Effect.raceFirst(process.exitCode.pipe(Effect.asVoid)), + ), + process.stdout.pipe(Stream.runForEach(input.stdout)), + process.stderr.pipe(Stream.runForEach(input.stderr)), + process.exitCode, + ], + { concurrency: "unbounded" }, + ), + (process) => process.cleanup, + ); + return { jobId, exitCode }; + }), + ); + }); + return { run, cleanup } satisfies Interface; + }); + +export const layer = (options: { + readonly stackId: string; + readonly root: string; + readonly cacheRoot: string; + readonly runtime: "native" | "docker" | "podman"; +}) => Layer.effect(Service, makeToolRunner(options).pipe(Effect.map(Service.of))); diff --git a/packages/stack/src/identity/Errors.ts b/packages/stack/src/identity/Errors.ts new file mode 100644 index 0000000000..f49ac054b1 --- /dev/null +++ b/packages/stack/src/identity/Errors.ts @@ -0,0 +1,18 @@ +import { Data } from "effect"; +import type { StackId } from "./StackId.ts"; + +export class InvalidStackIdentityError extends Data.TaggedError("InvalidStackIdentityError")<{ + readonly message: string; + readonly cause?: unknown; + readonly path?: string; + readonly reason?: string; + readonly stackId?: StackId; + readonly name?: string; +}> {} + +export class InvalidProjectRootError extends Data.TaggedError("InvalidProjectRootError")<{ + readonly message: string; + readonly cause?: unknown; + readonly projectRoot?: string; + readonly stateRoot?: string; +}> {} diff --git a/packages/stack/src/identity/GitBranchContext.ts b/packages/stack/src/identity/GitBranchContext.ts index 17051ede10..d335c12a29 100644 --- a/packages/stack/src/identity/GitBranchContext.ts +++ b/packages/stack/src/identity/GitBranchContext.ts @@ -1,6 +1,6 @@ import { Effect, FileSystem, Path } from "effect"; import type { PlatformError } from "effect/PlatformError"; -import { InvalidStackIdentityError } from "../public/Errors.ts"; +import { InvalidStackIdentityError } from "./Errors.ts"; const FULL_REF_PATTERN = /^refs\/.+$/; const OBJECT_ID_PATTERN = /^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$/; @@ -8,39 +8,41 @@ const OBJECT_ID_PATTERN = /^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$/; const invalidMetadata = (path: string, reason: string): InvalidStackIdentityError => new InvalidStackIdentityError({ path, reason, message: reason }); -const canonicalGitDirectory = ( - fs: FileSystem.FileSystem, - path: Path.Path, - checkoutRoot: string, - gitEntry: string, -): Effect.Effect => - Effect.gen(function* () { - const info = yield* fs.stat(gitEntry); - if (info.type === "Directory") { - return yield* fs.realPath(gitEntry); - } - if (info.type !== "File") { - return yield* invalidMetadata( - gitEntry, - "The .git entry is neither a directory nor a gitdir file", - ); - } +const canonicalGitDirectory = Effect.fn("Identity.canonicalGitDirectory")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + checkoutRoot: string, + gitEntry: string, + ): Effect.Effect => + Effect.gen(function* () { + const info = yield* fs.stat(gitEntry); + if (info.type === "Directory") { + return yield* fs.realPath(gitEntry); + } + if (info.type !== "File") { + return yield* invalidMetadata( + gitEntry, + "The .git entry is neither a directory nor a gitdir file", + ); + } - const content = yield* fs.readFileString(gitEntry); - const line = content - .split("\n") - .map((candidate) => candidate.trim()) - .find((candidate) => candidate.toLowerCase().startsWith("gitdir:")); - const target = line?.slice("gitdir:".length).trim(); - if (target === undefined || target.length === 0) { - return yield* invalidMetadata(gitEntry, "The .git file does not contain a gitdir target"); - } - return yield* fs.realPath(path.resolve(checkoutRoot, target)); - }).pipe( - Effect.catchTag("PlatformError", (error: PlatformError) => - Effect.fail(invalidMetadata(gitEntry, `Unable to read Git metadata: ${error.message}`)), + const content = yield* fs.readFileString(gitEntry); + const line = content + .split("\n") + .map((candidate) => candidate.trim()) + .find((candidate) => candidate.toLowerCase().startsWith("gitdir:")); + const target = line?.slice("gitdir:".length).trim(); + if (target === undefined || target.length === 0) { + return yield* invalidMetadata(gitEntry, "The .git file does not contain a gitdir target"); + } + return yield* fs.realPath(path.resolve(checkoutRoot, target)); + }).pipe( + Effect.catchTag("PlatformError", (error: PlatformError) => + Effect.fail(invalidMetadata(gitEntry, `Unable to read Git metadata: ${error.message}`)), + ), ), - ); +); const hasGitRepositoryMarkers = ( fs: FileSystem.FileSystem, @@ -56,84 +58,85 @@ const hasGitRepositoryMarkers = ( ); }); -const locateGitDirectory = ( - fs: FileSystem.FileSystem, - path: Path.Path, - canonicalProjectRoot: string, -): Effect.Effect => - Effect.gen(function* () { - let directory = canonicalProjectRoot; - while (true) { - const gitEntry = path.join(directory, ".git"); - if (yield* fs.exists(gitEntry)) { - const entryInfo = yield* fs.stat(gitEntry); - const gitDirectory = yield* canonicalGitDirectory(fs, path, directory, gitEntry); - if ( - entryInfo.type !== "Directory" || - (yield* hasGitRepositoryMarkers(fs, path, gitDirectory)) - ) { - return gitDirectory; +const locateGitDirectory = Effect.fn("Identity.locateGitDirectory")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + canonicalProjectRoot: string, + ): Effect.Effect => + Effect.gen(function* () { + let directory = canonicalProjectRoot; + while (true) { + const gitEntry = path.join(directory, ".git"); + if (yield* fs.exists(gitEntry)) { + const entryInfo = yield* fs.stat(gitEntry); + const gitDirectory = yield* canonicalGitDirectory(fs, path, directory, gitEntry); + if ( + entryInfo.type !== "Directory" || + (yield* hasGitRepositoryMarkers(fs, path, gitDirectory)) + ) { + return gitDirectory; + } } + const parent = path.dirname(directory); + if (parent === directory) { + return undefined; + } + directory = parent; } - const parent = path.dirname(directory); - if (parent === directory) { - return undefined; - } - directory = parent; - } - }).pipe( - Effect.catchTag("PlatformError", (error: PlatformError) => - Effect.fail( - invalidMetadata(canonicalProjectRoot, `Unable to inspect Git metadata: ${error.message}`), + }).pipe( + Effect.catchTag("PlatformError", (error: PlatformError) => + Effect.fail( + invalidMetadata(canonicalProjectRoot, `Unable to inspect Git metadata: ${error.message}`), + ), ), ), - ); +); -const resolveBranchContext = ( - fs: FileSystem.FileSystem, - path: Path.Path, - gitDirectory: string, -): Effect.Effect => - Effect.gen(function* () { - const headPath = path.join(gitDirectory, "HEAD"); - const head = (yield* fs.readFileString(headPath)).trim(); - if (head.length === 0) { - return yield* invalidMetadata(headPath, "HEAD is empty"); - } - if (head.startsWith("ref:")) { - const ref = head.slice("ref:".length).trim(); - if (!FULL_REF_PATTERN.test(ref)) { - return yield* invalidMetadata(headPath, "HEAD must name a full symbolic ref under refs/"); +const resolveBranchContext = Effect.fn("Identity.resolveBranchContext")( + ( + fs: FileSystem.FileSystem, + path: Path.Path, + gitDirectory: string, + ): Effect.Effect => + Effect.gen(function* () { + const headPath = path.join(gitDirectory, "HEAD"); + const head = (yield* fs.readFileString(headPath)).trim(); + if (head.length === 0) { + return yield* invalidMetadata(headPath, "HEAD is empty"); + } + if (head.startsWith("ref:")) { + const ref = head.slice("ref:".length).trim(); + if (!FULL_REF_PATTERN.test(ref)) { + return yield* invalidMetadata(headPath, "HEAD must name a full symbolic ref under refs/"); + } + return ref; } - return ref; - } - if (!OBJECT_ID_PATTERN.test(head)) { - return yield* invalidMetadata( - headPath, - "Detached HEAD must contain a 40- or 64-character hexadecimal Git object id", - ); - } - return "detached"; - }).pipe( - Effect.catchTag("PlatformError", (error: PlatformError) => - Effect.fail( - invalidMetadata(path.join(gitDirectory, "HEAD"), `Unable to read HEAD: ${error.message}`), + if (!OBJECT_ID_PATTERN.test(head)) { + return yield* invalidMetadata( + headPath, + "Detached HEAD must contain a 40- or 64-character hexadecimal Git object id", + ); + } + return "detached"; + }).pipe( + Effect.catchTag("PlatformError", (error: PlatformError) => + Effect.fail( + invalidMetadata(path.join(gitDirectory, "HEAD"), `Unable to read HEAD: ${error.message}`), + ), ), ), - ); +); /** Resolves Git metadata without spawning Git or writing identity markers. */ -export const resolveGitBranchContext = ( +export const resolveGitBranchContext = Effect.fn("Identity.resolveGitBranchContext")(function* ( canonicalProjectRoot: string, -): Effect.Effect< - string | undefined, - InvalidStackIdentityError, - FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { +) { + return yield* Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const gitDirectory = yield* locateGitDirectory(fs, path, canonicalProjectRoot); if (gitDirectory === undefined) return undefined; return yield* resolveBranchContext(fs, path, gitDirectory); }); +}); diff --git a/packages/stack/src/identity/Identity.ts b/packages/stack/src/identity/Identity.ts index 422ebd83ba..bef867a950 100644 --- a/packages/stack/src/identity/Identity.ts +++ b/packages/stack/src/identity/Identity.ts @@ -1,7 +1,7 @@ -import { Effect, FileSystem, Path, Schema, Crypto } from "effect"; +import { Crypto, Effect, FileSystem, Schema } from "effect"; import type { PlatformError } from "effect/PlatformError"; -import { InvalidProjectRootError, InvalidStackIdentityError } from "../public/Errors.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; +import { InvalidProjectRootError, InvalidStackIdentityError } from "./Errors.ts"; +import { StackIdSchema } from "./StackId.ts"; import { resolveGitBranchContext } from "./GitBranchContext.ts"; export interface StackIdentity { @@ -36,10 +36,10 @@ const encodeTuple = (identity: StackIdentity): Uint8Array => { }; /** Hashes the complete identity tuple using length-delimited UTF-8 fields. */ -export const deriveStackId = ( +export const deriveStackId = Effect.fn("Identity.deriveStackId")(function* ( identity: StackIdentity, -): Effect.Effect => - Effect.gen(function* () { +) { + return yield* Effect.gen(function* () { const crypto = yield* Crypto.Crypto; const digest = yield* crypto.digest("SHA-256", encodeTuple(identity)); let hexadecimal = ""; @@ -63,16 +63,13 @@ export const deriveStackId = ( ), ), ); +}); /** Resolves a canonical Git or ordinary-folder identity without writing state. */ -export const resolveStackIdentity = ( +export const resolveStackIdentity = Effect.fn("Identity.resolveStackIdentity")(function* ( options: ResolveStackIdentityOptions, -): Effect.Effect< - StackIdentity, - InvalidProjectRootError | InvalidStackIdentityError, - FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { +) { + return yield* Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const projectRoot = options.projectRoot; if (projectRoot.trim().length === 0) { @@ -112,3 +109,4 @@ export const resolveStackIdentity = ( ), ), ); +}); diff --git a/packages/stack/src/public/StackId.ts b/packages/stack/src/identity/StackId.ts similarity index 84% rename from packages/stack/src/public/StackId.ts rename to packages/stack/src/identity/StackId.ts index f13f2f9dc5..04d460536c 100644 --- a/packages/stack/src/public/StackId.ts +++ b/packages/stack/src/identity/StackId.ts @@ -7,7 +7,7 @@ export type StackId = string & { readonly [StackIdTypeId]: "StackId" }; const SHA256_HEX = /^[0-9a-f]{64}$/; -export const isStackId = (value: string): value is StackId => SHA256_HEX.test(value); +const isStackId = (value: string): value is StackId => SHA256_HEX.test(value); export const StackIdSchema = Schema.String.pipe( Schema.refine((value): value is StackId => isStackId(value), { diff --git a/packages/stack/src/identity/identity.integration.test.ts b/packages/stack/src/identity/identity.integration.test.ts index 41c8e8305c..61248ca5d6 100644 --- a/packages/stack/src/identity/identity.integration.test.ts +++ b/packages/stack/src/identity/identity.integration.test.ts @@ -1,9 +1,8 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Crypto, Effect, Exit, FileSystem, Path, Schema } from "effect"; -import { InvalidStackIdentityError } from "../public/Errors.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; +import { Crypto, Effect, Exit, FileSystem, Path } from "effect"; +import { InvalidStackIdentityError } from "./Errors.ts"; +import type { StackId } from "./StackId.ts"; import { GitSetupError, runGit } from "../../tests/helpers/git.ts"; import { deriveStackId, resolveStackIdentity, type StackIdentity } from "./Identity.ts"; @@ -298,42 +297,4 @@ describe("deterministic stack identity and state paths", () => { }), ), ); - - it.live("rejects non-digest ids and names every state path below the exact identity root", () => - withScope( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-paths-" }); - const stateRoot = path.join(root, "state"); - const project = path.join(root, "project"); - yield* fs.makeDirectory(stateRoot); - yield* fs.makeDirectory(project); - const identity = yield* resolveStackIdentity({ projectRoot: project }); - const id = yield* stackId(identity); - const paths = yield* resolveStackPaths({ stateRoot, stackId: id }); - - expect(paths.stackRoot).toBe(path.join(stateRoot, id)); - expect(paths.stateDocument).toBe(path.join(stateRoot, id, "state.json")); - expect(paths.data).toBe(path.join(stateRoot, id, "data")); - expect(paths.logs).toBe(path.join(stateRoot, id, "logs")); - expect(paths.runtime).toBe(path.join(stateRoot, id, "runtime")); - expect(paths.controlMetadata).toBe(path.join(stateRoot, id, "control.json")); - for (const value of Object.values(paths)) { - const relative = path.relative(paths.stackRoot, value); - expect( - relative === "" || (relative !== ".." && !relative.startsWith(`..${path.sep}`)), - ).toBe(true); - } - - const invalid = yield* Schema.decodeEffect(StackIdSchema)("stack_local").pipe(Effect.exit); - expect(Exit.isFailure(invalid)).toBe(true); - const unsafe = yield* resolveStackPaths({ - stateRoot, - stackId: "../outside" as StackId, - }).pipe(Effect.exit); - expect(Exit.isFailure(unsafe)).toBe(true); - }), - ), - ); }); diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index a0ffbc0e41..ace2e539a7 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -1,64 +1,338 @@ -export { - createStack, - openStack, - findStack, - listStacks, - discoverStacks, - inspectStack, - createEphemeralPostgres, -} from "./public/PromiseStack.ts"; -export type { - PromiseStack, - PromiseStackConfig, - PromiseInspectStackOptions, - PromiseStartStackOptions, - PromisePrepareStackOptions, - PromiseCreateEphemeralPostgresOptions, - PromiseEphemeralPostgres, - CreateStackOptions, - FindStackOptions, - ListStacksOptions, - StackDiscoveryIssue, - StackDiscoveryResult, - PreparedCapability, -} from "./public/PromiseStack.ts"; -export type { - CapabilityName, - CapabilityStatus, - StackLifecycle, - DesiredStackLifecycle, - NetworkPort, - StackEndpoint, - StackStatus, - StackRecovery, - ArtifactPreparationState, - ArtifactPreparationStatus, - StackDescriptor, - StackInspection, -} from "./public/index.ts"; -export { StackIdSchema, isStackId } from "./public/StackId.ts"; -export type { StackId } from "./public/StackId.ts"; -export { StackRuntimeSchema, RuntimeEngineSchema } from "./public/Runtime.ts"; -export type { StackRuntime, RuntimeEngine, StackRuntimePreference } from "./public/Runtime.ts"; -export { - StackEndpointsSchema, - StackRecoverySchema, - CapabilityVersionsSchema, - ArtifactPreparationStateSchema, - ArtifactPreparationStatusSchema, -} from "./public/Status.ts"; -export { - CapabilityNameSchema, - CapabilityStatusSchema, - ActivationModeSchema, -} from "./public/Capability.ts"; -export { PreparationModeSchema } from "./public/Config.ts"; -export type { PreparationMode } from "./public/Config.ts"; -export { - LogCursorSchema, - LogQuerySchema, - StackLogBatchSchema, - StackLogEntrySchema, -} from "./public/Logs.ts"; -export type { LogCursor, LogQuery, StackLogBatch, StackLogEntry } from "./public/Logs.ts"; -export * from "./public/Errors.ts"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { Effect, Layer, ManagedRuntime, Schema, Stream } from "effect"; +import * as StackEffect from "./effect.ts"; +import { StackError } from "./Rpc.ts"; +import { + ServiceCreation as CreationSchema, + type ServiceCreation as EffectCreation, +} from "./services/Catalog.ts"; +import type { PostgresTool } from "./Tools.ts"; + +export { postgres } from "./Tools.ts"; +export { StackError } from "./Rpc.ts"; +type DatabaseCreation = Extract; +/** Plain configuration accepted by non-Effect callers. */ +export type ServiceCreation = + | Exclude + | (Omit & { + readonly config: Omit< + DatabaseCreation["config"], + "databasePassword" | "jwtSecret" | "rootKey" + > & { + readonly databasePassword: string; + readonly jwtSecret: string; + readonly rootKey?: string; + }; + }); +const creationJson = Schema.toCodecJson(CreationSchema); +const decodeCreation = (creation: ServiceCreation) => + Schema.decodeEffect(creationJson)(creation).pipe( + Effect.mapError((cause) => new StackError({ operation: "config", message: cause.message })), + ); +export type { CompositionConfig } from "./Orchestrator.ts"; +export type { Observation } from "./Rpc.ts"; +export type { DatabaseSnapshot } from "./services/DatabaseSnapshot.ts"; +export type { PgProveOptions } from "./effect.ts"; +export type { SupabaseCompositionOptions } from "./effect.ts"; +export type { CreateOptions, OpenOptions, StackLocations } from "./effect.ts"; + +const clientLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); +type Runtime = ReturnType; +const makeRuntime = () => ManagedRuntime.make(clientLayer); +type Kind = ServiceCreation["service"]; +/** Optional cancellation ends the caller's wait, or its attached tool job. */ +export interface CallOptions { + readonly signal?: AbortSignal; +} +export type CompositionSupabaseOptions = StackEffect.SupabaseCompositionOptions & CallOptions; +/** An individual service; closing the client does not stop this process. */ +export interface ServiceInstance { + readonly id: string; + readonly service: K; + readonly start: (options?: CallOptions) => Promise; + readonly ready: (options?: CallOptions) => Promise; + readonly stop: (options?: CallOptions) => Promise; + readonly restart: ( + input?: Pick, "config">, + options?: CallOptions, + ) => Promise; + readonly destroy: (options?: CallOptions) => Promise; + readonly prepare: (options?: CallOptions) => Promise; + readonly status: (options?: CallOptions) => Promise; + readonly followStatus: () => AsyncIterable; + readonly logs: () => AsyncIterable<{ + readonly stream: "stdout" | "stderr"; + readonly bytes: Uint8Array; + }>; + readonly credentials: ( + options?: CallOptions & { readonly from?: "host" | "runtime" }, + ) => Promise>>; +} +/** Database storage operations require a stopped instance with wake disabled. */ +export interface DatabaseInstance extends ServiceInstance<"database"> { + readonly exportSnapshot: ( + destination: string, + options?: CallOptions, + ) => Promise; + readonly restoreSnapshot: ( + source: string, + options?: CallOptions, + ) => Promise; + /** Removes database-owned data while preserving the instance registration. */ + readonly resetData: (options?: CallOptions) => Promise; +} +/** Creation preserves the selected service's available operations. */ +export type ServiceInstances = { + [K in Kind]: K extends "database" ? DatabaseInstance : ServiceInstance; +}; +type AnyInstance = ServiceInstances[Kind]; +/** Streaming inputs and awaited output sinks for an attached finite tool. */ +export interface ToolOptions extends CallOptions { + readonly args?: ReadonlyArray; + readonly env?: Readonly>; + readonly pgProve?: StackEffect.PgProveOptions; + readonly stdin?: AsyncIterable; + readonly stdout: (bytes: Uint8Array) => void | Promise; + readonly stderr: (bytes: Uint8Array) => void | Promise; +} + +const adapt = (handle: StackEffect.Stack, runtime: Runtime) => { + const run = (effect: Effect.Effect, options?: CallOptions) => + runtime.runPromise(effect, options); + const activeIterators = new Set<() => Promise>(); + let clientClosePromise: Promise | undefined; + const iterable = (stream: Stream.Stream): AsyncIterable => ({ + [Symbol.asyncIterator]() { + if (clientClosePromise !== undefined) throw new Error("Stack client is closed"); + const iterator = runtime + .runSync(Stream.toAsyncIterableEffect(stream)) + [Symbol.asyncIterator](); + const iteratorReturn = iterator.return; + const iteratorThrow = iterator.throw; + let iteratorClosePromise: Promise> | undefined; + let dispose: () => Promise; + const close = (): Promise> => { + if (iteratorClosePromise !== undefined) return iteratorClosePromise; + const promise: Promise> = ( + iteratorReturn === undefined + ? Promise.resolve>({ done: true, value: undefined }) + : iteratorReturn(undefined) + ).finally(() => activeIterators.delete(dispose)); + iteratorClosePromise = promise; + return promise; + }; + dispose = () => close().then(() => undefined); + activeIterators.add(dispose); + return { + next: (value?: unknown) => + iterator.next(value).then( + (result) => { + if (result.done) { + activeIterators.delete(dispose); + } + return result; + }, + (error) => { + activeIterators.delete(dispose); + return Promise.reject(error); + }, + ), + return: close, + ...(iteratorThrow === undefined + ? {} + : { + throw: (error?: unknown) => + iteratorThrow(error).then( + (result) => { + if (result.done) { + activeIterators.delete(dispose); + } + return result; + }, + (failure) => { + activeIterators.delete(dispose); + return Promise.reject(failure); + }, + ), + }), + }; + }, + }); + const common = (service: StackEffect.ServiceInstance): ServiceInstance => ({ + id: service.id, + service: service.service, + start: (options) => run(service.start, options), + ready: (options) => run(service.ready, options), + stop: (options) => run(service.stop, options), + restart: (input, options) => + run( + input === undefined + ? service.restart() + : Schema.decodeUnknownEffect(creationJson)({ + service: service.service, + config: input.config, + }).pipe( + Effect.mapError( + (cause) => new StackError({ operation: "restart", message: cause.message }), + ), + Effect.flatMap((decoded) => { + const matchesKind = ( + candidate: EffectCreation, + ): candidate is EffectCreation & + Pick, "config"> => + candidate.service === service.service; + return matchesKind(decoded) + ? service.restart(decoded) + : Effect.fail( + new StackError({ + operation: "restart", + message: "Service kind cannot change", + }), + ); + }), + ), + options, + ), + destroy: (options) => run(service.destroy, options), + prepare: (options) => run(service.prepare, options), + status: (options) => run(service.status, options), + followStatus: () => iterable(service.followStatus), + logs: () => iterable(service.logs), + credentials: (options) => run(service.credentials(options), options), + }); + function instance(service: StackEffect.ServiceInstances[K]): ServiceInstances[K]; + function instance(service: StackEffect.ServiceInstances[Kind]): AnyInstance { + switch (service.service) { + case "database": + return { + ...common(service), + exportSnapshot: (destination, options) => + run(service.exportSnapshot(destination), options), + restoreSnapshot: (source, options) => run(service.restoreSnapshot(source), options), + resetData: (options) => run(service.resetData, options), + }; + case "rest": + return common(service); + case "auth": + return common(service); + case "realtime": + return common(service); + case "storage": + return common(service); + case "imgproxy": + return common(service); + case "functions": + return common(service); + case "studio": + return common(service); + case "pgmeta": + return common(service); + case "mail": + return common(service); + case "analytics": + return common(service); + case "vector": + return common(service); + case "pooler": + return common(service); + } + } + function create( + creation: Input, + options?: CallOptions, + ): Promise; + function create(creation: ServiceCreation, options?: CallOptions): Promise { + return run( + decodeCreation(creation).pipe(Effect.flatMap(handle.services.create), Effect.map(instance)), + options, + ); + } + const sinkError = (cause: unknown) => + new StackError({ + operation: "tool-stream", + message: cause instanceof Error ? cause.message : String(cause), + }); + return { + id: handle.id, + services: { + create, + get: (id: string, options?: CallOptions) => + run(handle.services.get(id).pipe(Effect.map(instance)), options), + list: (options?: CallOptions) => + run(handle.services.list.pipe(Effect.map((services) => services.map(instance))), options), + }, + composition: { + supabase: (services: ReadonlyArray, options?: CompositionSupabaseOptions) => + run( + Effect.forEach(services, decodeCreation).pipe( + Effect.flatMap((decoded) => handle.composition.supabase(decoded, options)), + Effect.map((instances) => instances.map(instance)), + ), + options, + ), + configure: (config: StackEffect.CompositionConfig, options?: CallOptions) => + run(handle.composition.configure(config), options), + describe: (options?: CallOptions) => run(handle.composition.describe, options), + start: (options?: CallOptions) => run(handle.composition.start, options), + stop: (options?: CallOptions) => run(handle.composition.stop, options), + restart: (options?: CallOptions) => run(handle.composition.restart, options), + }, + stop: (options?: CallOptions) => run(handle.stop, options), + destroy: (options?: CallOptions) => run(handle.destroy, options), + close: () => { + if (clientClosePromise !== undefined) return clientClosePromise; + clientClosePromise = Promise.allSettled( + [...activeIterators].map((dispose) => dispose()), + ).then(() => runtime.dispose()); + return clientClosePromise; + }, + tools: { + run: (tool: PostgresTool, options: ToolOptions) => + run( + handle.tools.run(tool, { + args: options.args, + env: options.env, + pgProve: options.pgProve, + ...(options.stdin === undefined + ? {} + : { stdin: Stream.fromAsyncIterable(options.stdin, sinkError) }), + stdout: (bytes) => + Effect.tryPromise({ + try: () => Promise.resolve(options.stdout(bytes)), + catch: sinkError, + }), + stderr: (bytes) => + Effect.tryPromise({ + try: () => Promise.resolve(options.stderr(bytes)), + catch: sinkError, + }), + }), + options, + ), + }, + }; +}; +/** A Promise client whose close operation leaves the detached owner running. */ +export type Stack = ReturnType; + +const acquire = ( + effect: ReturnType, + options?: CallOptions, +): Promise => { + const runtime = makeRuntime(); + return runtime.runPromise(effect, options).then( + (handle) => adapt(handle, runtime), + (error: unknown) => runtime.dispose().then(() => Promise.reject(error)), + ); +}; +/** Registers a new stack identity. */ +export const create = ( + options: StackEffect.CreateOptions, + callOptions?: CallOptions, +): Promise => acquire(StackEffect.create(options), callOptions); +/** Opens an existing stack without launching its services. */ +export const open = (options: StackEffect.OpenOptions, callOptions?: CallOptions): Promise => + acquire(StackEffect.open(options), callOptions); +/** Discovers saved stacks and separately reports their live-owner availability. */ +export const discover = (options: Pick) => + Effect.runPromise(StackEffect.discover(options).pipe(Effect.provide(clientLayer))); diff --git a/packages/stack/src/internal/dispatch-markers.ts b/packages/stack/src/internal/dispatch-markers.ts index 1ec20e6898..5dfa8f5ff9 100644 --- a/packages/stack/src/internal/dispatch-markers.ts +++ b/packages/stack/src/internal/dispatch-markers.ts @@ -1,5 +1,9 @@ -/** Private argv marker used when a compiled CLI dispatches an embedded Supervisor. */ -export const SUPERVISOR_DISPATCH_SENTINEL = "__supabase_stack_supervisor__" as const; - /** Private argv marker used when a compiled CLI dispatches its embedded native launcher. */ export const NATIVE_PROCESS_DISPATCH_SENTINEL = "__supabase_stack_native__" as const; + +/** Private argv marker used when a compiled CLI dispatches its embedded stack owner. */ +export const HOST_PROCESS_DISPATCH_SENTINEL = "__supabase_stack_host__" as const; + +/** Identifies Bun's virtual module roots in source and standalone builds. */ +export const isBunVirtualPath = (value: string): boolean => + /(?:^|[\\/])(?:\$bunfs|~BUN)(?:[\\/]|$)/.test(value); diff --git a/packages/stack/src/internal/dispatch-markers.unit.test.ts b/packages/stack/src/internal/dispatch-markers.unit.test.ts new file mode 100644 index 0000000000..9bf78333f5 --- /dev/null +++ b/packages/stack/src/internal/dispatch-markers.unit.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; +import { isBunVirtualPath } from "./dispatch-markers.ts"; + +describe("isBunVirtualPath", () => { + it.each([ + "file:///$bunfs/root/entry.ts", + "file:///B:/~BUN/root/entry.ts", + "B:\\~BUN\\root\\entry.ts", + ])("recognizes %s", (value) => { + expect(isBunVirtualPath(value)).toBe(true); + }); + + it("does not treat source files as virtual", () => { + expect(isBunVirtualPath("file:///workspace/src/entry.ts")).toBe(false); + }); +}); diff --git a/packages/stack/src/internal/dispatch.integration.test.ts b/packages/stack/src/internal/dispatch.integration.test.ts new file mode 100644 index 0000000000..2d0008a0f5 --- /dev/null +++ b/packages/stack/src/internal/dispatch.integration.test.ts @@ -0,0 +1,116 @@ +import { NodeServices } from "@effect/platform-node"; +import { Context, Data, Effect, FileSystem, Layer, Path, Schema, Stream } from "effect"; +import { expect, it } from "@effect/vitest"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; +import { HostEndpoint } from "../HostProcess.ts"; +import * as State from "../State.ts"; + +const fixturePath = fileURLToPath( + new URL("../../tests/compiled-dispatch-fixture.ts", import.meta.url), +); +class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} + +const spawnFixture = Effect.fnUntraced(function* (executable: string, args: ReadonlyArray) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner.spawn( + ChildProcess.make(executable, args, { + stdout: "pipe", + stderr: "pipe", + stdin: "ignore", + forceKillAfter: "2 seconds", + }), + ); +}); + +const readOutput = (child: ChildProcessSpawner.ChildProcessHandle) => + Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: 3 }, + ); + +const compileFixture = (executable: string) => + Effect.gen(function* () { + const child = yield* spawnFixture("bun", [ + "build", + fixturePath, + "--compile", + `--outfile=${executable}`, + ]); + const [, stderr, exitCode] = yield* readOutput(child); + if (Number(exitCode) !== 0) + return yield* new FixtureError({ message: `Fixture build failed: ${stderr}` }); + }); + +it.live("dispatches compiled owner and native launchers through the production boundary", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-compiled-dispatch-" }); + const executable = path.join(root, "dispatch-fixture"); + yield* compileFixture(executable); + const stateRoot = path.join(root, "state"); + const cacheRoot = path.join(root, "cache"); + const stateContext = yield* Layer.build(State.layer({ root: stateRoot })); + const state = Context.get(stateContext, State.Service); + const stackId = "compiled-dispatch"; + yield* state.save({ + id: stackId, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "compiled" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + + const runOwner = (mode: "owner" | "stop") => + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawnFixture(executable, [mode, stateRoot, cacheRoot, stackId]); + const [stdout, stderr, exitCode] = yield* readOutput(child); + return { + endpoint: yield* Schema.decodeEffect(Schema.fromJsonString(HostEndpoint))( + stdout.trim(), + ), + stderr, + exitCode, + }; + }), + ); + yield* Effect.addFinalizer(() => runOwner("stop").pipe(Effect.ignore)); + const first = yield* runOwner("owner"); + expect(first.exitCode, first.stderr).toBe(0); + const reopened = yield* runOwner("owner"); + expect(reopened.exitCode, reopened.stderr).toBe(0); + expect(reopened.endpoint).toEqual(first.endpoint); + const stopped = yield* runOwner("stop"); + expect(stopped.exitCode, stopped.stderr).toBe(0); + + const missing = yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawnFixture(executable, ["owner", stateRoot, cacheRoot, "missing"]); + return yield* readOutput(child); + }), + ); + expect(missing[2]).toBe(1); + expect(missing[1]).toContain("Stack is not registered"); + + const pidMarker = path.join(root, "native-pid"); + const native = yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawnFixture(executable, ["native", pidMarker]); + return yield* readOutput(child); + }), + ); + expect(native[2], native[1]).toBe(0); + expect(native[0]).toContain("native-ready\nnative-stopped\n"); + const workloadPid = Number(yield* fs.readFileString(pidMarker)); + expect(() => process.kill(workloadPid, 0)).toThrow(); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/internal/dispatch.ts b/packages/stack/src/internal/dispatch.ts new file mode 100644 index 0000000000..8af8959b38 --- /dev/null +++ b/packages/stack/src/internal/dispatch.ts @@ -0,0 +1,27 @@ +import { + HOST_PROCESS_DISPATCH_SENTINEL, + NATIVE_PROCESS_DISPATCH_SENTINEL, +} from "./dispatch-markers.ts"; + +/** Runs the embedded owner when a compiled CLI receives its private dispatch marker. */ +export const runHostProcessIfDispatched = (argv: ReadonlyArray): Promise => { + if (argv[0] !== HOST_PROCESS_DISPATCH_SENTINEL) return Promise.resolve(false); + return import("./host-process.ts") + .then(({ runHostProcess }) => runHostProcess(argv.slice(1))) + .then( + () => true, + () => { + process.exitCode = 1; + return true; + }, + ); +}; + +/** Runs the embedded native launcher when a compiled CLI receives its private dispatch marker. */ +export const runNativeProcessIfDispatched = (argv: ReadonlyArray): Promise => { + if (argv[0] !== NATIVE_PROCESS_DISPATCH_SENTINEL) return Promise.resolve(false); + return import("../runtime/native-launcher.ts").then(({ runNativeLauncher }) => { + runNativeLauncher(); + return true; + }); +}; diff --git a/packages/stack/src/internal/host-process.ts b/packages/stack/src/internal/host-process.ts new file mode 100644 index 0000000000..c2b439775e --- /dev/null +++ b/packages/stack/src/internal/host-process.ts @@ -0,0 +1,95 @@ +import { Cause, Effect, Option, Schema } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- readiness is an inherited launcher descriptor. +import { closeSync, writeSync } from "node:fs"; +import { runStackHost, StackHostError, type StackHostOptions } from "../StackHost.ts"; + +const causeCode = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null) return undefined; + if ("code" in cause && typeof cause.code === "string") return cause.code; + if ("cause" in cause) return causeCode(cause.cause); + return undefined; +}; + +const writeLine = (value: unknown) => + Effect.gen(function* () { + const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))( + value, + ).pipe( + Effect.mapError( + (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + ), + ); + yield* Effect.try({ + try: () => writeSync(3, Buffer.from(`${serialized}\n`, "utf8")), + catch: (cause) => new StackHostError({ operation: "ready", message: String(cause), cause }), + }); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + try { + closeSync(3); + } catch { + // The parent may close the inherited descriptor during cancellation. + } + }), + ), + ); + +const options = ( + args: ReadonlyArray, + report: (value: unknown) => Effect.Effect, +): Effect.Effect => { + if (args.length !== 3) + return Effect.fail( + new StackHostError({ + operation: "startup", + message: "Expected stateRoot, cacheRoot and stackId", + }), + ); + const stateRoot = args[0]; + const cacheRoot = args[1]; + const stackId = args[2]; + if (stateRoot === undefined || cacheRoot === undefined || stackId === undefined) + return Effect.fail( + new StackHostError({ operation: "startup", message: "Missing host argument" }), + ); + return Effect.succeed({ + stateRoot, + cacheRoot, + stackId, + onReady: (endpoint) => report({ type: "ready", endpoint }), + }); +}; + +const program = (args: ReadonlyArray) => + Effect.gen(function* () { + let reported = false; + const report = (value: unknown) => + Effect.suspend(() => { + if (reported) return Effect.void; + reported = true; + return writeLine(value); + }); + const host = yield* options(args, report); + yield* runStackHost(host).pipe( + Effect.catchCause((cause) => { + const failure = Option.getOrUndefined(Cause.findErrorOption(cause)); + const reason = causeCode(failure?.cause) === "EADDRINUSE" ? "bind-conflict" : undefined; + return report({ + type: "error", + message: String(cause), + ...(reason === undefined ? {} : { reason }), + }).pipe(Effect.exit, Effect.andThen(Effect.failCause(cause))); + }), + ); + }); + +/** Runs the owner process with its state root, artifact cache and stack identity. */ +export const runHostProcess = (args: ReadonlyArray): Promise => + Effect.runPromise(program(args)); + +if (import.meta.main) { + void runHostProcess(process.argv.slice(2)).catch(() => { + process.exitCode = 1; + }); +} diff --git a/packages/stack/src/internal/postgres-artifact.ts b/packages/stack/src/internal/postgres-artifact.ts new file mode 100644 index 0000000000..3a3121a83e --- /dev/null +++ b/packages/stack/src/internal/postgres-artifact.ts @@ -0,0 +1,7 @@ +/** Artifact preparation shared with the CLI's stack-independent PostgreSQL clients. */ +export { + ArtifactError, + postgresVersion, + prepareNativeArtifact, + resolveArtifact, +} from "../Artifacts.ts"; diff --git a/packages/stack/src/internal/supervisor-process.ts b/packages/stack/src/internal/supervisor-process.ts deleted file mode 100644 index ea04f26e40..0000000000 --- a/packages/stack/src/internal/supervisor-process.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { - NATIVE_PROCESS_DISPATCH_SENTINEL, - SUPERVISOR_DISPATCH_SENTINEL, -} from "./dispatch-markers.ts"; - -export { - NATIVE_PROCESS_DISPATCH_SENTINEL, - SUPERVISOR_DISPATCH_SENTINEL, -} from "./dispatch-markers.ts"; - -/** - * Supported process-entrypoint seam for embedders such as the CLI binary. - * Returns true when the argv is the supervisor child dispatch, otherwise the - * caller should continue with its normal command entrypoint. - */ -export const runSupervisorProcessIfDispatched = (argv: ReadonlyArray): Promise => { - if (argv[0] !== SUPERVISOR_DISPATCH_SENTINEL) return Promise.resolve(false); - return import("../entrypoints/supervisor-node.ts") - .then(({ runSupervisorProcess }) => runSupervisorProcess(argv.slice(1))) - .then(() => true); -}; - -/** - * Runs the embedded native launcher when a compiled CLI receives its private - * dispatch marker. Returns false for ordinary CLI argv so callers can continue - * with their normal command entrypoint. - */ -export const runNativeProcessIfDispatched = (argv: ReadonlyArray): Promise => { - if (argv[0] !== NATIVE_PROCESS_DISPATCH_SENTINEL) return Promise.resolve(false); - return import("../runtime/native-launcher.ts") - .then(({ runNativeLauncher }) => runNativeLauncher()) - .then(() => true); -}; diff --git a/packages/stack/src/model/CapabilityModule.ts b/packages/stack/src/model/CapabilityModule.ts deleted file mode 100644 index c9d04e49bf..0000000000 --- a/packages/stack/src/model/CapabilityModule.ts +++ /dev/null @@ -1,112 +0,0 @@ -import type { Redacted, Schema } from "effect"; -import type { CapabilityName, ActivationMode } from "../public/Capability.ts"; -import type { PortField } from "../public/Status.ts"; -import { catalogReleaseFor } from "./WorkloadCatalog.ts"; - -/** Logical artifact descriptors. Download/image resolution belongs to preparation. */ -export interface NativeArtifact { - readonly kind: "native"; - readonly release: string; -} - -export interface ContainerArtifact { - readonly kind: "container"; - readonly image: string; -} - -export interface WorkloadSpec { - readonly name: string; - readonly capability: CapabilityName; - /** Closed internal lifecycle marker for work that must run before dependents are ready. */ - readonly bootstrap?: "database"; - readonly dependencies: ReadonlyArray; - readonly readiness: Readonly<{ readonly portField?: PortField }>; - /** Exact artifacts selected for this workload release. */ - readonly artifacts: Readonly<{ - readonly native: NativeArtifact; - readonly container: ContainerArtifact; - }>; -} - -/** Settings shape persisted by the compiler, where absent and secret leaves are materialized. */ -type MaterializedValue = [T] extends [Redacted.Redacted] - ? { readonly slot: string } - : T extends ReadonlyArray - ? ReadonlyArray> - : T extends object - ? { - readonly [Key in keyof T]-?: - | MaterializedValue> - | (undefined extends T[Key] ? null : never); - } - : T; - -export type MaterializedSettings = MaterializedValue; - -export interface CapabilityModule { - readonly name: CapabilityName; - readonly settings: Schema.Schema; - readonly defaultSettings: Settings; - readonly defaultEnabled: boolean; - readonly defaultActivation: ActivationMode; - /** Default traffic idle timeout; false means this capability is not auto-stopped. */ - readonly defaultIdleTimeoutSeconds?: number | false; - readonly dependencies: ReadonlyArray; - readonly defaultVersion: string; - /** Release selectors are the only source of workload versions and artifacts. */ - readonly releases: Readonly>; - readonly routes: ReadonlyArray< - Readonly<{ readonly listener: PortField; readonly protocol: "http" | "tcp" }> - >; - /** Classifies every Redacted settings leaf; module contracts, not callers, own this policy. */ - readonly secretPolicy: (path: string) => "managed" | "passthrough"; - /** Managed slots required by the artifact even when omitted from user config. */ - readonly managedSecretSlots: ReadonlyArray; - /** Optional settings normalization. Omitted modules use their merged settings unchanged. */ - readonly materialize?: (settings: Settings, projectRoot: string) => Settings; - /** Selects private companion workloads from fully materialized settings. */ - readonly selectWorkloads?: ( - settings: MaterializedSettings, - workloads: ReadonlyArray, - ) => ReadonlyArray; -} - -export interface CapabilityRelease { - readonly version: string; - readonly workloads: ReadonlyArray; -} - -export const release = ( - version: string, - workloads: ReadonlyArray, -): CapabilityRelease => ({ version, workloads }); - -export const workload = ( - name: string, - capability: CapabilityName, - options: { - readonly bootstrap?: WorkloadSpec["bootstrap"]; - readonly dependencies?: ReadonlyArray; - readonly readiness?: WorkloadSpec["readiness"]; - readonly version?: string; - } = {}, -): WorkloadSpec => { - const workloadId = `${capability}:${name}`; - const selected = catalogReleaseFor(workloadId, options.version); - if (selected === undefined) - throw new Error( - `Missing workload artifact release: ${workloadId}${options.version === undefined ? "" : `@${options.version}`}`, - ); - const artifacts = { - native: { kind: "native" as const, release: selected.version }, - container: { kind: "container" as const, image: selected.containerImage }, - }; - return { - name, - capability, - ...(options.bootstrap === undefined ? {} : { bootstrap: options.bootstrap }), - dependencies: options.dependencies ?? [], - readiness: options.readiness ?? {}, - artifacts, - }; -}; diff --git a/packages/stack/src/model/Compiler.ts b/packages/stack/src/model/Compiler.ts deleted file mode 100644 index 9bebe92eb2..0000000000 --- a/packages/stack/src/model/Compiler.ts +++ /dev/null @@ -1,752 +0,0 @@ -import { Duration, Effect, Path, Redacted, Schema } from "effect"; -import { InvalidStackConfigError, StackVersionUnsupportedError } from "../public/Errors.ts"; -import { StackConfigSchema, type StackConfig, type PreparationMode } from "../public/Config.ts"; -import type { JwtSigning } from "../public/Config.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import type { PortField } from "../public/Status.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import { - AuthModule, - DatabaseModule, - FunctionsModule, - MailModule, - PoolerModule, - RealtimeModule, - RestModule, - StorageModule, - parseFileSize, - StudioModule, - AnalyticsModule, - parseGoDuration, -} from "./capabilities/index.ts"; -import { resolveThirdPartyIssuer } from "./capabilities/auth-third-party.ts"; -import { - CAPABILITY_MODULES, - createExecutionPlan, - type ExecutionPlan, - type MaterializedCapabilities, - type MaterializedCapability, -} from "./ExecutionPlan.ts"; -import type { CapabilityModule, MaterializedSettings } from "./CapabilityModule.ts"; -import type { SecretGenerator, SecretJwtSigning } from "../state/SecretStore.ts"; -import { AUTH_JWT_SECRET_SLOT, DATABASE_INTERNAL_PASSWORD_SLOT } from "../state/SecretStore.ts"; - -interface SecretSlot { - readonly slot: string; -} - -export interface StackDefinition { - readonly preparation: PreparationMode; - readonly capabilities: MaterializedCapabilities; - readonly listeners: Readonly>; - readonly security: Readonly<{ - readonly jwt: Readonly<{ - readonly issuer: string | null; - readonly signing: MaterializedJwtSigning; - }>; - }>; -} - -type MaterializedJwtSigning = - | { readonly kind: "symmetric"; readonly secret: SecretSlot } - | { readonly kind: "jwks-file"; readonly path: string } - | null; - -interface MaterializedListener { - readonly enabled: boolean; - readonly address: string; - readonly port: "automatic" | number; -} - -export interface SecretSlotInput { - readonly slot: string; - readonly policy: "managed" | "passthrough"; - readonly value?: Redacted.Redacted; - /** Private lifecycle-only generator metadata; never materialized into StackDefinition. */ - readonly generator?: SecretGenerator; -} - -/** Internal credentials are not user settings but still need durable managed slots. */ -const INTERNAL_MANAGED_SECRET_SLOTS = [DATABASE_INTERNAL_PASSWORD_SLOT] as const; - -export interface CompiledStack { - readonly definition: StackDefinition; - readonly secrets: ReadonlyArray; - readonly executionPlan: ExecutionPlan; -} - -export interface PreviousCompilation { - readonly definition: StackDefinition; -} - -export interface CompileStackInput { - readonly projectRoot: string; - readonly runtime: StackRuntime; - readonly config?: StackConfig; -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && - value !== null && - !Array.isArray(value) && - !Redacted.isRedacted(value); - -const canonical = (value: unknown): string => { - if (Redacted.isRedacted(value)) return '{"$secret":true}'; - if (value === undefined) return "null"; - if (value === null || typeof value !== "object") return JSON.stringify(value); - if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; - if (isRecord(value)) { - return `{${Object.keys(value) - .sort() - .map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`) - .join(",")}}`; - } - return JSON.stringify(value); -}; - -function merge(defaults: T, supplied: unknown): T; -function merge(defaults: unknown, supplied: unknown): unknown { - if (!isRecord(defaults) || !isRecord(supplied)) - return supplied === undefined ? defaults : supplied; - const keys = new Set([...Object.keys(defaults), ...Object.keys(supplied)]); - const result: Record = {}; - for (const key of keys) result[key] = merge(defaults[key], supplied[key]); - return result; -} - -function materializeAbsence(value: T): T; -function materializeAbsence(value: unknown): unknown { - if (value === undefined) return null; - if (Array.isArray(value)) return value.map(materializeAbsence); - if (isRecord(value)) { - const result: Record = {}; - for (const key of Object.keys(value).sort()) result[key] = materializeAbsence(value[key]); - return result; - } - return value; -} - -function slotsFor( - value: T, - path: string, - slots: SecretSlotInput[], - policyForPath?: (path: string) => "managed" | "passthrough", -): MaterializedSettings; -function slotsFor( - value: unknown, - path: string, - slots: SecretSlotInput[], - policyForPath: (path: string) => "managed" | "passthrough" = () => "passthrough", -): unknown { - if (Redacted.isRedacted(value)) { - slots.push({ slot: `secret:${path}`, policy: policyForPath(path), value }); - return { slot: `secret:${path}` }; - } - if (Array.isArray(value)) - return value.map((entry, index) => slotsFor(entry, `${path}.${index}`, slots, policyForPath)); - if (isRecord(value)) { - const result: Record = {}; - for (const key of Object.keys(value).sort()) - result[key] = slotsFor(value[key], `${path}.${key}`, slots, policyForPath); - return result; - } - return value; -} - -const canonicalJwtSecret = ( - config: StackConfig, -): Effect.Effect | undefined, InvalidStackConfigError> => - Effect.gen(function* () { - const auth = config.capabilities?.auth; - const authSecret = - auth !== undefined && "settings" in auth ? auth.settings?.jwt_secret : undefined; - const signing = config.security?.jwt?.signing; - const signingSecret = signing?.kind === "symmetric" ? signing.secret : undefined; - if ( - authSecret !== undefined && - signingSecret !== undefined && - Redacted.value(authSecret) !== Redacted.value(signingSecret) - ) - return yield* new InvalidStackConfigError({ - message: "Auth and stack JWT signing secrets must match in symmetric mode", - }); - return signingSecret ?? authSecret; - }); - -const ensureCanonicalJwtSlot = ( - slots: SecretSlotInput[], - value: Redacted.Redacted | undefined, -): void => { - const existingIndex = slots.findIndex((entry) => entry.slot === AUTH_JWT_SECRET_SLOT); - if (existingIndex < 0) { - slots.push({ - slot: AUTH_JWT_SECRET_SLOT, - policy: "managed", - ...(value === undefined ? {} : { value }), - }); - return; - } - const existing = slots[existingIndex]; - if (existing !== undefined && existing.value === undefined && value !== undefined) - slots[existingIndex] = { ...existing, value }; -}; - -const attachAuthSecretGenerators = ( - slots: SecretSlotInput[], - projectRoot: string, - signing: JwtSigning | undefined, -): void => { - const jwtSigning: SecretJwtSigning = - signing?.kind === "jwks-file" - ? { kind: "jwks-file", projectRoot, path: signing.path } - : { kind: "symmetric" }; - for (let index = 0; index < slots.length; index++) { - const entry = slots[index]; - if (entry === undefined || entry.policy !== "managed") continue; - const generator = - entry.slot === "secret:auth.settings.publishable_key" - ? ({ kind: "publishable-key" } satisfies SecretGenerator) - : entry.slot === "secret:auth.settings.secret_key" - ? ({ kind: "secret-key" } satisfies SecretGenerator) - : entry.slot === AUTH_JWT_SECRET_SLOT - ? ({ kind: "jwt-secret" } satisfies SecretGenerator) - : entry.slot === "secret:auth.settings.anon_key" - ? ({ kind: "jwt-token", role: "anon", signing: jwtSigning } satisfies SecretGenerator) - : entry.slot === "secret:auth.settings.service_role_key" - ? ({ - kind: "jwt-token", - role: "service_role", - signing: jwtSigning, - } satisfies SecretGenerator) - : undefined; - if (generator !== undefined) slots[index] = { ...entry, generator }; - } -}; - -const managedRandomBase64urlGenerators: Readonly> = { - "secret:analytics.settings.api_key": { kind: "random-base64url", bytes: 32 }, - "secret:pooler.settings.encryption_key": { kind: "random-base64url", bytes: 24 }, - "secret:pooler.settings.secret_key_base": { kind: "random-base64url", bytes: 48 }, - // Realtime encrypts with AES-128-ECB and consumes this value as a raw 16-byte key. - // Twelve random bytes encode to exactly sixteen unpadded base64url characters. - "secret:realtime.settings.db_enc_key": { kind: "random-base64url", bytes: 12 }, - "secret:realtime.settings.secret_key_base": { kind: "random-base64url", bytes: 48 }, -}; - -const attachManagedRandomSecretGenerators = (slots: SecretSlotInput[]): void => { - for (let index = 0; index < slots.length; index++) { - const entry = slots[index]; - if (entry === undefined || entry.policy !== "managed") continue; - const generator = managedRandomBase64urlGenerators[entry.slot]; - if (generator !== undefined) slots[index] = { ...entry, generator }; - } -}; - -const setMaterializedPath = ( - value: unknown, - parts: ReadonlyArray, - slot: SecretSlot, -): void => { - if (!isRecord(value) || parts.length === 0) return; - const head = parts[0]; - if (head === undefined) return; - const tail = parts.slice(1); - if (tail.length === 0) { - value[head] = slot; - return; - } - if (!isRecord(value[head])) value[head] = {}; - setMaterializedPath(value[head], tail, slot); -}; - -const ensureManagedSlots = ( - settings: MaterializedSettings, - module: CapabilityModule, - enabled: boolean, - slots: SecretSlotInput[], -): MaterializedSettings => { - if (!enabled || module.managedSecretSlots.length === 0) return settings; - const result = settings; - for (const path of module.managedSecretSlots) { - const slot = `secret:${path}`; - const existing = slots.find((candidate) => candidate.slot === slot); - if (existing === undefined) slots.push({ slot, policy: "managed" }); - setMaterializedPath(result, path.split(".").slice(2), { slot }); - } - return result; -}; - -const materializeListener = (value: unknown, enabledByDefault: boolean): MaterializedListener => { - if (!isRecord(value)) - return { enabled: enabledByDefault, address: "127.0.0.1", port: "automatic" }; - if (value.enabled === false) return { enabled: false, address: "127.0.0.1", port: "automatic" }; - const address = - typeof value.address === "string" && value.address.length > 0 ? value.address : "127.0.0.1"; - const port = typeof value.port === "number" ? value.port : "automatic"; - return { enabled: true, address, port }; -}; - -const extract = (value: unknown, key: string): unknown => - isRecord(value) ? value[key] : undefined; - -function materializeFunctionsRoot( - settings: T, - projectRoot: string, - path: Path.Path, -): Effect.Effect; -function materializeFunctionsRoot( - settings: unknown, - projectRoot: string, - path: Path.Path, -): Effect.Effect { - if (!isRecord(settings)) return Effect.succeed(settings); - const supplied = settings.functions_root; - const root = - typeof supplied === "string" && supplied.length > 0 ? supplied : "supabase/functions"; - const resolvedProject = path.resolve(projectRoot); - const resolvedRoot = path.resolve(resolvedProject, root); - const relative = path.relative(resolvedProject, resolvedRoot); - if ( - path.isAbsolute(root) || - relative === ".." || - relative.startsWith(`..${path.sep}`) || - path.isAbsolute(relative) - ) { - return Effect.fail( - new InvalidStackConfigError({ - message: "functions_root must remain inside projectRoot", - functionsRoot: root, - }), - ); - } - return Effect.succeed({ ...settings, functions_root: resolvedRoot }); -} - -const decodeConfig = (config: unknown): Effect.Effect => - Schema.decodeUnknownEffect(StackConfigSchema)(config, { onExcessProperty: "error" }).pipe( - Effect.mapError( - (error) => - new InvalidStackConfigError({ message: `Invalid stack configuration: ${String(error)}` }), - ), - ); - -const validateStorageFileSizes = ( - config: StackConfig, -): Effect.Effect => { - const storage = config.capabilities?.storage; - if (storage === undefined || !("settings" in storage) || storage.settings === undefined) - return Effect.void; - const values = [ - storage.settings.file_size_limit, - ...Object.values(storage.settings.buckets ?? {}).map((bucket) => bucket.file_size_limit), - ]; - for (const value of values) { - if (value !== undefined && parseFileSize(value) === undefined) - return Effect.fail(new InvalidStackConfigError({ message: "Invalid storage file size" })); - } - return Effect.void; -}; - -const validateDatabaseHealthTimeout = ( - config: StackConfig, -): Effect.Effect => { - const database = config.capabilities?.database; - if (database === undefined || !("settings" in database) || database.settings === undefined) - return Effect.void; - const value = database.settings.health_timeout; - if (value === undefined) return Effect.void; - return Effect.try({ - try: () => parseGoDuration(value), - catch: (cause) => - new InvalidStackConfigError({ - message: `Invalid database health_timeout: ${value}`, - setting: "capabilities.database.settings.health_timeout", - cause, - }), - }).pipe( - Effect.flatMap((duration) => - Duration.isNegative(duration) || Duration.isZero(duration) - ? Effect.fail( - new InvalidStackConfigError({ - message: `Invalid database health_timeout: ${value}; duration must be positive`, - setting: "capabilities.database.settings.health_timeout", - }), - ) - : Effect.void, - ), - ); -}; - -const validatePoolerKeys = (config: StackConfig): Effect.Effect => { - const pooler = config.capabilities?.pooler; - if (pooler === undefined || !("settings" in pooler) || pooler.settings === undefined) - return Effect.void; - const settings = pooler.settings; - const validate = ( - value: Redacted.Redacted | undefined, - expectedLength: number, - field: string, - ): Effect.Effect => { - if (value === undefined) return Effect.void; - const text = Redacted.value(value); - if ( - text.length !== expectedLength || - [...text].some((character) => character < "!" || character > "~") - ) - return Effect.fail( - new InvalidStackConfigError({ - message: `Invalid pooler ${field}: expected ${expectedLength} printable ASCII characters`, - setting: `capabilities.pooler.settings.${field}`, - }), - ); - return Effect.void; - }; - return Effect.gen(function* () { - yield* validate(settings.encryption_key, 32, "encryption_key"); - yield* validate(settings.secret_key_base, 64, "secret_key_base"); - }); -}; - -const validateFunctionKeys = (config: unknown): Effect.Effect => { - const capabilities = isRecord(config) ? config.capabilities : undefined; - const capability = isRecord(capabilities) ? capabilities.functions : undefined; - const settings = isRecord(capability) ? capability.settings : undefined; - if (!isRecord(settings)) return Effect.void; - const functions = settings.functions; - if (!isRecord(functions)) return Effect.void; - for (const [slug, value] of Object.entries(functions)) { - if (!/^[a-zA-Z0-9_-]+$/.test(slug)) - return Effect.fail( - new InvalidStackConfigError({ - message: `Invalid function slug: ${slug}`, - function: slug, - }), - ); - if (!isRecord(value)) continue; - const env = value.env; - if (!isRecord(env)) continue; - for (const name of Object.keys(env)) - if (!/^[A-Z_][A-Z0-9_]*$/.test(name)) - return Effect.fail( - new InvalidStackConfigError({ - message: `Invalid function environment name: ${name}`, - function: slug, - environment: name, - }), - ); - } - return Effect.void; -}; - -const releaseFor = ( - module: CapabilityModule, - raw: unknown, - previousVersion?: string, -): Effect.Effect => { - const selected = extract(raw, "version"); - const selector = - typeof selected === "string" ? selected : (previousVersion ?? module.defaultVersion); - // The CLI config exposes PostgreSQL as a major selector (`15`, `17`), while the catalog - // persists a concrete release; resolve a major against the catalog here so the CLI never - // needs to know artifact patch IDs. - const majorSelector = module.name === "database" && /^\d+$/.test(selector); - const resolvedSelector = majorSelector - ? (() => { - const sameMajor = (version: string | undefined): boolean => - version !== undefined && version.split(".", 1)[0] === selector; - if (sameMajor(previousVersion) && previousVersion !== undefined) return previousVersion; - if (sameMajor(module.defaultVersion)) return module.defaultVersion; - return Object.keys(module.releases).find( - (version) => version.includes(".") && sameMajor(version), - ); - })() - : selector; - const release = resolvedSelector === undefined ? undefined : module.releases[resolvedSelector]; - if (release !== undefined) return Effect.succeed(release.version); - return Effect.fail( - new StackVersionUnsupportedError({ - capability: module.name, - version: selector, - message: `Unsupported ${module.name} version: ${selector}`, - }), - ); -}; - -const enabledSettings = ( - name: CapabilityName, - raw: unknown, -): { - enabled: boolean; - activation: "eager" | "lazy"; - idleTimeoutSeconds: number | false; - settings: unknown; - raw: unknown; -} => { - const defaultIdleTimeout = CAPABILITY_MODULES[name].defaultIdleTimeoutSeconds ?? false; - if (name === "database") - return { - enabled: true, - activation: "eager", - idleTimeoutSeconds: false, - settings: extract(raw, "settings") ?? {}, - raw, - }; - if (raw === undefined || raw === null) { - const module = CAPABILITY_MODULES[name]; - return { - enabled: module.defaultEnabled, - activation: module.defaultActivation, - idleTimeoutSeconds: module.defaultActivation === "lazy" ? defaultIdleTimeout : false, - settings: module.defaultSettings, - raw: {}, - }; - } - if (extract(raw, "enabled") === false) - return { - enabled: false, - activation: CAPABILITY_MODULES[name].defaultActivation, - idleTimeoutSeconds: false, - settings: CAPABILITY_MODULES[name].defaultSettings, - raw, - }; - const activation = extract(raw, "activation"); - const idleTimeoutSeconds = extract(raw, "idleTimeoutSeconds"); - const selectedActivation = - activation === "eager" || activation === "lazy" - ? activation - : CAPABILITY_MODULES[name].defaultActivation; - return { - enabled: true, - activation: selectedActivation, - idleTimeoutSeconds: - selectedActivation === "eager" - ? false - : idleTimeoutSeconds === false - ? false - : defaultIdleTimeout === false - ? false - : typeof idleTimeoutSeconds === "number" && - Number.isFinite(idleTimeoutSeconds) && - idleTimeoutSeconds > 0 - ? idleTimeoutSeconds - : defaultIdleTimeout, - settings: extract(raw, "settings") ?? {}, - raw, - }; -}; - -const materializeCapability = ( - module: CapabilityModule, - raw: unknown, - projectRoot: string, - path: Path.Path, - slots: SecretSlotInput[], - normalizeFunctions: boolean, - previousVersion?: string, -): Effect.Effect< - MaterializedCapability, - InvalidStackConfigError | StackVersionUnsupportedError, - Path.Path -> => { - const selected = enabledSettings(module.name, raw); - const mergedInput = merge(module.defaultSettings, selected.settings); - const materialized = module.materialize?.(mergedInput, projectRoot) ?? mergedInput; - const normalized = normalizeFunctions - ? materializeFunctionsRoot(materialized, projectRoot, path) - : Effect.succeed(materialized); - return Effect.gen(function* () { - const normalizedSettings = yield* normalized; - const merged = materializeAbsence(normalizedSettings); - const slotted = slotsFor(merged, `${module.name}.settings`, slots, module.secretPolicy); - const completeSettings = ensureManagedSlots(slotted, module, selected.enabled, slots); - const version = yield* releaseFor(module, selected.raw, previousVersion); - return { - enabled: selected.enabled, - activation: selected.activation, - idleTimeoutSeconds: selected.idleTimeoutSeconds, - version, - settings: completeSettings, - }; - }); -}; - -const planForDefinition = ( - runtime: StackRuntime, - definition: StackDefinition, -): Effect.Effect => - createExecutionPlan(runtime, definition.capabilities); - -/** Rebuilds the private execution plan from a persisted, fully materialized definition. */ -export const rebuildExecutionPlan = ( - runtime: StackRuntime, - definition: StackDefinition, -): Effect.Effect => - planForDefinition(runtime, definition); - -export const compileStack = ( - input: CompileStackInput, - previous?: PreviousCompilation, -): Effect.Effect< - CompiledStack, - InvalidStackConfigError | StackVersionUnsupportedError, - Path.Path -> => - Effect.gen(function* () { - const path = yield* Path.Path; - yield* validateFunctionKeys(input.config ?? {}); - const config = yield* decodeConfig(input.config ?? {}); - yield* validateDatabaseHealthTimeout(config); - yield* validatePoolerKeys(config); - yield* validateStorageFileSizes(config); - const jwtSecret = yield* canonicalJwtSecret(config); - const rawCapabilities = isRecord(config.capabilities) ? config.capabilities : {}; - const slots: SecretSlotInput[] = []; - for (const slot of INTERNAL_MANAGED_SECRET_SLOTS) slots.push({ slot, policy: "managed" }); - const databaseResult = yield* materializeCapability( - DatabaseModule, - extract(rawCapabilities, "database"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.database.version, - ); - const restResult = yield* materializeCapability( - RestModule, - extract(rawCapabilities, "rest"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.rest.version, - ); - const authResult = yield* materializeCapability( - AuthModule, - extract(rawCapabilities, "auth"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.auth.version, - ); - const thirdParty = resolveThirdPartyIssuer(authResult.settings); - if (!thirdParty.ok) - return yield* new InvalidStackConfigError({ - message: thirdParty.message, - ...(thirdParty.provider === undefined ? {} : { provider: thirdParty.provider }), - }); - const realtimeResult = yield* materializeCapability( - RealtimeModule, - extract(rawCapabilities, "realtime"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.realtime.version, - ); - const storageResult = yield* materializeCapability( - StorageModule, - extract(rawCapabilities, "storage"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.storage.version, - ); - const functionsResult = yield* materializeCapability( - FunctionsModule, - extract(rawCapabilities, "functions"), - input.projectRoot, - path, - slots, - true, - previous?.definition.capabilities.functions.version, - ); - const studioResult = yield* materializeCapability( - StudioModule, - extract(rawCapabilities, "studio"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.studio.version, - ); - const mailResult = yield* materializeCapability( - MailModule, - extract(rawCapabilities, "mail"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.mail.version, - ); - const analyticsResult = yield* materializeCapability( - AnalyticsModule, - extract(rawCapabilities, "analytics"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.analytics.version, - ); - const poolerResult = yield* materializeCapability( - PoolerModule, - extract(rawCapabilities, "pooler"), - input.projectRoot, - path, - slots, - false, - previous?.definition.capabilities.pooler.version, - ); - const capabilities = { - database: databaseResult, - rest: restResult, - auth: authResult, - realtime: realtimeResult, - storage: storageResult, - functions: functionsResult, - studio: studioResult, - mail: mailResult, - analytics: analyticsResult, - pooler: poolerResult, - }; - const rawListeners = isRecord(config.listeners) ? config.listeners : {}; - const listeners = { - api: materializeListener(rawListeners.api, true), - database: materializeListener(rawListeners.database, true), - pooler: materializeListener(rawListeners.pooler, true), - studio: materializeListener(rawListeners.studio, true), - mailUi: materializeListener(rawListeners.mailUi, true), - smtp: materializeListener(rawListeners.smtp, false), - pop3: materializeListener(rawListeners.pop3, false), - functionsInspector: materializeListener(rawListeners.functionsInspector, false), - } satisfies Record; - const rawJwt = config.security?.jwt; - ensureCanonicalJwtSlot(slots, jwtSecret); - attachAuthSecretGenerators(slots, input.projectRoot, rawJwt?.signing); - attachManagedRandomSecretGenerators(slots); - const security = { - jwt: { - issuer: rawJwt?.issuer ?? null, - signing: - rawJwt?.signing?.kind === "jwks-file" - ? slotsFor(rawJwt.signing, "security.jwt.signing", slots) - : { kind: "symmetric" as const, secret: { slot: AUTH_JWT_SECRET_SLOT } }, - }, - }; - const definition: StackDefinition = { - preparation: config.preparation ?? "background", - capabilities, - listeners, - security, - }; - const executionPlan = yield* planForDefinition(input.runtime, definition); - return { definition, secrets: slots, executionPlan }; - }); - -export const canonicalize = canonical; - -/** Compares two complete materialized definitions by their canonical schema representation. */ -export const sameDefinition = (left: StackDefinition, right: StackDefinition): boolean => - canonical(left) === canonical(right); diff --git a/packages/stack/src/model/Exclusions.ts b/packages/stack/src/model/Exclusions.ts deleted file mode 100644 index f357d9524e..0000000000 --- a/packages/stack/src/model/Exclusions.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { CAPABILITY_NAMES, type CapabilityName } from "../public/Capability.ts"; -import type { StackConfig } from "../public/Config.ts"; -import { CAPABILITY_MODULES } from "./ExecutionPlan.ts"; - -export type ExcludableCapabilityName = Exclude; - -/** Disable requested optional capabilities and their transitive dependents. */ -export const excludeStackCapabilities = ( - config: StackConfig, - exclusions: readonly ExcludableCapabilityName[], -): StackConfig => { - if (exclusions.length === 0) return config; - - const disabled = new Set(exclusions); - let changed = true; - while (changed) { - changed = false; - for (const name of CAPABILITY_NAMES) { - if (name === "database" || disabled.has(name)) continue; - if (CAPABILITY_MODULES[name].dependencies.some((dependency) => disabled.has(dependency))) { - disabled.add(name); - changed = true; - } - } - } - - const capabilities = { ...config.capabilities }; - for (const name of CAPABILITY_NAMES) { - if (name === "database" || !disabled.has(name)) continue; - capabilities[name] = { enabled: false }; - } - return { ...config, capabilities }; -}; diff --git a/packages/stack/src/model/ExecutionPlan.ts b/packages/stack/src/model/ExecutionPlan.ts deleted file mode 100644 index 4c94ffc051..0000000000 --- a/packages/stack/src/model/ExecutionPlan.ts +++ /dev/null @@ -1,312 +0,0 @@ -import { CAPABILITY_NAMES, type CapabilityName } from "../public/Capability.ts"; -import type { PortField } from "../public/Status.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import { - AuthModule, - DatabaseModule, - FunctionsModule, - MailModule, - PoolerModule, - RealtimeModule, - RestModule, - StorageModule, - StudioModule, - AnalyticsModule, -} from "./capabilities/index.ts"; -import type { - WorkloadSpec, - NativeArtifact, - ContainerArtifact, - MaterializedSettings, -} from "./CapabilityModule.ts"; -import { Effect } from "effect"; -import { InvalidStackConfigError } from "../public/Errors.ts"; -import type { AnalyticsSettings } from "./capabilities/analytics.ts"; -import type { AuthSettings } from "./capabilities/auth.ts"; -import type { DatabaseSettings } from "./capabilities/database.ts"; -import type { FunctionsSettings } from "./capabilities/functions.ts"; -import type { MailSettings } from "./capabilities/mail.ts"; -import type { PoolerSettings } from "./capabilities/pooler.ts"; -import type { RealtimeSettings } from "./capabilities/realtime.ts"; -import type { RestSettings } from "./capabilities/rest.ts"; -import type { StorageSettings } from "./capabilities/storage.ts"; -import type { StudioSettings } from "./capabilities/studio.ts"; - -export const CAPABILITY_MODULES = { - database: DatabaseModule, - rest: RestModule, - auth: AuthModule, - realtime: RealtimeModule, - storage: StorageModule, - functions: FunctionsModule, - studio: StudioModule, - mail: MailModule, - analytics: AnalyticsModule, - pooler: PoolerModule, -}; - -export interface PlannedWorkload { - readonly id: string; - readonly capability: CapabilityName; - readonly bootstrap?: WorkloadSpec["bootstrap"]; - readonly dependencies: ReadonlyArray; - readonly readiness: WorkloadSpec["readiness"]; - readonly artifacts: Readonly<{ - readonly native: NativeArtifact; - readonly container: ContainerArtifact; - }>; - readonly selected: NativeArtifact | ContainerArtifact; -} - -export interface ExecutionPlan { - readonly runtime: StackRuntime; - /** Materialized lazy/eager policy consumed by the Supervisor gateway seam. */ - readonly activation: Readonly<{ [Name in CapabilityName]: "eager" | "lazy" }>; - readonly startOrder: ReadonlyArray; - readonly dependencies: Readonly<{ [Name in CapabilityName]: ReadonlyArray }>; - readonly routes: ReadonlyArray< - Readonly<{ - readonly capability: CapabilityName; - readonly listener: PortField; - readonly protocol: "http" | "tcp"; - }> - >; - readonly workloads: ReadonlyArray; -} - -export interface MaterializedCapability { - readonly enabled: boolean; - readonly activation: "eager" | "lazy"; - readonly idleTimeoutSeconds: number | false; - readonly version: string; - readonly settings: MaterializedSettings; -} - -export interface MaterializedCapabilities { - readonly database: MaterializedCapability; - readonly rest: MaterializedCapability; - readonly auth: MaterializedCapability; - readonly realtime: MaterializedCapability; - readonly storage: MaterializedCapability; - readonly functions: MaterializedCapability; - readonly studio: MaterializedCapability; - readonly mail: MaterializedCapability; - readonly analytics: MaterializedCapability; - readonly pooler: MaterializedCapability; -} - -/** Return the requested capabilities and every transitive dependency. */ -export const dependencyClosure = ( - plan: ExecutionPlan, - roots: Iterable, -): Set => { - const closure = new Set(); - const visit = (name: CapabilityName): void => { - if (closure.has(name)) return; - closure.add(name); - for (const dependency of plan.dependencies[name]) visit(dependency); - }; - for (const root of roots) visit(root); - return closure; -}; - -export const eagerCapabilities = (plan: ExecutionPlan): Set => { - return dependencyClosure( - plan, - CAPABILITY_NAMES.filter((name) => plan.activation[name] === "eager"), - ); -}; - -export const activeExecutionPlan = ( - plan: ExecutionPlan, - active: ReadonlySet, -): ExecutionPlan => ({ - ...plan, - workloads: plan.workloads.filter((workload) => active.has(workload.capability)), - startOrder: plan.startOrder.filter((name) => active.has(name)), -}); - -const selectedWorkloads = ( - name: CapabilityName, - modules: typeof CAPABILITY_MODULES, - capabilities: MaterializedCapabilities, - workloads: ReadonlyArray, -): ReadonlyArray => { - switch (name) { - case "database": - return ( - modules.database.selectWorkloads?.(capabilities.database.settings, workloads) ?? workloads - ); - case "rest": - return modules.rest.selectWorkloads?.(capabilities.rest.settings, workloads) ?? workloads; - case "auth": - return modules.auth.selectWorkloads?.(capabilities.auth.settings, workloads) ?? workloads; - case "realtime": - return ( - modules.realtime.selectWorkloads?.(capabilities.realtime.settings, workloads) ?? workloads - ); - case "storage": - return ( - modules.storage.selectWorkloads?.(capabilities.storage.settings, workloads) ?? workloads - ); - case "functions": - return ( - modules.functions.selectWorkloads?.(capabilities.functions.settings, workloads) ?? workloads - ); - case "studio": - return modules.studio.selectWorkloads?.(capabilities.studio.settings, workloads) ?? workloads; - case "mail": - return modules.mail.selectWorkloads?.(capabilities.mail.settings, workloads) ?? workloads; - case "analytics": - return ( - modules.analytics.selectWorkloads?.(capabilities.analytics.settings, workloads) ?? workloads - ); - case "pooler": - return modules.pooler.selectWorkloads?.(capabilities.pooler.settings, workloads) ?? workloads; - } -}; - -export const createExecutionPlan = ( - runtime: StackRuntime, - capabilities: MaterializedCapabilities, - modules: typeof CAPABILITY_MODULES = CAPABILITY_MODULES, -): Effect.Effect => { - const dependencyMap = { - database: modules.database.dependencies, - rest: modules.rest.dependencies, - auth: modules.auth.dependencies, - realtime: modules.realtime.dependencies, - storage: modules.storage.dependencies, - functions: modules.functions.dependencies, - studio: modules.studio.dependencies, - mail: modules.mail.dependencies, - analytics: modules.analytics.dependencies, - pooler: modules.pooler.dependencies, - } satisfies { [Name in CapabilityName]: ReadonlyArray }; - for (const name of CAPABILITY_NAMES) { - if (!capabilities[name].enabled) continue; - for (const dependency of dependencyMap[name]) { - if (!capabilities[dependency].enabled) { - return Effect.fail( - new InvalidStackConfigError({ - message: `${name} requires disabled capability ${dependency}`, - capability: name, - dependency, - }), - ); - } - } - } - const start: CapabilityName[] = []; - const visited = new Set(); - const visitingCapabilities = new Set(); - let capabilityGraphError: InvalidStackConfigError | undefined; - const visit = (name: CapabilityName): void => { - if (visitingCapabilities.has(name)) { - capabilityGraphError = new InvalidStackConfigError({ - message: `Capability dependency cycle detected at ${name}`, - capability: name, - }); - return; - } - if (visited.has(name) || !capabilities[name].enabled) return; - visitingCapabilities.add(name); - visited.add(name); - for (const dependency of dependencyMap[name]) { - if (!Object.hasOwn(modules, dependency)) { - capabilityGraphError = new InvalidStackConfigError({ - message: `Unknown capability dependency ${dependency}`, - capability: name, - dependency, - }); - continue; - } - visit(dependency); - } - visitingCapabilities.delete(name); - start.push(name); - }; - for (const name of CAPABILITY_NAMES) visit(name); - if (capabilityGraphError !== undefined) return Effect.fail(capabilityGraphError); - const routes = CAPABILITY_NAMES.flatMap((name) => - capabilities[name].enabled - ? modules[name].routes.map((route) => ({ capability: name, ...route })) - : [], - ); - const declaredWorkloads: PlannedWorkload[] = []; - for (const name of CAPABILITY_NAMES) { - if (!capabilities[name].enabled) continue; - const release = modules[name].releases[capabilities[name].version]; - if (release === undefined) - return Effect.fail( - new InvalidStackConfigError({ - message: `Missing ${name} release ${capabilities[name].version}`, - capability: name, - version: capabilities[name].version, - }), - ); - const selectedEntries = selectedWorkloads(name, modules, capabilities, release.workloads); - for (const entry of selectedEntries) { - const id = `${name}:${entry.name}`; - declaredWorkloads.push({ - id, - capability: name, - ...(entry.bootstrap === undefined ? {} : { bootstrap: entry.bootstrap }), - dependencies: entry.dependencies, - readiness: entry.readiness, - artifacts: entry.artifacts, - selected: runtime.kind === "native" ? entry.artifacts.native : entry.artifacts.container, - }); - } - } - const byId = new Map(declaredWorkloads.map((entry) => [entry.id, entry])); - const workloadOrder: typeof declaredWorkloads = []; - const visiting = new Set(); - const visitedWorkloads = new Set(); - let graphError: InvalidStackConfigError | undefined; - const visitWorkload = (id: string): void => { - if (visitedWorkloads.has(id)) return; - if (visiting.has(id)) { - graphError = new InvalidStackConfigError({ - message: `Workload dependency cycle detected at ${id}`, - workload: id, - }); - return; - } - const entry = byId.get(id); - if (entry === undefined) { - graphError = new InvalidStackConfigError({ - message: `Missing private workload dependency ${id}`, - workload: id, - }); - return; - } - visiting.add(id); - for (const dependency of entry.dependencies) visitWorkload(dependency); - visiting.delete(id); - visitedWorkloads.add(id); - workloadOrder.push(entry); - }; - for (const entry of declaredWorkloads) visitWorkload(entry.id); - if (graphError !== undefined) return Effect.fail(graphError); - const activation = { - database: capabilities.database.activation, - rest: capabilities.rest.activation, - auth: capabilities.auth.activation, - realtime: capabilities.realtime.activation, - storage: capabilities.storage.activation, - functions: capabilities.functions.activation, - studio: capabilities.studio.activation, - mail: capabilities.mail.activation, - analytics: capabilities.analytics.activation, - pooler: capabilities.pooler.activation, - } satisfies { [Name in CapabilityName]: "eager" | "lazy" }; - return Effect.succeed({ - runtime, - activation, - startOrder: start, - dependencies: dependencyMap, - routes, - workloads: workloadOrder, - }); -}; diff --git a/packages/stack/src/model/WorkloadCatalog.ts b/packages/stack/src/model/WorkloadCatalog.ts deleted file mode 100644 index f778fa8a0d..0000000000 --- a/packages/stack/src/model/WorkloadCatalog.ts +++ /dev/null @@ -1,264 +0,0 @@ -import type { PlannedWorkload } from "./ExecutionPlan.ts"; -import { Effect } from "effect"; -import { StackPreparationError } from "../public/Errors.ts"; - -export type NativeTarget = "darwin-arm64" | "linux-amd64" | "linux-arm64"; - -export interface NativeWorkloadArtifact { - readonly provider: "supabase/slim-services"; - readonly service: string; - readonly version: string; - readonly releaseTag: string; - readonly target: NativeTarget; - readonly archive: "tar.zst"; - readonly assetName: string; - readonly downloadUrl: string; - readonly manifestUrl: string; - readonly checksumUrl: string; - readonly requiredRuntimePaths: ReadonlyArray; - readonly executablePath: string; -} - -export interface WorkloadCatalogEntry { - readonly service: string; - readonly defaultVersion: string; - /** Supported exact versions mapped to their matching container image. */ - readonly releases: Readonly>; - readonly requiredRuntimePaths: ReadonlyArray; - readonly executablePath: string; - /** Stable, stack-local DNS identity used by container dependants. */ - readonly containerAlias: string; -} - -const native = ( - service: string, - defaultVersion: string, - containerImage: string, - executablePath: string, - requiredRuntimePaths: ReadonlyArray = [executablePath], - options: { - readonly additionalReleases?: Readonly>; - readonly containerAlias?: string; - } = {}, -): WorkloadCatalogEntry => ({ - service, - defaultVersion, - releases: { - [defaultVersion]: containerImage, - ...options.additionalReleases, - }, - requiredRuntimePaths, - executablePath, - containerAlias: options.containerAlias ?? `supabase-${service}`, -}); - -/** - * The single authoritative private workload identity table. - * - * Pins track the slim-services release feed (see - * `docs/adr/0017-simplified-managed-stack-architecture.md`), not the Dockerfile, and diverge - * from it on purpose — do not "reconcile" the two. Maintained by - * `.github/workflows/sync-stack-workload-catalog.yml`. - */ -const workloadCatalog = { - "database:database": native( - "postgres", - "17.6.1.168", - "ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:0ad32960ba2cb95167e1be115c6f84b303f39361772eb137c6fdae6cf0093fc3", - "bin/supabase-postgres-start", - ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/psql"], - { - additionalReleases: { - "15.14.1.168": - "ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:da2cae5bba640d819a70718522a6270544f12ab846e64b36d899144de1096899", - }, - containerAlias: "supabase-database", - }, - ), - "rest:rest": native( - "postgrest", - "v16.2", - "ghcr.io/supabase/cli/postgrest:v16.2", - "bin/postgrest", - ["bin/postgrest"], - { containerAlias: "supabase-rest" }, - ), - "auth:auth": native("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth", [ - "bin/auth", - ]), - "realtime:realtime": native( - "realtime", - "v2.134.5", - "ghcr.io/supabase/cli/realtime:v2.134.5@sha256:7fb53cc6987085d739c7d161608505ed138d1895df884c3bdc2147cef444138a", - "bin/server", - ["bin/server", "bin/prepare"], - ), - "storage:storage": native( - "storage", - "v1.73.0", - "ghcr.io/supabase/cli/storage:v1.73.0@sha256:69590a75f916837641976d4018e5ead7c7d2c2305312d9bfb06d86aec8fb1cdd", - "bin/storage", - ["bin/storage", "bin/prepare"], - ), - "storage:imgproxy": native( - "imgproxy", - "v3.8.0", - "ghcr.io/supabase/cli/imgproxy:v3.8.0", - "bin/imgproxy", - ), - "functions:edge-runtime": native( - "edge-runtime", - "v1.76.2", - "ghcr.io/supabase/cli/edge-runtime:v1.76.2", - "bin/edge-runtime", - ["bin/edge-runtime"], - { containerAlias: "supabase-functions" }, - ), - "studio:studio": native( - "studio", - "2026.09.04-sha-5a67366", - "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9823a31668028f1846e87331bc21598d9cd74bcaa1466c72dab58c33c9c82720", - "bin/studio", - ["bin/studio"], - ), - "studio:pgmeta": native( - "pgmeta", - "v0.99.0", - "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:22d40cf766949cf9bd5183d9d46927a05a63743eb1d41f9e418e9db7d7470b29", - "bin/pgmeta", - ["bin/pgmeta"], - ), - "mail:mail": native( - "mailpit", - "v1.30.2", - "ghcr.io/supabase/cli/mailpit:v1.30.2", - "bin/mailpit", - ["bin/mailpit"], - { containerAlias: "supabase-mail" }, - ), - "analytics:analytics": native( - "analytics", - "v1.50.9", - "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:7db85cc6cb0cdeb4b71f2fadb49c0f9197bea0492daf7896f6ae69edad76d28e", - "bin/logflare", - ["bin/logflare", "bin/prepare"], - ), - "analytics:vector": native( - "vector", - "0.53.0", - "ghcr.io/supabase/cli/vector:0.53.0", - "bin/vector", - ["bin/vector", "share/doc/vector/config/vector.yaml"], - ), - "pooler:pooler": native( - "pooler", - "v2.9.12", - "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:12bb9dcb7ddace79bee173ccb7327c6646af2236679f3bd932a86b3a06479aac", - "bin/server", - ["bin/server", "bin/prepare", "bin/provision-tenant"], - ), -} satisfies Readonly>; - -export const WORKLOAD_CATALOG: Readonly> = workloadCatalog; -export type WorkloadId = keyof typeof workloadCatalog; - -export const targetForPlatform = (platform: { - readonly os: string; - readonly arch: string; -}): NativeTarget | undefined => { - if (platform.os === "darwin" && platform.arch === "arm64") return "darwin-arm64"; - if (platform.os === "linux" && platform.arch === "x64") return "linux-amd64"; - if (platform.os === "linux" && platform.arch === "arm64") return "linux-arm64"; - return undefined; -}; - -/** Returns the catalog entry for a workload. Known workload IDs are always present. */ -export function catalogEntryFor(workloadId: WorkloadId): WorkloadCatalogEntry; -export function catalogEntryFor(workloadId: string): WorkloadCatalogEntry | undefined; -export function catalogEntryFor(workloadId: string): WorkloadCatalogEntry | undefined { - return WORKLOAD_CATALOG[workloadId]; -} - -export interface WorkloadCatalogRelease { - readonly version: string; - readonly containerImage: string; -} - -/** Returns one exact native/container release pair for a workload. */ -export const catalogReleaseFor = ( - workloadId: string, - version?: string, -): WorkloadCatalogRelease | undefined => { - const entry = catalogEntryFor(workloadId); - if (entry === undefined) return undefined; - const selected = version ?? entry.defaultVersion; - const containerImage = entry.releases[selected]; - return containerImage === undefined ? undefined : { version: selected, containerImage }; -}; - -/** Resolves the container alias for a catalog workload identity. */ -export const containerAliasFor = (workloadId: WorkloadId): string => - workloadCatalog[workloadId].containerAlias; - -const artifactFor = ( - entry: WorkloadCatalogEntry, - release: WorkloadCatalogRelease, - target: NativeTarget, -): NativeWorkloadArtifact => { - const version = release.version; - const releaseTag = `${entry.service}-${version}`; - const assetName = `${releaseTag}-${target}`; - const base = `https://github.com/supabase/slim-services/releases/download/${releaseTag}`; - return { - provider: "supabase/slim-services", - service: entry.service, - version, - releaseTag, - target, - archive: "tar.zst", - assetName, - downloadUrl: `${base}/${assetName}.tar.zst`, - manifestUrl: `${base}/${assetName}.manifest.json`, - checksumUrl: `${base}/SHA256SUMS`, - requiredRuntimePaths: entry.requiredRuntimePaths, - executablePath: entry.executablePath, - }; -}; - -/** Effect-native resolver used by preparation/runtime boundaries. */ -export const resolveNativeArtifactForWorkload = ( - workload: Pick, - platform: { readonly os: string; readonly arch: string } = { - os: process.platform, - arch: process.arch, - }, -): Effect.Effect => { - const entry = catalogEntryFor(workload.id); - if (entry === undefined) - return Effect.fail( - new StackPreparationError({ - message: `Unknown workload catalog entry: ${workload.id}`, - workload: workload.id, - }), - ); - const target = targetForPlatform(platform); - if (target === undefined) - return Effect.fail( - new StackPreparationError({ - message: `Native runtime is unsupported on ${platform.os}/${platform.arch}`, - workload: workload.id, - platform: `${platform.os}/${platform.arch}`, - }), - ); - const release = catalogReleaseFor(workload.id, workload.artifacts.native.release); - if (release === undefined) - return Effect.fail( - new StackPreparationError({ - message: `Unsupported native release ${workload.artifacts.native.release}`, - workload: workload.id, - service: entry.service, - version: workload.artifacts.native.release, - }), - ); - return Effect.succeed(artifactFor(entry, release, target)); -}; diff --git a/packages/stack/src/model/capabilities/analytics.ts b/packages/stack/src/model/capabilities/analytics.ts deleted file mode 100644 index 062961d777..0000000000 --- a/packages/stack/src/model/capabilities/analytics.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("analytics:analytics").defaultVersion; - -const Secret = Schema.Redacted(Schema.String); -export const AnalyticsSettingsSchema = Schema.Struct({ - backend: Schema.optionalKey(Schema.Literals(["postgres", "bigquery"] as const)), - gcp_project_id: Schema.optionalKey(Schema.String), - gcp_project_number: Schema.optionalKey(Schema.String), - gcp_jwt_path: Schema.optionalKey(Schema.String), - api_key: Schema.optionalKey(Secret), -}); -export type AnalyticsSettings = Schema.Schema.Type; -export const AnalyticsModule: CapabilityModule = { - name: "analytics", - settings: AnalyticsSettingsSchema, - defaultSettings: { - backend: "postgres", - gcp_project_id: "local", - gcp_project_number: "0", - gcp_jwt_path: undefined, - api_key: undefined, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("analytics", "analytics", { - dependencies: ["database:database"], - readiness: { portField: "api" }, - }), - workload("vector", "analytics", { - dependencies: ["analytics:analytics"], - readiness: {}, - }), - ]), - }, - routes: [{ listener: "api", protocol: "http" }], - secretPolicy: () => "managed", - managedSecretSlots: ["analytics.settings.api_key"], -}; diff --git a/packages/stack/src/model/capabilities/auth-third-party.ts b/packages/stack/src/model/capabilities/auth-third-party.ts deleted file mode 100644 index ade33678bc..0000000000 --- a/packages/stack/src/model/capabilities/auth-third-party.ts +++ /dev/null @@ -1,116 +0,0 @@ -/** Closed mapping for the third-party issuer rules used by Auth.ResolveJWKS. */ - -type ThirdPartyProviderName = "firebase" | "auth0" | "aws_cognito" | "clerk" | "workos"; - -interface ThirdPartyIssuer { - readonly provider: ThirdPartyProviderName; - readonly issuer: string; -} - -interface ThirdPartyIssuerError { - readonly provider?: ThirdPartyProviderName; - readonly message: string; -} - -export type ThirdPartyIssuerResult = - | { readonly ok: true; readonly value?: ThirdPartyIssuer } - | ({ readonly ok: false } & ThirdPartyIssuerError); - -const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null && !Array.isArray(value); - -const text = (value: unknown): string => (typeof value === "string" ? value.trim() : ""); - -const enabled = (value: unknown): boolean => isRecord(value) && value.enabled === true; - -/** Applies Auth.ResolveJWKS' issuerURL and validation rules. */ -export const resolveThirdPartyIssuer = (settings: unknown): ThirdPartyIssuerResult => { - if (!isRecord(settings)) return { ok: true }; - const thirdParty = isRecord(settings.third_party) ? settings.third_party : settings; - const candidates: Array<{ readonly provider: ThirdPartyProviderName; readonly value: unknown }> = - []; - for (const candidate of [ - { provider: "firebase" as const, value: thirdParty.firebase }, - { provider: "auth0" as const, value: thirdParty.auth0 }, - { provider: "aws_cognito" as const, value: thirdParty.aws_cognito }, - { provider: "clerk" as const, value: thirdParty.clerk }, - { provider: "workos" as const, value: thirdParty.workos }, - ]) - if (enabled(candidate.value)) candidates.push(candidate); - if (candidates.length > 1) - return { - ok: false, - message: "Only one auth.third_party provider may be enabled at a time", - }; - const candidate = candidates[0]; - if (candidate === undefined) return { ok: true }; - const value = isRecord(candidate.value) ? candidate.value : {}; - switch (candidate.provider) { - case "firebase": { - const projectId = text(value.project_id); - return projectId.length === 0 - ? { ok: false, provider: candidate.provider, message: "Firebase project_id is required" } - : { - ok: true, - value: { - provider: candidate.provider, - issuer: `https://securetoken.google.com/${projectId}`, - }, - }; - } - case "auth0": { - const tenant = text(value.tenant); - if (tenant.length === 0) - return { ok: false, provider: candidate.provider, message: "Auth0 tenant is required" }; - const region = text(value.tenant_region); - return { - ok: true, - value: { - provider: candidate.provider, - issuer: `https://${tenant}.${region.length > 0 ? `${region}.` : ""}auth0.com`, - }, - }; - } - case "aws_cognito": { - const userPoolId = text(value.user_pool_id); - const region = text(value.user_pool_region); - if (userPoolId.length === 0) - return { - ok: false, - provider: candidate.provider, - message: "AWS Cognito user_pool_id is required", - }; - if (region.length === 0) - return { - ok: false, - provider: candidate.provider, - message: "AWS Cognito user_pool_region is required", - }; - return { - ok: true, - value: { - provider: candidate.provider, - issuer: `https://cognito-idp.${region}.amazonaws.com/${userPoolId}`, - }, - }; - } - case "clerk": { - const domain = text(value.domain); - if (domain.length === 0) - return { ok: false, provider: candidate.provider, message: "Clerk domain is required" }; - if (!/^(?:clerk(?:\.[a-z0-9-]+){2,}|(?:[a-z0-9-]+\.)+clerk\.accounts\.dev)$/u.test(domain)) - return { - ok: false, - provider: candidate.provider, - message: "Clerk domain has an invalid format", - }; - return { ok: true, value: { provider: candidate.provider, issuer: `https://${domain}` } }; - } - case "workos": { - const issuer = text(value.issuer_url); - return issuer.length === 0 - ? { ok: false, provider: candidate.provider, message: "WorkOS issuer_url is required" } - : { ok: true, value: { provider: candidate.provider, issuer } }; - } - } -}; diff --git a/packages/stack/src/model/capabilities/auth.ts b/packages/stack/src/model/capabilities/auth.ts deleted file mode 100644 index 567f399203..0000000000 --- a/packages/stack/src/model/capabilities/auth.ts +++ /dev/null @@ -1,428 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; -import { NetworkPortSchema } from "../../public/Status.ts"; - -const version = catalogEntryFor("auth:auth").defaultVersion; - -const Secret = Schema.Redacted(Schema.String); -const OptionalString = Schema.optionalKey(Schema.String); -const OptionalBoolean = Schema.optionalKey(Schema.Boolean); -const OptionalNumber = Schema.optionalKey(Schema.Finite); - -const ProviderSchema = Schema.Struct({ - enabled: OptionalBoolean, - client_id: OptionalString, - secret: Schema.optionalKey(Secret), - url: OptionalString, - redirect_uri: OptionalString, - skip_nonce_check: OptionalBoolean, - email_optional: OptionalBoolean, -}); - -const ExternalSchema = Schema.Struct({ - apple: Schema.optionalKey(ProviderSchema), - azure: Schema.optionalKey(ProviderSchema), - bitbucket: Schema.optionalKey(ProviderSchema), - discord: Schema.optionalKey(ProviderSchema), - facebook: Schema.optionalKey(ProviderSchema), - github: Schema.optionalKey(ProviderSchema), - gitlab: Schema.optionalKey(ProviderSchema), - google: Schema.optionalKey(ProviderSchema), - kakao: Schema.optionalKey(ProviderSchema), - keycloak: Schema.optionalKey(ProviderSchema), - linkedin_oidc: Schema.optionalKey(ProviderSchema), - notion: Schema.optionalKey(ProviderSchema), - twitch: Schema.optionalKey(ProviderSchema), - twitter: Schema.optionalKey(ProviderSchema), - x: Schema.optionalKey(ProviderSchema), - slack_oidc: Schema.optionalKey(ProviderSchema), - spotify: Schema.optionalKey(ProviderSchema), - workos: Schema.optionalKey(ProviderSchema), - zoom: Schema.optionalKey(ProviderSchema), -}); - -const HookSchema = Schema.Struct({ - enabled: OptionalBoolean, - uri: OptionalString, - secrets: Schema.optionalKey(Secret), -}); - -const EmailTemplateSchema = Schema.Struct({ - subject: OptionalString, - content_path: OptionalString, -}); -const EmailNotificationSchema = Schema.Struct({ - enabled: OptionalBoolean, - subject: OptionalString, - content_path: OptionalString, -}); -const SmtpSchema = Schema.Struct({ - enabled: OptionalBoolean, - host: OptionalString, - port: Schema.optionalKey(NetworkPortSchema), - user: OptionalString, - pass: Schema.optionalKey(Secret), - admin_email: OptionalString, - sender_name: OptionalString, -}); - -const Twilio = Schema.Struct({ - enabled: OptionalBoolean, - account_sid: OptionalString, - message_service_sid: OptionalString, - auth_token: Schema.optionalKey(Secret), -}); -const TwilioVerify = Schema.Struct({ - enabled: OptionalBoolean, - account_sid: OptionalString, - message_service_sid: OptionalString, - auth_token: Schema.optionalKey(Secret), -}); -const Messagebird = Schema.Struct({ - enabled: OptionalBoolean, - originator: OptionalString, - access_key: Schema.optionalKey(Secret), -}); -const Textlocal = Schema.Struct({ - enabled: OptionalBoolean, - sender: OptionalString, - api_key: Schema.optionalKey(Secret), -}); -const Vonage = Schema.Struct({ - enabled: OptionalBoolean, - from: OptionalString, - api_key: Schema.optionalKey(Secret), - api_secret: Schema.optionalKey(Secret), -}); - -const SmsSchema = Schema.Struct({ - enable_signup: OptionalBoolean, - enable_confirmations: OptionalBoolean, - template: OptionalString, - max_frequency: OptionalString, - twilio: Schema.optionalKey(Twilio), - twilio_verify: Schema.optionalKey(TwilioVerify), - messagebird: Schema.optionalKey(Messagebird), - textlocal: Schema.optionalKey(Textlocal), - vonage: Schema.optionalKey(Vonage), - test_otp: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), -}); - -const MfaSchema = Schema.Struct({ - totp: Schema.optionalKey( - Schema.Struct({ enroll_enabled: OptionalBoolean, verify_enabled: OptionalBoolean }), - ), - phone: Schema.optionalKey( - Schema.Struct({ - enroll_enabled: OptionalBoolean, - verify_enabled: OptionalBoolean, - otp_length: OptionalNumber, - template: OptionalString, - max_frequency: OptionalString, - }), - ), - web_authn: Schema.optionalKey( - Schema.Struct({ enroll_enabled: OptionalBoolean, verify_enabled: OptionalBoolean }), - ), - max_enrolled_factors: OptionalNumber, -}); - -const EmailSchema = Schema.Struct({ - enable_signup: OptionalBoolean, - double_confirm_changes: OptionalBoolean, - enable_confirmations: OptionalBoolean, - secure_password_change: OptionalBoolean, - max_frequency: OptionalString, - otp_length: OptionalNumber, - otp_expiry: OptionalNumber, - smtp: Schema.optionalKey(SmtpSchema), - template: Schema.optionalKey(Schema.Record(Schema.String, EmailTemplateSchema)), - notification: Schema.optionalKey(Schema.Record(Schema.String, EmailNotificationSchema)), -}); - -const EnabledFlag = Schema.Struct({ enabled: OptionalBoolean }); -const Web3Schema = Schema.Struct({ - solana: Schema.optionalKey(EnabledFlag), - ethereum: Schema.optionalKey(EnabledFlag), -}); -const CaptchaSchema = Schema.Struct({ - enabled: OptionalBoolean, - provider: Schema.optionalKey(Schema.Literals(["hcaptcha", "turnstile"] as const)), - secret: Schema.optionalKey(Secret), -}); -const RateLimitSchema = Schema.Struct({ - email_sent: OptionalNumber, - sms_sent: OptionalNumber, - anonymous_users: OptionalNumber, - token_refresh: OptionalNumber, - sign_in_sign_ups: OptionalNumber, - token_verifications: OptionalNumber, - web3: OptionalNumber, -}); -const SessionsSchema = Schema.Struct({ - timebox: OptionalString, - inactivity_timeout: OptionalString, -}); -const ThirdPartySchema = Schema.Struct({ - firebase: Schema.optionalKey( - Schema.Struct({ enabled: OptionalBoolean, project_id: OptionalString }), - ), - auth0: Schema.optionalKey( - Schema.Struct({ - enabled: OptionalBoolean, - tenant: OptionalString, - tenant_region: OptionalString, - }), - ), - aws_cognito: Schema.optionalKey( - Schema.Struct({ - enabled: OptionalBoolean, - user_pool_id: OptionalString, - user_pool_region: OptionalString, - }), - ), - clerk: Schema.optionalKey(Schema.Struct({ enabled: OptionalBoolean, domain: OptionalString })), - workos: Schema.optionalKey( - Schema.Struct({ enabled: OptionalBoolean, issuer_url: OptionalString }), - ), -}); - -export const AuthSettingsSchema = Schema.Struct({ - site_url: OptionalString, - additional_redirect_urls: Schema.optionalKey(Schema.Array(Schema.String)), - jwt_expiry: OptionalNumber, - jwt_issuer: OptionalString, - signing_keys_path: OptionalString, - enable_refresh_token_rotation: OptionalBoolean, - refresh_token_reuse_interval: OptionalNumber, - enable_manual_linking: OptionalBoolean, - enable_signup: OptionalBoolean, - enable_anonymous_sign_ins: OptionalBoolean, - minimum_password_length: OptionalNumber, - password_requirements: Schema.optionalKey( - Schema.Literals([ - "", - "letters_digits", - "lower_upper_letters_digits", - "lower_upper_letters_digits_symbols", - ] as const), - ), - publishable_key: Schema.optionalKey(Secret), - secret_key: Schema.optionalKey(Secret), - jwt_secret: Schema.optionalKey(Secret), - anon_key: Schema.optionalKey(Secret), - service_role_key: Schema.optionalKey(Secret), - rate_limit: Schema.optionalKey(RateLimitSchema), - captcha: Schema.optionalKey(CaptchaSchema), - hook: Schema.optionalKey( - Schema.Struct({ - mfa_verification_attempt: Schema.optionalKey(HookSchema), - password_verification_attempt: Schema.optionalKey(HookSchema), - custom_access_token: Schema.optionalKey(HookSchema), - send_sms: Schema.optionalKey(HookSchema), - send_email: Schema.optionalKey(HookSchema), - before_user_created: Schema.optionalKey(HookSchema), - }), - ), - mfa: Schema.optionalKey(MfaSchema), - sessions: Schema.optionalKey(SessionsSchema), - email: Schema.optionalKey(EmailSchema), - sms: Schema.optionalKey(SmsSchema), - external: Schema.optionalKey(ExternalSchema), - web3: Schema.optionalKey(Web3Schema), - oauth_server: Schema.optionalKey( - Schema.Struct({ - enabled: OptionalBoolean, - authorization_url_path: OptionalString, - allow_dynamic_registration: OptionalBoolean, - }), - ), - third_party: Schema.optionalKey(ThirdPartySchema), -}); -export type AuthSettings = Schema.Schema.Type; - -const providerDefaults = { - enabled: false, - client_id: "", - secret: undefined, - url: "", - redirect_uri: "", - skip_nonce_check: false, - email_optional: false, -}; -const providers = { - apple: providerDefaults, - azure: providerDefaults, - bitbucket: providerDefaults, - discord: providerDefaults, - facebook: providerDefaults, - github: providerDefaults, - gitlab: providerDefaults, - google: providerDefaults, - kakao: providerDefaults, - keycloak: providerDefaults, - linkedin_oidc: providerDefaults, - notion: providerDefaults, - twitch: providerDefaults, - twitter: providerDefaults, - x: providerDefaults, - slack_oidc: providerDefaults, - spotify: providerDefaults, - workos: providerDefaults, - zoom: providerDefaults, -} satisfies AuthSettings["external"]; - -const hooks = { - mfa_verification_attempt: { enabled: false, uri: undefined, secrets: undefined }, - password_verification_attempt: { enabled: false, uri: undefined, secrets: undefined }, - custom_access_token: { enabled: false, uri: undefined, secrets: undefined }, - send_sms: { enabled: false, uri: undefined, secrets: undefined }, - send_email: { enabled: false, uri: undefined, secrets: undefined }, - before_user_created: { enabled: false, uri: undefined, secrets: undefined }, -} satisfies AuthSettings["hook"]; - -const materializeAuthSettings = (settings: AuthSettings): AuthSettings => ({ - ...settings, - email: - settings.email === undefined - ? settings.email - : { - ...settings.email, - template: Object.fromEntries( - Object.entries(settings.email.template ?? {}).map(([name, template]) => [ - name, - { subject: undefined, content_path: undefined, ...template }, - ]), - ), - notification: Object.fromEntries( - Object.entries(settings.email.notification ?? {}).map(([name, notification]) => [ - name, - { enabled: false, subject: undefined, content_path: undefined, ...notification }, - ]), - ), - }, -}); - -export const AuthModule: CapabilityModule = { - name: "auth", - settings: AuthSettingsSchema, - defaultSettings: { - site_url: "http://127.0.0.1:3000", - additional_redirect_urls: ["https://127.0.0.1:3000"], - jwt_expiry: 3600, - jwt_issuer: undefined, - signing_keys_path: undefined, - enable_refresh_token_rotation: true, - refresh_token_reuse_interval: 10, - enable_manual_linking: false, - enable_signup: true, - enable_anonymous_sign_ins: false, - minimum_password_length: 6, - password_requirements: "", - publishable_key: undefined, - secret_key: undefined, - jwt_secret: undefined, - anon_key: undefined, - service_role_key: undefined, - rate_limit: { - email_sent: 2, - sms_sent: 30, - anonymous_users: 30, - token_refresh: 150, - sign_in_sign_ups: 30, - token_verifications: 30, - web3: 30, - }, - captcha: { enabled: false, provider: undefined, secret: undefined }, - hook: hooks, - mfa: { - totp: { enroll_enabled: false, verify_enabled: false }, - phone: { - enroll_enabled: false, - verify_enabled: false, - otp_length: 6, - template: "Your code is {{ .Code }}", - max_frequency: "5s", - }, - web_authn: { enroll_enabled: false, verify_enabled: false }, - max_enrolled_factors: 10, - }, - sessions: { timebox: undefined, inactivity_timeout: undefined }, - email: { - enable_signup: true, - double_confirm_changes: true, - enable_confirmations: false, - secure_password_change: false, - max_frequency: "1s", - otp_length: 6, - otp_expiry: 3600, - smtp: { enabled: false }, - template: {}, - notification: {}, - }, - sms: { - enable_signup: false, - enable_confirmations: false, - template: "Your code is {{ .Code }}", - max_frequency: "5s", - twilio: { enabled: false, account_sid: "", message_service_sid: "", auth_token: undefined }, - twilio_verify: { - enabled: false, - account_sid: undefined, - message_service_sid: undefined, - auth_token: undefined, - }, - messagebird: { enabled: false, originator: undefined, access_key: undefined }, - textlocal: { enabled: false, sender: undefined, api_key: undefined }, - vonage: { enabled: false, from: undefined, api_key: undefined, api_secret: undefined }, - test_otp: undefined, - }, - external: providers, - web3: { solana: { enabled: false }, ethereum: { enabled: false } }, - oauth_server: { - enabled: false, - authorization_url_path: "/oauth/consent", - allow_dynamic_registration: false, - }, - third_party: { - firebase: { enabled: false }, - auth0: { enabled: false }, - aws_cognito: { enabled: false }, - clerk: { enabled: false }, - workos: { enabled: false }, - }, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultIdleTimeoutSeconds: 60, - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("auth", "auth", { - dependencies: ["database:database"], - readiness: { portField: "api" }, - }), - ]), - }, - routes: [{ listener: "api", protocol: "http" }], - secretPolicy: (path) => - [ - "auth.settings.publishable_key", - "auth.settings.secret_key", - "auth.settings.jwt_secret", - "auth.settings.anon_key", - "auth.settings.service_role_key", - ].includes(path) - ? "managed" - : "passthrough", - managedSecretSlots: [ - "auth.settings.publishable_key", - "auth.settings.secret_key", - "auth.settings.jwt_secret", - "auth.settings.anon_key", - "auth.settings.service_role_key", - ], - materialize: materializeAuthSettings, -}; diff --git a/packages/stack/src/model/capabilities/database.ts b/packages/stack/src/model/capabilities/database.ts deleted file mode 100644 index e7053b2eb2..0000000000 --- a/packages/stack/src/model/capabilities/database.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { Duration, Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const settingsFields = { - effective_cache_size: Schema.optionalKey(Schema.String), - logical_decoding_work_mem: Schema.optionalKey(Schema.String), - maintenance_work_mem: Schema.optionalKey(Schema.String), - max_connections: Schema.optionalKey(Schema.Finite), - max_locks_per_transaction: Schema.optionalKey(Schema.Finite), - max_parallel_maintenance_workers: Schema.optionalKey(Schema.Finite), - max_parallel_workers: Schema.optionalKey(Schema.Finite), - max_parallel_workers_per_gather: Schema.optionalKey(Schema.Finite), - max_replication_slots: Schema.optionalKey(Schema.Finite), - max_slot_wal_keep_size: Schema.optionalKey(Schema.String), - max_standby_archive_delay: Schema.optionalKey(Schema.String), - max_standby_streaming_delay: Schema.optionalKey(Schema.String), - max_wal_size: Schema.optionalKey(Schema.String), - max_wal_senders: Schema.optionalKey(Schema.Finite), - max_worker_processes: Schema.optionalKey(Schema.Finite), - session_replication_role: Schema.optionalKey( - Schema.Literals(["origin", "replica", "local"] as const), - ), - shared_buffers: Schema.optionalKey(Schema.String), - statement_timeout: Schema.optionalKey(Schema.String), - track_activity_query_size: Schema.optionalKey(Schema.String), - track_commit_timestamp: Schema.optionalKey(Schema.Boolean), - wal_keep_size: Schema.optionalKey(Schema.String), - wal_sender_timeout: Schema.optionalKey(Schema.String), - work_mem: Schema.optionalKey(Schema.String), -}; - -const settingsSchema = Schema.Struct(settingsFields); -export const DatabaseSettingsSchema = Schema.Struct({ - health_timeout: Schema.optionalKey(Schema.String), - settings: Schema.optionalKey(settingsSchema), -}); -export type DatabaseSettings = Schema.Schema.Type; - -export const DEFAULT_DATABASE_HEALTH_TIMEOUT = "2m"; - -const MAX_GO_DURATION_NS = 9_223_372_036_854_775_807n; -const GO_DURATION_UNITS = { - ns: 1n, - us: 1_000n, - µs: 1_000n, - μs: 1_000n, - ms: 1_000_000n, - s: 1_000_000_000n, - m: 60_000_000_000n, - h: 3_600_000_000_000n, -} as const; - -/** Parses the Go duration syntax used by persisted database configuration. */ -export const parseGoDuration = (value: string): Duration.Duration => { - const original = value; - let input = value; - const negative = input.startsWith("-"); - if (input.startsWith("-") || input.startsWith("+")) input = input.slice(1); - if (input === "0") return Duration.zero; - if (input.length === 0) throw new Error(`invalid duration "${original}"`); - - let total = 0n; - while (input.length > 0) { - const number = /^(?:\d+(?:\.\d*)?|\.\d+)/u.exec(input); - if (number === null) throw new Error(`invalid duration "${original}"`); - const token = number[0]; - const dot = token.indexOf("."); - const whole = dot === -1 ? token : token.slice(0, dot); - const fraction = dot === -1 ? "" : token.slice(dot + 1); - input = input.slice(token.length); - - let unit: keyof typeof GO_DURATION_UNITS | undefined; - for (const candidate of ["ns", "us", "µs", "μs", "ms", "s", "m", "h"] as const) { - if (input.startsWith(candidate)) { - unit = candidate; - input = input.slice(candidate.length); - break; - } - } - if (unit === undefined) throw new Error(`invalid duration "${original}"`); - const unitNs = GO_DURATION_UNITS[unit]; - const wholeNs = BigInt(whole) * unitNs; - const fractionNs = - fraction.length === 0 ? 0n : (BigInt(fraction) * unitNs) / 10n ** BigInt(fraction.length); - total += wholeNs + fractionNs; - if (total > MAX_GO_DURATION_NS) throw new Error(`duration out of range "${original}"`); - } - if (negative) return Duration.nanos(-total); - return Duration.nanos(total); -}; - -const defaults: DatabaseSettings = { - health_timeout: DEFAULT_DATABASE_HEALTH_TIMEOUT, - settings: { - effective_cache_size: undefined, - logical_decoding_work_mem: undefined, - maintenance_work_mem: undefined, - max_connections: undefined, - max_locks_per_transaction: undefined, - max_parallel_maintenance_workers: undefined, - max_parallel_workers: undefined, - max_parallel_workers_per_gather: undefined, - max_replication_slots: undefined, - max_slot_wal_keep_size: undefined, - max_standby_archive_delay: undefined, - max_standby_streaming_delay: undefined, - max_wal_size: undefined, - max_wal_senders: undefined, - max_worker_processes: undefined, - session_replication_role: undefined, - shared_buffers: undefined, - statement_timeout: undefined, - track_activity_query_size: undefined, - track_commit_timestamp: undefined, - wal_keep_size: undefined, - wal_sender_timeout: undefined, - work_mem: undefined, - }, -}; - -const databaseCatalog = catalogEntryFor("database:database"); -const databaseReleases = Object.fromEntries( - Object.keys(databaseCatalog.releases).flatMap((version) => { - const selected = release(version, [ - workload("database", "database", { - bootstrap: "database", - readiness: { portField: "database" }, - version, - }), - ]); - return [ - [version, selected], - [version.split(".")[0], selected], - ]; - }), -); - -export const DatabaseModule: CapabilityModule = { - name: "database", - settings: DatabaseSettingsSchema, - defaultSettings: defaults, - defaultEnabled: true, - defaultActivation: "eager", - defaultVersion: databaseCatalog.defaultVersion, - dependencies: [], - releases: databaseReleases, - routes: [{ listener: "database", protocol: "tcp" }], - secretPolicy: () => "passthrough", - managedSecretSlots: [], -}; diff --git a/packages/stack/src/model/capabilities/functions.ts b/packages/stack/src/model/capabilities/functions.ts deleted file mode 100644 index 5b1d472f0b..0000000000 --- a/packages/stack/src/model/capabilities/functions.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("functions:edge-runtime").defaultVersion; - -const Secret = Schema.Redacted(Schema.String); -const FunctionSlug = Schema.String.check(Schema.isPattern(/^[a-zA-Z0-9_-]+$/)); -const EnvName = Schema.String.check(Schema.isPattern(/^[A-Z_][A-Z0-9_]*$/)); -const FunctionOverrideSchema = Schema.Struct({ - enabled: Schema.optionalKey(Schema.Boolean), - verify_jwt: Schema.optionalKey(Schema.Boolean), - import_map: Schema.optionalKey(Schema.String), - entrypoint: Schema.optionalKey(Schema.String), - static_files: Schema.optionalKey(Schema.Array(Schema.String)), - env: Schema.optionalKey(Schema.Record(EnvName, Secret)), -}); - -const EdgeRuntimeSettingsSchema = Schema.Struct({ - policy: Schema.optionalKey(Schema.Literals(["oneshot", "per_worker"] as const)), - deno_version: Schema.optionalKey(Schema.Finite), - /** Defaults applied to functions discovered after the stack starts. */ - verify_jwt_default: Schema.optionalKey(Schema.Boolean), - import_map_default: Schema.optionalKey(Schema.String), - secrets: Schema.optionalKey(Schema.Record(Schema.String, Secret)), -}); - -export const FunctionsInspectorSettingsSchema = Schema.Struct({ - /** Expose the Edge Runtime inspector as run, break-on-start, or wait-for-debugger. */ - mode: Schema.optionalKey(Schema.Literals(["run", "brk", "wait"] as const)), - /** Also create an inspector for the main worker. */ - main: Schema.optionalKey(Schema.Boolean), -}); -export type FunctionsInspectorSettings = Schema.Schema.Type< - typeof FunctionsInspectorSettingsSchema ->; - -export const FunctionsSettingsSchema = Schema.Struct({ - functions_root: Schema.optionalKey(Schema.String), - /** Edge Runtime behavior; capability enabled and functionsInspector port are public fields. */ - edge_runtime: Schema.optionalKey(EdgeRuntimeSettingsSchema), - inspector: Schema.optionalKey(FunctionsInspectorSettingsSchema), - functions: Schema.optionalKey(Schema.Record(FunctionSlug, FunctionOverrideSchema)), -}); -export type FunctionsSettings = Schema.Schema.Type; - -export const FunctionsModule: CapabilityModule = { - name: "functions", - settings: FunctionsSettingsSchema, - defaultSettings: { - functions_root: "supabase/functions", - edge_runtime: { - policy: "per_worker", - deno_version: 2, - secrets: {}, - }, - inspector: undefined, - functions: {}, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("edge-runtime", "functions", { - dependencies: ["database:database"], - readiness: { portField: "functionsInspector" }, - }), - ]), - }, - routes: [ - { listener: "api", protocol: "http" }, - { listener: "functionsInspector", protocol: "http" }, - ], - secretPolicy: () => "passthrough", - managedSecretSlots: [], - materialize: (settings) => ({ - ...settings, - functions: Object.fromEntries( - Object.entries(settings.functions ?? {}).map(([name, fn]) => [ - name, - { - enabled: fn.enabled ?? true, - verify_jwt: fn.verify_jwt ?? true, - import_map: fn.import_map ?? "", - entrypoint: fn.entrypoint ?? "", - static_files: fn.static_files ?? [], - env: fn.env ?? {}, - }, - ]), - ), - }), -}; diff --git a/packages/stack/src/model/capabilities/index.ts b/packages/stack/src/model/capabilities/index.ts deleted file mode 100644 index 9c52193f21..0000000000 --- a/packages/stack/src/model/capabilities/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export { AuthModule, AuthSettingsSchema } from "./auth.ts"; -export { DatabaseModule, DatabaseSettingsSchema, parseGoDuration } from "./database.ts"; -export { FunctionsModule, FunctionsSettingsSchema } from "./functions.ts"; -export { MailModule, MailSettingsSchema } from "./mail.ts"; -export { PoolerModule, PoolerSettingsSchema } from "./pooler.ts"; -export { RealtimeModule, RealtimeSettingsSchema } from "./realtime.ts"; -export { RestModule, RestSettingsSchema } from "./rest.ts"; -export { parseFileSize, StorageModule, StorageSettingsSchema } from "./storage.ts"; -export { StudioModule, StudioSettingsSchema } from "./studio.ts"; -export { AnalyticsModule, AnalyticsSettingsSchema } from "./analytics.ts"; diff --git a/packages/stack/src/model/capabilities/mail.ts b/packages/stack/src/model/capabilities/mail.ts deleted file mode 100644 index 1bb38c126c..0000000000 --- a/packages/stack/src/model/capabilities/mail.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("mail:mail").defaultVersion; - -export const MailSettingsSchema = Schema.Struct({ - admin_email: Schema.optionalKey(Schema.String), - sender_name: Schema.optionalKey(Schema.String), -}); -export type MailSettings = Schema.Schema.Type; -export const MailModule: CapabilityModule = { - name: "mail", - settings: MailSettingsSchema, - defaultSettings: { admin_email: "admin@email.com", sender_name: "Admin" }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultVersion: version, - dependencies: [], - releases: { - [version]: release(version, [ - workload("mail", "mail", { - readiness: { portField: "mailUi" }, - }), - ]), - }, - routes: [ - { listener: "mailUi", protocol: "http" }, - { listener: "smtp", protocol: "tcp" }, - { listener: "pop3", protocol: "tcp" }, - ], - secretPolicy: () => "passthrough", - managedSecretSlots: [], -}; diff --git a/packages/stack/src/model/capabilities/pooler.ts b/packages/stack/src/model/capabilities/pooler.ts deleted file mode 100644 index 77668aec38..0000000000 --- a/packages/stack/src/model/capabilities/pooler.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("pooler:pooler").defaultVersion; - -const Secret = Schema.Redacted(Schema.String); -export const PoolerSettingsSchema = Schema.Struct({ - pool_mode: Schema.optionalKey(Schema.Literals(["transaction", "session"] as const)), - tenant_id: Schema.optionalKey(Schema.String), - encryption_key: Schema.optionalKey(Secret), - secret_key_base: Schema.optionalKey(Secret), - default_pool_size: Schema.optionalKey(Schema.Finite), - max_client_conn: Schema.optionalKey(Schema.Finite), -}); -export type PoolerSettings = Schema.Schema.Type; -export const PoolerModule: CapabilityModule = { - name: "pooler", - settings: PoolerSettingsSchema, - defaultSettings: { - pool_mode: "transaction", - tenant_id: "pooler-dev", - encryption_key: undefined, - secret_key_base: undefined, - default_pool_size: 20, - max_client_conn: 100, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultIdleTimeoutSeconds: 60, - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("pooler", "pooler", { - dependencies: ["database:database"], - readiness: { portField: "pooler" }, - }), - ]), - }, - routes: [{ listener: "pooler", protocol: "tcp" }], - secretPolicy: (path) => - path === "pooler.settings.encryption_key" || path === "pooler.settings.secret_key_base" - ? "managed" - : "passthrough", - managedSecretSlots: ["pooler.settings.encryption_key", "pooler.settings.secret_key_base"], -}; diff --git a/packages/stack/src/model/capabilities/realtime.ts b/packages/stack/src/model/capabilities/realtime.ts deleted file mode 100644 index 29903a6458..0000000000 --- a/packages/stack/src/model/capabilities/realtime.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("realtime:realtime").defaultVersion; - -const Secret = Schema.Redacted(Schema.String); - -export const RealtimeSettingsSchema = Schema.Struct({ - ip_version: Schema.optionalKey(Schema.Literals(["IPv4", "IPv6"] as const)), - max_header_length: Schema.optionalKey(Schema.Finite), - db_enc_key: Schema.optionalKey(Secret), - secret_key_base: Schema.optionalKey(Secret), -}); -export type RealtimeSettings = Schema.Schema.Type; - -export const RealtimeModule: CapabilityModule = { - name: "realtime", - settings: RealtimeSettingsSchema, - defaultSettings: { - ip_version: "IPv4", - max_header_length: 4096, - db_enc_key: undefined, - secret_key_base: undefined, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultIdleTimeoutSeconds: 60, - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("realtime", "realtime", { - dependencies: ["database:database"], - readiness: { portField: "api" }, - }), - ]), - }, - routes: [{ listener: "api", protocol: "http" }], - secretPolicy: (path) => - path === "realtime.settings.db_enc_key" || path === "realtime.settings.secret_key_base" - ? "managed" - : "passthrough", - managedSecretSlots: ["realtime.settings.db_enc_key", "realtime.settings.secret_key_base"], -}; diff --git a/packages/stack/src/model/capabilities/rest.ts b/packages/stack/src/model/capabilities/rest.ts deleted file mode 100644 index f15f4c3f03..0000000000 --- a/packages/stack/src/model/capabilities/rest.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("rest:rest").defaultVersion; - -export const RestSettingsSchema = Schema.Struct({ - schemas: Schema.optionalKey(Schema.Array(Schema.String)), - extra_search_path: Schema.optionalKey(Schema.Array(Schema.String)), - max_rows: Schema.optionalKey(Schema.Finite), - external_url: Schema.optionalKey(Schema.String), -}); -export type RestSettings = Schema.Schema.Type; - -export const RestModule: CapabilityModule = { - name: "rest", - settings: RestSettingsSchema, - defaultSettings: { - schemas: ["public", "graphql_public"], - extra_search_path: ["public", "extensions"], - max_rows: 1000, - external_url: undefined, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultIdleTimeoutSeconds: 60, - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("rest", "rest", { - dependencies: ["database:database"], - readiness: { portField: "api" }, - }), - ]), - }, - routes: [{ listener: "api", protocol: "http" }], - secretPolicy: () => "passthrough", - managedSecretSlots: [], -}; diff --git a/packages/stack/src/model/capabilities/storage.ts b/packages/stack/src/model/capabilities/storage.ts deleted file mode 100644 index e187af6216..0000000000 --- a/packages/stack/src/model/capabilities/storage.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("storage:storage").defaultVersion; - -const Bucket = Schema.Struct({ - public: Schema.optionalKey(Schema.Boolean), - file_size_limit: Schema.optionalKey(Schema.Union([Schema.String, Schema.Finite])), - allowed_mime_types: Schema.optionalKey(Schema.Array(Schema.String)), - objects_path: Schema.optionalKey(Schema.String), -}); -const Secret = Schema.Redacted(Schema.String); -const Vector = Schema.Struct({ - enabled: Schema.optionalKey(Schema.Boolean), - max_buckets: Schema.optionalKey(Schema.Finite), - max_indexes: Schema.optionalKey(Schema.Finite), - buckets: Schema.optionalKey(Schema.Record(Schema.String, Schema.Struct({}))), -}); -export const StorageSettingsSchema = Schema.Struct({ - file_size_limit: Schema.optionalKey(Schema.Union([Schema.String, Schema.Finite])), - image_transformation: Schema.optionalKey( - Schema.Struct({ enabled: Schema.optionalKey(Schema.Boolean) }), - ), - buckets: Schema.optionalKey(Schema.Record(Schema.String, Bucket)), - s3_protocol: Schema.optionalKey( - Schema.Struct({ - enabled: Schema.optionalKey(Schema.Boolean), - region: Schema.optionalKey(Schema.String), - access_key_id: Schema.optionalKey(Schema.String), - secret_access_key: Schema.optionalKey(Secret), - }), - ), - vector: Schema.optionalKey(Vector), -}); -export type StorageSettings = Schema.Schema.Type; - -const FILE_SIZE_UNITS: Readonly> = { - B: 1n, - KB: 1_000n, - MB: 1_000_000n, - GB: 1_000_000_000n, - TB: 1_000_000_000_000n, - KiB: 1_024n, - MiB: 1_048_576n, - GiB: 1_073_741_824n, - TiB: 1_099_511_627_776n, -}; -const MAX_FILE_SIZE = BigInt(Number.MAX_SAFE_INTEGER); - -/** Convert one non-negative byte size into its exact decimal representation. */ -export const parseFileSize = (value: string | number): string | undefined => { - if (typeof value === "number" && (!Number.isFinite(value) || value < 0)) return undefined; - const match = /^([0-9]+(?:\.[0-9]+)?)[ \t]*(B|KB|MB|GB|TB|KiB|MiB|GiB|TiB)?$/u.exec( - String(value).trim(), - ); - if (match === null) return undefined; - const amount = match[1]; - const unit = match[2] ?? "B"; - if (amount === undefined) return undefined; - const factor = FILE_SIZE_UNITS[unit]; - if (factor === undefined) return undefined; - const [whole, fraction = ""] = amount.split("."); - if (whole === undefined) return undefined; - const scale = 10n ** BigInt(fraction.length); - const scaled = (BigInt(whole) * scale + BigInt(fraction || "0")) * factor; - if (scaled % scale !== 0n) return undefined; - const bytes = scaled / scale; - return bytes > MAX_FILE_SIZE ? undefined : bytes.toString(); -}; - -const bucketDefaults = { - public: false, - file_size_limit: "50MiB", - allowed_mime_types: [], - objects_path: "", -}; - -export const StorageModule: CapabilityModule = { - name: "storage", - settings: StorageSettingsSchema, - defaultSettings: { - file_size_limit: "50MiB", - image_transformation: { enabled: true }, - buckets: {}, - s3_protocol: { - enabled: true, - region: "local", - access_key_id: "625729a08b95bf1b7ff351a663f3a23c", - // Managed by the compiler so every stack receives a fresh credential. - secret_access_key: undefined, - }, - vector: { enabled: true, max_buckets: 10, max_indexes: 5, buckets: {} }, - }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultVersion: version, - dependencies: ["database"], - releases: { - [version]: release(version, [ - workload("storage", "storage", { - dependencies: ["database:database"], - readiness: { portField: "api" }, - }), - workload("imgproxy", "storage", { - readiness: {}, - }), - ]), - }, - routes: [{ listener: "api", protocol: "http" }], - secretPolicy: (path) => - path === "storage.settings.s3_protocol.secret_access_key" ? "managed" : "passthrough", - managedSecretSlots: ["storage.settings.s3_protocol.secret_access_key"], - selectWorkloads: (settings, workloads) => { - const enabled = settings.image_transformation?.enabled === true; - return workloads - .filter((entry) => enabled || entry.name !== "imgproxy") - .map((entry) => - entry.name === "storage" && enabled - ? { - ...entry, - dependencies: [...entry.dependencies, "storage:imgproxy"], - } - : entry, - ); - }, - materialize: (settings) => ({ - ...settings, - buckets: Object.fromEntries( - Object.entries(settings.buckets ?? {}).map(([name, bucket]) => [ - name, - { ...bucketDefaults, ...bucket }, - ]), - ), - }), -}; diff --git a/packages/stack/src/model/capabilities/studio.ts b/packages/stack/src/model/capabilities/studio.ts deleted file mode 100644 index d5e87d2027..0000000000 --- a/packages/stack/src/model/capabilities/studio.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Schema } from "effect"; -import { release, workload, type CapabilityModule } from "../CapabilityModule.ts"; -import { catalogEntryFor } from "../WorkloadCatalog.ts"; - -const version = catalogEntryFor("studio:studio").defaultVersion; - -const Secret = Schema.Redacted(Schema.String); -export const StudioSettingsSchema = Schema.Struct({ - api_url: Schema.optionalKey(Schema.String), - openai_api_key: Schema.optionalKey(Secret), -}); -export type StudioSettings = Schema.Schema.Type; -export const StudioModule: CapabilityModule = { - name: "studio", - settings: StudioSettingsSchema, - defaultSettings: { api_url: "", openai_api_key: undefined }, - defaultEnabled: true, - defaultActivation: "lazy", - defaultIdleTimeoutSeconds: 60, - defaultVersion: version, - dependencies: ["rest"], - releases: { - [version]: release(version, [ - workload("studio", "studio", { - dependencies: ["studio:pgmeta"], - readiness: { portField: "studio" }, - }), - workload("pgmeta", "studio", { - dependencies: ["database:database"], - readiness: {}, - }), - ]), - }, - routes: [{ listener: "studio", protocol: "http" }], - secretPolicy: () => "passthrough", - managedSecretSlots: [], -}; diff --git a/packages/stack/src/model/catalog.integration.test.ts b/packages/stack/src/model/catalog.integration.test.ts deleted file mode 100644 index 65fd8b2232..0000000000 --- a/packages/stack/src/model/catalog.integration.test.ts +++ /dev/null @@ -1,322 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Option } from "effect"; -import { compileStack } from "./Compiler.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import { workload } from "./CapabilityModule.ts"; -import { - catalogEntryFor, - catalogReleaseFor, - resolveNativeArtifactForWorkload, - targetForPlatform, - WORKLOAD_CATALOG, -} from "./WorkloadCatalog.ts"; -import { StackPreparationError } from "../public/Errors.ts"; - -const databaseCatalog = catalogEntryFor("database:database"); -const defaultDatabaseMajor = databaseCatalog.defaultVersion.split(".")[0]; - -const compile = (config: Parameters[0]["config"]) => - compileStack({ projectRoot: "/tmp/catalog-project", runtime: { kind: "native" }, config }).pipe( - Effect.provide(NodeServices.layer), - ); - -const compileContainer = (config: Parameters[0]["config"]) => - compileStack({ - projectRoot: "/tmp/catalog-project", - runtime: { kind: "container", engine: "docker" }, - config, - }).pipe(Effect.provide(NodeServices.layer)); - -const expectWorkload = (value: T | undefined, description: string): T => { - expect(value, description).toBeDefined(); - if (value === undefined) throw new Error(`Expected ${description}`); - return value; -}; - -describe("complete workload catalog", () => { - it("uses the catalog release for both runtime artifact identities", () => { - const selected = workload("rest", "rest"); - const catalog = catalogReleaseFor("rest:rest"); - expect(selected.artifacts).toEqual({ - native: { kind: "native", release: catalog?.version }, - container: { kind: "container", image: catalog?.containerImage }, - }); - }); - - it("matches the executable paths shipped by slim-services archives", () => { - const expected = { - "database:database": "bin/supabase-postgres-start", - "rest:rest": "bin/postgrest", - "auth:auth": "bin/auth", - "realtime:realtime": "bin/server", - "storage:storage": "bin/storage", - "storage:imgproxy": "bin/imgproxy", - "functions:edge-runtime": "bin/edge-runtime", - "studio:studio": "bin/studio", - "studio:pgmeta": "bin/pgmeta", - "mail:mail": "bin/mailpit", - "analytics:analytics": "bin/logflare", - "analytics:vector": "bin/vector", - "pooler:pooler": "bin/server", - } as const; - for (const [id, executable] of Object.entries(expected)) { - expect(WORKLOAD_CATALOG[id]?.executablePath).toBe(executable); - expect(WORKLOAD_CATALOG[id]?.requiredRuntimePaths).toContain(executable); - } - expect(WORKLOAD_CATALOG["realtime:realtime"]?.requiredRuntimePaths).toEqual([ - "bin/server", - "bin/prepare", - ]); - expect(WORKLOAD_CATALOG["database:database"]?.requiredRuntimePaths).toEqual([ - "bin/supabase-postgres-start", - "bin/pg_dump", - "bin/pg_dumpall", - "bin/psql", - ]); - expect(WORKLOAD_CATALOG["storage:storage"]?.requiredRuntimePaths).toEqual([ - "bin/storage", - "bin/prepare", - ]); - expect(WORKLOAD_CATALOG["analytics:analytics"]?.requiredRuntimePaths).toEqual([ - "bin/logflare", - "bin/prepare", - ]); - expect(WORKLOAD_CATALOG["analytics:vector"]?.requiredRuntimePaths).toEqual([ - "bin/vector", - "share/doc/vector/config/vector.yaml", - ]); - expect(WORKLOAD_CATALOG["pooler:pooler"]?.requiredRuntimePaths).toEqual([ - "bin/server", - "bin/prepare", - "bin/provision-tenant", - ]); - }); - - it.live("keeps persisted capability versions aligned with catalog releases", () => - Effect.gen(function* () { - const defaults = yield* compile({}); - for (const name of CAPABILITY_NAMES) { - const capability = defaults.definition.capabilities[name]; - if (!capability.enabled) continue; - const workload = - defaults.executionPlan.workloads.find((entry) => entry.id === `${name}:${name}`) ?? - defaults.executionPlan.workloads.find((entry) => entry.capability === name); - const selected = expectWorkload(workload, `${name} workload`); - expect(selected.artifacts.native.release).toBe(capability.version); - } - - const databaseAlias = yield* compile({ - capabilities: { database: { version: defaultDatabaseMajor } }, - }); - const database = databaseAlias.executionPlan.workloads.find( - (entry) => entry.id === "database:database", - ); - const selected = expectWorkload(database, "database workload"); - expect(databaseAlias.definition.capabilities.database.version).toBe( - selected.artifacts.native.release, - ); - expect(selected.artifacts.native.release).toBe(databaseCatalog.defaultVersion); - }), - ); - - it.live("resolves every declared workload to a slim-services native release", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - storage: { settings: { image_transformation: { enabled: true } } }, - analytics: { settings: {} }, - }, - }); - expect(result.executionPlan.workloads.length).toBeGreaterThan(10); - for (const workload of result.executionPlan.workloads) { - const artifact = yield* resolveNativeArtifactForWorkload(workload); - expect(artifact.provider).toBe("supabase/slim-services"); - expect(artifact.downloadUrl).toContain( - "github.com/supabase/slim-services/releases/download", - ); - expect(artifact.requiredRuntimePaths.length).toBeGreaterThan(0); - expect(artifact.executablePath).toBeDefined(); - } - }), - ); - - it.live("uses canonical native release tags for companion workloads", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - storage: { settings: { image_transformation: { enabled: true } } }, - analytics: { settings: {} }, - }, - }); - for (const id of ["studio:pgmeta", "analytics:vector"] as const) { - const catalog = WORKLOAD_CATALOG[id]; - const selectedCatalog = expectWorkload(catalog, `${id} catalog entry`); - const workload = result.executionPlan.workloads.find((entry) => entry.id === id); - const selectedWorkload = expectWorkload(workload, `${id} workload`); - expect(selectedWorkload.artifacts.native.release).toBe(selectedCatalog.defaultVersion); - const artifact = yield* resolveNativeArtifactForWorkload(selectedWorkload, { - os: "darwin", - arch: "arm64", - }); - expect(artifact.releaseTag).toBe( - `${selectedCatalog.service}-${selectedCatalog.defaultVersion}`, - ); - expect(artifact.downloadUrl).toBe( - `https://github.com/supabase/slim-services/releases/download/${selectedCatalog.service}-${selectedCatalog.defaultVersion}/${selectedCatalog.service}-${selectedCatalog.defaultVersion}-darwin-arm64.tar.zst`, - ); - } - }), - ); - - it.live("enables companion workloads by default and honors explicit disablement", () => - Effect.gen(function* () { - const defaults = yield* compile({}); - expect(defaults.executionPlan.workloads.some(({ id }) => id === "storage:imgproxy")).toBe( - true, - ); - expect(defaults.executionPlan.workloads.some(({ id }) => id === "analytics:vector")).toBe( - true, - ); - expect(defaults.executionPlan.workloads).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - id: "storage:storage", - dependencies: ["database:database", "storage:imgproxy"], - }), - ]), - ); - - const enabled = yield* compile({ - capabilities: { - storage: { settings: { image_transformation: { enabled: true } } }, - analytics: { settings: {} }, - }, - }); - expect(enabled.executionPlan.workloads.some(({ id }) => id === "storage:imgproxy")).toBe( - true, - ); - expect(enabled.executionPlan.workloads.some(({ id }) => id === "analytics:vector")).toBe( - true, - ); - expect(enabled.executionPlan.workloads).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - id: "storage:storage", - dependencies: ["database:database", "storage:imgproxy"], - }), - expect.objectContaining({ id: "storage:imgproxy", dependencies: [] }), - expect.objectContaining({ - id: "analytics:analytics", - dependencies: ["database:database"], - }), - expect.objectContaining({ - id: "analytics:vector", - dependencies: ["analytics:analytics"], - }), - ]), - ); - - const disabled = yield* compile({ - capabilities: { - storage: { settings: { image_transformation: { enabled: false } } }, - analytics: { enabled: false }, - studio: { enabled: false }, - }, - }); - expect(disabled.executionPlan.workloads.some(({ id }) => id === "storage:imgproxy")).toBe( - false, - ); - expect(disabled.executionPlan.workloads.some(({ id }) => id === "analytics:vector")).toBe( - false, - ); - }), - ); - - it.live("selects qualified slim mirror images in container plans", () => - Effect.gen(function* () { - const result = yield* compileContainer({ - capabilities: { - storage: { settings: { image_transformation: { enabled: true } } }, - analytics: { settings: {} }, - }, - }); - const images = new Map( - result.executionPlan.workloads.map((entry) => [entry.id, entry.artifacts.container.image]), - ); - expect(images.get("mail:mail")).toBe(catalogReleaseFor("mail:mail")?.containerImage); - expect(images.get("storage:imgproxy")).toBe( - catalogReleaseFor("storage:imgproxy")?.containerImage, - ); - expect(images.get("analytics:vector")).toBe( - catalogReleaseFor("analytics:vector")?.containerImage, - ); - }), - ); - - it.live("derives native database artifacts from the planned release", () => - Effect.gen(function* () { - const compiled = yield* compile({ - capabilities: { database: { version: defaultDatabaseMajor } }, - }); - const database = compiled.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const artifact = yield* resolveNativeArtifactForWorkload( - expectWorkload(database, "database workload"), - { - os: "linux", - arch: "x64", - }, - ); - expect(artifact.version).toBe(databaseCatalog.defaultVersion); - expect(artifact.downloadUrl).toContain( - `postgres-${databaseCatalog.defaultVersion}-linux-amd64.tar.zst`, - ); - }), - ); - - it.live("rejects an exact native release absent from the catalog", () => - Effect.gen(function* () { - const compiled = yield* compile({}); - const database = compiled.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const selected = expectWorkload(database, "database workload"); - const unsupported = { - ...selected, - artifacts: { - ...selected.artifacts, - native: { ...selected.artifacts.native, release: "99.0.0" }, - }, - }; - const failed = yield* resolveNativeArtifactForWorkload(unsupported, { - os: "linux", - arch: "x64", - }).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - if (!Exit.isFailure(failed)) throw new Error("expected unsupported release to fail"); - const error = Option.getOrUndefined(Cause.findErrorOption(failed.cause)); - expect(error).toBeInstanceOf(StackPreparationError); - }), - ); - - it.live("rejects unsupported native targets before selecting an artifact", () => { - expect(targetForPlatform({ os: "win32", arch: "x64" })).toBeUndefined(); - expect(targetForPlatform({ os: "darwin", arch: "x64" })).toBeUndefined(); - return Effect.gen(function* () { - const compiled = yield* compile({}); - const database = compiled.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const failed = yield* resolveNativeArtifactForWorkload( - expectWorkload(database, "database workload"), - { os: "win32", arch: "x64" }, - ).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - if (!Exit.isFailure(failed)) throw new Error("expected unsupported platform to fail"); - const error = Option.getOrUndefined(Cause.findErrorOption(failed.cause)); - expect(error).toBeInstanceOf(StackPreparationError); - }); - }); -}); diff --git a/packages/stack/src/model/compiler.integration.test.ts b/packages/stack/src/model/compiler.integration.test.ts deleted file mode 100644 index 9bda95a4f0..0000000000 --- a/packages/stack/src/model/compiler.integration.test.ts +++ /dev/null @@ -1,942 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Option, Redacted } from "effect"; -import { InvalidStackConfigError, StackVersionUnsupportedError } from "../public/Errors.ts"; -import { canonicalize, compileStack, rebuildExecutionPlan, sameDefinition } from "./Compiler.ts"; -import { resolveThirdPartyIssuer } from "./capabilities/auth-third-party.ts"; -import { DEFAULT_DATABASE_HEALTH_TIMEOUT } from "./capabilities/database.ts"; -import { catalogEntryFor } from "./WorkloadCatalog.ts"; -import { excludeStackCapabilities } from "./Exclusions.ts"; - -const layer = NodeServices.layer; -const compile = ( - config: Parameters[0]["config"], - runtime: Parameters[0]["runtime"] = { kind: "native" }, - previous?: Parameters[1], -) => - compileStack({ projectRoot: "/tmp/supabase-project", runtime, config }, previous).pipe( - Effect.provide(layer), - ); - -const failureOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("closed capability compiler", () => { - it.live("compiles every optional exclusion and closes Studio dependents", () => - Effect.gen(function* () { - for (const name of [ - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", - ] as const) { - const config = { capabilities: { rest: { settings: { max_rows: 42 } } } }; - const excluded = excludeStackCapabilities(config, [name]); - expect(config.capabilities?.rest).toEqual({ settings: { max_rows: 42 } }); - const result = yield* compile(excluded); - expect(result.definition.capabilities[name].enabled).toBe(false); - expect(result.definition.capabilities.rest.settings.max_rows).toBe(42); - if (name === "analytics") expect(result.definition.capabilities.studio.enabled).toBe(true); - } - const combined = excludeStackCapabilities({}, ["analytics", "pooler"]); - const result = yield* compile(combined); - expect(result.definition.capabilities.studio.enabled).toBe(true); - expect(result.definition.capabilities.rest.enabled).toBe(true); - expect(result.definition.capabilities.analytics.enabled).toBe(false); - expect(result.definition.capabilities.pooler.enabled).toBe(false); - expect(result.definition.capabilities.auth.enabled).toBe(true); - expect(excludeStackCapabilities({}, [])).toEqual({}); - const studioExcluded = yield* compile(excludeStackCapabilities({}, ["studio"])); - expect(studioExcluded.definition.capabilities.rest.enabled).toBe(true); - expect(studioExcluded.definition.capabilities.analytics.enabled).toBe(true); - const analyticsOff = yield* compile({ - capabilities: { analytics: { enabled: false } }, - }); - expect(analyticsOff.definition.capabilities.studio.enabled).toBe(true); - expect(analyticsOff.definition.capabilities.analytics.enabled).toBe(false); - expect(analyticsOff.executionPlan.workloads.some(({ id }) => id === "studio:studio")).toBe( - true, - ); - expect( - analyticsOff.executionPlan.workloads.some(({ id }) => id === "analytics:analytics"), - ).toBe(false); - }), - ); - - it.live("accepts named storage byte limit formats during compilation", () => - Effect.gen(function* () { - for (const input of ["50MiB", "1.5KB", "2 GiB"]) { - const result = yield* compile({ - capabilities: { storage: { settings: { file_size_limit: input } } }, - }); - expect(result.definition.capabilities.storage.settings.file_size_limit).toBe(input); - } - }), - ); - - it.live("rejects invalid storage byte limits during compilation", () => - Effect.gen(function* () { - for (const value of [-1, "not-a-size", "999999999999999999TiB", "1.1B"]) { - const result = yield* compile({ - capabilities: { storage: { settings: { file_size_limit: value } } }, - }).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - } - }), - ); - it.live("materializes a non-default database setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { database: { settings: { health_timeout: "5m" } } }, - }); - expect(result.definition.capabilities.database.settings).toMatchObject({ - health_timeout: "5m", - }); - }), - ); - - it.live("materializes the default database health timeout", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(result.definition.capabilities.database.settings.health_timeout).toBe( - DEFAULT_DATABASE_HEALTH_TIMEOUT, - ); - }), - ); - - it.live("defaults preparation to background with lazy optional activation", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(result.definition.preparation).toBe("background"); - expect(result.executionPlan.activation).toEqual({ - database: "eager", - rest: "lazy", - auth: "lazy", - realtime: "lazy", - storage: "lazy", - functions: "lazy", - studio: "lazy", - mail: "lazy", - analytics: "lazy", - pooler: "lazy", - }); - expect(result.definition.capabilities.pooler).toMatchObject({ - enabled: true, - activation: "lazy", - }); - expect(result.definition.listeners.pooler.enabled).toBe(true); - }), - ); - - it.live("disables pooler workloads while retaining its listener", () => - Effect.gen(function* () { - const disabledPooler = yield* compile({ capabilities: { pooler: { enabled: false } } }); - expect(disabledPooler.definition.capabilities.pooler.enabled).toBe(false); - expect(disabledPooler.definition.listeners.pooler.enabled).toBe(true); - }), - ); - - it.live("accepts on-demand preparation", () => - Effect.gen(function* () { - const onDemand = yield* compile({ preparation: "on-demand" }); - expect(onDemand.definition.preparation).toBe("on-demand"); - }), - ); - - it.live("rejects an invalid preparation mode during compilation", () => - Effect.gen(function* () { - const invalid = yield* compile({ preparation: "invalid" } as never).pipe(Effect.exit); - expect(failureOf(invalid)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("rejects an invalid database health timeout during compilation", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { database: { settings: { health_timeout: "not-a-duration" } } }, - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - const zero = yield* compile({ - capabilities: { database: { settings: { health_timeout: "0" } } }, - }).pipe(Effect.exit); - expect(failureOf(zero)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("materializes a non-default rest setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { rest: { settings: { schemas: ["private"] } } }, - }); - expect(result.definition.capabilities.rest.settings).toMatchObject({ schemas: ["private"] }); - }), - ); - - it.live("rejects settings without a local runtime consumer", () => - Effect.gen(function* () { - const unsupported = yield* compile({ - capabilities: { - rest: { - settings: { auto_expose_new_tables: true, tls: { enabled: true } }, - }, - storage: { settings: { analytics: { enabled: true } } }, - }, - } as never).pipe(Effect.exit); - expect(failureOf(unsupported)).toBeInstanceOf(InvalidStackConfigError); - - const defaults = yield* compile({}); - expect(defaults.definition.capabilities.rest.settings).not.toHaveProperty( - "auto_expose_new_tables", - ); - expect(defaults.definition.capabilities.rest.settings).not.toHaveProperty("tls"); - expect(defaults.definition.capabilities.storage.settings).not.toHaveProperty("analytics"); - }), - ); - - it.live("persists defaults consumed by workload runtimes", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(result.definition.capabilities.rest.settings).toMatchObject({ - schemas: ["public", "graphql_public"], - extra_search_path: ["public", "extensions"], - max_rows: 1000, - }); - expect(result.definition.capabilities.auth.settings).toMatchObject({ - site_url: "http://127.0.0.1:3000", - jwt_expiry: 3600, - }); - expect(result.definition.capabilities.mail.settings).toEqual({ - admin_email: "admin@email.com", - sender_name: "Admin", - }); - expect(result.definition.capabilities.analytics.settings).toMatchObject({ - backend: "postgres", - gcp_project_id: "local", - gcp_project_number: "0", - }); - expect(result.definition.capabilities.realtime.settings).toMatchObject({ - max_header_length: 4096, - }); - expect(result.definition.capabilities.functions.settings).toMatchObject({ - functions_root: "/tmp/supabase-project/supabase/functions", - edge_runtime: { policy: "per_worker", deno_version: 2 }, - }); - }), - ); - - it.live("materializes a non-default auth setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - auth: { - settings: { - site_url: "https://example.test", - secret_key: Redacted.make("secret-value"), - }, - }, - }, - }); - expect(result.definition.capabilities.auth.settings).toMatchObject({ - site_url: "https://example.test", - }); - expect(canonicalize(result.definition)).not.toContain("secret-value"); - expect(canonicalize(result.executionPlan)).not.toContain("secret-value"); - const supplied = result.secrets.find( - (entry) => entry.slot === "secret:auth.settings.secret_key", - ); - expect(supplied?.policy).toBe("managed"); - expect(Redacted.isRedacted(supplied?.value)).toBe(true); - for (const slot of [ - "secret:database.internal.password", - "secret:auth.settings.publishable_key", - "secret:auth.settings.jwt_secret", - "secret:auth.settings.anon_key", - "secret:auth.settings.service_role_key", - ]) { - expect(result.secrets.find((entry) => entry.slot === slot)?.policy).toBe("managed"); - } - expect(canonicalize(result.executionPlan)).not.toContain("secret-value"); - }), - ); - - it.live("persists the closed storage S3 defaults", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(result.definition.capabilities.storage.settings.s3_protocol).toEqual({ - enabled: true, - region: "local", - access_key_id: "625729a08b95bf1b7ff351a663f3a23c", - secret_access_key: { slot: "secret:storage.settings.s3_protocol.secret_access_key" }, - }); - expect(result.secrets).toContainEqual( - expect.objectContaining({ - slot: "secret:storage.settings.s3_protocol.secret_access_key", - policy: "managed", - }), - ); - expect( - result.secrets.some( - (entry) => entry.slot === "secret:storage.settings.s3_protocol.access_key_id", - ), - ).toBe(false); - }), - ); - - it.live("uses one canonical JWT secret slot for symmetric signing", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(result.definition.security.jwt.signing).toEqual({ - kind: "symmetric", - secret: { slot: "secret:auth.settings.jwt_secret" }, - }); - expect(result.secrets.filter((entry) => entry.slot.includes("jwt"))).toEqual([ - expect.objectContaining({ slot: "secret:auth.settings.jwt_secret", policy: "managed" }), - ]); - }), - ); - - it.live("accepts either symmetric JWT secret spelling and deduplicates the slot", () => - Effect.gen(function* () { - const fromAuth = yield* compile({ - capabilities: { auth: { settings: { jwt_secret: Redacted.make("auth-secret") } } }, - }); - expect(fromAuth.definition.security.jwt.signing).toEqual({ - kind: "symmetric", - secret: { slot: "secret:auth.settings.jwt_secret" }, - }); - expect(fromAuth.secrets.filter((entry) => entry.slot.includes("jwt"))).toEqual([ - expect.objectContaining({ - slot: "secret:auth.settings.jwt_secret", - policy: "managed", - value: Redacted.make("auth-secret"), - }), - ]); - - const fromSecurity = yield* compile({ - security: { jwt: { signing: { kind: "symmetric", secret: Redacted.make("top-secret") } } }, - }); - expect(fromSecurity.definition.capabilities.auth.settings.jwt_secret).toEqual({ - slot: "secret:auth.settings.jwt_secret", - }); - expect(fromSecurity.secrets.filter((entry) => entry.slot.includes("jwt"))).toEqual([ - expect.objectContaining({ - slot: "secret:auth.settings.jwt_secret", - policy: "managed", - value: Redacted.make("top-secret"), - }), - ]); - - const equal = yield* compile({ - capabilities: { auth: { settings: { jwt_secret: Redacted.make("equal-secret") } } }, - security: { - jwt: { signing: { kind: "symmetric", secret: Redacted.make("equal-secret") } }, - }, - }); - expect(equal.secrets.filter((entry) => entry.slot.includes("jwt"))).toHaveLength(1); - }), - ); - - it.live("rejects conflicting symmetric JWT secret spellings without leaking values", () => - Effect.gen(function* () { - const exit = yield* compile({ - capabilities: { auth: { settings: { jwt_secret: Redacted.make("auth-secret") } } }, - security: { - jwt: { signing: { kind: "symmetric", secret: Redacted.make("top-secret") } }, - }, - }).pipe(Effect.exit); - const error = failureOf(exit); - expect(error).toBeInstanceOf(InvalidStackConfigError); - expect(String(error)).not.toContain("auth-secret"); - expect(String(error)).not.toContain("top-secret"); - }), - ); - - it.live("retains Auth's local JWT secret when signing uses a JWKS file", () => - Effect.gen(function* () { - const result = yield* compile({ - security: { jwt: { signing: { kind: "jwks-file", path: "jwt.json" } } }, - }); - expect(result.definition.security.jwt.signing).toEqual({ - kind: "jwks-file", - path: "jwt.json", - }); - expect(result.secrets.filter((entry) => entry.slot.includes("jwt"))).toEqual([ - expect.objectContaining({ slot: "secret:auth.settings.jwt_secret", policy: "managed" }), - ]); - expect( - result.secrets.find((entry) => entry.slot === "secret:auth.settings.anon_key")?.generator, - ).toEqual({ - kind: "jwt-token", - role: "anon", - signing: { - kind: "jwks-file", - projectRoot: "/tmp/supabase-project", - path: "jwt.json", - }, - }); - }), - ); - - it("derives each supported third-party issuer", () => { - const cases = [ - [ - { firebase: { enabled: true, project_id: "project-42" } }, - "https://securetoken.google.com/project-42", - ], - [ - { auth0: { enabled: true, tenant: "tenant", tenant_region: "eu" } }, - "https://tenant.eu.auth0.com", - ], - [ - { - aws_cognito: { enabled: true, user_pool_id: "eu_pool", user_pool_region: "eu-west-1" }, - }, - "https://cognito-idp.eu-west-1.amazonaws.com/eu_pool", - ], - [ - { clerk: { enabled: true, domain: "example.clerk.accounts.dev" } }, - "https://example.clerk.accounts.dev", - ], - [ - { workos: { enabled: true, issuer_url: "https://login.example.test" } }, - "https://login.example.test", - ], - ] as const; - for (const [settings, issuer] of cases) { - const result = resolveThirdPartyIssuer(settings); - expect(result).toEqual({ ok: true, value: expect.objectContaining({ issuer }) }); - } - }); - - it.live("rejects conflicting or incomplete third-party issuer settings", () => - Effect.gen(function* () { - const invalid = yield* compile({ - capabilities: { - auth: { - settings: { - third_party: { - firebase: { enabled: true, project_id: "project-42" }, - auth0: { enabled: true, tenant: "tenant" }, - }, - }, - }, - }, - }).pipe(Effect.exit); - expect(failureOf(invalid)).toBeInstanceOf(InvalidStackConfigError); - expect(resolveThirdPartyIssuer({ firebase: { enabled: true } })).toMatchObject({ - ok: false, - provider: "firebase", - }); - expect( - resolveThirdPartyIssuer({ clerk: { enabled: true, domain: "not-a-clerk-domain" } }), - ).toMatchObject({ ok: false, provider: "clerk" }); - }), - ); - - it.live("materializes a non-default realtime setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { realtime: { settings: { ip_version: "IPv6" } } }, - }); - expect(result.definition.capabilities.realtime.settings).toMatchObject({ - ip_version: "IPv6", - }); - }), - ); - - it.live("materializes a non-default storage bucket", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { storage: { settings: { buckets: { avatars: { public: true } } } } }, - }); - expect(result.definition.capabilities.storage.settings).toMatchObject({ - buckets: { avatars: { public: true } }, - }); - }), - ); - - it.live("normalizes the functions root and preserves a function override", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - functions: { - settings: { - functions_root: "./supabase/functions", - functions: { hello: { verify_jwt: false } }, - }, - }, - }, - }); - expect(result.definition.capabilities.functions.settings).toMatchObject({ - functions_root: "/tmp/supabase-project/supabase/functions", - functions: { hello: { verify_jwt: false } }, - }); - }), - ); - - it.live("materializes a non-default studio setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { studio: { settings: { api_url: "http://localhost:9999" } } }, - }); - expect(result.definition.capabilities.studio.settings).toMatchObject({ - api_url: "http://localhost:9999", - }); - }), - ); - - it.live("materializes a non-default mail setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { mail: { settings: { sender_name: "Tests" } } }, - }); - expect(result.definition.capabilities.mail.settings).toMatchObject({ sender_name: "Tests" }); - }), - ); - - it.live("materializes a non-default analytics setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { analytics: { settings: { backend: "bigquery" } } }, - }); - expect(result.definition.capabilities.analytics.settings).toMatchObject({ - backend: "bigquery", - }); - }), - ); - - it.live("creates a managed Analytics access key when omitted", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(result.definition.capabilities.analytics.settings).toMatchObject({ - api_key: { slot: "secret:analytics.settings.api_key" }, - }); - expect(result.secrets).toContainEqual({ - slot: "secret:analytics.settings.api_key", - policy: "managed", - generator: { kind: "random-base64url", bytes: 32 }, - }); - }), - ); - - it.live("manages a supplied Analytics access key", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { analytics: { settings: { api_key: Redacted.make("custom-api-key") } } }, - }); - expect(result.secrets).toContainEqual({ - slot: "secret:analytics.settings.api_key", - policy: "managed", - value: Redacted.make("custom-api-key"), - generator: { kind: "random-base64url", bytes: 32 }, - }); - expect(result.definition.capabilities.analytics.settings.api_key).toEqual({ - slot: "secret:analytics.settings.api_key", - }); - }), - ); - - it.live("materializes a non-default pooler setting", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { pooler: { enabled: true, settings: { pool_mode: "session" } } }, - }); - expect(result.definition.capabilities.pooler.settings).toMatchObject({ - pool_mode: "session", - }); - expect(result.definition.capabilities.pooler.enabled).toBe(true); - }), - ); - - it.live("attaches artifact-compatible generators for managed pooler keys", () => - Effect.gen(function* () { - const result = yield* compile({ capabilities: { pooler: { enabled: true } } }); - expect(result.secrets).toContainEqual({ - slot: "secret:pooler.settings.encryption_key", - policy: "managed", - generator: { kind: "random-base64url", bytes: 24 }, - }); - expect(result.secrets).toContainEqual({ - slot: "secret:pooler.settings.secret_key_base", - policy: "managed", - generator: { kind: "random-base64url", bytes: 48 }, - }); - }), - ); - - it.live("rejects pooler key values that violate the artifact contract", () => - Effect.gen(function* () { - const invalidEncryption = yield* compile({ - capabilities: { - pooler: { - enabled: true, - settings: { encryption_key: Redacted.make("too-short") }, - }, - }, - }).pipe(Effect.exit); - expect(failureOf(invalidEncryption)).toBeInstanceOf(InvalidStackConfigError); - expect(failureOf(invalidEncryption)?.setting).toBe( - "capabilities.pooler.settings.encryption_key", - ); - - const invalidSecretBase = yield* compile({ - capabilities: { - pooler: { - enabled: true, - settings: { secret_key_base: Redacted.make("!".repeat(63)) }, - }, - }, - }).pipe(Effect.exit); - expect(failureOf(invalidSecretBase)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("persists every capability and optional absence as null", () => - Effect.gen(function* () { - const result = yield* compile({}); - expect(Object.keys(result.definition.capabilities)).toHaveLength(10); - expect(result.definition.capabilities.rest.settings).toMatchObject({ external_url: null }); - expect(result.definition.capabilities.functions.settings).toMatchObject({ - edge_runtime: { secrets: {} }, - functions: {}, - }); - }), - ); - - it.live("persists and reuses the managed database password slot", () => - Effect.gen(function* () { - const first = yield* compile({}); - const initial = first.secrets.filter( - (entry) => entry.slot === "secret:database.internal.password", - ); - expect(initial).toHaveLength(1); - const second = yield* compile( - {}, - { kind: "native" }, - { - definition: first.definition, - }, - ); - expect( - second.secrets.filter((entry) => entry.slot === "secret:database.internal.password"), - ).toHaveLength(1); - expect(second.secrets[0]?.slot).toBe(initial[0]?.slot); - }), - ); - - it.live("rejects unknown fields at the public compiler boundary", () => - Effect.gen(function* () { - const exit = yield* compile({ - capabilities: { rest: { settings: { typo: true } } }, - } as never).pipe(Effect.exit); - expect(failureOf(exit)).toBeInstanceOf(InvalidStackConfigError); - const edgeEnabled = yield* compile({ - capabilities: { functions: { settings: { edge_runtime: { enabled: false } } } }, - } as never).pipe(Effect.exit); - expect(failureOf(edgeEnabled)).toBeInstanceOf(InvalidStackConfigError); - const inspectorPort = yield* compile({ - capabilities: { functions: { settings: { edge_runtime: { inspector_port: 8083 } } } }, - } as never).pipe(Effect.exit); - expect(failureOf(inspectorPort)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("keeps record key insertion order out of materialized definitions", () => - Effect.gen(function* () { - const one = yield* compile({ - capabilities: { - storage: { settings: { buckets: { a: { public: true }, b: { public: false } } } }, - }, - }); - const two = yield* compile({ - capabilities: { - storage: { settings: { buckets: { b: { public: false }, a: { public: true } } } }, - }, - }); - expect(sameDefinition(one.definition, two.definition)).toBe(true); - }), - ); - - it.live("materializes omitted and explicit defaults identically", () => - Effect.gen(function* () { - const omitted = yield* compile({}); - const explicit = yield* compile({ capabilities: { rest: { enabled: true } } }); - expect(sameDefinition(omitted.definition, explicit.definition)).toBe(true); - }), - ); - - it.live("accepts an equivalent materialized definition from a previous compilation", () => - Effect.gen(function* () { - const first = yield* compile({ - capabilities: { rest: { settings: { schemas: ["private"] } } }, - }); - const second = yield* compile({ - capabilities: { rest: { settings: { schemas: ["private"] } } }, - }); - const reused = yield* compile( - { capabilities: { rest: { settings: { schemas: ["private"] } } } }, - { kind: "native" }, - first, - ); - expect(sameDefinition(first.definition, second.definition)).toBe(true); - expect(sameDefinition(reused.definition, first.definition)).toBe(true); - }), - ); - - it.live("rejects an unsupported database version before output", () => - Effect.gen(function* () { - const result = yield* compile({ capabilities: { database: { version: "99" } } }).pipe( - Effect.exit, - ); - expect(failureOf(result)).toBeInstanceOf(StackVersionUnsupportedError); - }), - ); - - it.live("keeps both native and container artifacts while selecting one runtime", () => - Effect.gen(function* () { - const native = yield* compile({}, { kind: "native" }); - const container = yield* compile({}, { kind: "container", engine: "docker" }); - expect(native.executionPlan.workloads[0]?.artifacts.native.kind).toBe("native"); - expect(native.executionPlan.workloads[0]?.artifacts.container.kind).toBe("container"); - expect(container.executionPlan.workloads[0]?.selected.kind).toBe("container"); - }), - ); - - it.live("compiles database bootstrap as a marker on the database workload", () => - Effect.gen(function* () { - const result = yield* compile({}); - const database = result.executionPlan.workloads.filter((w) => w.capability === "database"); - expect(database).toHaveLength(1); - expect(database[0]?.id).toBe("database:database"); - expect(database[0]?.bootstrap).toBe("database"); - }), - ); - - it.live("rejects a functions root that escapes the project", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { functions: { settings: { functions_root: "../outside" } } }, - }).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("reports dependency closure errors", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { rest: { enabled: false }, studio: { enabled: true } }, - }).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("resolves the supported database major to its supported release", () => - Effect.gen(function* () { - for (const [release, image] of Object.entries( - catalogEntryFor("database:database").releases, - )) { - const major = release.split(".")[0]; - const result = yield* compile({ capabilities: { database: { version: major } } }); - expect(result.definition.capabilities.database.version).toBe(release); - expect( - result.executionPlan.workloads.find((w) => w.id === "database:database")?.artifacts, - ).toEqual({ - native: { kind: "native", release }, - container: { - kind: "container", - image, - }, - }); - } - }), - ); - - it.live("prefers a compatible previous database release for a major selector", () => - Effect.gen(function* () { - const previous = yield* compile({ - capabilities: { database: { version: "15.14.1.168" } }, - }); - const selected = yield* compile( - { capabilities: { database: { version: "15" } } }, - { kind: "native" }, - previous, - ); - expect(selected.definition.capabilities.database.version).toBe("15.14.1.168"); - }), - ); - - it.live("uses the default release when a supported major matches it", () => - Effect.gen(function* () { - const result = yield* compile({ capabilities: { database: { version: "17" } } }); - expect(result.definition.capabilities.database.version).toBe("17.6.1.168"); - }), - ); - - it.live("rejects an unknown non-database release", () => - Effect.gen(function* () { - const result = yield* compile({ capabilities: { rest: { version: "not-real" } } }).pipe( - Effect.exit, - ); - expect(failureOf(result)).toBeInstanceOf(StackVersionUnsupportedError); - }), - ); - - it.live("rejects unsupported historical database releases", () => - Effect.gen(function* () { - for (const version of ["13", "13.3.0", "14"]) { - const result = yield* compile({ capabilities: { database: { version } } }).pipe( - Effect.exit, - ); - expect(failureOf(result)).toBeInstanceOf(StackVersionUnsupportedError); - } - }), - ); - - it.live("rejects an empty non-database release", () => - Effect.gen(function* () { - const result = yield* compile({ capabilities: { rest: { version: "" } } }).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(StackVersionUnsupportedError); - }), - ); - - it.live("compiles supplied configuration against persisted release pins", () => - Effect.gen(function* () { - const first = yield* compile({ - capabilities: { rest: { settings: { schemas: ["current"] } } }, - }); - const persisted = { - ...first.definition, - capabilities: { - ...first.definition.capabilities, - rest: { - ...first.definition.capabilities.rest, - settings: { ...first.definition.capabilities.rest.settings, schemas: ["persisted"] }, - }, - }, - }; - const compiled = yield* compile( - { capabilities: { rest: { settings: { schemas: ["current"] } } } }, - { kind: "native" }, - { definition: persisted }, - ); - expect(compiled.definition.capabilities.rest.settings.schemas).toEqual(["current"]); - expect(compiled.definition.capabilities.rest.version).toBe( - persisted.capabilities.rest.version, - ); - }), - ); - - it.live("rejects a persisted definition whose release is no longer supported", () => - Effect.gen(function* () { - const first = yield* compile({}); - const persisted = { - ...first.definition, - capabilities: { - ...first.definition.capabilities, - rest: { ...first.definition.capabilities.rest, version: "not-real" }, - }, - }; - const result = yield* compile( - {}, - { kind: "native" }, - { - definition: persisted, - }, - ).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(StackVersionUnsupportedError); - }), - ); - - it.live("validates persisted capability closure before rebuilding its plan", () => - Effect.gen(function* () { - const first = yield* compile({}); - const persisted = { - ...first.definition, - capabilities: { - ...first.definition.capabilities, - rest: { ...first.definition.capabilities.rest, enabled: false }, - }, - }; - const result = yield* rebuildExecutionPlan({ kind: "native" }, persisted).pipe( - Effect.provide(layer), - Effect.exit, - ); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("materializes defaults for supplied records", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - storage: { settings: { buckets: { avatars: { public: true } } } }, - functions: { settings: { functions: { hello: { verify_jwt: false } } } }, - }, - }); - expect(result.definition.capabilities.storage.settings.buckets).toEqual({ - avatars: { - public: true, - file_size_limit: "50MiB", - allowed_mime_types: [], - objects_path: "", - }, - }); - expect(result.definition.capabilities.functions.settings.functions).toEqual({ - hello: { - enabled: true, - verify_jwt: false, - import_map: "", - entrypoint: "", - static_files: [], - env: {}, - }, - }); - }), - ); - - it.live("uses pool_mode and rejects the old mode spelling", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { pooler: { settings: { pool_mode: "session" } } }, - }); - expect(result.definition.capabilities.pooler.settings.pool_mode).toBe("session"); - const invalid = yield* compile({ - capabilities: { pooler: { settings: { mode: "session" } } }, - } as never).pipe(Effect.exit); - expect(failureOf(invalid)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("rejects invalid function slugs and environment names", () => - Effect.gen(function* () { - const invalidSlug = yield* compile({ - capabilities: { functions: { settings: { functions: { "bad.slug": {} } } } }, - } as never).pipe(Effect.exit); - expect(failureOf(invalidSlug)).toBeInstanceOf(InvalidStackConfigError); - const invalidEnv = yield* compile({ - capabilities: { - functions: { - settings: { functions: { hello: { env: { "bad-name": Redacted.make("x") } } } }, - }, - }, - } as never).pipe(Effect.exit); - expect(failureOf(invalidEnv)).toBeInstanceOf(InvalidStackConfigError); - }), - ); - - it.live("enforces network port bounds", () => - Effect.gen(function* () { - for (const port of [0, -1, 1.5, 65536]) { - const invalid = yield* compile({ listeners: { api: { port } } } as never).pipe(Effect.exit); - expect(failureOf(invalid)).toBeInstanceOf(InvalidStackConfigError); - } - for (const port of [1, 65535]) { - const valid = yield* compile({ listeners: { api: { port } } }); - expect(valid.definition.listeners.api.port).toBe(port); - } - }), - ); -}); diff --git a/packages/stack/src/model/database-bootstrap.integration.test.ts b/packages/stack/src/model/database-bootstrap.integration.test.ts deleted file mode 100644 index 457cc058fc..0000000000 --- a/packages/stack/src/model/database-bootstrap.integration.test.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Redacted, Semaphore } from "effect"; -import { - DatabaseBootstrapError, - type DatabaseSession, - runDatabaseBootstrap, -} from "./DatabaseBootstrap.ts"; - -const ROLE_NAMES = [ - "postgres", - "authenticator", - "pgbouncer", - "supabase_auth_admin", - "supabase_storage_admin", - "supabase_replication_admin", - "supabase_read_only_user", -] as const; - -type SettingValue = string | number; -type BootstrapState = { - readonly schemas: Set; - readonly schemaOwners: Map; - readonly passwords: Map; - readonly settings: Map; -}; - -const makeSession = (options: { readonly failOnce?: "role" | "setting" } = {}) => - Effect.gen(function* () { - const lock = yield* Semaphore.make(1); - const state: BootstrapState = { - schemas: new Set(), - schemaOwners: new Map(), - passwords: new Map(), - settings: new Map(), - }; - const operations: string[] = []; - let transactions = 0; - let failed = false; - - const session: DatabaseSession = { - execute: () => Effect.void, - transaction: (use) => - lock.withPermit( - Effect.gen(function* () { - transactions += 1; - const pending: BootstrapState = { - schemas: new Set(state.schemas), - schemaOwners: new Map(state.schemaOwners), - passwords: new Map(state.passwords), - settings: new Map(state.settings), - }; - const shouldFail = (kind: typeof options.failOnce) => { - if (options.failOnce !== kind || failed) return false; - failed = true; - return true; - }; - const transaction = { - execute: (statement: string) => - Effect.gen(function* () { - yield* Effect.sync(() => operations.push(statement)); - if (statement.includes("CREATE SCHEMA IF NOT EXISTS _realtime;")) - yield* Effect.sync(() => pending.schemas.add("_realtime")); - if (statement.includes("ALTER SCHEMA _realtime OWNER TO postgres;")) - yield* Effect.sync(() => pending.schemaOwners.set("_realtime", "postgres")); - }), - setRolePasswords: ( - roles: ReadonlyArray, - password: Redacted.Redacted, - ) => - Effect.gen(function* () { - yield* Effect.sync(() => - roles.forEach((role) => operations.push(`ALTER ROLE ${role} PASSWORD`)), - ); - if (shouldFail("role")) - return yield* new DatabaseBootstrapError({ - message: "role failed secret-password", - }); - yield* Effect.sync(() => - roles.forEach((role) => pending.passwords.set(role, Redacted.value(password))), - ); - }), - setDatabaseSettings: ( - settings: ReadonlyArray<{ - readonly name: string; - readonly value: Redacted.Redacted | number; - }>, - ) => - Effect.gen(function* () { - yield* Effect.sync(() => - settings.forEach((setting) => - operations.push(`ALTER DATABASE postgres SET ${setting.name}`), - ), - ); - if (shouldFail("setting")) - return yield* new DatabaseBootstrapError({ - message: "setting failed secret-jwt", - }); - yield* Effect.sync(() => - settings.forEach((setting) => - pending.settings.set( - setting.name, - typeof setting.value === "number" - ? setting.value - : Redacted.value(setting.value), - ), - ), - ); - }), - }; - yield* use(transaction); - state.schemas.clear(); - pending.schemas.forEach((schema) => state.schemas.add(schema)); - state.schemaOwners.clear(); - pending.schemaOwners.forEach((owner, schema) => state.schemaOwners.set(schema, owner)); - state.passwords.clear(); - pending.passwords.forEach((password, role) => state.passwords.set(role, password)); - state.settings.clear(); - pending.settings.forEach((value, name) => state.settings.set(name, value)); - }), - ), - }; - return { - session, - state, - operations, - get transactions() { - return transactions; - }, - }; - }); - -const options = (password: string, jwtSecret: string, jwtExpiry = 3600) => ({ - databasePassword: Redacted.make(password), - jwtSecret: Redacted.make(jwtSecret), - jwtExpiry, -}); - -describe("database bootstrap", () => { - it.live("reconciles the schema, login roles, and settings on every invocation", () => - Effect.gen(function* () { - const runtime = yield* makeSession(); - yield* runDatabaseBootstrap(runtime.session, options("password-a", "jwt-a")); - runtime.state.schemaOwners.set("_realtime", "other"); - yield* runDatabaseBootstrap(runtime.session, options("password-b", "jwt-b", 7200)); - - expect(runtime.state.schemas).toEqual(new Set(["_realtime"])); - expect(runtime.state.schemaOwners).toEqual(new Map([["_realtime", "postgres"]])); - expect([...runtime.state.passwords.entries()]).toEqual( - ROLE_NAMES.map((role) => [role, "password-b"]), - ); - expect(runtime.state.settings).toEqual( - new Map([ - ["app.settings.jwt_secret", "jwt-b"], - ["app.settings.jwt_exp", 7200], - ]), - ); - expect(runtime.transactions).toBe(2); - expect(runtime.operations[0]).toContain("pg_advisory_xact_lock"); - const operationText = runtime.operations.join(" "); - expect(operationText).not.toContain("password-a"); - expect(operationText).not.toContain("password-b"); - expect(operationText).not.toContain("jwt-a"); - expect(operationText).not.toContain("jwt-b"); - }), - ); - - it.live("rolls back every bootstrap change after a transaction failure", () => - Effect.gen(function* () { - const runtime = yield* makeSession({ failOnce: "role" }); - const first = yield* runDatabaseBootstrap( - runtime.session, - options("secret-password", "secret-jwt"), - ).pipe(Effect.exit); - expect(Exit.isFailure(first)).toBe(true); - if (Exit.isFailure(first)) expect(Cause.pretty(first.cause)).not.toContain("secret-password"); - expect(runtime.state.schemas).toEqual(new Set()); - expect(runtime.state.schemaOwners).toEqual(new Map()); - expect(runtime.state.passwords).toEqual(new Map()); - expect(runtime.state.settings).toEqual(new Map()); - }), - ); - - it.live("can retry bootstrap successfully after a transaction failure", () => - Effect.gen(function* () { - const runtime = yield* makeSession({ failOnce: "role" }); - const first = yield* runDatabaseBootstrap( - runtime.session, - options("secret-password", "secret-jwt"), - ).pipe(Effect.exit); - expect(Exit.isFailure(first)).toBe(true); - yield* runDatabaseBootstrap(runtime.session, options("secret-password", "secret-jwt")); - expect(runtime.state.schemas).toEqual(new Set(["_realtime"])); - expect(runtime.state.schemaOwners).toEqual(new Map([["_realtime", "postgres"]])); - expect(runtime.state.passwords.size).toBe(ROLE_NAMES.length); - expect(runtime.state.settings.get("app.settings.jwt_secret")).toBe("secret-jwt"); - }), - ); - - it.live("maps setting failures without exposing JWT material", () => - Effect.gen(function* () { - const runtime = yield* makeSession({ failOnce: "setting" }); - const result = yield* runDatabaseBootstrap( - runtime.session, - options("secret-password", "secret-jwt"), - ).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - expect(Cause.pretty(result.cause)).not.toContain("secret-password"); - expect(Cause.pretty(result.cause)).not.toContain("secret-jwt"); - } - expect(runtime.state.schemas).toEqual(new Set()); - expect(runtime.state.schemaOwners).toEqual(new Map()); - expect(runtime.state.passwords).toEqual(new Map()); - expect(runtime.state.settings).toEqual(new Map()); - }), - ); -}); diff --git a/packages/stack/src/model/idle-config.integration.test.ts b/packages/stack/src/model/idle-config.integration.test.ts deleted file mode 100644 index 2ec057019a..0000000000 --- a/packages/stack/src/model/idle-config.integration.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Option } from "effect"; -import { InvalidStackConfigError } from "../public/Errors.ts"; -import { compileStack } from "./Compiler.ts"; - -const compile = (config: Parameters[0]["config"]) => - compileStack({ projectRoot: "/tmp/supabase-project", runtime: { kind: "native" }, config }).pipe( - Effect.provide(NodeServices.layer), - ); - -const failureOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("traffic idle timeout configuration", () => { - it.live("defaults supported lazy capabilities to a 60 second timeout", () => - Effect.gen(function* () { - const result = yield* compile({}); - for (const name of ["rest", "auth", "realtime", "studio", "pooler"] as const) { - expect(result.definition.capabilities[name]).toMatchObject({ - activation: "lazy", - idleTimeoutSeconds: 60, - }); - } - }), - ); - - it.live("allows disabling idle stopping explicitly", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - rest: { idleTimeoutSeconds: false }, - auth: { idleTimeoutSeconds: false }, - realtime: { idleTimeoutSeconds: false }, - studio: { idleTimeoutSeconds: false }, - pooler: { idleTimeoutSeconds: false }, - }, - }); - for (const name of ["rest", "auth", "realtime", "studio", "pooler"] as const) - expect(result.definition.capabilities[name].idleTimeoutSeconds).toBe(false); - }), - ); - - it.live("persists a positive finite timeout override", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { rest: { idleTimeoutSeconds: 15.5 } }, - }); - expect(result.definition.capabilities.rest.idleTimeoutSeconds).toBe(15.5); - }), - ); - - it.live("does not idle-stop eager capabilities", () => - Effect.gen(function* () { - const result = yield* compile({ - capabilities: { - rest: { activation: "eager", idleTimeoutSeconds: 15 }, - studio: { activation: "eager" }, - }, - }); - expect(result.definition.capabilities.rest).toMatchObject({ - activation: "eager", - idleTimeoutSeconds: false, - }); - expect(result.definition.capabilities.studio).toMatchObject({ - activation: "eager", - idleTimeoutSeconds: false, - }); - }), - ); - - it.live("keeps unsupported services opted out of idle stopping", () => - Effect.gen(function* () { - const defaults = yield* compile({}); - for (const name of ["database", "storage", "functions", "mail", "analytics"] as const) - expect(defaults.definition.capabilities[name].idleTimeoutSeconds).toBe(false); - - for (const name of ["database", "storage", "functions", "mail", "analytics"] as const) { - const result = yield* compile({ - capabilities: { [name]: { idleTimeoutSeconds: 30 } }, - } as never).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - } - }), - ); - - it.live("rejects zero, negative, and non-finite timeouts", () => - Effect.gen(function* () { - for (const value of [0, -1, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY, Number.NaN]) { - const result = yield* compile({ - capabilities: { rest: { idleTimeoutSeconds: value } }, - } as never).pipe(Effect.exit); - expect(failureOf(result)).toBeInstanceOf(InvalidStackConfigError); - } - }), - ); -}); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 7f2e1977f9..15571092e8 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -1,6 +1,7 @@ import { Crypto, Effect, FileSystem, Option, Path, PlatformError, Predicate, Schema } from "effect"; +import { HttpClient } from "effect/unstable/http"; import { ChildProcessSpawner } from "effect/unstable/process"; -import { ArtifactIntegrityError, StackPreparationError } from "../public/Errors.ts"; +import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; import { validateRelativePath, validateSha256 } from "./Integrity.ts"; /** A concrete artifact identity. `key` may contain subdirectories but never an absolute or traversing path. */ @@ -15,12 +16,14 @@ export interface ArtifactRequest { /** * The source is the only download/archive boundary: it writes an unpacked artifact tree - * below `destination` after verifying the downloaded archive digest. The store stays - * independent of transport and archive formats. + * below `destination` after verifying the downloaded archive digest. HTTP transport remains + * supplied by the caller through the Effect HttpClient service. */ export interface ArtifactSource { /** Resolves the published digest only when the store has no valid cached artifact. */ - readonly checksum: (request: ArtifactRequest) => Effect.Effect; + readonly checksum: ( + request: ArtifactRequest, + ) => Effect.Effect; readonly materialize: ( request: ArtifactRequest, destination: string, @@ -28,8 +31,12 @@ export interface ArtifactSource { onProgress?: (state: "downloading" | "preparing") => void, ) => Effect.Effect< void, - StackPreparationError | ArtifactIntegrityError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawner.ChildProcessSpawner + PreparationError | ArtifactIntegrityError, + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | HttpClient.HttpClient >; } @@ -38,7 +45,7 @@ export interface ArtifactStoreOptions { readonly source: ArtifactSource; } -export interface PreparedArtifact { +interface PreparedArtifact { readonly key: string; /** Installed artifact directory. Required runtime paths are relative to this directory. */ readonly path: string; @@ -48,14 +55,7 @@ export interface PreparedArtifact { readonly outcome: "cached" | "downloaded"; } -export type ArtifactStoreError = StackPreparationError | ArtifactIntegrityError; - -export interface ArtifactStore { - readonly prepare: ( - request: ArtifactRequest, - onProgress?: (state: "downloading" | "preparing") => void, - ) => Effect.Effect; -} +type ArtifactStoreError = PreparationError | ArtifactIntegrityError; const ARTIFACT_FORMAT = "supabase-stack-artifact-v3"; const METADATA_NAME = ".artifact.json"; @@ -70,7 +70,7 @@ type InspectedArtifactPath = { }; const artifactError = (message: string, fields: Readonly> = {}) => - new StackPreparationError({ ...fields, message }); + new PreparationError({ ...fields, message }); const metadataError = (message: string, fields: Readonly> = {}) => new ArtifactIntegrityError({ ...fields, message }); @@ -92,7 +92,7 @@ const mapFs = ( path: string, operation: string, effect: Effect.Effect, -): Effect.Effect => +): Effect.Effect => effect.pipe( Effect.mapError((cause) => artifactError( @@ -119,7 +119,7 @@ const isNotFound = (cause: unknown): cause is PlatformError.PlatformError => const isMissingArtifactRoot = (error: ArtifactIntegrityError): boolean => Predicate.hasProperty(error, "cause") && isNotFound(error.cause); -const validateKey = (key: string): Effect.Effect => +const validateKey = (key: string): Effect.Effect => validateRelativePath(key, "artifact key").pipe( Effect.flatMap(() => /^[A-Za-z0-9][A-Za-z0-9._-]*(?:[\\/][A-Za-z0-9][A-Za-z0-9._-]*)*$/u.test(key) @@ -128,7 +128,7 @@ const validateKey = (key: string): Effect.Effect => ), ); -const validateRequest = (request: ArtifactRequest): Effect.Effect => +const validateRequest = (request: ArtifactRequest): Effect.Effect => Effect.gen(function* () { yield* validateKey(request.key); const seen = new Set(); @@ -160,7 +160,7 @@ const metadataFor = ( ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), }); -const encodeMetadata = (metadata: ArtifactMetadata): Effect.Effect => +const encodeMetadata = (metadata: ArtifactMetadata): Effect.Effect => Schema.encodeEffect(Schema.fromJsonString(ArtifactMetadataSchema))(metadata).pipe( Effect.mapError((cause) => artifactError(`Unable to encode artifact metadata: ${String(cause)}`), @@ -193,7 +193,7 @@ const ensureDirectory = ( path: Path.Path, directory: string, canonicalRoot: string, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const root = path.resolve(canonicalRoot); const resolved = path.resolve(directory); @@ -538,7 +538,7 @@ const writeBytesSync = ( fs: FileSystem.FileSystem, path: string, bytes: Uint8Array, -): Effect.Effect => +): Effect.Effect => Effect.scoped( Effect.gen(function* () { const file = yield* fs.open(path, { flag: "wx", mode: 0o600 }).pipe( @@ -569,16 +569,13 @@ const writeMetadataSync = ( fs: FileSystem.FileSystem, path: string, metadata: ArtifactMetadata, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const encoded = yield* encodeMetadata(metadata); yield* writeBytesSync(fs, path, new TextEncoder().encode(encoded)); }); -const cleanup = ( - fs: FileSystem.FileSystem, - path: string, -): Effect.Effect => +const cleanup = (fs: FileSystem.FileSystem, path: string): Effect.Effect => fs .remove(path, { recursive: true, force: true }) .pipe( @@ -587,7 +584,7 @@ const cleanup = ( ), ); -const makeArtifactOperation = ( +const makeArtifactOperation = Effect.fn("ArtifactStore.operation")(function* ( fs: FileSystem.FileSystem, path: Path.Path, crypto: Crypto.Crypto, @@ -596,8 +593,8 @@ const makeArtifactOperation = ( source: ArtifactSource, request: ArtifactRequest, onProgress?: (state: "downloading" | "preparing") => void, -): Effect.Effect => - Effect.gen(function* () { +) { + return yield* Effect.gen(function* () { const target = path.resolve(cacheRoot, request.key); const targetParent = path.dirname(target); yield* ensureDirectory(fs, path, targetParent, cacheRoot); @@ -783,65 +780,60 @@ const makeArtifactOperation = ( outcome: "downloaded" as const, }; }); +}); -export const makeArtifactStore = ( +export const makeArtifactStore = Effect.fn("ArtifactStore.makeStore")(function* ( options: ArtifactStoreOptions, -): Effect.Effect< - ArtifactStore, - StackPreparationError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; - if (options.cacheRoot.trim().length === 0) - return yield* artifactError("Artifact cache root must not be blank"); - const requestedRoot = path.resolve(options.cacheRoot); - yield* mapFs( - requestedRoot, - "create artifact cache root", - fs.makeDirectory(requestedRoot, { recursive: true, mode: 0o700 }), - ); - const cacheRoot = yield* fs.realPath(requestedRoot).pipe( - Effect.mapError((cause) => - artifactError(`Unable to resolve artifact cache root: ${cause.message}`, { - path: requestedRoot, - cause, - }), - ), - ); - const rootInfo = yield* fs.stat(cacheRoot).pipe( - Effect.mapError((cause) => - artifactError(`Unable to inspect artifact cache root: ${cause.message}`, { - path: cacheRoot, - cause, - }), - ), +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + if (options.cacheRoot.trim().length === 0) + return yield* artifactError("Artifact cache root must not be blank"); + const requestedRoot = path.resolve(options.cacheRoot); + yield* mapFs( + requestedRoot, + "create artifact cache root", + fs.makeDirectory(requestedRoot, { recursive: true, mode: 0o700 }), + ); + const cacheRoot = yield* fs.realPath(requestedRoot).pipe( + Effect.mapError((cause) => + artifactError(`Unable to resolve artifact cache root: ${cause.message}`, { + path: requestedRoot, + cause, + }), + ), + ); + const rootInfo = yield* fs.stat(cacheRoot).pipe( + Effect.mapError((cause) => + artifactError(`Unable to inspect artifact cache root: ${cause.message}`, { + path: cacheRoot, + cause, + }), + ), + ); + if (rootInfo.type !== "Directory") + return yield* artifactError("Artifact cache root must be a directory", { path: cacheRoot }); + yield* mapFs(cacheRoot, "secure artifact cache root", fs.chmod(cacheRoot, 0o700)); + const prepare = Effect.fn("ArtifactStore.prepare")(function* ( + request: ArtifactRequest, + onProgress?: (state: "downloading" | "preparing") => void, + ) { + yield* validateRequest(request); + const target = path.resolve(cacheRoot, request.key); + if (!pathWithin(cacheRoot, target, path.sep)) + return yield* artifactError("Artifact key escapes cache root", { key: request.key }); + return yield* makeArtifactOperation( + fs, + path, + crypto, + childProcessSpawner, + cacheRoot, + options.source, + request, + onProgress, ); - if (rootInfo.type !== "Directory") - return yield* artifactError("Artifact cache root must be a directory", { path: cacheRoot }); - yield* mapFs(cacheRoot, "secure artifact cache root", fs.chmod(cacheRoot, 0o700)); - const prepare = ( - request: ArtifactRequest, - onProgress?: (state: "downloading" | "preparing") => void, - ) => - Effect.gen(function* () { - yield* validateRequest(request); - const target = path.resolve(cacheRoot, request.key); - if (!pathWithin(cacheRoot, target, path.sep)) - return yield* artifactError("Artifact key escapes cache root", { key: request.key }); - return yield* makeArtifactOperation( - fs, - path, - crypto, - childProcessSpawner, - cacheRoot, - options.source, - request, - onProgress, - ); - }); - return { prepare }; }); + return { prepare }; +}); diff --git a/packages/stack/src/preparation/Errors.ts b/packages/stack/src/preparation/Errors.ts new file mode 100644 index 0000000000..ce5272c419 --- /dev/null +++ b/packages/stack/src/preparation/Errors.ts @@ -0,0 +1,25 @@ +import { Data } from "effect"; + +interface PreparationErrorFields { + readonly message: string; + readonly cause?: unknown; + readonly field?: string; + readonly value?: unknown; + readonly service?: string; + readonly version?: string; + readonly platform?: string; + readonly target?: string; + readonly path?: string; + readonly key?: string; +} + +export class PreparationError extends Data.TaggedError( + "PreparationError", +) {} + +export class ArtifactIntegrityError extends Data.TaggedError("ArtifactIntegrityError")< + PreparationErrorFields & { + readonly expected?: string; + readonly actual?: string; + } +> {} diff --git a/packages/stack/src/preparation/Integrity.ts b/packages/stack/src/preparation/Integrity.ts index ec000030bf..381aef46ca 100644 --- a/packages/stack/src/preparation/Integrity.ts +++ b/packages/stack/src/preparation/Integrity.ts @@ -1,5 +1,5 @@ import { Crypto, Effect } from "effect"; -import { ArtifactIntegrityError, StackPreparationError } from "../public/Errors.ts"; +import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; const SHA256 = /^[0-9a-f]{64}$/i; @@ -7,25 +7,23 @@ const integrityError = (message: string, fields: Readonly => - SHA256.test(value) - ? Effect.succeed(value.toLowerCase()) - : Effect.fail( - integrityError("Artifact SHA-256 must be exactly 64 hexadecimal characters", { - expected: value, - }), - ); +export const validateSha256 = Effect.fn("Integrity.validateSha256")(function* (value: string) { + if (SHA256.test(value)) return value.toLowerCase(); + return yield* integrityError("Artifact SHA-256 must be exactly 64 hexadecimal characters", { + expected: value, + }); +}); /** Converts a digest returned by the Effect Crypto service to its canonical hexadecimal form. */ export const digestHex = (digest: Uint8Array): string => Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); /** Computes and verifies a SHA-256 digest without collapsing Crypto failures. */ -export const verifySha256 = ( +export const verifySha256 = Effect.fn("Integrity.verifySha256")(function* ( bytes: Uint8Array, expected: string, -): Effect.Effect => - Effect.gen(function* () { +) { + yield* Effect.gen(function* () { const canonical = yield* validateSha256(expected); const crypto = yield* Crypto.Crypto; const digest = yield* crypto @@ -42,12 +40,13 @@ export const verifySha256 = ( actual, }); }); +}); /** Validates one relative artifact path before it is joined to a cache root. */ -export const validateRelativePath = ( +export const validateRelativePath = Effect.fn("Integrity.validateRelativePath")(function* ( value: string, field: string, -): Effect.Effect => { +) { const segments = value.split(/[\\/]/u); if ( value.length === 0 || @@ -59,12 +58,10 @@ export const validateRelativePath = ( segment.length === 0 || segment === "." || segment === ".." || segment.includes(":"), ) ) - return Effect.fail( - new StackPreparationError({ - field, - value, - message: `${field} must be a non-empty relative path without traversal`, - }), - ); - return Effect.void; -}; + return yield* new PreparationError({ + field, + value, + message: `${field} must be a non-empty relative path without traversal`, + }); + return; +}); diff --git a/packages/stack/src/preparation/RuntimeArtifacts.ts b/packages/stack/src/preparation/RuntimeArtifacts.ts deleted file mode 100644 index eac0dee437..0000000000 --- a/packages/stack/src/preparation/RuntimeArtifacts.ts +++ /dev/null @@ -1,280 +0,0 @@ -import { Crypto, Effect, FileSystem, Path } from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { - resolveNativeArtifactForWorkload, - type NativeWorkloadArtifact, -} from "../model/WorkloadCatalog.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import type { ArtifactPreparationStatus } from "../public/Status.ts"; -import { - ContainerEngineError, - ContainerPullError, - StackPreparationError, -} from "../public/Errors.ts"; -import { - makeArtifactStore, - type ArtifactStore, - type ArtifactStoreError, - type PreparedArtifact, -} from "./ArtifactStore.ts"; -import { makeSlimServicesSource } from "./SlimServicesSource.ts"; -import type { ContainerEngine } from "../runtime/ContainerEngine.ts"; -import { - resolveContainerEngine, - type ContainerEngineResolverShape, -} from "../runtime/ContainerEngineResolver.ts"; - -/** Result of preparing one workload's immutable runtime artifact. */ -export interface PreparedWorkloadArtifact { - readonly workloadId: string; - readonly capability: PlannedWorkload["capability"]; - readonly version: string; - readonly outcome: "cached" | "downloaded" | "pulled"; - /** Native installation root; container preparation has no root. */ - readonly artifactRoot?: string; - readonly executablePath?: string; - readonly image?: string; -} - -export type RuntimeArtifactPreparationProgress = ArtifactPreparationStatus; -type RuntimeArtifactPreparationProgressListener = ( - progress: RuntimeArtifactPreparationProgress, -) => void; - -export interface RuntimeArtifactPreparer { - readonly prepare: ( - runtime: StackRuntime, - workload: PlannedWorkload, - onProgress?: RuntimeArtifactPreparationProgressListener, - ) => Effect.Effect; -} - -export type RuntimeArtifactPreparationError = - | ArtifactStoreError - | StackPreparationError - | ContainerEngineError - | ContainerPullError; - -export interface RuntimeArtifactPreparerOptions { - readonly native?: { - readonly store: ArtifactStore; - /** Registers catalog metadata for the source that resolves a cache-miss request. */ - readonly onArtifactResolved?: (artifact: NativeWorkloadArtifact) => void; - readonly platform?: { readonly os: string; readonly arch: string }; - }; - readonly containerEngine?: ContainerEngine; -} - -const error = (message: string, fields: Readonly> = {}) => - new StackPreparationError({ message, ...fields }); - -const artifactKey = (artifact: NativeWorkloadArtifact): string => - `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; - -const containerVersion = (image: string): string => { - const separator = image.lastIndexOf(":"); - return separator > image.lastIndexOf("/") ? image.slice(separator + 1) : image; -}; - -/** - * Prepares only immutable native/container artifacts. No containers, processes, - * ports, secrets, or runtime resources are created by this boundary. - */ -export const makeRuntimeArtifactPreparer = ( - options: RuntimeArtifactPreparerOptions, -): RuntimeArtifactPreparer => { - const prepare = ( - runtime: StackRuntime, - workload: PlannedWorkload, - onProgress?: RuntimeArtifactPreparationProgressListener, - ): Effect.Effect => - Effect.suspend(() => { - const report = (state: ArtifactPreparationStatus["state"], cause?: unknown): void => { - onProgress?.({ - workloadId: workload.id, - capability: workload.capability, - state, - ...(cause === undefined - ? {} - : { - error: - cause instanceof Error - ? cause.message - : typeof cause === "string" - ? cause - : (JSON.stringify(cause) ?? "Unknown preparation failure"), - }), - }); - }; - report("preparing"); - if (runtime.kind === "native") { - const native = options.native; - if (native === undefined) - return Effect.fail( - error("Native runtime has no configured artifact store", { workload: workload.id }), - ); - if (workload.selected.kind !== "native") - return Effect.fail( - error("Native runtime received a container workload artifact", { - workload: workload.id, - }), - ); - return Effect.gen(function* () { - const artifact = yield* resolveNativeArtifactForWorkload(workload, native.platform); - native.onArtifactResolved?.(artifact); - const request = { - key: artifactKey(artifact), - requiredRuntimePaths: artifact.requiredRuntimePaths, - executablePath: artifact.executablePath, - }; - const prepared = yield* native.store.prepare(request, (state) => report(state)); - report("ready"); - return nativeResult(workload, artifact, prepared); - }).pipe(Effect.tapError((cause) => Effect.sync(() => report("failed", cause)))); - } - if (workload.selected.kind !== "container") - return Effect.fail( - error("Container runtime received a native workload artifact", { - workload: workload.id, - }), - ); - const engine = options.containerEngine; - if (engine === undefined) - return Effect.fail( - error("Container runtime has no configured container engine", { - workload: workload.id, - engine: runtime.engine, - }), - ); - if (engine.kind !== runtime.engine) - return Effect.fail( - error("Configured container engine does not match runtime", { - workload: workload.id, - engine: runtime.engine, - configuredEngine: engine.kind, - }), - ); - const image = workload.selected.image; - return engine.probe.pipe( - Effect.andThen(engine.inspectImage(image)), - Effect.flatMap((inspection) => - inspection.present - ? Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: containerVersion(image), - outcome: "cached", - image, - }) - : Effect.sync(() => report("downloading")).pipe( - Effect.andThen(engine.pullImage(image)), - Effect.mapError( - (cause) => - new ContainerPullError({ - message: `Unable to pull container image ${image}`, - workload: workload.id, - cause, - }), - ), - Effect.as({ - workloadId: workload.id, - capability: workload.capability, - version: containerVersion(image), - outcome: "pulled" as const, - image, - }), - ), - ), - Effect.tap(() => Effect.sync(() => report("ready"))), - Effect.mapError((cause) => - cause instanceof ContainerPullError - ? cause - : new ContainerEngineError({ - message: - cause instanceof Error ? cause.message : "Container engine operation failed", - engine: runtime.engine, - cause, - }), - ), - Effect.tapError((cause) => Effect.sync(() => report("failed", cause))), - ); - }); - return { prepare }; -}; - -const nativeResult = ( - workload: PlannedWorkload, - artifact: NativeWorkloadArtifact, - prepared: PreparedArtifact, -): PreparedWorkloadArtifact => ({ - workloadId: workload.id, - capability: workload.capability, - version: artifact.version, - outcome: prepared.outcome, - artifactRoot: prepared.path, - executablePath: prepared.executablePath, -}); - -/** - * Production constructor. The artifact cache is shared below stateRoot rather - * than inside a stack's data directory; container engines are selected from - * persisted runtime state by prepare(). - */ -export const makeProductionRuntimeArtifactPreparer = (options: { - readonly stateRoot: string; - /** Optional cache root shared across disposable stack state roots. */ - readonly artifactCacheRoot?: string; - readonly runtime?: StackRuntime; - /** Optional already-selected engine owned by the production runtime. */ - readonly containerEngine?: ContainerEngine; - /** Test seam for caller-owned preparation when no engine was supplied. */ - readonly containerEngineResolver?: ContainerEngineResolverShape; -}): Effect.Effect< - RuntimeArtifactPreparer, - RuntimeArtifactPreparationError, - Path.Path | FileSystem.FileSystem | Crypto.Crypto | ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - if (options.stateRoot.trim().length === 0) - return yield* error("Artifact cache state root must not be blank"); - const path = yield* Path.Path; - if (options.artifactCacheRoot !== undefined && options.artifactCacheRoot.trim().length === 0) - return yield* error("Artifact cache root must not be blank"); - const cacheRoot = - options.artifactCacheRoot === undefined - ? path.join(path.resolve(options.stateRoot), "artifacts") - : path.resolve(options.artifactCacheRoot); - const artifacts = new Map(); - const store = - options.runtime?.kind === "container" - ? undefined - : yield* makeArtifactStore({ - cacheRoot, - source: makeSlimServicesSource((request) => artifacts.get(request.key)), - }); - const registerArtifact = (artifact: NativeWorkloadArtifact) => { - artifacts.set(artifactKey(artifact), artifact); - }; - const selectedEngine = - options.runtime?.kind === "container" ? options.runtime.engine : undefined; - const containerEngine = - options.containerEngine ?? - (selectedEngine === undefined - ? undefined - : yield* resolveContainerEngine(selectedEngine, options.containerEngineResolver).pipe( - Effect.mapError( - (cause) => - new ContainerEngineError({ - message: `Unable to configure ${selectedEngine} artifact engine`, - engine: selectedEngine, - cause, - }), - ), - )); - const preparer = makeRuntimeArtifactPreparer({ - ...(store === undefined ? {} : { native: { store, onArtifactResolved: registerArtifact } }), - ...(containerEngine === undefined ? {} : { containerEngine }), - }); - return preparer; - }); diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index 27653ddcd4..8e9a7a48e4 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -1,126 +1,113 @@ -import { Effect, FileSystem, Layer, Path, Schema, Stream } from "effect"; +import { Effect, FileSystem, Path, Schema, Stream } from "effect"; import { NodeStream } from "@effect/platform-node"; -import { - FetchHttpClient, - HttpClient, - HttpClientError, - HttpClientResponse, -} from "effect/unstable/http"; +import { HttpClient } from "effect/unstable/http"; import { createZstdDecompress } from "node:zlib"; import { createHash } from "node:crypto"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; -import type { NativeWorkloadArtifact } from "../model/WorkloadCatalog.ts"; -import { StackPreparationError } from "../public/Errors.ts"; +import { PreparationError } from "./Errors.ts"; -type Fetcher = (input: string, init?: RequestInit) => Promise; +export interface SlimServicesArtifact { + readonly provider: "supabase/slim-services"; + readonly service: string; + readonly version: string; + readonly releaseTag: string; + readonly target: "darwin-arm64" | "linux-amd64" | "linux-arm64"; + readonly archive: "tar.zst"; + readonly assetName: string; + readonly downloadUrl: string; + readonly manifestUrl: string; + readonly checksumUrl: string; + readonly requiredRuntimePaths: ReadonlyArray; + readonly executablePath: string; +} export interface ZstdDecompressor { readonly decompress: ( compressedPath: string, outputPath: string, - ) => Effect.Effect; + ) => Effect.Effect; } export interface TarBoundary { readonly list: ( archivePath: string, - ) => Effect.Effect; + ) => Effect.Effect; readonly links: ( archivePath: string, - ) => Effect.Effect; + ) => Effect.Effect; readonly extract: ( archivePath: string, destination: string, - ) => Effect.Effect; + ) => Effect.Effect; } /** The system tar boundary is argv-based so archive paths never enter a shell string. */ export const systemTarBoundary: TarBoundary = { - list: (archivePath) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - return yield* spawner - .string(ChildProcess.make("tar", ["-tf", archivePath])) - .pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: "tar listing failed", cause }), - ), - ); - }), - links: (archivePath) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - return yield* spawner - .string(ChildProcess.make("tar", ["-tvf", archivePath])) - .pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: "tar link listing failed", cause }), - ), - ); - }), - extract: (archivePath, destination) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - return yield* spawner - .exitCode(ChildProcess.make("tar", ["-xf", archivePath, "-C", destination])) - .pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: "tar extraction failed", cause }), - ), - ); - }), -}; -const transport = (fetchRequest?: Fetcher) => - fetchRequest === undefined - ? FetchHttpClient.layer - : Layer.succeed( - HttpClient.HttpClient, - HttpClient.make((request, url, signal) => - Effect.tryPromise({ - try: () => - fetchRequest(url.href, { signal, method: request.method, headers: request.headers }), - catch: (cause) => - new HttpClientError.HttpClientError({ - reason: new HttpClientError.TransportError({ request, cause }), - }), - }).pipe(Effect.map((response) => HttpClientResponse.fromWeb(request, response))), + list: Effect.fn("SlimServicesSource.tarList")(function* (archivePath) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner + .string(ChildProcess.make("tar", ["-tf", archivePath])) + .pipe( + Effect.mapError((cause) => new PreparationError({ message: "tar listing failed", cause })), + ); + }), + links: Effect.fn("SlimServicesSource.tarLinks")(function* (archivePath) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner + .string(ChildProcess.make("tar", ["-tvf", archivePath])) + .pipe( + Effect.mapError( + (cause) => new PreparationError({ message: "tar link listing failed", cause }), ), ); - + }), + extract: Effect.fn("SlimServicesSource.tarExtract")(function* (archivePath, destination) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner + .exitCode(ChildProcess.make("tar", ["-xf", archivePath, "-C", destination])) + .pipe( + Effect.mapError( + (cause) => new PreparationError({ message: "tar extraction failed", cause }), + ), + ); + }), +}; const responseFor = (url: string) => - HttpClient.get(url).pipe( + Effect.flatMap(HttpClient.HttpClient, (client) => + HttpClient.followRedirects(client).get(url), + ).pipe( Effect.flatMap((response) => Effect.gen(function* () { if (response.status < 200 || response.status >= 300) - return yield* new StackPreparationError({ message: `HTTP ${response.status}` }); + return yield* new PreparationError({ message: `HTTP ${response.status}` }); return response; }), ), ); -const fetchBytes = ( - url: string, - request?: Fetcher, -): Effect.Effect => - responseFor(url).pipe( +const fetchBytes = Effect.fn("SlimServicesSource.fetchBytes")(function* (url: string) { + return yield* responseFor(url).pipe( Effect.flatMap((response) => response.arrayBuffer), Effect.map((bytes) => new Uint8Array(bytes)), Effect.mapError( - (cause) => new StackPreparationError({ message: `Unable to download ${url}`, cause }), + (cause) => new PreparationError({ message: `Unable to download ${url}`, cause }), ), - Effect.provide(transport(request)), ); +}); const nodeZstdDecompressor: ZstdDecompressor = { - decompress: (compressedPath, outputPath) => - Effect.gen(function* () { + decompress: Effect.fn("SlimServicesSource.decompress")(function* ( + compressedPath: string, + outputPath: string, + ) { + return yield* Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; yield* fs.stream(compressedPath).pipe( NodeStream.pipeThroughDuplex({ evaluate: () => createZstdDecompress(), onError: (cause) => - new StackPreparationError({ + new PreparationError({ message: "Unable to decompress slim-services archive", cause, }), @@ -130,54 +117,47 @@ const nodeZstdDecompressor: ZstdDecompressor = { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: "Unable to decompress slim-services archive", cause, }), ), - ), + ); + }), }; -const downloadToFile = ( +const downloadToFile = Effect.fn("SlimServicesSource.downloadToFile")(function* ( url: string, destination: string, - request: Fetcher | undefined, expectedSha256: string, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const response = yield* responseFor(url); - const hash = yield* Effect.try({ - try: () => createHash("sha256"), - catch: (cause) => - new StackPreparationError({ message: "Unable to initialize archive digest", cause }), - }); - yield* response.stream.pipe( - Stream.tap((chunk) => - Effect.try({ - try: () => { - hash.update(chunk); - }, - catch: (cause) => new StackPreparationError({ message: "Unable to hash archive", cause }), - }), - ), - Stream.run(fs.sink(destination, { mode: 0o600 })), - ); - const actual = yield* Effect.try({ - try: () => hash.digest("hex"), - catch: (cause) => - new StackPreparationError({ message: "Unable to finish archive digest", cause }), - }); - if (actual !== expectedSha256.toLowerCase()) - return yield* new StackPreparationError({ - message: `expected ${expectedSha256}, got ${actual}`, - }); - }).pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: `Unable to download ${url}`, cause }), +) { + const fs = yield* FileSystem.FileSystem; + const response = yield* responseFor(url); + const hash = yield* Effect.try({ + try: () => createHash("sha256"), + catch: (cause) => + new PreparationError({ message: "Unable to initialize archive digest", cause }), + }); + yield* response.stream.pipe( + Stream.tap((chunk) => + Effect.try({ + try: () => { + hash.update(chunk); + }, + catch: (cause) => new PreparationError({ message: "Unable to hash archive", cause }), + }), ), - Effect.provide(transport(request)), + Stream.run(fs.sink(destination, { mode: 0o600 })), ); + const actual = yield* Effect.try({ + try: () => hash.digest("hex"), + catch: (cause) => new PreparationError({ message: "Unable to finish archive digest", cause }), + }); + if (actual !== expectedSha256.toLowerCase()) + return yield* new PreparationError({ + message: `expected ${expectedSha256}, got ${actual}`, + }); +}); const checksumFor = (contents: string, archiveName: string): string | undefined => contents @@ -185,17 +165,16 @@ const checksumFor = (contents: string, archiveName: string): string | undefined .map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu)) .find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]; -export const slimServicesChecksum = ( - artifact: NativeWorkloadArtifact, - request?: Fetcher, -): Effect.Effect => - fetchBytes(artifact.checksumUrl, request).pipe( +export const slimServicesChecksum = Effect.fn("SlimServicesSource.checksum")(function* ( + artifact: SlimServicesArtifact, +) { + return yield* fetchBytes(artifact.checksumUrl).pipe( Effect.map((bytes) => new TextDecoder().decode(bytes)), Effect.flatMap((contents) => { const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) ? Effect.fail( - new StackPreparationError({ + new PreparationError({ message: "Slim-services checksum is missing", service: artifact.service, version: artifact.version, @@ -204,6 +183,7 @@ export const slimServicesChecksum = ( : Effect.succeed(checksum.toLowerCase()); }), ); +}); const unsafeArchivePath = (value: string): boolean => { const normalized = value.trim(); @@ -250,7 +230,7 @@ const validateExtractedTree = ( fs: FileSystem.FileSystem, path: Path.Path, destination: string, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const root = yield* fs.realPath(destination); const entries = yield* fs.readDirectory(destination, { recursive: true }); @@ -258,16 +238,16 @@ const validateExtractedTree = ( const candidate = path.join(destination, entry); const resolved = yield* fs.realPath(candidate); if (pathEscapesRoot(path, root, resolved)) - return yield* new StackPreparationError({ + return yield* new PreparationError({ message: "Slim-services archive entry escapes its staging directory", path: entry, }); } }).pipe( Effect.mapError((error) => - error instanceof StackPreparationError + error instanceof PreparationError ? error - : new StackPreparationError({ + : new PreparationError({ message: "Unable to validate extracted slim-services archive", cause: error, }), @@ -275,28 +255,23 @@ const validateExtractedTree = ( ); export const makeSlimServicesSource = ( - resolve: (request: ArtifactRequest) => NativeWorkloadArtifact | undefined, - fetchRequest?: Fetcher, + resolve: (request: ArtifactRequest) => SlimServicesArtifact | undefined, tarBoundary: TarBoundary = systemTarBoundary, decompressor: ZstdDecompressor = nodeZstdDecompressor, ): ArtifactSource => { - const resolveArtifact = ( + const resolveArtifact = Effect.fn("SlimServicesSource.resolveArtifact")(function* ( request: ArtifactRequest, - ): Effect.Effect => { + ) { const artifact = resolve(request); if (artifact === undefined) - return Effect.fail( - new StackPreparationError({ message: `No slim-services source for ${request.key}` }), - ); - return Effect.succeed(artifact); - }; + return yield* new PreparationError({ message: `No slim-services source for ${request.key}` }); + return artifact; + }); return { checksum: (request) => - resolveArtifact(request).pipe( - Effect.flatMap((artifact) => slimServicesChecksum(artifact, fetchRequest)), - ), - materialize: (request, destination, expectedSha256, onProgress) => - Effect.gen(function* () { + resolveArtifact(request).pipe(Effect.flatMap((artifact) => slimServicesChecksum(artifact))), + materialize: Effect.fn("SlimServicesSource.materialize")( + function* (request, destination, expectedSha256, onProgress) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const compressedPath = path.join(destination, ".slim-services.tar.zst"); @@ -307,59 +282,53 @@ export const makeSlimServicesSource = ( ]); return yield* Effect.gen(function* () { const artifact = yield* resolveArtifact(request); - const manifestBytes = yield* fetchBytes(artifact.manifestUrl, fetchRequest); + const manifestBytes = yield* fetchBytes(artifact.manifestUrl); const manifestText = new TextDecoder().decode(manifestBytes); - const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( + const manifestSchema = Schema.Struct({ + service: Schema.String, + version: Schema.String, + target: Schema.Literals(["darwin-arm64", "linux-amd64", "linux-arm64"]), + entrypoint: Schema.optionalKey(Schema.Array(Schema.String)), + cmd: Schema.optionalKey(Schema.Array(Schema.String)), + }); + const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( manifestText, ).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: "Slim-services manifest is invalid", cause, }), ), ); if ( - typeof manifest !== "object" || - manifest === null || - !("service" in manifest) || - !("version" in manifest) || - !("target" in manifest) || manifest.service !== artifact.service || manifest.version !== artifact.version || manifest.target !== artifact.target ) - return yield* new StackPreparationError({ + return yield* new PreparationError({ message: "Slim-services manifest does not match the catalog artifact", service: artifact.service, version: artifact.version, target: artifact.target, }); - const entrypoint = "entrypoint" in manifest ? manifest.entrypoint : undefined; - const command = "cmd" in manifest ? manifest.cmd : undefined; + const entrypoint = manifest.entrypoint; + const command = manifest.cmd; if ( - (entrypoint !== undefined && - (!Array.isArray(entrypoint) || - !entrypoint.every((value) => typeof value === "string") || - entrypoint.some(unsafeManifestCommand))) || - (command !== undefined && - (!Array.isArray(command) || - !command.every((value) => typeof value === "string") || - command.some(unsafeManifestCommand))) + (entrypoint !== undefined && entrypoint.some(unsafeManifestCommand)) || + (command !== undefined && command.some(unsafeManifestCommand)) ) - return yield* new StackPreparationError({ + return yield* new PreparationError({ message: "Slim-services manifest command is invalid", service: artifact.service, version: artifact.version, }); yield* Effect.sync(() => onProgress?.("downloading")).pipe( - Effect.andThen( - downloadToFile(artifact.downloadUrl, compressedPath, fetchRequest, expectedSha256), - ), + Effect.andThen(downloadToFile(artifact.downloadUrl, compressedPath, expectedSha256)), Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: "Unable to download slim-services archive", service: artifact.service, version: artifact.version, @@ -372,7 +341,7 @@ export const makeSlimServicesSource = ( const members = yield* tarBoundary.list(archivePath).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: "Unable to list slim-services archive", cause, }), @@ -383,14 +352,14 @@ export const makeSlimServicesSource = ( .map((member) => member.trim()) .find(unsafeArchivePath); if (unsafeMember !== undefined) - return yield* new StackPreparationError({ + return yield* new PreparationError({ message: "Slim-services archive contains an unsafe path", path: unsafeMember, }); const links = yield* tarBoundary.links(archivePath).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: "Unable to inspect slim-services archive links", cause, }), @@ -409,25 +378,26 @@ export const makeSlimServicesSource = ( }) .find((target): target is string => target !== undefined); if (unsafeLink !== undefined) - return yield* new StackPreparationError({ + return yield* new PreparationError({ message: "Slim-services archive contains an unsafe link target", path: unsafeLink, }); const exitCode = yield* tarBoundary.extract(archivePath, destination).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: "Unable to extract slim-services archive", cause, }), ), ); if (exitCode !== 0) - return yield* new StackPreparationError({ + return yield* new PreparationError({ message: `Slim-services archive extraction exited with code ${exitCode}`, }); yield* validateExtractedTree(fs, path, destination); }).pipe(Effect.ensuring(cleanup)); - }), + }, + ), }; }; diff --git a/packages/stack/src/preparation/artifacts.integration.test.ts b/packages/stack/src/preparation/artifacts.integration.test.ts index fe2d291726..6eace4a368 100644 --- a/packages/stack/src/preparation/artifacts.integration.test.ts +++ b/packages/stack/src/preparation/artifacts.integration.test.ts @@ -1,4 +1,4 @@ -import { NodeServices } from "@effect/platform-node"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Cause, @@ -8,14 +8,15 @@ import { Exit, Fiber, FileSystem, + Layer, Option, PlatformError, } from "effect"; -import { ArtifactIntegrityError, StackPreparationError } from "../public/Errors.ts"; +import { ArtifactIntegrityError, PreparationError } from "./Errors.ts"; import { makeArtifactStore, type ArtifactRequest, type ArtifactSource } from "./ArtifactStore.ts"; import { verifySha256 } from "./Integrity.ts"; -const layer = NodeServices.layer; +const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); const archive = new TextEncoder().encode("archive"); const archiveSha256 = "0eb3e36bfb24dcd9bb1d1bece1531216b59539a8fde17ee80224af0653c92aa3"; const request: ArtifactRequest = { @@ -45,7 +46,7 @@ const sourceWriting = (bytes: Uint8Array = archive): ArtifactSource => ({ Effect.mapError((cause) => cause instanceof ArtifactIntegrityError ? cause - : new StackPreparationError({ + : new PreparationError({ message: `materialization failed: ${cause instanceof Error ? cause.message : String(cause)}`, cause, }), @@ -85,7 +86,6 @@ describe("verified native artifact preparation", () => { expect(prepared.outcome).toBe("downloaded"); expect(yield* fs.exists(`${prepared.path}/bin/postgres`)).toBe(true); expect((yield* fs.stat(`${prepared.path}/bin/postgres`)).mode & 0o111).not.toBe(0); - expect(yield* fs.exists(`${prepared.path}/.artifact-source`)).toBe(false); }), ), ); @@ -205,7 +205,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -284,7 +284,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -386,7 +386,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -443,7 +443,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -481,7 +481,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -522,7 +522,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -568,7 +568,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -602,7 +602,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -648,7 +648,7 @@ describe("verified native artifact preparation", () => { }).pipe( Effect.mapError( (cause) => - new StackPreparationError({ + new PreparationError({ message: `materialization failed: ${cause.message}`, cause, }), @@ -689,8 +689,8 @@ describe("verified native artifact preparation", () => { const pathExit = yield* store .prepare({ ...request, requiredRuntimePaths: ["bin/../escape"] }) .pipe(Effect.exit); - expect(errorOf(keyExit)).toBeInstanceOf(StackPreparationError); - expect(errorOf(pathExit)).toBeInstanceOf(StackPreparationError); + expect(errorOf(keyExit)).toBeInstanceOf(PreparationError); + expect(errorOf(pathExit)).toBeInstanceOf(PreparationError); expect(called).toBe(false); }), ), @@ -717,7 +717,7 @@ describe("verified native artifact preparation", () => { const store = yield* makeArtifactStore({ cacheRoot: root, source }); const exit = yield* store.prepare({ ...request, key: "nested/postgres" }).pipe(Effect.exit); - expect(errorOf(exit)).toBeInstanceOf(StackPreparationError); + expect(errorOf(exit)).toBeInstanceOf(PreparationError); expect(called).toBe(false); expect(yield* fs.exists(`${outside}/postgres`)).toBe(false); }), diff --git a/packages/stack/src/preparation/runtime-artifacts.integration.test.ts b/packages/stack/src/preparation/runtime-artifacts.integration.test.ts deleted file mode 100644 index 471ac1a399..0000000000 --- a/packages/stack/src/preparation/runtime-artifacts.integration.test.ts +++ /dev/null @@ -1,409 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, FileSystem, Option, Path, Stream } from "effect"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { - makeArtifactStore, - type ArtifactRequest, - type ArtifactSource, - type ArtifactStore, - type PreparedArtifact, -} from "./ArtifactStore.ts"; -import type { - ContainerEngine, - ContainerNetworkSpec, - ContainerVolumeSpec, - ContainerContainerSpec, -} from "../runtime/ContainerEngine.ts"; -import { - makeProductionRuntimeArtifactPreparer, - makeRuntimeArtifactPreparer, -} from "./RuntimeArtifacts.ts"; -import { ContainerEngineError, StackPreparationError } from "../public/Errors.ts"; -import { ContainerEngineProtocolError } from "../runtime/ContainerEngine.ts"; -import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; - -const databaseRelease = catalogReleaseFor("database:database"); -if (databaseRelease === undefined) throw new Error("Missing default database release"); - -const nativeWorkload = (selected: PlannedWorkload["selected"]): PlannedWorkload => ({ - id: "database:database", - capability: "database", - dependencies: [], - readiness: {}, - artifacts: { - native: { kind: "native", release: databaseRelease.version }, - container: { - kind: "container", - image: databaseRelease.containerImage, - }, - }, - selected, -}); - -const prepared = (request: ArtifactRequest): PreparedArtifact => ({ - key: request.key, - path: "/tmp/prepared-database", - sha256: "a".repeat(64), - requiredRuntimePaths: request.requiredRuntimePaths, - executablePath: request.executablePath, - outcome: "downloaded", -}); - -const archiveSha256 = "0eb3e36bfb24dcd9bb1d1bece1531216b59539a8fde17ee80224af0653c92aa3"; - -const nativeSource = (): ArtifactSource => ({ - checksum: () => Effect.succeed(archiveSha256), - materialize: (request, destination) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - for (const relative of request.requiredRuntimePaths) { - const target = path.join(destination, relative); - if (relative === request.executablePath) { - yield* fs.makeDirectory(path.dirname(target), { recursive: true }); - yield* fs.writeFileString(target, "native executable"); - } else { - yield* fs.makeDirectory(target, { recursive: true }); - } - } - return; - }).pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: "native fixture failed", cause }), - ), - ), -}); - -const containerEngine = ( - present: boolean, - calls: string[], - kind: ContainerEngine["kind"] = "docker", -): ContainerEngine => ({ - kind, - preflight: Effect.succeed({ host: "host.docker.internal" }), - probe: Effect.sync(() => { - calls.push("probe"); - }), - inspectImage: (image) => - Effect.sync(() => { - calls.push(`inspect:${image}`); - return { present }; - }), - pullImage: (image) => - Effect.sync(() => { - calls.push(`pull:${image}`); - }), - listResources: () => Effect.succeed([]), - createNetwork: (_spec: ContainerNetworkSpec) => Effect.die("unused"), - removeNetwork: () => Effect.void, - createVolume: (_spec: ContainerVolumeSpec) => Effect.die("unused"), - removeVolume: () => Effect.void, - createContainer: (_spec: ContainerContainerSpec) => Effect.die("unused"), - copyToContainer: () => Effect.die("unused"), - startContainer: () => Effect.void, - waitContainer: () => Effect.succeed(0), - stopContainer: () => Effect.void, - removeContainer: () => Effect.void, - streamLogs: () => Stream.empty, -}); - -describe("runtime artifact preparation", () => { - it.live("does not create a native artifact store for a container runtime", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-artifact-runtime-" }); - yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: root, - runtime: { kind: "container", engine: "podman" }, - }); - expect(yield* fs.exists(path.join(root, "artifacts"))).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("creates the default native artifact cache below state root", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-artifact-native-" }); - yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: root, - runtime: { kind: "native" }, - }); - expect(yield* fs.exists(path.join(root, "artifacts"))).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("places a native artifact cache in an explicit shared root", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-artifact-shared-" }); - const sharedRoot = path.join(root, "shared-artifacts"); - const isolatedStateRoot = path.join(root, "isolated-state"); - yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: isolatedStateRoot, - artifactCacheRoot: sharedRoot, - runtime: { kind: "native" }, - }); - expect(yield* fs.exists(sharedRoot)).toBe(true); - expect(yield* fs.exists(path.join(isolatedStateRoot, "artifacts"))).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("surfaces selected-engine resolver failures as container engine errors", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-artifact-engine-" }); - const resolver = { - isInstalled: () => Effect.succeed(false), - resolve: () => - Effect.fail( - new ContainerEngineProtocolError({ - operation: "probe", - message: "podman is not installed", - }), - ), - }; - const exit = yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: root, - runtime: { kind: "container", engine: "podman" }, - containerEngineResolver: resolver, - }).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - const error = Exit.isFailure(exit) - ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) - : undefined; - expect(error).toBeInstanceOf(ContainerEngineError); - expect(error).toMatchObject({ engine: "podman" }); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it("resolves native catalog metadata and one ArtifactRequest", () => { - const requests: ArtifactRequest[] = []; - const store: ArtifactStore = { - prepare: (request) => - Effect.sync(() => { - requests.push(request); - return prepared(request); - }), - }; - const runtime = makeRuntimeArtifactPreparer({ - native: { - store, - platform: { os: "darwin", arch: "arm64" }, - }, - }); - const result = Effect.runSync( - runtime.prepare( - { kind: "native" }, - nativeWorkload({ kind: "native", release: databaseRelease.version }), - ), - ); - expect(result.outcome).toBe("downloaded"); - expect(result.version).toBe(databaseRelease.version); - expect(requests).toHaveLength(1); - expect(requests[0]?.key).toContain( - `slim-services/postgres/${databaseRelease.version}/darwin-arm64`, - ); - }); - - it.live("reuses a published native artifact without contacting its source again", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-artifact-offline-" }); - const firstStore = yield* makeArtifactStore({ - cacheRoot: root, - source: nativeSource(), - }); - const workload = nativeWorkload({ - kind: "native", - release: databaseRelease.version, - }); - const firstPreparer = makeRuntimeArtifactPreparer({ - native: { - store: firstStore, - platform: { os: "darwin", arch: "arm64" }, - }, - }); - const first = yield* firstPreparer.prepare({ kind: "native" }, workload); - expect(first.outcome).toBe("downloaded"); - const offline: ArtifactSource = { - checksum: () => - Effect.fail(new StackPreparationError({ message: "checksum network unavailable" })), - materialize: () => - Effect.fail(new StackPreparationError({ message: "artifact network unavailable" })), - }; - const secondStore = yield* makeArtifactStore({ cacheRoot: root, source: offline }); - const secondPreparer = makeRuntimeArtifactPreparer({ - native: { - store: secondStore, - platform: { os: "darwin", arch: "arm64" }, - }, - }); - const second = yield* secondPreparer.prepare({ kind: "native" }, workload); - expect(second.outcome).toBe("cached"); - expect(second.artifactRoot).toBe(first.artifactRoot); - expect(yield* fs.exists(`${first.artifactRoot}/.artifact.json`)).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("does not trust malformed native cache metadata while offline", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-artifact-offline-invalid-", - }); - const firstStore = yield* makeArtifactStore({ - cacheRoot: root, - source: nativeSource(), - }); - const workload = nativeWorkload({ - kind: "native", - release: databaseRelease.version, - }); - const preparer = makeRuntimeArtifactPreparer({ - native: { - store: firstStore, - platform: { os: "darwin", arch: "arm64" }, - }, - }); - const first = yield* preparer.prepare({ kind: "native" }, workload); - yield* fs.writeFileString(`${first.artifactRoot}/.artifact.json`, "malformed"); - - const offline: ArtifactSource = { - checksum: () => - Effect.fail(new StackPreparationError({ message: "checksum network unavailable" })), - materialize: () => - Effect.fail(new StackPreparationError({ message: "artifact network unavailable" })), - }; - const secondStore = yield* makeArtifactStore({ cacheRoot: root, source: offline }); - const exit = yield* makeRuntimeArtifactPreparer({ - native: { - store: secondStore, - platform: { os: "darwin", arch: "arm64" }, - }, - }) - .prepare({ kind: "native" }, workload) - .pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - const error = Exit.isFailure(exit) - ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) - : undefined; - expect(error).toBeInstanceOf(StackPreparationError); - expect(error).toMatchObject({ message: "checksum network unavailable" }); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it("probes and pulls an absent container image without creating or starting a container", () => { - const calls: string[] = []; - const engine = containerEngine(false, calls); - const runtime = makeRuntimeArtifactPreparer({ - native: { store: { prepare: () => Effect.die("unused") } }, - containerEngine: engine, - }); - const result = Effect.runSync( - runtime.prepare( - { kind: "container", engine: "docker" }, - nativeWorkload({ - kind: "container", - image: databaseRelease.containerImage, - }), - ), - ); - expect(result.outcome).toBe("pulled"); - expect(calls).toEqual([ - "probe", - `inspect:${databaseRelease.containerImage}`, - `pull:${databaseRelease.containerImage}`, - ]); - }); - - it("uses the persisted container engine identity", () => { - const podmanCalls: string[] = []; - const runtime = makeRuntimeArtifactPreparer({ - native: { store: { prepare: () => Effect.die("unused") } }, - containerEngine: containerEngine(true, podmanCalls, "podman"), - }); - Effect.runSync( - runtime.prepare( - { kind: "container", engine: "podman" }, - nativeWorkload({ - kind: "container", - image: databaseRelease.containerImage, - }), - ), - ); - expect(podmanCalls).toContain("probe"); - }); - - it("reports unavailable container engines separately from artifact preparation", () => { - const engine = containerEngine(true, []); - const unavailable: ContainerEngine = { - ...engine, - probe: Effect.fail( - new ContainerEngineProtocolError({ - operation: "probe", - message: "docker daemon is unavailable", - }), - ), - }; - const runtime = makeRuntimeArtifactPreparer({ - containerEngine: unavailable, - }); - const exit = Effect.runSyncExit( - runtime.prepare( - { kind: "container", engine: "docker" }, - nativeWorkload({ - kind: "container", - image: databaseRelease.containerImage, - }), - ), - ); - expect(Exit.isFailure(exit)).toBe(true); - const error = Exit.isFailure(exit) - ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) - : undefined; - expect(error).toBeInstanceOf(ContainerEngineError); - expect(error).toMatchObject({ engine: "docker", message: "docker daemon is unavailable" }); - }); - - it("fails strict runtime/artifact mismatches before touching a store or engine", () => { - let called = false; - const runtime = makeRuntimeArtifactPreparer({ - native: { - store: { - prepare: () => - Effect.sync(() => { - called = true; - return prepared({ key: "x", requiredRuntimePaths: [] }); - }), - }, - }, - }); - const exit = Effect.runSyncExit( - runtime.prepare( - { kind: "native" }, - nativeWorkload({ - kind: "container", - image: databaseRelease.containerImage, - }), - ), - ); - expect(Exit.isFailure(exit)).toBe(true); - expect(called).toBe(false); - }); -}); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 2ad2be58c6..9b7d882013 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -1,17 +1,20 @@ -import { NodeServices } from "@effect/platform-node"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Crypto, Deferred, Effect, Exit, Fiber, FileSystem, Option } from "effect"; +import { Cause, Crypto, Deferred, Effect, Exit, Fiber, FileSystem, Layer, Option } from "effect"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- the redirect test owns a local native listener. +import { createServer, type Server } from "node:http"; import { zstdCompress } from "node:zlib"; +import { FetchHttpClient } from "effect/unstable/http"; import { makeArtifactStore, type ArtifactRequest } from "./ArtifactStore.ts"; import { digestHex } from "./Integrity.ts"; import { makeSlimServicesSource, slimServicesChecksum, systemTarBoundary, + type SlimServicesArtifact, type ZstdDecompressor, } from "./SlimServicesSource.ts"; -import type { NativeWorkloadArtifact } from "../model/WorkloadCatalog.ts"; -import { StackPreparationError } from "../public/Errors.ts"; +import { PreparationError } from "./Errors.ts"; const waitForAbort = (signal?: AbortSignal | null): Promise => Effect.runPromise(Effect.never, { signal: signal ?? undefined }); @@ -19,7 +22,7 @@ const waitForAbort = (signal?: AbortSignal | null): Promise => const waitForRelease = (released: Deferred.Deferred): Promise => Effect.runPromise(Deferred.await(released)); -const artifact: NativeWorkloadArtifact = { +const artifact: SlimServicesArtifact = { provider: "supabase/slim-services", service: "demo", version: "v1.0.0", @@ -110,7 +113,53 @@ type FetchLike = ( const requestUrl = (input: Parameters[0]): string => typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; +const withFetch = (fetcher: FetchLike, effect: Effect.Effect) => + effect.pipe( + Effect.provide(FetchHttpClient.layer), + Effect.provideService(FetchHttpClient.Fetch, fetcher), + ); + describe("slim-services artifact source", () => { + it.live("follows release redirects through the supplied Node HTTP client", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => + // oxlint-disable-next-line effecttsgo/new-promise -- node server listen exposes a callback lifecycle. + new Promise((resolve, reject) => { + const value = createServer((request, response) => { + if (request.url === "/redirect") { + response.statusCode = 302; + response.setHeader("location", "/checksums"); + } + response.end("a".repeat(64) + " demo-v1.0.0-linux-amd64.tar.zst\n"); + }); + value.once("error", reject); + value.listen(0, "127.0.0.1", () => resolve(value)); + }), + catch: (cause) => + new PreparationError({ message: "Unable to start redirect server", cause }), + }), + (value) => + Effect.callback((resume) => { + value.close(() => resume(Effect.asVoid(Effect.succeed(true)))); + return Effect.asVoid(Effect.succeed(true)); + }), + ); + const address = server.address(); + if (address === null || typeof address === "string") + return yield* Effect.die("redirect server did not expose a port"); + const redirected = { + ...artifact, + checksumUrl: `http://127.0.0.1:${address.port}/redirect`, + }; + const checksum = yield* slimServicesChecksum(redirected); + expect(checksum).toBe("a".repeat(64)); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ), + ); + it.live("verifies checksums and extracts a manifest-matched archive using injected fetch", () => Effect.scoped( Effect.gen(function* () { @@ -129,11 +178,11 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(archive)); }; - expect(yield* slimServicesChecksum(artifact, fetcher)).toBe(expected); - const source = makeSlimServicesSource(() => artifact, fetcher); + expect(yield* withFetch(fetcher, slimServicesChecksum(artifact))).toBe(expected); + const source = makeSlimServicesSource(() => artifact); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-source-" }); - yield* source.materialize(request, destination, expected); + yield* withFetch(fetcher, source.materialize(request, destination, expected)); expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); }).pipe(Effect.provide(NodeServices.layer)), ), @@ -157,11 +206,14 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(archive)); }; - const source = makeSlimServicesSource(() => artifact, fetcher); + const source = makeSlimServicesSource(() => artifact); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-unsafe-" }); - const failed = yield* source.materialize(request, destination, expected).pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackPreparationError); + const failed = yield* withFetch( + fetcher, + source.materialize(request, destination, expected).pipe(Effect.exit), + ); + expect(errorOf(failed)).toBeInstanceOf(PreparationError); expect(yield* fs.exists(`${destination}/outside`)).toBe(false); }).pipe(Effect.provide(NodeServices.layer)), ), @@ -192,10 +244,9 @@ describe("slim-services artifact source", () => { }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-links-" }); - yield* makeSlimServicesSource(() => artifact, fetcher).materialize( - request, - destination, - expected, + yield* withFetch( + fetcher, + makeSlimServicesSource(() => artifact).materialize(request, destination, expected), ); expect(yield* fs.readFileString(`${destination}/bin/current`)).toBe("demo"); @@ -215,10 +266,13 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(malformed)); }; - const failed = yield* makeSlimServicesSource(() => artifact, malformedFetcher) - .materialize(request, destination, malformedDigest) - .pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackPreparationError); + const failed = yield* withFetch( + malformedFetcher, + makeSlimServicesSource(() => artifact) + .materialize(request, destination, malformedDigest) + .pipe(Effect.exit), + ); + expect(errorOf(failed)).toBeInstanceOf(PreparationError); }).pipe(Effect.provide(NodeServices.layer)), ), ); @@ -250,10 +304,13 @@ describe("slim-services artifact source", () => { const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-link-escape-", }); - const failed = yield* makeSlimServicesSource(() => artifact, fetcher) - .materialize(request, destination, expected) - .pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackPreparationError); + const failed = yield* withFetch( + fetcher, + makeSlimServicesSource(() => artifact) + .materialize(request, destination, expected) + .pipe(Effect.exit), + ); + expect(errorOf(failed)).toBeInstanceOf(PreparationError); expect(yield* fs.exists(`${destination}/outside`)).toBe(false); }).pipe(Effect.provide(NodeServices.layer)), ), @@ -285,10 +342,13 @@ describe("slim-services artifact source", () => { prefix: "slim-services-interrupt-", }); const fiber = yield* Effect.forkChild( - makeSlimServicesSource(() => artifact, fetcher).materialize( - request, - destination, - "0".repeat(64), + withFetch( + fetcher, + makeSlimServicesSource(() => artifact).materialize( + request, + destination, + "0".repeat(64), + ), ), { startImmediately: true }, ); @@ -327,13 +387,13 @@ describe("slim-services artifact source", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-integrity-", }); - const source = makeSlimServicesSource(() => artifact, fetcher); + const source = makeSlimServicesSource(() => artifact); const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const failed = yield* store.prepare(request).pipe(Effect.exit); + const failed = yield* withFetch(fetcher, store.prepare(request).pipe(Effect.exit)); expect(Exit.isFailure(failed)).toBe(true); expect(yield* fs.exists(`${root}/demo/v1`)).toBe(false); validChecksum = true; - const prepared = yield* store.prepare(request); + const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); }).pipe(Effect.provide(NodeServices.layer)), ), @@ -381,10 +441,13 @@ describe("slim-services artifact source", () => { const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-stream-" }); const fiber = yield* Effect.forkChild( - makeSlimServicesSource(() => artifact, fetcher).materialize( - request, - destination, - "0".repeat(64), + withFetch( + fetcher, + makeSlimServicesSource(() => artifact).materialize( + request, + destination, + "0".repeat(64), + ), ), { startImmediately: true }, ); @@ -429,12 +492,14 @@ describe("slim-services artifact source", () => { const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-zstd-" }); const fiber = yield* Effect.forkChild( - makeSlimServicesSource( - () => artifact, + withFetch( fetcher, - systemTarBoundary, - decompressor, - ).materialize(request, destination, expected), + makeSlimServicesSource(() => artifact, systemTarBoundary, decompressor).materialize( + request, + destination, + expected, + ), + ), { startImmediately: true }, ); yield* Deferred.await(started); @@ -466,10 +531,9 @@ describe("slim-services artifact source", () => { }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-pax-" }); - yield* makeSlimServicesSource(() => artifact, fetcher).materialize( - request, - destination, - expected, + yield* withFetch( + fetcher, + makeSlimServicesSource(() => artifact).materialize(request, destination, expected), ); expect(yield* fs.exists(`${destination}/${longName}`)).toBe(true); }).pipe(Effect.provide(NodeServices.layer)), @@ -498,11 +562,11 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(archive)); }; - const source = makeSlimServicesSource(() => artifact, fetcher); + const source = makeSlimServicesSource(() => artifact); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-" }); const store = yield* makeArtifactStore({ cacheRoot: root, source }); - const prepared = yield* store.prepare(request); + const prepared = yield* withFetch(fetcher, store.prepare(request)); expect(prepared.outcome).toBe("downloaded"); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); expect(yield* fs.exists(`${prepared.path}/.artifact.json`)).toBe(true); diff --git a/packages/stack/src/public.e2e.test.ts b/packages/stack/src/public.e2e.test.ts new file mode 100644 index 0000000000..b97c803e66 --- /dev/null +++ b/packages/stack/src/public.e2e.test.ts @@ -0,0 +1,423 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, expectTypeOf, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Path, Redacted, Schema, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { HttpClient } from "effect/unstable/http"; +import { tmpdir } from "node:os"; +import { open, postgres } from "./effect.ts"; +import * as PromiseStack from "./index.ts"; +import { assertOwnerExited, captureOwnerPid } from "../tests/owner.ts"; + +for (const runtime of ["node", "bun"] as const) { + it.live( + `${runtime}: Promise stop and destroy confirm the launching client's owner has exited`, + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: `stack-owner-${runtime}-` }); + const child = yield* ChildProcess.make( + runtime === "bun" ? process.execPath : "node", + [ + new URL("../tests/owner-exit-client.ts", import.meta.url).pathname, + root, + `${tmpdir()}/supabase-stack-artifacts`, + ], + { stdin: "ignore", stdout: "pipe", stderr: "pipe" }, + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(Number(code), stderr).toBe(0); + expect(stdout).toContain("owner-exit-confirmed"); + }).pipe( + Effect.scoped, + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + { timeout: 120_000 }, + ); +} + +it.live( + "keeps a service alive after its Promise client exits and reconnects through Effect", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-public-e2e-" }); + const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make( + process.execPath, + [new URL("../tests/public-client-fixture.ts", import.meta.url).pathname, root, cacheRoot], + { stdin: "ignore" }, + ), + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText(), + Stream.runFold( + () => "", + (text, chunk) => text + chunk, + ), + ), + child.stderr.pipe( + Stream.decodeText(), + Stream.runFold( + () => "", + (text, chunk) => text + chunk, + ), + ), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(Number(code), stderr).toBe(0); + const identity = yield* Schema.decodeEffect( + Schema.fromJsonString(Schema.Struct({ stackId: Schema.String, instanceId: Schema.String })), + )(stdout.trim()); + const locations = { + stateRoot: `${root}/state`, + cacheRoot, + id: identity.stackId, + }; + const stack = yield* open(locations); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + const pid = yield* captureOwnerPid(locations, stack.id); + yield* stack.destroy; + yield* assertOwnerExited(pid); + }).pipe(Effect.catchCause(Effect.die)), + ); + const mail = yield* stack.services.get(identity.instanceId); + expect((yield* mail.status).lifecycle).toBe("running"); + const credentials = yield* mail.credentials(); + expect(credentials.url).toBeDefined(); + const http = yield* HttpClient.HttpClient; + const response = yield* http.get(`${credentials.url}/api/v1/messages`); + expect(response.status).toBe(200); + yield* mail.stop; + expect((yield* mail.status).lifecycle).toBe("stopped"); + yield* mail.start; + yield* mail.ready; + expect((yield* mail.credentials()).url).toBe(credentials.url); + + const stdoutChunks: Array = []; + const tool = yield* stack.tools.run(postgres.psql({ major: 17 }), { + args: ["--version"], + stdout: (bytes) => + Effect.sync(() => { + stdoutChunks.push(bytes); + }), + stderr: () => Effect.void, + }); + expect(tool.exitCode).toBe(0); + expect(stdoutChunks.map((bytes) => new TextDecoder().decode(bytes)).join("")).toContain( + "psql (PostgreSQL) 17", + ); + const database = yield* stack.services.create({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("public-pgprove-password"), + jwtSecret: Redacted.make("public-pgprove-jwt-secret"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* database.start; + yield* database.ready; + const databaseCredentials = yield* database.credentials(); + const databaseUrl = databaseCredentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("Public pgProve URL missing"); + const pgProveRoot = path.join(root, "public-pgprove-tests"); + yield* fs.makeDirectory(pgProveRoot, { recursive: true }); + yield* fs.writeFileString(path.join(pgProveRoot, "main.sql"), "\\ir included.sql\n"); + yield* fs.writeFileString(path.join(pgProveRoot, "included.sql"), "\\i nested.sql\n"); + yield* fs.writeFileString( + path.join(pgProveRoot, "nested.sql"), + "SELECT plan(1);\nSELECT pass('public pgProve');\nSELECT * FROM finish();\n", + ); + const extension = yield* stack.tools.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "-c", "CREATE EXTENSION IF NOT EXISTS pgtap"], + stdout: () => Effect.void, + stderr: () => Effect.void, + }); + expect(extension.exitCode).toBe(0); + const pgProveOutput: Array = []; + const pgProve = yield* stack.tools.run(postgres.pgProve({ major: 17 }), { + args: ["--dbname", databaseUrl, "--ext", ".sql", "main.sql", "--verbose"], + pgProve: { + mounts: [{ source: pgProveRoot, target: "/tests" }], + cwd: pgProveRoot, + workingDir: undefined, + }, + stdout: (bytes) => Effect.sync(() => pgProveOutput.push(bytes)), + stderr: () => Effect.void, + }); + expect(pgProve.exitCode).toBe(0); + expect(pgProveOutput.map((bytes) => new TextDecoder().decode(bytes)).join("")).toContain( + "public pgProve", + ); + expect(yield* fs.readFileString(path.join(pgProveRoot, "nested.sql"))).toContain( + "public pgProve", + ); + const promiseClient = yield* Effect.tryPromise(() => PromiseStack.open(locations)); + yield* Effect.acquireUseRelease( + Effect.succeed(promiseClient), + (client) => + Effect.gen(function* () { + const status = yield* Effect.tryPromise(() => + client.services.get(mail.id).then((service) => service.status()), + ); + expect(status.lifecycle).toBe("running"); + const database = yield* Effect.tryPromise(() => + client.services.create({ + service: "database", + config: { + version: "17", + databasePassword: "plain-secret", + jwtSecret: "plain-jwt-secret", + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }), + ); + expectTypeOf(database).toEqualTypeOf(); + yield* Effect.tryPromise(() => + database.restart({ + config: { + version: "17", + databasePassword: "replacement-secret", + jwtSecret: "replacement-jwt", + jwtExpiry: 3600, + }, + }), + ); + yield* Effect.tryPromise(() => database.ready()); + const urls = yield* Effect.tryPromise(() => database.credentials({ from: "runtime" })); + const databaseUrl = urls.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("Database URL missing"); + const promiseExtension = yield* Effect.tryPromise(() => + client.tools.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "-c", "CREATE EXTENSION IF NOT EXISTS pgtap"], + stdout: () => {}, + stderr: () => {}, + }), + ); + expect(promiseExtension.exitCode).toBe(0); + const promiseProveOutput: Array = []; + const promiseProveError: Array = []; + const promiseProve = yield* Effect.tryPromise(() => + client.tools.run(postgres.pgProve({ major: 17 }), { + args: ["--dbname", databaseUrl, "--ext", ".sql", "main.sql", "--verbose"], + pgProve: { + mounts: [{ source: pgProveRoot, target: "/tests" }], + cwd: pgProveRoot, + workingDir: undefined, + }, + stdout: (bytes) => { + promiseProveOutput.push(bytes); + }, + stderr: (bytes) => { + promiseProveError.push(bytes); + }, + }), + ); + expect( + promiseProve.exitCode, + promiseProveError.map((bytes) => new TextDecoder().decode(bytes)).join(""), + ).toBe(0); + expect( + promiseProveOutput.map((bytes) => new TextDecoder().decode(bytes)).join(""), + ).toContain("public pgProve"); + const sqlOutput: Array = []; + const sql = yield* Effect.tryPromise(() => + client.tools.run(postgres.psql({ major: 17 }), { + args: ["--dbname", databaseUrl, "-At"], + stdin: Stream.toAsyncIterable( + Stream.make(new TextEncoder().encode("SELECT 42;\n")), + ), + stdout: (bytes) => { + sqlOutput.push(bytes); + }, + stderr: () => {}, + }), + ); + expect(sql.exitCode).toBe(0); + expect( + sqlOutput + .map((bytes) => new TextDecoder().decode(bytes)) + .join("") + .trim(), + ).toBe("42"); + const sourceCredentials = yield* Effect.tryPromise(() => + database.credentials({ from: "runtime" }), + ); + if (sourceCredentials.databaseUrl === undefined) + return yield* Effect.die("Snapshot source URL missing"); + const sourceUrl = sourceCredentials.databaseUrl; + const seed = yield* Effect.tryPromise(() => + client.tools.run(postgres.psql({ major: 17 }), { + args: [ + "--dbname", + sourceUrl, + "-c", + "CREATE TABLE snapshot_rows (value text NOT NULL); INSERT INTO snapshot_rows VALUES ('roundtrip');", + ], + stdout: () => {}, + stderr: () => {}, + }), + ); + expect(seed.exitCode).toBe(0); + yield* Effect.tryPromise(() => database.stop()); + const archive = path.join(root, "public-snapshot.tar"); + yield* Effect.tryPromise(() => database.exportSnapshot(archive)); + const restored = yield* Effect.tryPromise(() => + client.services.create({ + service: "database", + config: { + version: "17", + databasePassword: "restored-secret", + jwtSecret: "restored-jwt-secret", + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }), + ); + yield* Effect.tryPromise(() => restored.restoreSnapshot(archive)); + yield* Effect.tryPromise(() => restored.start()); + yield* Effect.tryPromise(() => restored.ready()); + const restoredCredentials = yield* Effect.tryPromise(() => + restored.credentials({ from: "runtime" }), + ); + if (restoredCredentials.databaseUrl === undefined) + return yield* Effect.die("Restored snapshot URL missing"); + const restoredUrl = restoredCredentials.databaseUrl; + const restoredOutput: Array = []; + const restoredQuery = yield* Effect.tryPromise(() => + client.tools.run(postgres.psql({ major: 17 }), { + args: ["--dbname", restoredUrl, "-Atc", "SELECT value FROM snapshot_rows"], + stdout: (bytes) => { + restoredOutput.push(bytes); + }, + stderr: () => {}, + }), + ); + expect(restoredQuery.exitCode).toBe(0); + expect( + restoredOutput + .map((bytes) => new TextDecoder().decode(bytes)) + .join("") + .trim(), + ).toBe("roundtrip"); + expect((yield* stack.services.list).map((service) => service.id)).toContain( + database.id, + ); + const selected = yield* Effect.tryPromise(() => + client.composition.supabase([ + { service: "mail", config: {}, endpoints: { http: { port: "auto" } } }, + ]), + ); + const selectedMail = selected[0]; + if (selectedMail === undefined) return yield* Effect.die("Composition member missing"); + yield* Effect.tryPromise(() => client.composition.start()); + expect((yield* Effect.tryPromise(() => selectedMail.status())).wakeEnabled).toBe(true); + const selectedUrl = yield* Effect.tryPromise(() => selectedMail.credentials()); + expect((yield* http.get(`${selectedUrl.url}/api/v1/messages`)).status).toBe(200); + yield* Effect.tryPromise(() => client.composition.stop()); + expect((yield* Effect.tryPromise(() => selectedMail.status())).lifecycle).toBe( + "stopped", + ); + const reused = yield* Effect.tryPromise(() => + client.composition.supabase( + [{ service: "mail", config: {}, endpoints: { http: { port: "auto" } } }], + { reuseIds: [selectedMail.id] }, + ), + ); + expect(reused[0]?.id).toBe(selectedMail.id); + yield* Effect.tryPromise(() => client.composition.start()); + expect((yield* Effect.tryPromise(() => selectedMail.status())).wakeEnabled).toBe(true); + expect((yield* http.get(`${selectedUrl.url}/api/v1/messages`)).status).toBe(200); + expect((yield* Effect.tryPromise(() => selectedMail.status())).lifecycle).toBe( + "running", + ); + expect((yield* mail.status).lifecycle).toBe("running"); + }), + (client) => Effect.promise(() => client.close()), + ); + }).pipe( + Effect.scoped, + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + { timeout: 180_000 }, +); + +it.live( + "closes Promise observation iterators while leaving the owner usable", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-public-observation-" }); + const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + const locations = { stateRoot: `${root}/state`, cacheRoot }; + const stack = yield* Effect.tryPromise(() => + PromiseStack.create({ ...locations, projectRoot: root, runtime: "native" }), + ); + const owner = yield* open({ ...locations, id: stack.id }); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + const pid = yield* captureOwnerPid(locations, owner.id); + yield* owner.destroy; + yield* assertOwnerExited(pid); + }).pipe(Effect.catchCause(Effect.die)), + ); + yield* Effect.addFinalizer(() => + Effect.tryPromise(() => stack.close()).pipe( + Effect.catch((cause) => Effect.logError("Promise stack close failed", cause)), + ), + ); + const mail = yield* Effect.tryPromise(() => + stack.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" }, smtp: { port: "auto" }, pop3: { port: "auto" } }, + }), + ); + yield* Effect.tryPromise(() => mail.start()); + yield* Effect.tryPromise(() => mail.ready()); + const attached = yield* Effect.tryPromise(() => stack.services.get(mail.id)); + const observations = attached.followStatus()[Symbol.asyncIterator](); + const firstObservation = yield* Effect.tryPromise(() => observations.next()); + expect(firstObservation.done).toBe(false); + const pendingObservation = observations.next(); + yield* Effect.tryPromise(() => stack.close()); + expect((yield* Effect.tryPromise(() => pendingObservation)).done).toBe(true); + expect(() => attached.followStatus()[Symbol.asyncIterator]()).toThrow(); + const reopened = yield* Effect.tryPromise(() => + PromiseStack.open({ ...locations, id: stack.id }), + ); + yield* Effect.acquireUseRelease( + Effect.succeed(reopened), + (client) => + Effect.tryPromise(() => client.services.get(mail.id)).pipe( + Effect.flatMap((reopenedMail) => + Effect.tryPromise(() => reopenedMail.status()).pipe( + Effect.map((status) => expect(status.lifecycle).toBe("running")), + ), + ), + ), + (client) => Effect.tryPromise(() => client.close()), + ); + }).pipe( + Effect.scoped, + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + { timeout: 60_000 }, +); diff --git a/packages/stack/src/public/Capability.ts b/packages/stack/src/public/Capability.ts deleted file mode 100644 index 9037b5f18a..0000000000 --- a/packages/stack/src/public/Capability.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { Schema } from "effect"; - -export const CAPABILITY_NAMES = [ - "database", - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", -] as const; - -export const CapabilityNameSchema = Schema.Literals(CAPABILITY_NAMES); -export type CapabilityName = Schema.Schema.Type; - -export const CapabilityStateSchema = Schema.Literals([ - "disabled", - "dormant", - "starting", - "ready", - "stopping", - "stopped", - "failed", -] as const); -export type CapabilityState = Schema.Schema.Type; - -export const ActivationModeSchema = Schema.Literals(["eager", "lazy"] as const); -export type ActivationMode = Schema.Schema.Type; - -export const CapabilityStatusSchema = Schema.Struct({ - name: CapabilityNameSchema, - activation: ActivationModeSchema, - state: CapabilityStateSchema, - error: Schema.optionalKey(Schema.String), -}); -export type CapabilityStatus = Schema.Schema.Type; diff --git a/packages/stack/src/public/Config.ts b/packages/stack/src/public/Config.ts deleted file mode 100644 index aa163dfc80..0000000000 --- a/packages/stack/src/public/Config.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { Schema } from "effect"; -import { DatabaseSettingsSchema, type DatabaseSettings } from "../model/capabilities/database.ts"; -import { RestSettingsSchema, type RestSettings } from "../model/capabilities/rest.ts"; -import { AuthSettingsSchema, type AuthSettings } from "../model/capabilities/auth.ts"; -import { RealtimeSettingsSchema, type RealtimeSettings } from "../model/capabilities/realtime.ts"; -import { StorageSettingsSchema, type StorageSettings } from "../model/capabilities/storage.ts"; -import { - FunctionsSettingsSchema, - FunctionsInspectorSettingsSchema, - type FunctionsSettings, - type FunctionsInspectorSettings, -} from "../model/capabilities/functions.ts"; -import { StudioSettingsSchema, type StudioSettings } from "../model/capabilities/studio.ts"; -import { MailSettingsSchema, type MailSettings } from "../model/capabilities/mail.ts"; -import { - AnalyticsSettingsSchema, - type AnalyticsSettings, -} from "../model/capabilities/analytics.ts"; -import { PoolerSettingsSchema, type PoolerSettings } from "../model/capabilities/pooler.ts"; -import { ActivationModeSchema, type ActivationMode } from "./Capability.ts"; -import { NetworkPortSchema, PORT_FIELDS, type PortField } from "./Status.ts"; - -export const PreparationModeSchema = Schema.Literals(["background", "on-demand"] as const); -type PreparationMode = Schema.Schema.Type; - -export const ListenerConfigSchema = Schema.Union([ - Schema.Struct({ enabled: Schema.Literal(false) }), - Schema.Struct({ - enabled: Schema.optionalKey(Schema.Literal(true)), - address: Schema.optionalKey(Schema.String), - port: Schema.optionalKey(NetworkPortSchema), - }), -]); -export type ListenerConfig = Schema.Schema.Type; - -const capabilityFields = (settings: S) => ({ - enabled: Schema.optionalKey(Schema.Literal(true)), - activation: Schema.optionalKey(ActivationModeSchema), - version: Schema.optionalKey(Schema.String), - settings: Schema.optionalKey(settings), -}); - -const optionalCapability = (settings: S) => - Schema.Union([ - Schema.Struct({ enabled: Schema.Literal(false) }), - Schema.Struct(capabilityFields(settings)), - ]); - -const retirableCapability = (settings: S) => - Schema.Union([ - Schema.Struct({ enabled: Schema.Literal(false) }), - Schema.Struct({ - ...capabilityFields(settings), - idleTimeoutSeconds: Schema.optionalKey( - Schema.Union([Schema.Finite.check(Schema.isGreaterThan(0)), Schema.Literal(false)]), - ), - }), - ]); - -export const DatabaseCapabilityConfigSchema = Schema.Struct({ - // PostgreSQL accepts an exact catalog release or a major selector such as - // "15"/"17". Major selectors are resolved to a concrete catalog release by - // the compiler, preserving a compatible previous pin when one exists. - version: Schema.optionalKey(Schema.String), - settings: Schema.optionalKey(DatabaseSettingsSchema), -}); -export const StackCapabilitiesConfigSchema = Schema.Struct({ - database: Schema.optionalKey(DatabaseCapabilityConfigSchema), - rest: Schema.optionalKey(retirableCapability(RestSettingsSchema)), - auth: Schema.optionalKey(retirableCapability(AuthSettingsSchema)), - realtime: Schema.optionalKey(retirableCapability(RealtimeSettingsSchema)), - storage: Schema.optionalKey(optionalCapability(StorageSettingsSchema)), - functions: Schema.optionalKey(optionalCapability(FunctionsSettingsSchema)), - studio: Schema.optionalKey(retirableCapability(StudioSettingsSchema)), - mail: Schema.optionalKey(optionalCapability(MailSettingsSchema)), - analytics: Schema.optionalKey(optionalCapability(AnalyticsSettingsSchema)), - pooler: Schema.optionalKey(retirableCapability(PoolerSettingsSchema)), -}); - -const listenerFields = { - [PORT_FIELDS[0]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[1]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[2]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[3]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[4]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[5]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[6]]: Schema.optionalKey(ListenerConfigSchema), - [PORT_FIELDS[7]]: Schema.optionalKey(ListenerConfigSchema), -} satisfies { readonly [Name in PortField]: Schema.Top }; -export const StackListenersConfigSchema = Schema.Struct(listenerFields); - -export const JwtSigningSchema = Schema.Union([ - Schema.Struct({ kind: Schema.Literal("symmetric"), secret: Schema.Redacted(Schema.String) }), - Schema.Struct({ kind: Schema.Literal("jwks-file"), path: Schema.String }), -]); -export type JwtSigning = Schema.Schema.Type; -export const StackSecurityConfigSchema = Schema.Struct({ - jwt: Schema.optionalKey( - Schema.Struct({ - issuer: Schema.optionalKey(Schema.String), - signing: Schema.optionalKey(JwtSigningSchema), - }), - ), -}); - -export const StackConfigSchema = Schema.Struct({ - preparation: Schema.optionalKey(PreparationModeSchema), - capabilities: Schema.optionalKey(StackCapabilitiesConfigSchema), - listeners: Schema.optionalKey(StackListenersConfigSchema), - security: Schema.optionalKey(StackSecurityConfigSchema), -}); -export type StackConfig = Schema.Schema.Type; -export type StackCapabilitiesConfig = Schema.Schema.Type; -export type StackSecurityConfig = Schema.Schema.Type; -export { - DatabaseSettingsSchema, - RestSettingsSchema, - AuthSettingsSchema, - RealtimeSettingsSchema, - StorageSettingsSchema, - FunctionsSettingsSchema, - FunctionsInspectorSettingsSchema, - StudioSettingsSchema, - MailSettingsSchema, - AnalyticsSettingsSchema, - PoolerSettingsSchema, -}; -export type { - DatabaseSettings, - RestSettings, - AuthSettings, - RealtimeSettings, - StorageSettings, - FunctionsSettings, - FunctionsInspectorSettings, - StudioSettings, - MailSettings, - AnalyticsSettings, - PoolerSettings, - ActivationMode, - PreparationMode, -}; -export { NetworkPortSchema } from "./Status.ts"; diff --git a/packages/stack/src/public/Credentials.ts b/packages/stack/src/public/Credentials.ts deleted file mode 100644 index 8b78a3e631..0000000000 --- a/packages/stack/src/public/Credentials.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Schema } from "effect"; -import type * as Redacted from "effect/Redacted"; - -const EffectDatabaseCredentialsSchema = Schema.Struct({ - url: Schema.RedactedFromValue(Schema.String), - password: Schema.RedactedFromValue(Schema.String), -}); - -const EffectApiCredentialsSchema = Schema.Struct({ - publishableKey: Schema.String, - secretKey: Schema.RedactedFromValue(Schema.String), - anonJwt: Schema.String, - serviceRoleJwt: Schema.RedactedFromValue(Schema.String), -}); - -const EffectStorageCredentialsSchema = Schema.Struct({ - endpoint: Schema.String, - region: Schema.String, - accessKeyId: Schema.String, - secretAccessKey: Schema.RedactedFromValue(Schema.String), -}); - -export const EffectStackCredentialsSchema = Schema.Struct({ - database: EffectDatabaseCredentialsSchema, - api: Schema.optionalKey(EffectApiCredentialsSchema), - storage: Schema.optionalKey(EffectStorageCredentialsSchema), -}); -export interface EffectStackCredentials { - readonly database: { - readonly url: Redacted.Redacted; - readonly password: Redacted.Redacted; - }; - readonly api?: { - readonly publishableKey: string; - readonly secretKey: Redacted.Redacted; - readonly anonJwt: string; - readonly serviceRoleJwt: Redacted.Redacted; - }; - readonly storage?: { - readonly endpoint: string; - readonly region: string; - readonly accessKeyId: string; - readonly secretAccessKey: Redacted.Redacted; - }; -} - -export const PromiseStackCredentialsSchema = Schema.Struct({ - database: Schema.Struct({ - url: Schema.String, - password: Schema.String, - }), - api: Schema.optionalKey( - Schema.Struct({ - publishableKey: Schema.String, - secretKey: Schema.String, - anonJwt: Schema.String, - serviceRoleJwt: Schema.String, - }), - ), - storage: Schema.optionalKey( - Schema.Struct({ - endpoint: Schema.String, - region: Schema.String, - accessKeyId: Schema.String, - secretAccessKey: Schema.String, - }), - ), -}); -export type PromiseStackCredentials = Schema.Schema.Type; diff --git a/packages/stack/src/public/EffectStack.ts b/packages/stack/src/public/EffectStack.ts deleted file mode 100644 index 96c8f477ea..0000000000 --- a/packages/stack/src/public/EffectStack.ts +++ /dev/null @@ -1,1425 +0,0 @@ -import { - Cause, - Context, - Crypto, - Deferred, - Effect, - Exit, - FileSystem, - Fiber, - Match, - Option, - Path, - Predicate, - Redacted, - Result, - Schedule, - Schema, - Stream, -} from "effect"; -import type { RpcClientError } from "effect/unstable/rpc/RpcClientError"; -import type { SocketError } from "effect/unstable/socket/Socket"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -import type { StackIdentity } from "../identity/Identity.ts"; -import { resolveStackIdentity, deriveStackId } from "../identity/Identity.ts"; -import { - compileStack, - rebuildExecutionPlan, - sameDefinition, - type SecretSlotInput, - type StackDefinition, -} from "../model/Compiler.ts"; -import { dependencyClosure, type ExecutionPlan } from "../model/ExecutionPlan.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { toPersistedIdentity } from "../state/StackState.ts"; -import { - isMissingStateRemnantError, - makeStackStateStore, - withRegistryLock, - type StackStateStore, -} from "../state/StackStateStore.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { StackIdSchema, type StackId } from "./StackId.ts"; -import type { StackRuntime, StackRuntimePreference } from "./Runtime.ts"; -import type { StackConfig } from "./Config.ts"; -import { - type ArtifactPreparationStatus, - type StackStatus, - type StackDescriptor, - type StackInspection, -} from "./Status.ts"; -import { CAPABILITY_NAMES, type CapabilityName } from "./Capability.ts"; -import type { LogQuery, StackLogBatch, StackLogEntry } from "./Logs.ts"; -import type { EffectStackCredentials } from "./Credentials.ts"; -import { - InvalidStackIdentityError, - InvalidProjectRootError, - InvalidStackConfigError, - StackVersionUnsupportedError, - StackDestructionError, - StackNotFoundError, - StackNotRunningError, - StackOwnershipConflictError, - StackRuntimeMismatchError, - StackLifecycleConflictError, - StackPreparationError, - ArtifactIntegrityError, - ContainerPullError, - StackSecretMismatchError, - InvalidJwtSigningMaterialError, - StackRuntimeError, - StackCleanupError, - ContainerEngineError, - EphemeralPostgresError, - RequiresActivatedProcessError, - StackStateInvalidError, - StackStateFormatUnsupportedError, - StackUpgradeRequiredError, - StackMustBeStoppedError, - PortAllocationError, - PortUnavailableError, - GatewayActivationError, - InvalidLogCursorError, - PostgresClientError, - type CreateStackError, - type OpenStackError, - type StackDiscoveryError, - type StackStatusError, - type StackCredentialsError, - type PrepareStackError, - type StackStartError, - type StackStopError, - type StackLogsError, - type DestroyStackError, - type ResetDatabaseError, - type StackError, - type StackErrorTag, - isStackError, - isStackErrorTag, - PREPARE_STACK_ERROR_TAGS, - STACK_STATUS_ERROR_TAGS, - STACK_CREDENTIALS_ERROR_TAGS, - STACK_START_ERROR_TAGS, - STACK_STOP_ERROR_TAGS, - STACK_LOGS_ERROR_TAGS, - DESTROY_STACK_ERROR_TAGS, - RESET_DATABASE_ERROR_TAGS, -} from "./Errors.ts"; -import { - ownerLockExists, - readOwnerMetadata, - type OwnerMetadata, - type StackRuntimeEnvironmentValue, -} from "../state/Ownership.ts"; -import { makeControlClient } from "../control/ControlServer.ts"; -import { - isMaintenanceTransportFailure, - MaintenanceProtocolError, -} from "../control/MaintenanceProtocol.ts"; -import { STACK_RPC_RELEASE, type StackRpcError, type StackRpcClient } from "../control/StackRpc.ts"; -import { - ensureSupervisor, - defaultRuntimeEnvironment, - StackRuntimeEnvironment, -} from "../supervisor/Launcher.ts"; -import { - ContainerEngineResolver, - selectDefaultRuntimeSelection, - nativeRuntimeBlockedForUid, - NATIVE_ROOT_UNSUPPORTED_MESSAGE, - type ContainerEngineResolverShape, -} from "../runtime/ContainerEngineResolver.ts"; -import { formatStopTimeoutMessage } from "../runtime/Diagnostics.ts"; -import { statusForSnapshot } from "../supervisor/StatusProjection.ts"; -import type { SupervisorSnapshot } from "../supervisor/SupervisorState.ts"; -import { EMPTY_LOG_CURSOR, readRetainedLogs, selectLogBatch } from "../supervisor/LogStore.ts"; -import { - makeProductionRuntimeArtifactPreparer, - type PreparedWorkloadArtifact, -} from "../preparation/RuntimeArtifacts.ts"; - -export interface StartStackOptions { - readonly config?: StackConfig; -} -export interface PrepareStackOptions { - readonly config?: StackConfig; - readonly capabilities?: ReadonlyArray; - /** Synchronous progress observer for this caller-owned preparation. */ - readonly onProgress?: (status: ArtifactPreparationStatus) => void; -} -export interface CreateStackOptions { - readonly projectRoot: string; - readonly name?: string; - readonly runtime?: StackRuntimePreference; -} -export interface FindStackOptions { - readonly projectRoot: string; - readonly name?: string; -} -export interface ListStacksOptions { - readonly projectRoot?: string; -} - -export interface InspectStackOptions { - readonly config?: StackConfig; -} -export interface PreparedCapability { - readonly capability: CapabilityName; - readonly version: string; - readonly outcome: "cached" | "downloaded" | "pulled"; -} - -export interface PrepareStackResult { - readonly capabilities: ReadonlyArray; -} - -export interface EffectStack { - readonly id: StackId; - readonly status: Effect.Effect; - readonly credentials: Effect.Effect; - readonly prepare: ( - options?: PrepareStackOptions, - ) => Effect.Effect; - readonly start: (options?: StartStackOptions) => Effect.Effect; - readonly stop: Effect.Effect; - readonly destroy: Effect.Effect; - readonly resetDatabase: Effect.Effect; - readonly logs: (query?: LogQuery) => Effect.Effect; - readonly followLogs: (query?: LogQuery) => Stream.Stream; - /** Present when auto-select persisted native because the Docker daemon was down. */ - readonly dockerFallbackNotice?: string; -} - -const optionOf = (value: A | undefined): Option.Option => - value === undefined ? Option.none() : Option.some(value); - -const descriptor = (state: PersistedStackState, id: StackId): StackDescriptor => ({ - id, - projectRoot: state.identity.projectRoot, - name: state.identity.stackName, - branchContext: state.identity.branchContext, - runtime: state.runtime, - desiredLifecycle: state.desiredLifecycle, -}); - -const environment = () => - Effect.serviceOption(StackRuntimeEnvironment).pipe( - Effect.flatMap((configured) => - Option.isSome(configured) ? Effect.succeed(configured.value) : defaultRuntimeEnvironment, - ), - ); - -const isCapabilityName = (value: unknown): value is CapabilityName => - typeof value === "string" && CAPABILITY_NAMES.some((name) => name === value); - -type ControlError = - | StackRpcError - | RpcClientError - | SocketError - | MaintenanceProtocolError - | StackError; - -const stackErrorFactories = { - InvalidStackIdentityError: (message: string) => new InvalidStackIdentityError({ message }), - InvalidProjectRootError: (message: string) => new InvalidProjectRootError({ message }), - InvalidStackConfigError: (message: string) => new InvalidStackConfigError({ message }), - StackVersionUnsupportedError: (message: string) => new StackVersionUnsupportedError({ message }), - StackNotFoundError: (message: string) => new StackNotFoundError({ message }), - StackOwnershipConflictError: (message: string) => new StackOwnershipConflictError({ message }), - StackRuntimeMismatchError: (message: string) => new StackRuntimeMismatchError({ message }), - StackNotRunningError: (message: string) => new StackNotRunningError({ message }), - StackMustBeStoppedError: (message: string) => new StackMustBeStoppedError({ message }), - StackLifecycleConflictError: (message: string) => new StackLifecycleConflictError({ message }), - StackStateInvalidError: (message: string) => new StackStateInvalidError({ message }), - InvalidLogCursorError: (message: string) => new InvalidLogCursorError({ message }), - StackStateFormatUnsupportedError: (message: string) => - new StackStateFormatUnsupportedError({ message }), - StackUpgradeRequiredError: (message: string) => new StackUpgradeRequiredError({ message }), - StackSecretMismatchError: (message: string) => new StackSecretMismatchError({ message }), - InvalidJwtSigningMaterialError: (message: string) => - new InvalidJwtSigningMaterialError({ message }), - PortAllocationError: (message: string) => new PortAllocationError({ message }), - PortUnavailableError: (message: string) => new PortUnavailableError({ message }), - GatewayActivationError: (message: string) => new GatewayActivationError({ message }), - StackPreparationError: (message: string) => new StackPreparationError({ message }), - ArtifactIntegrityError: (message: string) => new ArtifactIntegrityError({ message }), - ContainerPullError: (message: string) => new ContainerPullError({ message }), - StackRuntimeError: (message: string) => new StackRuntimeError({ message }), - StackCleanupError: (message: string) => new StackCleanupError({ message }), - ContainerEngineError: (message: string) => new ContainerEngineError({ message }), - StackDestructionError: (message: string) => new StackDestructionError({ message }), - EphemeralPostgresError: (message: string) => new EphemeralPostgresError({ message }), - RequiresActivatedProcessError: (message: string) => - new RequiresActivatedProcessError({ message, capability: "unknown" }), - PostgresClientError: (message: string) => new PostgresClientError({ message }), -} satisfies Record StackError>; - -const isOwnerUnreachable = (error: unknown): boolean => - Predicate.isTagged(error, "RpcClientError") || - Predicate.isTagged(error, "SocketError") || - isMaintenanceTransportFailure(error); - -const errorForRpc = (error: ControlError): StackError => { - if (isStackError(error)) return error; - if (isOwnerUnreachable(error)) - return new StackOwnershipConflictError({ - message: `Stack owner is unreachable: ${error.message}`, - }); - if ( - typeof error === "object" && - error !== null && - "tag" in error && - "message" in error && - typeof error.tag === "string" && - typeof error.message === "string" - ) { - if (isStackErrorTag(error.tag)) return stackErrorFactories[error.tag](error.message); - return new StackStateInvalidError({ message: error.message }); - } - return new StackStateInvalidError({ message: error.message }); -}; - -const isNarrowError = >( - error: StackError, - tags: Tags, -): error is Extract => tags.some((tag) => tag === error._tag); - -const narrowError = >( - error: ControlError, - tags: Tags, - fallback: (message: string) => Extract, -): Extract => { - const mapped = errorForRpc(error); - return isNarrowError(mapped, tags) ? mapped : fallback(mapped.message); -}; - -const statusError = (error: ControlError): StackStatusError => - narrowError(error, STACK_STATUS_ERROR_TAGS, (message) => new StackStateInvalidError({ message })); -const credentialsError = (error: ControlError): StackCredentialsError => - narrowError( - error, - STACK_CREDENTIALS_ERROR_TAGS, - (message) => new StackNotRunningError({ message }), - ); -const startError = (error: ControlError): StackStartError => - narrowError(error, STACK_START_ERROR_TAGS, (message) => new StackStateInvalidError({ message })); -const stopError = (error: ControlError): StackStopError => - narrowError( - error, - STACK_STOP_ERROR_TAGS, - (message) => new StackLifecycleConflictError({ message }), - ); -const logsError = (error: ControlError): StackLogsError => - narrowError(error, STACK_LOGS_ERROR_TAGS, (message) => new StackStateInvalidError({ message })); -const destroyError = (error: ControlError): DestroyStackError => - narrowError(error, DESTROY_STACK_ERROR_TAGS, (message) => new StackDestructionError({ message })); -const resetDatabaseError = (error: ControlError): ResetDatabaseError => - narrowError( - error, - RESET_DATABASE_ERROR_TAGS, - (message) => new StackStateInvalidError({ message }), - ); - -/** Internal control-transport seam used by public lifecycle integration tests. */ -export interface HandleDependencies { - readonly resolveOwner: ( - launch: boolean, - ) => Effect.Effect, StackError>; - readonly readOfflineState: Effect.Effect, StackError>; - readonly readPersistedState: Effect.Effect, StackError>; - readonly readLogs: (query?: LogQuery) => Effect.Effect; - readonly waitForRelease: Effect.Effect; - readonly prepare: ( - options?: PrepareStackOptions, - ) => Effect.Effect; -} - -/** @internal Owner metadata together with whether this handle launched the owner. */ -export interface OwnerResolution { - readonly owner: OwnerMetadata; - readonly launched: boolean; -} - -export const makeHandle = (id: StackId, options: HandleDependencies): Effect.Effect => - Effect.sync(() => { - const isStoppedState = (state: PersistedStackState): boolean => - state.desiredLifecycle === "stopped" || state.desiredLifecycle === "unconfigured"; - const stackNotFound = () => new StackNotFoundError({ message: "Stack state was not found" }); - type ResolvedClient = { - readonly client: ReturnType; - readonly resolution: OwnerResolution; - }; - const resolveClient = ( - launch: boolean, - protocol: "rpc" | "maintenance" = "rpc", - ): Effect.Effect => - options.resolveOwner(launch).pipe( - Effect.flatMap((resolution): Effect.Effect => - Option.isSome(resolution) - ? protocol === "rpc" && resolution.value.owner.rpcRelease !== STACK_RPC_RELEASE - ? Effect.fail( - new StackUpgradeRequiredError({ - message: `Stack owner release ${resolution.value.owner.rpcRelease} requires stop before start`, - expectedRelease: STACK_RPC_RELEASE, - actualRelease: resolution.value.owner.rpcRelease, - }), - ) - : Effect.succeed({ - resolution: resolution.value, - client: makeControlClient(resolution.value.owner.endpoint, { - stackId: id, - ownerSessionId: resolution.value.owner.ownerSessionId, - rpcRelease: - protocol === "rpc" ? STACK_RPC_RELEASE : resolution.value.owner.rpcRelease, - }), - }) - : Effect.fail( - new StackOwnershipConflictError({ message: "No Supervisor owns this stack" }), - ), - ), - ); - const stopExactOwner = (owner: OwnerMetadata): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const client = makeControlClient(owner.endpoint, { - stackId: id, - ownerSessionId: owner.ownerSessionId, - rpcRelease: owner.rpcRelease, - }); - const stop = yield* Effect.exit(client.stop); - if ( - Exit.isSuccess(stop) && - !stop.value.ok && - stop.value.error.tag === "operation-failed" && - stop.value.error.stackErrorTag === "StackLifecycleConflictError" - ) { - return; - } - const release = yield* Effect.exit(options.waitForRelease); - let cause: Cause.Cause = Cause.empty; - if (Exit.isFailure(stop)) { - cause = Cause.combine( - cause, - Cause.fail( - new StackCleanupError({ - message: "Unable to stop freshly launched Supervisor", - cause: stop.cause, - }), - ), - ); - } else if (!stop.value.ok) { - cause = Cause.combine( - cause, - Cause.fail( - new StackCleanupError({ - message: stop.value.error.message, - cause: stop.value.error, - }), - ), - ); - } - if (Exit.isFailure(release)) { - cause = Cause.combine( - cause, - Cause.fail( - new StackCleanupError({ - message: "Freshly launched Supervisor did not release ownership", - cause: release.cause, - }), - ), - ); - } - if (cause.reasons.length > 0) return yield* Effect.failCause(cause); - }), - ); - const shouldCleanupFreshOwner = (result: Exit.Exit): boolean => { - if (Exit.isSuccess(result)) return false; - if (Cause.hasInterruptsOnly(result.cause)) return true; - const failure = Cause.findErrorOption(result.cause); - return Option.isSome(failure) && Predicate.isTagged(failure.value, "RpcClientError"); - }; - const cleanupLaunchedOwner = ( - resolution: OwnerResolution, - result: Exit.Exit, - ): Effect.Effect => - resolution.launched && shouldCleanupFreshOwner(result) - ? Effect.uninterruptible( - stopExactOwner(resolution.owner).pipe( - Effect.catchCause((cleanupCause) => - Effect.fail( - new StackCleanupError({ - message: "Unable to clean up freshly launched Supervisor", - cause: Cause.combine( - Exit.isFailure(result) ? result.cause : Cause.empty, - cleanupCause, - ), - }), - ), - ), - ), - ) - : Effect.void; - const invoke = ( - call: (rpc: StackRpcClient) => Effect.Effect, - mapError: (error: ControlError) => E, - launch = false, - ): Effect.Effect => { - const rpcCall: Effect.Effect = resolveClient( - launch, - ).pipe( - Effect.flatMap(({ client, resolution }) => - Effect.scoped(client.rpc.pipe(Effect.flatMap(call))).pipe( - Effect.onExit((result) => - launch ? cleanupLaunchedOwner(resolution, result) : Effect.void, - ), - ), - ), - ); - const mapped: Effect.Effect = rpcCall.pipe(Effect.mapError(mapError)); - return mapped; - }; - const destroyAndAwaitOwner: Effect.Effect = resolveClient(true).pipe( - Effect.mapError(destroyError), - Effect.flatMap(({ client, resolution }) => - Effect.gen(function* () { - const ownerConnected = yield* Deferred.make(); - const ownerWatch = client.awaitClose( - Deferred.succeed(ownerConnected, undefined).pipe(Effect.asVoid), - ); - const ownerFiber = yield* Effect.forkChild(ownerWatch, { startImmediately: true }); - const ownerReady = Deferred.await(ownerConnected).pipe( - Effect.raceFirst( - Fiber.join(ownerFiber).pipe( - Effect.flatMap(() => - Effect.fail( - new StackDestructionError({ - message: "Unable to observe Supervisor control connection", - }), - ), - ), - Effect.mapError( - (cause) => - new StackDestructionError({ - message: "Unable to observe Supervisor control connection", - cause, - }), - ), - ), - ), - ); - const destroyAttempt = ownerReady.pipe( - Effect.andThen( - Effect.scoped(client.rpc.pipe(Effect.flatMap((rpc) => rpc.destroy(undefined)))), - ), - Effect.onExit((attempt) => cleanupLaunchedOwner(resolution, attempt)), - Effect.mapError(destroyError), - ); - const result = yield* Effect.exit( - destroyAttempt.pipe( - // The owner closes its control server only after all workload cleanup has completed. - // Await the exact preface-only socket instead of decoding a terminal RPC stream Exit. - Effect.andThen( - Fiber.join(ownerFiber).pipe( - Effect.mapError( - (cause) => - new StackDestructionError({ - message: "Unable to observe Supervisor shutdown completion", - cause, - }), - ), - ), - ), - ), - ); - if (Exit.isFailure(result)) { - yield* Fiber.interrupt(ownerFiber); - } - return yield* result; - }), - ), - ); - const destroy: Effect.Effect = Effect.suspend(() => - options.readPersistedState.pipe( - Effect.mapError(destroyError), - Effect.flatMap((state) => - Option.isNone(state) ? Effect.fail(stackNotFound()) : destroyAndAwaitOwner, - ), - ), - ); - const resetDatabase: Effect.Effect = Effect.suspend( - (): Effect.Effect => - invoke((rpc) => rpc.resetDatabase(undefined), resetDatabaseError).pipe( - Effect.catchTag("StackOwnershipConflictError", (ownershipError) => { - const offline: Effect.Effect = options.readOfflineState.pipe( - Effect.mapError(resetDatabaseError), - Effect.flatMap((state): Effect.Effect => - Option.isNone(state) - ? Effect.fail(stackNotFound()) - : isStoppedState(state.value) - ? Effect.fail( - new StackNotRunningError({ - stackId: id, - message: "Stack is not running", - }), - ) - : Effect.fail(ownershipError), - ), - Effect.catchTag("StackOwnershipConflictError", () => Effect.fail(ownershipError)), - ); - return offline; - }), - ), - ); - const status: Effect.Effect = Effect.suspend( - (): Effect.Effect => { - const rpcStatus = invoke((rpc) => rpc.status(undefined), statusError); - return rpcStatus.pipe( - Effect.catchTag("StackOwnershipConflictError", (ownershipError) => - options.readOfflineState.pipe( - Effect.mapError(statusError), - Effect.flatMap((state): Effect.Effect => { - if (Option.isNone(state)) return Effect.fail(stackNotFound()); - if (isStoppedState(state.value)) { - const fallback: SupervisorSnapshot = { - stack: { _tag: "stopped", session: "initialized" }, - sessionId: Symbol("offline-status"), - plan: undefined, - capabilities: new Map(), - }; - return statusForSnapshot(id, state.value, { _tag: "unavailable" }, fallback); - } - return Effect.fail( - new StackOwnershipConflictError({ message: "No Supervisor owns this stack" }), - ); - }), - Effect.catchTag("StackOwnershipConflictError", () => Effect.fail(ownershipError)), - ), - ), - ); - }, - ); - const credentials: Effect.Effect = - Effect.suspend((): Effect.Effect => - invoke((rpc) => rpc.credentials(undefined), credentialsError).pipe( - Effect.catchTag("StackOwnershipConflictError", (ownershipError) => { - const offline: Effect.Effect = - options.readOfflineState.pipe( - Effect.mapError(credentialsError), - Effect.flatMap((state): Effect.Effect => - Option.isNone(state) - ? Effect.fail(stackNotFound()) - : isStoppedState(state.value) - ? Effect.fail( - new StackNotRunningError({ - stackId: id, - message: "Stack is not running", - }), - ) - : Effect.fail(ownershipError), - ), - Effect.catchTag("StackOwnershipConflictError", () => Effect.fail(ownershipError)), - ); - return offline; - }), - ), - ); - const start = (startOptions?: StartStackOptions) => { - return invoke( - (rpc) => - startOptions?.config === undefined - ? rpc.start({}) - : rpc.start({ config: startOptions.config }), - startError, - true, - ).pipe( - Effect.tapError(() => - options.readPersistedState.pipe( - Effect.flatMap((state) => - Option.isSome(state) && isStoppedState(state.value) - ? options.waitForRelease.pipe(Effect.ignore) - : Effect.void, - ), - Effect.ignore, - ), - ), - ); - }; - const logsStateError = (error: StackError): StackLogsError => - isNarrowError(error, STACK_LOGS_ERROR_TAGS) - ? error - : new StackStateInvalidError({ message: error.message, cause: error }); - const stopOwner = (owner: ReturnType) => - Effect.gen(function* () { - // Subscribe to the owner control connection before sending stop so a - // fast shutdown cannot race the close witness. - const closeFiber = yield* Effect.forkChild(owner.awaitClose(), { startImmediately: true }); - const response = yield* owner.stop.pipe(Effect.exit); - if (Exit.isFailure(response)) { - yield* Fiber.interrupt(closeFiber); - return yield* Effect.failCause(response.cause); - } - if (!response.value.ok) { - yield* Fiber.interrupt(closeFiber); - if ( - response.value.error.tag === "operation-failed" && - response.value.error.stackErrorTag !== undefined && - isStackErrorTag(response.value.error.stackErrorTag) - ) { - return yield* stackErrorFactories[response.value.error.stackErrorTag]( - response.value.error.message, - ); - } - return yield* new StackLifecycleConflictError({ message: response.value.error.message }); - } - yield* Fiber.join(closeFiber).pipe(Effect.ignore); - yield* options.waitForRelease; - }).pipe( - Effect.mapError(stopError), - Effect.timeoutOrElse({ - duration: "60 seconds", - orElse: () => - status.pipe( - Effect.map((value) => - value.capabilities - .filter( - (capability) => - capability.state === "ready" || - capability.state === "starting" || - capability.state === "stopping", - ) - .map((capability) => capability.name), - ), - Effect.timeoutOrElse({ - duration: "2 seconds", - orElse: () => Effect.succeed>([]), - }), - Effect.orElseSucceed(() => [] as ReadonlyArray), - Effect.flatMap((running) => - Effect.fail( - new StackLifecycleConflictError({ - message: formatStopTimeoutMessage(running), - }), - ), - ), - ), - }), - ); - const launchAndStop = resolveClient(true, "maintenance").pipe( - Effect.mapError(stopError), - Effect.flatMap(({ client }) => stopOwner(client)), - ); - const stop: Effect.Effect = Effect.suspend(() => - resolveClient(false, "maintenance").pipe( - Effect.mapError(stopError), - Effect.flatMap(({ client }) => stopOwner(client)), - Effect.catchTag("StackOwnershipConflictError", () => - options.readOfflineState.pipe( - Effect.mapError(stopError), - Effect.flatMap((state) => - Option.isSome(state) && isStoppedState(state.value) ? Effect.void : launchAndStop, - ), - // Ownership artifacts that block the offline fast path may be - // stale. Let ensureSupervisor arbitrate the lease; a live owner - // remains protected and returns a typed conflict. - Effect.catchTag("StackOwnershipConflictError", () => launchAndStop), - ), - ), - ), - ); - const prepare = ( - prepareOptions?: PrepareStackOptions, - ): Effect.Effect => options.prepare(prepareOptions); - const logs = (query?: LogQuery): Effect.Effect => - invoke((rpc) => rpc.logs(query ?? {}), logsError).pipe( - Effect.catchTag("StackOwnershipConflictError", (ownershipError) => { - const ownerStopped = options.readPersistedState.pipe(Effect.mapError(logsStateError)); - return ownerStopped.pipe( - Effect.flatMap((state) => { - if (Option.isNone(state)) return Effect.fail(stackNotFound()); - const teardown = isStoppedState(state.value); - if (!teardown) return Effect.fail(ownershipError); - return Effect.suspend(() => - // During an owner stop the control socket can close before its metadata/lease are - // released. Re-check ownership before each read and retry only that typed - // transition; a live owner or other log failure remains visible. - options.readLogs(query), - ).pipe( - Effect.retry({ - schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ times: 200 })), - while: (error) => error instanceof StackOwnershipConflictError, - }), - ); - }), - ); - }), - ); - return { - id, - status, - credentials, - prepare, - start, - stop, - destroy, - resetDatabase, - logs, - followLogs: (query) => - Stream.paginate({ cursor: query?.cursor, first: true }, ({ cursor, first }) => { - const { cursor: _initialCursor, tail: _tail, ...baseQuery } = query ?? {}; - const pollQuery = { - ...baseQuery, - ...(first && query?.tail !== undefined ? { tail: query.tail } : {}), - ...(cursor === undefined || cursor.opaque === EMPTY_LOG_CURSOR.opaque - ? {} - : { cursor }), - }; - const request = logs(pollQuery); - const delayed = first - ? request - : Effect.schedule(Effect.void, Schedule.duration("100 millis")).pipe( - Effect.andThen(request), - ); - return delayed.pipe( - Effect.map( - (batch) => - [ - batch.entries, - batch.running - ? Option.some({ cursor: batch.cursor, first: false }) - : Option.none(), - ] as const, - ), - ); - }), - } satisfies EffectStack; - }); - -const stateInitial = (identity: StackIdentity, runtime: StackRuntime): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity: toPersistedIdentity(identity), - runtime, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, -}); - -type ChildProcessSpawnerValue = Context.Service.Shape< - typeof ChildProcessSpawner.ChildProcessSpawner ->; - -const handleDependencies = (options: { - readonly environment: StackRuntimeEnvironmentValue; - readonly store: StackStateStore; - readonly id: StackId; - readonly fileSystem: FileSystem.FileSystem; - readonly path: Path.Path; - readonly crypto: Crypto.Crypto; - readonly spawner: ChildProcessSpawnerValue; - readonly containerEngineResolver?: ContainerEngineResolverShape; -}) => { - const provide = ( - effect: Effect.Effect, - ) => - effect.pipe( - Effect.provideService(FileSystem.FileSystem, options.fileSystem), - Effect.provideService(Path.Path, options.path), - Effect.provideService(Crypto.Crypto, options.crypto), - ); - const resolveOwner = (launch: boolean) => - launch - ? Effect.scoped( - ensureSupervisor({ - stackId: options.id, - stateStore: options.store, - environment: options.environment, - }).pipe( - Effect.provideService(FileSystem.FileSystem, options.fileSystem), - Effect.provideService(Path.Path, options.path), - Effect.provideService(Crypto.Crypto, options.crypto), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, options.spawner), - ), - ).pipe(Effect.map((resolution) => Option.some(resolution))) - : provide( - readOwnerMetadata(options.environment.stateRoot, options.id, options.environment), - ).pipe( - Effect.map((owner) => - owner === undefined ? Option.none() : Option.some({ owner, launched: false }), - ), - ); - const ensureOffline = () => - provide(readOwnerMetadata(options.environment.stateRoot, options.id, options.environment)).pipe( - Effect.flatMap((owner) => - owner !== undefined - ? Effect.fail( - new StackOwnershipConflictError({ message: "A Supervisor still owns this stack" }), - ) - : provide(ownerLockExists(options.environment.stateRoot, options.id)).pipe( - Effect.flatMap((locked) => - locked - ? Effect.fail( - new StackOwnershipConflictError({ - message: "The stack ownership lease is still held", - }), - ) - : Effect.void, - ), - ), - ), - ); - const readOfflineState = ensureOffline().pipe( - Effect.andThen(provide(options.store.read(options.id))), - Effect.map(optionOf), - ); - const readPersistedState = provide(options.store.read(options.id)).pipe(Effect.map(optionOf)); - const directPrepareError = (cause: unknown): PrepareStackError => { - if (isStackError(cause) && isNarrowError(cause, PREPARE_STACK_ERROR_TAGS)) return cause; - return new StackPreparationError({ - stackId: options.id, - message: cause instanceof Error ? cause.message : String(cause), - cause, - }); - }; - const prepare = ( - prepareOptions?: PrepareStackOptions, - ): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const state = yield* provide(options.store.read(options.id)).pipe( - Effect.mapError(directPrepareError), - ); - if (state === undefined) - return yield* new StackStateInvalidError({ - stackId: options.id, - message: "Stack state is missing", - }); - let definition: StackDefinition; - let plan: ExecutionPlan; - if (prepareOptions?.config === undefined && state.definition !== undefined) { - definition = state.definition; - plan = yield* rebuildExecutionPlan(state.runtime, definition); - } else { - const compiled = yield* compileStack( - { - projectRoot: state.identity.projectRoot, - runtime: state.runtime, - config: prepareOptions?.config, - }, - state.definition === undefined ? undefined : { definition: state.definition }, - ).pipe(Effect.provideService(Path.Path, options.path)); - definition = compiled.definition; - plan = compiled.executionPlan; - } - const selected = new Set(); - if (prepareOptions?.capabilities === undefined) { - for (const name of CAPABILITY_NAMES) - if (definition.capabilities[name].enabled) selected.add(name); - } else { - const requested: CapabilityName[] = []; - for (const name of prepareOptions.capabilities) { - if (!isCapabilityName(name)) - return yield* new StackPreparationError({ - stackId: options.id, - capability: String(name), - message: `Unknown capability ${String(name)}`, - }); - requested.push(name); - } - for (const name of dependencyClosure(plan, requested)) { - if (!definition.capabilities[name].enabled) - return yield* new InvalidStackConfigError({ - stackId: options.id, - capability: name, - message: `Capability ${name} is disabled`, - }); - selected.add(name); - } - } - const workloads = plan.workloads.filter((workload) => selected.has(workload.capability)); - for (const workload of workloads) - prepareOptions?.onProgress?.({ - workloadId: workload.id, - capability: workload.capability, - state: "queued", - }); - const preparer = yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: options.environment.stateRoot, - runtime: state.runtime, - ...(options.environment.artifactCacheRoot === undefined - ? {} - : { artifactCacheRoot: options.environment.artifactCacheRoot }), - ...(options.containerEngineResolver === undefined - ? {} - : { containerEngineResolver: options.containerEngineResolver }), - }).pipe( - Effect.provideService(FileSystem.FileSystem, options.fileSystem), - Effect.provideService(Path.Path, options.path), - Effect.provideService(Crypto.Crypto, options.crypto), - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, options.spawner), - ); - const artifacts = yield* Effect.forEach( - workloads, - (workload) => preparer.prepare(state.runtime, workload, prepareOptions?.onProgress), - { concurrency: "unbounded" }, - ); - const byCapability = new Map>(); - for (const artifact of artifacts) { - const existing = byCapability.get(artifact.capability) ?? []; - byCapability.set(artifact.capability, [...existing, artifact]); - } - return { - capabilities: plan.startOrder - .filter((name) => selected.has(name)) - .map((name): PreparedCapability => { - const outcome: PreparedCapability["outcome"] = - state.runtime.kind === "native" - ? byCapability.get(name)?.some((entry) => entry.outcome === "downloaded") - ? "downloaded" - : "cached" - : byCapability.get(name)?.some((entry) => entry.outcome === "pulled") - ? "pulled" - : "cached"; - return { capability: name, version: definition.capabilities[name].version, outcome }; - }), - }; - }), - ).pipe(Effect.mapError(directPrepareError)); - const readLogs = (query?: LogQuery) => - ensureOffline().pipe( - Effect.andThen( - resolveStackPaths({ stateRoot: options.environment.stateRoot, stackId: options.id }), - ), - Effect.provideService(Path.Path, options.path), - Effect.flatMap((paths) => - readRetainedLogs( - options.fileSystem, - paths.logs, - query?.cursor === undefined || query.cursor.opaque === EMPTY_LOG_CURSOR.opaque - ? undefined - : { cursor: query.cursor }, - ).pipe( - Effect.map((scanned) => { - const selected = selectLogBatch(scanned, query); - return { - ...selected, - running: false, - } satisfies StackLogBatch; - }), - ), - ), - Effect.mapError((error) => - error instanceof StackOwnershipConflictError || error instanceof InvalidLogCursorError - ? error - : new StackStateInvalidError({ message: error.message, cause: error }), - ), - ); - const waitForRelease = Effect.gen(function* () { - if ( - (yield* readOwnerMetadata(options.environment.stateRoot, options.id, options.environment)) !== - undefined - ) - return yield* new StackOwnershipConflictError({ - message: "Supervisor is still shutting down", - }); - if (yield* ownerLockExists(options.environment.stateRoot, options.id)) - return yield* new StackOwnershipConflictError({ - message: "Supervisor ownership lease is still held", - }); - }).pipe( - Effect.retry(Schedule.spaced("25 millis").pipe(Schedule.upTo({ times: 200 }))), - Effect.mapError((error) => new StackOwnershipConflictError({ message: error.message })), - Effect.provideService(FileSystem.FileSystem, options.fileSystem), - Effect.provideService(Path.Path, options.path), - Effect.provideService(Crypto.Crypto, options.crypto), - ); - return { - resolveOwner, - readOfflineState, - readPersistedState, - readLogs, - waitForRelease, - prepare, - }; -}; - -export const createStack = ( - options: CreateStackOptions, -): Effect.Effect< - EffectStack, - CreateStackError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => - Effect.gen(function* () { - const env = yield* environment(); - const identity = yield* resolveStackIdentity({ - projectRoot: options.projectRoot, - name: options.name, - }); - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - // Concurrent initial writes may expose temporary files before state.json is published. - const persisted = yield* withRegistryLock( - env.stateRoot, - store - .read(stackId) - .pipe( - Effect.catch((error) => - isMissingStateRemnantError(error) - ? Effect.map(Effect.void, () => undefined) - : Effect.fail(error), - ), - ), - ); - const resolverOption = yield* Effect.serviceOption(ContainerEngineResolver).pipe( - Effect.map(Option.getOrUndefined), - ); - let dockerFallbackNotice: string | undefined; - let requestedRuntime: StackRuntime; - if (options.runtime?.kind === "container") { - requestedRuntime = { kind: "container", engine: options.runtime.engine ?? "docker" }; - } else if (options.runtime?.kind === "native") { - requestedRuntime = { kind: "native" }; - } else if (persisted !== undefined) { - requestedRuntime = persisted.runtime; - } else { - const selected = yield* selectDefaultRuntimeSelection(resolverOption); - requestedRuntime = selected.runtime; - dockerFallbackNotice = selected.dockerFallbackNotice; - } - if ( - persisted === undefined && - requestedRuntime.kind === "native" && - nativeRuntimeBlockedForUid() - ) - return yield* new StackRuntimeError({ message: NATIVE_ROOT_UNSUPPORTED_MESSAGE }); - const current = yield* store.initialize(stackId, stateInitial(identity, requestedRuntime)); - const runtimeMismatch = - options.runtime !== undefined && - (current.runtime.kind !== requestedRuntime.kind || - (requestedRuntime.kind === "container" && - current.runtime.kind === "container" && - current.runtime.engine !== requestedRuntime.engine)); - if (runtimeMismatch) - return yield* new StackRuntimeMismatchError({ - message: "Stack runtime is immutable for an existing identity", - }); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const dependencies = handleDependencies({ - environment: env, - store, - id: stackId, - fileSystem: fs, - path, - crypto, - spawner, - containerEngineResolver: resolverOption, - }); - const handle = yield* makeHandle(stackId, dependencies); - return dockerFallbackNotice === undefined ? handle : { ...handle, dockerFallbackNotice }; - }); - -export const openStack = ( - id: StackId, -): Effect.Effect< - EffectStack, - OpenStackError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => - Effect.gen(function* () { - const env = yield* environment(); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(id); - if (state === undefined) - return yield* new StackNotFoundError({ stackId: id, message: "Stack state was not found" }); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const containerEngineResolver = yield* Effect.serviceOption(ContainerEngineResolver).pipe( - Effect.map(Option.getOrUndefined), - ); - const dependencies = handleDependencies({ - environment: env, - store, - id, - fileSystem: fs, - path, - crypto, - spawner, - containerEngineResolver, - }); - return yield* makeHandle(id, dependencies); - }); - -export const findStack = ( - options: FindStackOptions, -): Effect.Effect< - Option.Option, - StackDiscoveryError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const env = yield* environment(); - const identity = yield* resolveStackIdentity({ - projectRoot: options.projectRoot, - name: options.name, - }); - const id = yield* deriveStackId(identity); - const state = yield* (yield* makeStackStateStore({ stateRoot: env.stateRoot })).read(id); - return state === undefined ? Option.none() : Option.some(descriptor(state, id)); - }); - -export interface StackDiscoveryIssue { - readonly id: StackId; - readonly error: StackDiscoveryError; -} - -/** The managed stack registry with entry-level read errors retained for bulk operations. */ -export interface StackDiscoveryResult { - readonly stacks: ReadonlyArray; - readonly errors: ReadonlyArray; -} - -const enrichStackDiscoveryError = ( - entry: StackId, - error: Effect.Error>, -): StackDiscoveryError => { - const message = `Failed to read managed stack ${entry}: ${error.message}`; - return Match.value(error).pipe( - Match.tag( - "InvalidProjectRootError", - (value) => new InvalidProjectRootError({ ...value, message, cause: error }), - ), - Match.tag( - "StackStateInvalidError", - (value) => new StackStateInvalidError({ ...value, stackId: entry, message, cause: error }), - ), - Match.tag( - "StackStateFormatUnsupportedError", - (value) => new StackStateFormatUnsupportedError({ ...value, message, cause: error }), - ), - Match.exhaustive, - ); -}; - -export const discoverStacks = ( - options: ListStacksOptions = {}, -): Effect.Effect< - StackDiscoveryResult, - StackDiscoveryError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const env = yield* environment(); - const fs = yield* FileSystem.FileSystem; - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const projectRoot = - options.projectRoot === undefined - ? undefined - : yield* fs - .realPath(options.projectRoot) - .pipe( - Effect.mapError((error) => new InvalidProjectRootError({ message: error.message })), - ); - if ( - !(yield* fs - .exists(env.stateRoot) - .pipe(Effect.mapError((error) => new StackStateInvalidError({ message: error.message })))) - ) - return { stacks: [], errors: [] }; - const entries = yield* fs - .readDirectory(env.stateRoot) - .pipe(Effect.mapError((error) => new StackStateInvalidError({ message: error.message }))); - const stacks: StackDescriptor[] = []; - const errors: StackDiscoveryIssue[] = []; - for (const entry of entries) { - if (!Schema.is(StackIdSchema)(entry)) continue; - const result = yield* store.read(entry).pipe( - Effect.catchTag("StackStateInvalidError", (error) => - isMissingStateRemnantError(error) ? Effect.void : Effect.fail(error), - ), - Effect.result, - ); - if (Result.isFailure(result)) { - errors.push({ id: entry, error: enrichStackDiscoveryError(entry, result.failure) }); - continue; - } - const state = result.success; - if ( - state !== undefined && - (projectRoot === undefined || state.identity.projectRoot === projectRoot) - ) - stacks.push(descriptor(state, entry)); - } - return { stacks, errors }; - }); - -type ConfigDrift = NonNullable; - -const isPlainRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null && !Array.isArray(value); - -const definitionDiffPaths = ( - left: unknown, - right: unknown, - prefix: string, - paths: string[], -): void => { - if (Object.is(left, right)) return; - if ((left === undefined || left === null) && (right === undefined || right === null)) return; - if (Array.isArray(left) && Array.isArray(right)) { - if (left.length !== right.length) { - paths.push(prefix); - return; - } - for (let index = 0; index < left.length; index++) { - definitionDiffPaths(left[index], right[index], `${prefix}.${index}`, paths); - } - return; - } - if (Array.isArray(left) || Array.isArray(right)) { - paths.push(prefix); - return; - } - if (isPlainRecord(left) && isPlainRecord(right)) { - const keys = new Set([...Object.keys(left), ...Object.keys(right)]); - for (const key of keys) { - definitionDiffPaths(left[key], right[key], `${prefix}.${key}`, paths); - } - return; - } - paths.push(prefix); -}; - -const secretDriftPaths = ( - candidate: ReadonlyArray, - persisted: PersistedStackState["secrets"], -): ReadonlyArray => { - const paths: string[] = []; - const supplied = new Map(candidate.map((entry) => [entry.slot, entry])); - for (const entry of candidate) { - const old = persisted[entry.slot]; - if (old === undefined) { - if (entry.policy === "passthrough" || entry.value !== undefined) - paths.push(`secrets.${entry.slot}`); - continue; - } - if (old.policy !== entry.policy) { - paths.push(`secrets.${entry.slot}`); - continue; - } - if (entry.policy === "passthrough" || entry.value !== undefined) { - const value = entry.value === undefined ? undefined : Redacted.value(entry.value); - if (value !== old.value) paths.push(`secrets.${entry.slot}`); - } - } - for (const [slot, old] of Object.entries(persisted)) { - if (old.policy === "passthrough" && !supplied.has(slot)) paths.push(`secrets.${slot}`); - } - return paths; -}; - -const inspectConfigDrift = ( - state: PersistedStackState, - config: StackConfig, -): Effect.Effect => - Effect.gen(function* () { - const compiled = yield* compileStack( - { - projectRoot: state.identity.projectRoot, - runtime: state.runtime, - config, - }, - state.definition === undefined ? undefined : { definition: state.definition }, - ); - if (state.definition === undefined) - return { status: "unconfigured", paths: [] } satisfies ConfigDrift; - const paths: string[] = []; - if (!sameDefinition(state.definition, compiled.definition)) - definitionDiffPaths(state.definition, compiled.definition, "definition", paths); - paths.push(...secretDriftPaths(compiled.secrets, state.secrets)); - const uniquePaths = [...new Set(paths)].sort(); - return { - status: uniquePaths.length === 0 ? "unchanged" : "changed", - paths: uniquePaths, - } satisfies ConfigDrift; - }); - -export const listStacks = ( - options: ListStacksOptions = {}, -): Effect.Effect< - ReadonlyArray, - StackDiscoveryError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - discoverStacks(options).pipe( - Effect.flatMap(({ stacks, errors }) => { - const firstError = errors[0]; - return firstError === undefined ? Effect.succeed(stacks) : Effect.fail(firstError.error); - }), - ); - -export const inspectStack = ( - id: StackId, - options: InspectStackOptions = {}, -): Effect.Effect< - StackInspection, - StackNotFoundError | StackDiscoveryError | InvalidStackConfigError | StackVersionUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const env = yield* environment(); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(id); - if (state === undefined) - return yield* new StackNotFoundError({ stackId: id, message: "Stack state was not found" }); - const configDrift = - options.config === undefined ? undefined : yield* inspectConfigDrift(state, options.config); - const metadata = yield* readOwnerMetadata(env.stateRoot, id, env); - if (metadata === undefined) - return { - descriptor: descriptor(state, id), - owner: (yield* ownerLockExists(env.stateRoot, id)) ? "unreachable" : "absent", - ...(configDrift === undefined ? {} : { configDrift }), - }; - if (metadata.rpcRelease !== STACK_RPC_RELEASE) - return { - descriptor: descriptor(state, id), - owner: "incompatible", - ...(configDrift === undefined ? {} : { configDrift }), - }; - const status = yield* Effect.scoped( - Effect.gen(function* () { - const client = makeControlClient(metadata.endpoint, { - stackId: id, - ownerSessionId: metadata.ownerSessionId, - }); - const rpc = yield* client.rpc; - return yield* rpc.status(undefined); - }), - ).pipe(Effect.exit); - if (Exit.isFailure(status)) { - const failure = Cause.findErrorOption(status.cause); - if (Option.isSome(failure) && isOwnerUnreachable(failure.value)) - return { - descriptor: descriptor(state, id), - owner: "unreachable", - ...(configDrift === undefined ? {} : { configDrift }), - }; - return { - descriptor: descriptor(state, id), - owner: "running", - ...(configDrift === undefined ? {} : { configDrift }), - }; - } - return { - descriptor: descriptor(state, id), - owner: "running", - status: status.value, - ...(configDrift === undefined ? {} : { configDrift }), - }; - }); diff --git a/packages/stack/src/public/EphemeralPostgres.ts b/packages/stack/src/public/EphemeralPostgres.ts deleted file mode 100644 index dfcdaa2743..0000000000 --- a/packages/stack/src/public/EphemeralPostgres.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { Crypto, Effect, FileSystem, Path, Redacted, Scope } from "effect"; -import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -import { DatabaseModule } from "../model/capabilities/database.ts"; -import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; -import { createEphemeralPostgresCluster } from "../runtime/EphemeralPostgres.ts"; -import type { EphemeralPostgresCreateError, EphemeralPostgresError } from "./Errors.ts"; -import { StackVersionUnsupportedError } from "./Errors.ts"; -import type { StackRuntime, StackRuntimePreference } from "./Runtime.ts"; - -export interface EphemeralPostgresSettings { - readonly [key: string]: string | number | boolean | undefined; -} - -export interface CreateEphemeralPostgresOptions { - /** Omitted preference uses Docker when installed, otherwise native. */ - readonly runtime?: StackRuntimePreference; - /** Exact catalog release or major selector such as `"17"`. */ - readonly version?: string; - readonly port?: number; - readonly databasePassword: Redacted.Redacted; - readonly jwtSecret: Redacted.Redacted; - readonly jwtExpiry?: number; - readonly postgresSettings?: EphemeralPostgresSettings; - readonly healthTimeout?: string; - /** Stopped-cluster PGDATA tar to restore before the first start. */ - readonly restoreFrom?: string; - /** Cache identity written into the snapshot runtime marker. Restore without a key still accepts a keyless marker. */ - readonly snapshotKey?: string; -} - -export interface EphemeralPostgresRelease { - readonly version: string; - readonly image: string; -} - -export type EphemeralPostgresServices = - | ChildProcessSpawnerService - | Scope.Scope - | FileSystem.FileSystem - | Path.Path; - -export interface EffectEphemeralPostgres { - readonly host: string; - readonly port: number; - readonly version: string; - readonly runtime: StackRuntime; - /** Catalog identity hashed into CLI shadow-cache keys. */ - readonly artifactIdentity: string; - readonly url: Redacted.Redacted; - /** Prepared postgres artifact root when extras such as `pg_dump` may be present. */ - readonly nativeArtifactRoot?: string; - /** Present when auto-select persisted native because the Docker daemon was down. */ - readonly dockerFallbackNotice?: string; - /** Container network id so schema-init one-shots can join and dial `supabase-database:5432`. */ - readonly networkId?: string; - readonly start: Effect.Effect; - readonly stop: Effect.Effect; - readonly exportPgData: ( - tarPath: string, - snapshotKey?: string, - ) => Effect.Effect; -} - -/** Resolves a Postgres catalog release the same way stack compilation does. */ -export const resolveEphemeralPostgresRelease = ( - version?: string, -): Effect.Effect => { - const requested = version ?? DatabaseModule.defaultVersion; - // A running stack can outlive a catalog pin bump; dump/test still need a client. - const selected = - DatabaseModule.releases[requested] ?? - (requested.includes(".") ? DatabaseModule.releases[requested.split(".")[0] ?? ""] : undefined); - const release = - selected === undefined - ? catalogReleaseFor("database:database", requested) - : catalogReleaseFor("database:database", selected.version); - if (release === undefined) - return Effect.fail( - new StackVersionUnsupportedError({ - message: `Unsupported PostgreSQL version ${requested}`, - version: requested, - capability: "database", - }), - ); - return Effect.succeed({ version: release.version, image: release.containerImage }); -}; - -export const createEphemeralPostgres = ( - options: CreateEphemeralPostgresOptions, -): Effect.Effect< - EffectEphemeralPostgres, - EphemeralPostgresCreateError, - Scope.Scope | FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => createEphemeralPostgresCluster(options); diff --git a/packages/stack/src/public/Errors.ts b/packages/stack/src/public/Errors.ts deleted file mode 100644 index eab40bb8a3..0000000000 --- a/packages/stack/src/public/Errors.ts +++ /dev/null @@ -1,411 +0,0 @@ -import { Data, Predicate } from "effect"; -import type { StackId } from "./StackId.ts"; -import type { ContainerEngineKind } from "../runtime/ContainerEngine.ts"; -import type { StackRecovery } from "./Status.ts"; - -/** Common context present on every public stack error. */ -interface ErrorFields { - readonly message: string; - readonly cause?: unknown; -} - -export interface IdentityErrorFields extends ErrorFields { - readonly path?: string; - readonly reason?: string; - readonly stackId?: StackId; - readonly name?: string; -} - -export interface ProjectRootErrorFields extends ErrorFields { - readonly projectRoot?: string; - readonly stateRoot?: string; -} - -export interface ConfigErrorFields extends ErrorFields { - readonly stackId?: StackId; - readonly capability?: string; - readonly dependency?: string; - readonly version?: string; - readonly workload?: string; - readonly functionsRoot?: string; - readonly setting?: string; - readonly function?: string; - readonly environment?: string; - readonly provider?: string; - readonly target?: string; -} - -export class InvalidStackIdentityError extends Data.TaggedError( - "InvalidStackIdentityError", -) {} -export class InvalidProjectRootError extends Data.TaggedError( - "InvalidProjectRootError", -) {} -export class InvalidStackConfigError extends Data.TaggedError( - "InvalidStackConfigError", -) {} -export class StackVersionUnsupportedError extends Data.TaggedError( - "StackVersionUnsupportedError", -) {} - -export class StackNotFoundError extends Data.TaggedError("StackNotFoundError")< - ErrorFields & { readonly stackId?: StackId } -> {} -export class StackOwnershipConflictError extends Data.TaggedError("StackOwnershipConflictError")< - ErrorFields & { readonly stackId?: StackId } -> {} -export class StackRuntimeMismatchError extends Data.TaggedError( - "StackRuntimeMismatchError", -) {} - -export class StackNotRunningError extends Data.TaggedError("StackNotRunningError")< - ErrorFields & { readonly stackId?: StackId } -> {} -export class StackMustBeStoppedError extends Data.TaggedError("StackMustBeStoppedError")< - ErrorFields & { readonly slot?: string; readonly stackId?: StackId; readonly guidance?: string } -> {} -export class StackLifecycleConflictError extends Data.TaggedError("StackLifecycleConflictError")< - ErrorFields & { readonly stackId?: StackId; readonly recovery?: StackRecovery } -> {} - -export class StackStateInvalidError extends Data.TaggedError("StackStateInvalidError")< - ErrorFields & { - readonly stackId?: StackId; - readonly path?: string; - readonly code?: string; - readonly slot?: string; - } -> {} -export class InvalidLogCursorError extends Data.TaggedError("InvalidLogCursorError") {} -export class StackStateFormatUnsupportedError extends Data.TaggedError( - "StackStateFormatUnsupportedError", -) {} -export class StackUpgradeRequiredError extends Data.TaggedError("StackUpgradeRequiredError")< - ErrorFields & { readonly expectedRelease?: string; readonly actualRelease?: string } -> {} -export class StackSecretMismatchError extends Data.TaggedError("StackSecretMismatchError")< - ErrorFields & { readonly slot?: string } -> {} -export class InvalidJwtSigningMaterialError extends Data.TaggedError( - "InvalidJwtSigningMaterialError", -) {} - -export class PortAllocationError extends Data.TaggedError("PortAllocationError")< - ErrorFields & { readonly port?: number; readonly field?: string } -> {} -export class PortUnavailableError extends Data.TaggedError("PortUnavailableError")< - ErrorFields & { - readonly port?: number; - readonly field?: string; - readonly address?: string; - } -> {} - -export class GatewayActivationError extends Data.TaggedError("GatewayActivationError")< - ErrorFields & { - readonly capability?: string; - readonly workload?: string; - readonly recovery?: StackRecovery; - } -> {} - -export class StackPreparationError extends Data.TaggedError("StackPreparationError")< - ConfigErrorFields & { - readonly path?: string; - readonly field?: string; - readonly value?: unknown; - readonly service?: string; - readonly platform?: string; - readonly image?: string; - readonly key?: string; - readonly name?: string; - } -> {} -export class ArtifactIntegrityError extends Data.TaggedError("ArtifactIntegrityError")< - ErrorFields & { - readonly expected?: string; - readonly actual?: string; - readonly path?: string; - readonly key?: string; - } -> {} -export class ContainerPullError extends Data.TaggedError("ContainerPullError")< - ErrorFields & { readonly workload?: string; readonly image?: string } -> {} - -export class StackRuntimeError extends Data.TaggedError("StackRuntimeError")< - ErrorFields & { readonly stackId?: StackId; readonly workloadId?: string } -> {} -export class StackCleanupError extends Data.TaggedError("StackCleanupError") {} -export class ContainerEngineError extends Data.TaggedError("ContainerEngineError")< - ErrorFields & { readonly engine?: ContainerEngineKind } -> {} -export class StackDestructionError extends Data.TaggedError("StackDestructionError") {} -export class EphemeralPostgresError extends Data.TaggedError("EphemeralPostgresError")< - ErrorFields & { - readonly reason?: - | "not-stopped" - | "not-running" - | "snapshot" - | "restore-mismatch" - | "bootstrap" - | "destroy"; - readonly path?: string; - readonly version?: string; - } -> {} -export class RequiresActivatedProcessError extends Data.TaggedError( - "RequiresActivatedProcessError", -) {} -export class PostgresClientError extends Data.TaggedError("PostgresClientError")< - ErrorFields & { - readonly reason?: "missing-bin" | "spawn"; - readonly bin?: string; - readonly engine?: ContainerEngineKind; - } -> {} - -/** Stable wire tags for errors produced by the managed stack runtime. */ -export const STACK_ERROR_TAGS = [ - "InvalidStackIdentityError", - "InvalidProjectRootError", - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "StackNotFoundError", - "StackOwnershipConflictError", - "StackRuntimeMismatchError", - "StackNotRunningError", - "StackMustBeStoppedError", - "StackLifecycleConflictError", - "StackStateInvalidError", - "InvalidLogCursorError", - "StackStateFormatUnsupportedError", - "StackUpgradeRequiredError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - "PortAllocationError", - "PortUnavailableError", - "GatewayActivationError", - "StackPreparationError", - "ArtifactIntegrityError", - "ContainerPullError", - "StackRuntimeError", - "StackCleanupError", - "ContainerEngineError", - "StackDestructionError", - "EphemeralPostgresError", - "RequiresActivatedProcessError", - "PostgresClientError", -] as const; - -export type StackErrorTag = (typeof STACK_ERROR_TAGS)[number]; - -export const isStackErrorTag = (tag: string): tag is StackErrorTag => - STACK_ERROR_TAGS.some((candidate) => candidate === tag); - -export type StackError = - | InvalidStackIdentityError - | InvalidProjectRootError - | InvalidStackConfigError - | StackVersionUnsupportedError - | StackNotFoundError - | StackOwnershipConflictError - | StackRuntimeMismatchError - | StackNotRunningError - | StackMustBeStoppedError - | StackLifecycleConflictError - | StackStateInvalidError - | InvalidLogCursorError - | StackStateFormatUnsupportedError - | StackUpgradeRequiredError - | StackSecretMismatchError - | InvalidJwtSigningMaterialError - | PortAllocationError - | PortUnavailableError - | GatewayActivationError - | StackPreparationError - | ArtifactIntegrityError - | ContainerPullError - | StackRuntimeError - | StackCleanupError - | ContainerEngineError - | StackDestructionError - | EphemeralPostgresError - | RequiresActivatedProcessError - | PostgresClientError; - -export const isStackError = (value: unknown): value is StackError => - Predicate.hasProperty(value, "_tag") && - typeof value._tag === "string" && - isStackErrorTag(value._tag); - -type ErrorByTag = Extract; - -export const CREATE_STACK_ERROR_TAGS = [ - "InvalidStackIdentityError", - "InvalidProjectRootError", - "StackOwnershipConflictError", - "StackRuntimeMismatchError", - "ContainerEngineError", - "StackRuntimeError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", -] as const satisfies ReadonlyArray; -export type CreateStackError = ErrorByTag<(typeof CREATE_STACK_ERROR_TAGS)[number]>; - -export const OPEN_STACK_ERROR_TAGS = [ - "StackNotFoundError", - "StackOwnershipConflictError", - "StackRuntimeMismatchError", - "InvalidProjectRootError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", - "StackUpgradeRequiredError", -] as const satisfies ReadonlyArray; -export type OpenStackError = ErrorByTag<(typeof OPEN_STACK_ERROR_TAGS)[number]>; - -export const STACK_DISCOVERY_ERROR_TAGS = [ - "InvalidStackIdentityError", - "InvalidProjectRootError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", -] as const satisfies ReadonlyArray; -export type StackDiscoveryError = ErrorByTag<(typeof STACK_DISCOVERY_ERROR_TAGS)[number]>; - -export const STACK_STATUS_ERROR_TAGS = [ - "StackNotFoundError", - "StackOwnershipConflictError", - "StackLifecycleConflictError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", - "StackUpgradeRequiredError", -] as const satisfies ReadonlyArray; -export type StackStatusError = ErrorByTag<(typeof STACK_STATUS_ERROR_TAGS)[number]>; - -export const STACK_CREDENTIALS_ERROR_TAGS = [ - "InvalidStackConfigError", - "StackNotFoundError", - "StackNotRunningError", - "StackOwnershipConflictError", - "StackLifecycleConflictError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - "StackUpgradeRequiredError", -] as const satisfies ReadonlyArray; -export type StackCredentialsError = ErrorByTag<(typeof STACK_CREDENTIALS_ERROR_TAGS)[number]>; - -export const PREPARE_STACK_ERROR_TAGS = [ - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "InvalidProjectRootError", - "StackPreparationError", - "ArtifactIntegrityError", - "ContainerPullError", - "ContainerEngineError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", -] as const satisfies ReadonlyArray; -export type PrepareStackError = ErrorByTag<(typeof PREPARE_STACK_ERROR_TAGS)[number]>; - -export const STACK_START_ERROR_TAGS = [ - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "StackOwnershipConflictError", - "StackNotRunningError", - "StackMustBeStoppedError", - "StackLifecycleConflictError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", - "StackUpgradeRequiredError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - "PortAllocationError", - "PortUnavailableError", - "StackPreparationError", - "ArtifactIntegrityError", - "ContainerPullError", - "StackRuntimeError", - "StackCleanupError", - "ContainerEngineError", -] as const satisfies ReadonlyArray; -export type StackStartError = ErrorByTag<(typeof STACK_START_ERROR_TAGS)[number]>; - -/** Stable maintenance stop reports cleanup failures as lifecycle conflicts with their message. */ -export const STACK_STOP_ERROR_TAGS = [ - "StackOwnershipConflictError", - "StackLifecycleConflictError", - "StackStateInvalidError", - "StackCleanupError", -] as const satisfies ReadonlyArray; -export type StackStopError = ErrorByTag<(typeof STACK_STOP_ERROR_TAGS)[number]>; - -export const STACK_LOGS_ERROR_TAGS = [ - "StackNotFoundError", - "StackNotRunningError", - "StackStateInvalidError", - "InvalidLogCursorError", - "StackOwnershipConflictError", - "StackLifecycleConflictError", - "StackUpgradeRequiredError", -] as const satisfies ReadonlyArray; -export type StackLogsError = ErrorByTag<(typeof STACK_LOGS_ERROR_TAGS)[number]>; - -export const DESTROY_STACK_ERROR_TAGS = [ - "StackDestructionError", - "StackNotFoundError", - "StackOwnershipConflictError", - "StackLifecycleConflictError", - "ContainerEngineError", - "StackCleanupError", - "StackUpgradeRequiredError", -] as const satisfies ReadonlyArray; -export type DestroyStackError = ErrorByTag<(typeof DESTROY_STACK_ERROR_TAGS)[number]>; - -export const RESET_DATABASE_ERROR_TAGS = [ - "StackNotFoundError", - ...STACK_START_ERROR_TAGS, -] as const satisfies ReadonlyArray; -export type ResetDatabaseError = ErrorByTag<(typeof RESET_DATABASE_ERROR_TAGS)[number]>; - -export const EPHEMERAL_POSTGRES_ERROR_TAGS = [ - "EphemeralPostgresError", - "StackVersionUnsupportedError", - "PortUnavailableError", - "StackPreparationError", - "ArtifactIntegrityError", - "ContainerPullError", - "ContainerEngineError", - "StackRuntimeError", -] as const satisfies ReadonlyArray; -export type EphemeralPostgresCreateError = ErrorByTag< - (typeof EPHEMERAL_POSTGRES_ERROR_TAGS)[number] ->; - -export const POSTGRES_CLIENT_ERROR_TAGS = [ - "PostgresClientError", - "StackVersionUnsupportedError", - "StackPreparationError", - "ArtifactIntegrityError", - "ContainerPullError", - "ContainerEngineError", -] as const satisfies ReadonlyArray; -export type PostgresClientRunError = ErrorByTag<(typeof POSTGRES_CLIENT_ERROR_TAGS)[number]>; - -export const SCHEMA_INIT_ERROR_TAGS = [ - "RequiresActivatedProcessError", - "InvalidStackConfigError", - "StackVersionUnsupportedError", - "InvalidProjectRootError", - "InvalidStackIdentityError", - "StackPreparationError", - "ArtifactIntegrityError", - "ContainerPullError", - "ContainerEngineError", - "StackSecretMismatchError", - "InvalidJwtSigningMaterialError", - "StackRuntimeError", - "StackMustBeStoppedError", - "StackStateInvalidError", - "StackStateFormatUnsupportedError", -] as const satisfies ReadonlyArray; -export type SchemaInitError = ErrorByTag<(typeof SCHEMA_INIT_ERROR_TAGS)[number]>; diff --git a/packages/stack/src/public/Logs.ts b/packages/stack/src/public/Logs.ts deleted file mode 100644 index 0763e3d84b..0000000000 --- a/packages/stack/src/public/Logs.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Schema } from "effect"; -import { CapabilityNameSchema, type CapabilityName } from "./Capability.ts"; - -export const LogCursorSchema = Schema.Struct({ - opaque: Schema.String, -}); -export type LogCursor = Schema.Schema.Type; - -export const LogQuerySchema = Schema.Struct({ - capabilities: Schema.optionalKey(Schema.Array(CapabilityNameSchema)), - cursor: Schema.optionalKey(LogCursorSchema), - tail: Schema.optionalKey(Schema.Finite), -}); -export type LogQuery = Schema.Schema.Type; - -export const StackLogEntrySchema = Schema.Struct({ - cursor: LogCursorSchema, - timestamp: Schema.String, - source: Schema.Union([CapabilityNameSchema, Schema.Literals(["supervisor", "gateway"] as const)]), - stream: Schema.Literals(["stdout", "stderr", "internal"] as const), - message: Schema.String, -}); -export type StackLogEntry = Schema.Schema.Type; - -export const StackLogBatchSchema = Schema.Struct({ - entries: Schema.Array(StackLogEntrySchema), - cursor: LogCursorSchema, - running: Schema.Boolean, -}); -export type StackLogBatch = Schema.Schema.Type; - -export type StackLogSource = CapabilityName | "supervisor" | "gateway"; diff --git a/packages/stack/src/public/PostgresClient.ts b/packages/stack/src/public/PostgresClient.ts deleted file mode 100644 index eb87d3af3d..0000000000 --- a/packages/stack/src/public/PostgresClient.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type { Crypto, Effect, FileSystem, Path } from "effect"; -import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -import { runPostgresClient as runPreparedPostgresClient } from "../runtime/postgres-client.ts"; -import type { PostgresClientMount } from "../runtime/postgres-client-args.ts"; -import type { PostgresClientRunError } from "./Errors.ts"; -import type { StackRuntimePreference } from "./Runtime.ts"; - -export type { PostgresClientMount }; - -export interface RunPostgresClientOptions { - /** Exact catalog release or major selector such as `"17"`. */ - readonly version?: string; - /** Omitted preference uses Docker when installed, otherwise native. */ - readonly runtime?: StackRuntimePreference; - readonly argv: ReadonlyArray; - readonly env?: Readonly>; - readonly cwd?: string; - readonly mounts?: ReadonlyArray; - readonly network?: "host" | { readonly name: string }; - readonly extraHosts?: ReadonlyArray; - readonly securityOpt?: ReadonlyArray; - readonly onStdout: (chunk: Uint8Array) => Effect.Effect; - readonly teeStderr?: boolean; - readonly captureStderr?: boolean; -} - -export interface PostgresClientResult { - readonly exitCode: number; - readonly stderr: string; -} - -export type PostgresClientServices = - | ChildProcessSpawnerService - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto; - -/** Runs `argv` against a prepared catalog Postgres artifact or image. */ -export const runPostgresClient = ( - options: RunPostgresClientOptions, -): Effect.Effect => - runPreparedPostgresClient(options); diff --git a/packages/stack/src/public/PromiseStack.ts b/packages/stack/src/public/PromiseStack.ts deleted file mode 100644 index 20245d2e90..0000000000 --- a/packages/stack/src/public/PromiseStack.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { - Crypto, - Effect, - Exit, - FileSystem, - Layer, - Option, - Path, - Redacted, - Schema, - Scope, - Stream, -} from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import { - createStack as createEffectStack, - discoverStacks as discoverEffectStacks, - findStack as findEffectStack, - inspectStack as inspectEffectStack, - listStacks as listEffectStacks, - openStack as openEffectStack, - type EffectStack, - type CreateStackOptions, - type FindStackOptions, - type ListStacksOptions, - type StackDiscoveryResult, - type StackDiscoveryIssue, - type PrepareStackOptions, - type StartStackOptions, -} from "./EffectStack.ts"; -import type { StackConfig } from "./Config.ts"; -import { StackConfigSchema } from "./Config.ts"; -import { PromiseStackCredentialsSchema, type PromiseStackCredentials } from "./Credentials.ts"; -import type { LogQuery, StackLogBatch, StackLogEntry } from "./Logs.ts"; -import type { StackDescriptor, StackInspection, StackStatus } from "./Status.ts"; -import type { StackId } from "./StackId.ts"; -import type { PreparedCapability, PrepareStackResult } from "./EffectStack.ts"; -import { InvalidStackConfigError } from "./Errors.ts"; -import { StackRuntimeEnvironment, type StackRuntimeEnvironmentValue } from "../state/Ownership.ts"; -import { - createEphemeralPostgres as createEffectEphemeralPostgres, - type CreateEphemeralPostgresOptions, -} from "./EphemeralPostgres.ts"; -import type { StackRuntime } from "./Runtime.ts"; - -/** Recursively replaces Effect `Redacted` leaves with their plain value. */ -type Unredacted = - T extends Redacted.Redacted - ? Unredacted - : T extends readonly (infer Item)[] - ? ReadonlyArray> - : T extends object - ? { readonly [Key in keyof T]: Unredacted } - : T; - -export type PromiseStackConfig = Unredacted; -export type PromiseStartStackOptions = Omit & { - readonly config?: PromiseStackConfig; -}; - -export interface PromiseInspectStackOptions { - readonly config?: PromiseStackConfig; -} -export type PromisePrepareStackOptions = Omit & { - readonly config?: PromiseStackConfig; -}; - -export interface PromiseStack { - readonly id: StackId; - readonly status: () => Promise; - readonly credentials: () => Promise; - readonly prepare: (options?: PromisePrepareStackOptions) => Promise; - readonly start: (options?: PromiseStartStackOptions) => Promise; - readonly stop: () => Promise; - readonly destroy: () => Promise; - readonly resetDatabase: () => Promise; - readonly logs: (query?: LogQuery) => Promise; - readonly followLogs: (query?: LogQuery) => AsyncIterable; -} - -export type PromiseCreateEphemeralPostgresOptions = Omit< - CreateEphemeralPostgresOptions, - "databasePassword" | "jwtSecret" -> & { - readonly databasePassword: string; - readonly jwtSecret: string; -}; - -export interface PromiseEphemeralPostgres { - readonly host: string; - readonly port: number; - readonly version: string; - readonly runtime: StackRuntime; - readonly artifactIdentity: string; - readonly url: string; - readonly start: () => Promise; - readonly stop: () => Promise; - readonly exportPgData: (tarPath: string) => Promise; - readonly destroy: () => Promise; -} - -interface PromiseStackApi { - readonly createStack: (options: CreateStackOptions) => Promise; - readonly openStack: (id: StackId) => Promise; - readonly findStack: (options: FindStackOptions) => Promise; - readonly listStacks: (options?: ListStacksOptions) => Promise>; - readonly discoverStacks: (options?: ListStacksOptions) => Promise; - readonly inspectStack: ( - id: StackId, - options?: PromiseInspectStackOptions, - ) => Promise; - readonly createEphemeralPostgres: ( - options: PromiseCreateEphemeralPostgresOptions, - ) => Promise; -} - -type PlatformLayer = typeof NodeServices.layer; -type RuntimeRequirements = - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | ChildProcessSpawner.ChildProcessSpawner; - -// Turns plain JSON config values into the Redacted values the Effect handle expects. -const stackConfigJsonCodec = Schema.toCodecJson(StackConfigSchema); -const decodePromiseConfig = ( - input: PromiseStackConfig, -): Effect.Effect => - Schema.decodeEffect(stackConfigJsonCodec)(input, { onExcessProperty: "error" }).pipe( - Effect.mapError( - (cause) => - new InvalidStackConfigError({ - message: `Invalid stack config: ${String(cause)}`, - cause, - }), - ), - ); - -/** Recursively unwraps every Redacted value at the Promise boundary. */ -function unredact(input: T): Unredacted; -function unredact(input: unknown): unknown { - if (Redacted.isRedacted(input)) return unredact(Redacted.value(input)); - if (Array.isArray(input)) return input.map(unredact); - if (typeof input === "object" && input !== null) { - const output: Record = {}; - for (const [key, value] of Object.entries(input)) output[key] = unredact(value); - return output; - } - return input; -} - -const adaptStream = (stream: Stream.Stream): AsyncIterable => - Stream.toAsyncIterable(stream); - -/** Adapts an already-created Effect handle; exported for facade integration tests. */ -export const adaptEffectStack = (effectStack: EffectStack): PromiseStack => { - const invoke = (effect: Effect.Effect): Promise => Effect.runPromise(effect); - const withConfig = ( - options: { readonly config?: PromiseStackConfig } | undefined, - operation: (config?: StackConfig) => Effect.Effect, - ): Effect.Effect => - Effect.gen(function* () { - const config = - options?.config === undefined ? undefined : yield* decodePromiseConfig(options.config); - return yield* operation(config); - }); - return { - id: effectStack.id, - status: () => invoke(effectStack.status), - credentials: () => - invoke(effectStack.credentials).then((value) => - Schema.decodeSync(PromiseStackCredentialsSchema)(unredact(value)), - ), - prepare: (options) => - invoke( - withConfig(options, (config) => - effectStack.prepare( - options === undefined - ? undefined - : { - ...(options.capabilities === undefined - ? {} - : { capabilities: options.capabilities }), - ...(options.onProgress === undefined ? {} : { onProgress: options.onProgress }), - ...(config === undefined ? {} : { config }), - }, - ), - ), - ), - start: (options) => - invoke( - withConfig(options, (config) => - options === undefined - ? effectStack.start() - : effectStack.start(config === undefined ? {} : { config }), - ), - ), - stop: () => invoke(effectStack.stop), - destroy: () => invoke(effectStack.destroy), - resetDatabase: () => invoke(effectStack.resetDatabase), - logs: (query) => invoke(effectStack.logs(query)), - followLogs: (query) => adaptStream(effectStack.followLogs(query)), - }; -}; - -export const makePromiseApi = ( - platformLayer: PlatformLayer = NodeServices.layer, - runtimeEnvironment?: StackRuntimeEnvironmentValue, -): PromiseStackApi => { - const providedLayer = - runtimeEnvironment === undefined - ? platformLayer - : Layer.mergeAll(platformLayer, Layer.succeed(StackRuntimeEnvironment, runtimeEnvironment)); - const run = (effect: Effect.Effect): Promise => - Effect.runPromise(effect.pipe(Effect.provide(providedLayer))); - - const createOrOpen = ( - effect: Effect.Effect, - ): Promise => run(effect).then(adaptEffectStack); - return { - createStack: (options) => createOrOpen(createEffectStack(options)), - openStack: (id) => createOrOpen(openEffectStack(id)), - findStack: (options) => - run(findEffectStack(options)).then((value) => Option.getOrUndefined(value)), - listStacks: (options) => run(listEffectStacks(options)), - discoverStacks: (options) => run(discoverEffectStacks(options)), - inspectStack: (id, options) => - run( - options?.config === undefined - ? inspectEffectStack(id) - : decodePromiseConfig(options.config).pipe( - Effect.flatMap((config) => inspectEffectStack(id, { config })), - ), - ), - // Promise facade at the published edge; the Effect API owns cluster lifetime. - // oxlint-disable-next-line effecttsgo/async-function -- public Promise API - createEphemeralPostgres: async (options) => { - const scope = await Effect.runPromise(Scope.make()); - const close = () => - Effect.runPromise(Scope.close(scope, Exit.void).pipe(Effect.provide(providedLayer))); - const invoke = ( - effect: Effect.Effect, - ): Promise => - Effect.runPromise( - effect.pipe(Effect.provideService(Scope.Scope, scope), Effect.provide(providedLayer)), - ); - try { - const handle = await invoke( - createEffectEphemeralPostgres({ - ...options, - databasePassword: Redacted.make(options.databasePassword), - jwtSecret: Redacted.make(options.jwtSecret), - }), - ); - return { - host: handle.host, - port: handle.port, - version: handle.version, - runtime: handle.runtime, - artifactIdentity: handle.artifactIdentity, - url: Redacted.value(handle.url), - start: () => invoke(handle.start), - stop: () => invoke(handle.stop), - exportPgData: (tarPath) => invoke(handle.exportPgData(tarPath)), - destroy: close, - }; - } catch (cause) { - await close().catch(() => undefined); - throw cause; - } - }, - }; -}; - -const defaultApi = makePromiseApi(); -export const createStack = defaultApi.createStack; -export const openStack = defaultApi.openStack; -export const findStack = defaultApi.findStack; -export const listStacks = defaultApi.listStacks; -export const discoverStacks = defaultApi.discoverStacks; -export const inspectStack = defaultApi.inspectStack; -export const createEphemeralPostgres = defaultApi.createEphemeralPostgres; - -export type { - CreateStackOptions, - FindStackOptions, - ListStacksOptions, - PreparedCapability, - StackDiscoveryIssue, - StackDiscoveryResult, -}; diff --git a/packages/stack/src/public/Runtime.ts b/packages/stack/src/public/Runtime.ts deleted file mode 100644 index dc4197bfdd..0000000000 --- a/packages/stack/src/public/Runtime.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Schema } from "effect"; - -export const RuntimeEngineSchema = Schema.Literals(["docker", "podman"] as const); -export type RuntimeEngine = Schema.Schema.Type; - -export const StackRuntimeSchema = Schema.Union([ - Schema.Struct({ kind: Schema.Literal("native") }), - Schema.Struct({ kind: Schema.Literal("container"), engine: RuntimeEngineSchema }), -]); -export type StackRuntime = Schema.Schema.Type; - -export const StackRuntimePreferenceSchema = Schema.Union([ - Schema.Struct({ kind: Schema.Literal("native") }), - Schema.Struct({ - kind: Schema.Literal("container"), - engine: Schema.optionalKey(RuntimeEngineSchema), - }), -]); -export type StackRuntimePreference = Schema.Schema.Type; diff --git a/packages/stack/src/public/SchemaInit.ts b/packages/stack/src/public/SchemaInit.ts deleted file mode 100644 index 21d524f425..0000000000 --- a/packages/stack/src/public/SchemaInit.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { Crypto, Effect, FileSystem, Path, Redacted } from "effect"; -import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -import type { SchemaInitError } from "./Errors.ts"; -import type { StackConfig } from "./Config.ts"; -import type { StackRuntime } from "./Runtime.ts"; -import type { StackId } from "./StackId.ts"; -import { schemaInitWorkloads, schemaInitArtifactIdentity } from "../runtime/SchemaInit.ts"; -import type { ContainerEngine } from "../runtime/ContainerEngine.ts"; -import type { RuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; - -export { schemaInitArtifactIdentity }; - -export const SCHEMA_INIT_CAPABILITY_NAMES = [ - "auth", - "storage", - "realtime", - "analytics", - "pooler", -] as const; -export type SchemaInitCapabilityName = (typeof SCHEMA_INIT_CAPABILITY_NAMES)[number]; - -export interface SchemaInitSecrets { - readonly databasePassword: Redacted.Redacted; - readonly jwtSecret?: Redacted.Redacted; -} - -interface SchemaInitTargetBase { - readonly projectRoot: string; - readonly runtime: StackRuntime; - readonly config: StackConfig; - readonly databaseUrl: string; - readonly secrets: SchemaInitSecrets; -} - -export interface SchemaInitLiveTarget extends SchemaInitTargetBase { - readonly kind: "live"; - readonly stackId: StackId; -} - -export interface SchemaInitEphemeralTarget extends SchemaInitTargetBase { - readonly kind: "ephemeral"; - /** Container network of the throwaway Postgres cluster; one-shots join it and dial `supabase-database:5432`. */ - readonly networkId?: string; -} - -export type SchemaInitTarget = SchemaInitLiveTarget | SchemaInitEphemeralTarget; - -export interface SchemaInitOptions { - readonly containerEngine?: ContainerEngine; - readonly artifactPreparer?: RuntimeArtifactPreparer; - /** Host OS for Linux extra hosts so `host.docker.internal` resolves; DNS only, not URL rewrite. */ - readonly platform?: string; -} - -export type SchemaInitServices = - | ChildProcessSpawnerService - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto; - -/** Runs service-owned one-shots against a target Postgres without activating long-running processes. */ -export const schemaInit = ( - names: ReadonlyArray, - target: SchemaInitTarget, - options: SchemaInitOptions = {}, -): Effect.Effect => - schemaInitWorkloads(names, target, options); diff --git a/packages/stack/src/public/Status.ts b/packages/stack/src/public/Status.ts deleted file mode 100644 index c2d991b477..0000000000 --- a/packages/stack/src/public/Status.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { Effect, Schema, SchemaGetter } from "effect"; -import { StackIdSchema, type StackId } from "./StackId.ts"; -import { - ActivationModeSchema, - CAPABILITY_NAMES, - CapabilityNameSchema, - CapabilityStatusSchema, - type ActivationMode, - type CapabilityName, - type CapabilityStatus, -} from "./Capability.ts"; -import { StackRuntimeSchema, type StackRuntime } from "./Runtime.ts"; - -/** Recovery guidance exposed when a failed cleanup blocks new workload activation. */ -export const StackRecoverySchema = Schema.Struct({ - operation: Schema.Literals(["stop", "destroy"] as const), - message: Schema.String, -}); -export type StackRecovery = Schema.Schema.Type; - -export const StackLifecycleSchema = Schema.Literals([ - "unconfigured", - "stopped", - "starting", - "running", - "stopping", - "destroying", -] as const); -export type StackLifecycle = Schema.Schema.Type; - -export const DesiredStackLifecycleSchema = Schema.Literals([ - "unconfigured", - "stopped", - "running", - "destroying", -] as const); -export type DesiredStackLifecycle = Schema.Schema.Type; - -/** Listener keys are shared by the public status and closed config schemas. */ -export const PORT_FIELDS = [ - "api", - "database", - "pooler", - "studio", - "mailUi", - "smtp", - "pop3", - "functionsInspector", -] as const; -export type PortField = (typeof PORT_FIELDS)[number]; - -export const PORT_FIELD_PROTOCOL: Readonly> = { - api: "http", - database: "tcp", - pooler: "tcp", - studio: "http", - mailUi: "http", - smtp: "tcp", - pop3: "tcp", - functionsInspector: "http", -}; - -/** A concrete host/network port. Automatic assignment is represented by listeners, not here. */ -export const NetworkPortSchema = Schema.Int.check( - Schema.isBetween({ minimum: 1, maximum: 65_535 }), -); -export type NetworkPort = Schema.Schema.Type; - -export const StackEndpointSchema = Schema.Struct({ - protocol: Schema.Literals(["http", "tcp"] as const), - address: Schema.String, - port: NetworkPortSchema, - url: Schema.String, -}); -export type StackEndpoint = Schema.Schema.Type; - -export const StackEndpointsSchema = Schema.Struct({ - api: Schema.optionalKey(StackEndpointSchema), - database: Schema.optionalKey(StackEndpointSchema), - pooler: Schema.optionalKey(StackEndpointSchema), - studio: Schema.optionalKey(StackEndpointSchema), - mailUi: Schema.optionalKey(StackEndpointSchema), - smtp: Schema.optionalKey(StackEndpointSchema), - pop3: Schema.optionalKey(StackEndpointSchema), - functionsInspector: Schema.optionalKey(StackEndpointSchema), -}); - -export const CapabilityVersionsSchema = Schema.Struct({ - database: Schema.optionalKey(Schema.String), - rest: Schema.optionalKey(Schema.String), - auth: Schema.optionalKey(Schema.String), - realtime: Schema.optionalKey(Schema.String), - storage: Schema.optionalKey(Schema.String), - functions: Schema.optionalKey(Schema.String), - studio: Schema.optionalKey(Schema.String), - mail: Schema.optionalKey(Schema.String), - analytics: Schema.optionalKey(Schema.String), - pooler: Schema.optionalKey(Schema.String), -}); - -/** Observable state of preparing one workload artifact for a running stack. */ -export const ArtifactPreparationStateSchema = Schema.Literals([ - "queued", - "preparing", - "downloading", - "ready", - "failed", -] as const); -export type ArtifactPreparationState = Schema.Schema.Type; - -export const ArtifactPreparationStatusSchema = Schema.Struct({ - workloadId: Schema.String, - capability: CapabilityNameSchema, - state: ArtifactPreparationStateSchema, - error: Schema.optionalKey(Schema.String), -}); -export type ArtifactPreparationStatus = Schema.Schema.Type; - -const CompleteCapabilityStatusesSchema = Schema.Array(CapabilityStatusSchema).pipe( - Schema.decode({ - decode: SchemaGetter.checkEffect((capabilities) => - Effect.succeed( - capabilities.length === CAPABILITY_NAMES.length && - new Set(capabilities.map(({ name }) => name)).size === CAPABILITY_NAMES.length && - CAPABILITY_NAMES.every((name) => - capabilities.some((capability) => capability.name === name), - ) - ? undefined - : "Expected exactly one status for each public capability", - ), - ), - encode: SchemaGetter.passthrough(), - }), -); - -export const StackStatusSchema = Schema.Struct({ - id: StackIdSchema, - lifecycle: StackLifecycleSchema, - desiredLifecycle: DesiredStackLifecycleSchema, - runtime: StackRuntimeSchema, - endpoints: StackEndpointsSchema, - versions: CapabilityVersionsSchema, - capabilities: CompleteCapabilityStatusesSchema, - artifacts: Schema.Array(ArtifactPreparationStatusSchema), - recovery: Schema.optionalKey(StackRecoverySchema), -}); - -export interface StackStatus { - readonly id: StackId; - readonly lifecycle: StackLifecycle; - readonly desiredLifecycle: DesiredStackLifecycle; - readonly runtime: StackRuntime; - readonly endpoints: Readonly>>; - readonly versions: Readonly>>; - readonly capabilities: ReadonlyArray; - readonly artifacts: ReadonlyArray; - readonly recovery?: StackRecovery; -} - -export const StackDescriptorSchema = Schema.Struct({ - id: StackIdSchema, - projectRoot: Schema.String, - name: Schema.String, - branchContext: Schema.String, - runtime: StackRuntimeSchema, - desiredLifecycle: DesiredStackLifecycleSchema, -}); -export type StackDescriptor = Schema.Schema.Type; - -export const StackInspectionSchema = Schema.Struct({ - descriptor: StackDescriptorSchema, - owner: Schema.Literals(["running", "absent", "unreachable", "incompatible"] as const), - status: Schema.optionalKey(StackStatusSchema), - configDrift: Schema.optionalKey( - Schema.Struct({ - status: Schema.Literals(["unchanged", "changed", "unconfigured"] as const), - paths: Schema.Array(Schema.String), - }), - ), -}); -export type StackInspection = Schema.Schema.Type; - -export type { ActivationMode }; -export { ActivationModeSchema }; diff --git a/packages/stack/src/public/Testing.ts b/packages/stack/src/public/Testing.ts deleted file mode 100644 index 1b20b414ca..0000000000 --- a/packages/stack/src/public/Testing.ts +++ /dev/null @@ -1,290 +0,0 @@ -import { tmpdir } from "node:os"; -import { NodeServices } from "@effect/platform-node"; -import { Cause, Data, Effect, Exit, FileSystem, Path } from "effect"; -import { defaultRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { makePromiseApi, type PromiseStack, type PromiseStackConfig } from "./PromiseStack.ts"; -import type { CreateStackOptions } from "./EffectStack.ts"; -import type { StackRuntimeEnvironmentValue } from "../state/Ownership.ts"; - -class TestStackReadinessError extends Data.TaggedError("TestStackReadinessError")<{ - readonly message: string; - readonly cause?: unknown; -}> {} - -class TestStackOperationError extends Data.TaggedError("TestStackOperationError")<{ - readonly message: string; - readonly cause: unknown; -}> {} - -const call = (operation: () => Promise) => - Effect.tryPromise({ - try: operation, - catch: (cause) => - new TestStackOperationError({ - message: cause instanceof Error ? cause.message : String(cause), - cause, - }), - }); - -const run = (program: Effect.Effect): Promise => - Effect.runPromiseExit(program).then((exit) => { - if (Exit.isSuccess(exit)) return exit.value; - const error = Cause.squash(exit.cause); - throw error instanceof TestStackOperationError ? error.cause : error; - }); -export interface CreateTestStackOptions { - readonly config?: PromiseStackConfig; - readonly name?: string; - readonly runtime?: CreateStackOptions["runtime"]; - /** - * Populates the isolated project root before the managed stack is created. - * Setup failures remove the root and never create a stack handle. - */ - readonly setupProject?: (projectRoot: string) => Promise; -} - -export type TestStack = PromiseStack & - AsyncDisposable & { - /** Managed state root shared with ordinary package and CLI stacks. */ - readonly stateRoot: string; - }; - -export interface TestStackOperations { - readonly createRoot: () => Promise; - readonly createStack: ( - options: CreateStackOptions, - environment?: StackRuntimeEnvironmentValue, - ) => Promise; - readonly removeRoot: (root: string) => Promise; -} - -type TestStackOperationsOverrides = Partial; - -const createTestProjectRoot = Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const projectsRoot = path.join( - path.dirname((yield* defaultRuntimeEnvironment).stateRoot), - "test-projects", - ); - yield* fs.makeDirectory(projectsRoot, { recursive: true }); - return yield* fs.makeTempDirectory({ directory: projectsRoot, prefix: "supabase-stack-test-" }); -}); - -const defaultOperations: TestStackOperations = { - createRoot: () => run(createTestProjectRoot.pipe(Effect.provide(NodeServices.layer))), - createStack: (options, environment) => - makePromiseApi(NodeServices.layer, environment).createStack(options), - removeRoot: (root) => - run( - Effect.flatMap(FileSystem.FileSystem, (fs) => - fs.remove(root, { recursive: true, force: true }), - ).pipe(Effect.provide(NodeServices.layer)), - ), -}; - -// Sharing immutable native artifacts keeps disposable test roots inexpensive. -const testRuntimeEnvironment = Effect.gen(function* () { - const path = yield* Path.Path; - return { - ...(yield* defaultRuntimeEnvironment), - artifactCacheRoot: path.join(tmpdir(), "supabase-stack-test-artifacts"), - } satisfies StackRuntimeEnvironmentValue; -}); - -const validateStartedStatus = ( - initial: Awaited>, - config: PromiseStackConfig | undefined, -) => { - const disabledCapabilities = new Set( - Object.entries(config?.capabilities ?? {}).flatMap(([name, capability]) => - capability !== undefined && "enabled" in capability && capability.enabled === false - ? [name] - : [], - ), - ); - const configuredListeners = Object.entries(config?.listeners ?? {}).flatMap( - ([name, listener]) => { - if (listener === undefined || ("enabled" in listener && listener.enabled === false)) { - return []; - } - return [name]; - }, - ); - const ready = (status: typeof initial) => - status.lifecycle === "running" && - status.capabilities.every( - (capability) => - disabledCapabilities.has(capability.name) || - capability.state === "disabled" || - capability.state === "ready" || - capability.state === "dormant" || - (status.lifecycle === "running" && - capability.state === "stopping" && - capability.activation === "lazy"), - ) && - configuredListeners.every((name) => - Object.entries(status.endpoints).some( - ([endpointName, endpoint]) => endpointName === name && endpoint !== undefined, - ), - ); - const terminalFailure = (status: typeof initial): TestStackReadinessError | undefined => { - const failed = status.capabilities.find( - (capability) => - !disabledCapabilities.has(capability.name) && - (capability.state === "failed" || - (status.lifecycle === "running" && capability.state === "stopped")), - ); - if (failed !== undefined) { - return new TestStackReadinessError({ - message: - failed.error === undefined - ? `Capability ${failed.name} ${failed.state} before stack became ready` - : failed.error, - }); - } - if ( - status.lifecycle === "stopped" || - status.lifecycle === "destroying" || - status.lifecycle === "unconfigured" || - status.lifecycle === "stopping" - ) { - return new TestStackReadinessError({ - message: `Stack lifecycle ${status.lifecycle} before stack became ready`, - }); - } - return undefined; - }; - if (ready(initial)) return Effect.void; - const failure = terminalFailure(initial); - return Effect.fail( - failure ?? - new TestStackReadinessError({ - message: `Stack did not become ready after start (lifecycle ${initial.lifecycle})`, - }), - ); -}; - -const STARTUP_DIAGNOSTIC_LOG_TAIL = 50; - -const withStartupDiagnostics = ( - stack: PromiseStack, - primary: TestStackOperationError | TestStackReadinessError, -) => - Effect.gen(function* () { - const [snapshot, recentLogs] = yield* Effect.all( - [ - call(() => stack.status()).pipe(Effect.catch(() => Effect.undefined)), - call(() => stack.logs({ tail: STARTUP_DIAGNOSTIC_LOG_TAIL })).pipe( - Effect.catch(() => Effect.undefined), - ), - ], - { concurrency: 2 }, - ); - const capabilityStates = - snapshot === undefined - ? "unavailable" - : snapshot.capabilities - .map( - ({ name, state, error }) => - `${name}=${state}${error === undefined ? "" : ` (${error})`}`, - ) - .join(", "); - const logs = - recentLogs === undefined - ? "unavailable" - : recentLogs.entries - .slice(-STARTUP_DIAGNOSTIC_LOG_TAIL) - .map(({ source, stream, message }) => `${source}/${stream}: ${message}`) - .join("\n") || "none"; - return new TestStackReadinessError({ - message: [ - primary.message, - `lifecycle=${snapshot?.lifecycle ?? "unavailable"}`, - `capabilities=${capabilityStates}`, - `recent logs:\n${logs}`, - ].join("; "), - cause: primary instanceof TestStackOperationError ? primary.cause : primary, - }); - }); - -const cleanup = ( - stack: PromiseStack | undefined, - root: string, - operations: TestStackOperations, - primary?: TestStackOperationError | TestStackReadinessError, -) => - Effect.gen(function* () { - if (stack !== undefined) { - const failure = yield* call(() => stack.destroy()).pipe( - Effect.match({ - onSuccess: () => undefined, - onFailure: (error) => error, - }), - ); - // Failed destruction leaves the root available for recovery of durable state. - if (failure !== undefined) - return yield* new TestStackReadinessError({ - message: `${(primary ?? failure).message}; retained test stack root ${root}`, - cause: primary ?? failure.cause, - }); - } - yield* call(() => operations.removeRoot(root)).pipe( - Effect.mapError((error) => primary ?? error), - ); - if (primary !== undefined) return yield* primary; - }); - -/** Internal seam used by integration tests; the package testing barrel exports only createTestStack. */ -export const createTestStackWith = ( - options: CreateTestStackOptions = {}, - operations: TestStackOperations | TestStackOperationsOverrides = defaultOperations, -): Promise => - run( - Effect.gen(function* () { - const resolvedOperations = { ...defaultOperations, ...operations }; - const projectRoot = yield* call(() => resolvedOperations.createRoot()); - let stack: PromiseStack | undefined; - return yield* Effect.gen(function* () { - if (options.setupProject !== undefined) { - const setup = options.setupProject; - yield* call(() => setup(projectRoot)); - } - const runtimeEnvironment = yield* testRuntimeEnvironment; - const resource = yield* call(() => - resolvedOperations.createStack( - { - projectRoot, - name: options.name, - runtime: options.runtime, - }, - runtimeEnvironment, - ), - ); - stack = resource; - yield* call(() => - resource.start(options.config === undefined ? undefined : { config: options.config }), - ).pipe( - Effect.flatMap((started) => validateStartedStatus(started, options.config)), - Effect.catch((error) => - withStartupDiagnostics(resource, error).pipe(Effect.flatMap(Effect.fail)), - ), - ); - return { - ...resource, - stateRoot: runtimeEnvironment.stateRoot, - [Symbol.asyncDispose]: () => run(cleanup(resource, projectRoot, resolvedOperations)), - } satisfies TestStack; - }).pipe( - Effect.catch((error) => - cleanup(stack, projectRoot, resolvedOperations, error).pipe( - Effect.andThen(Effect.fail(error)), - ), - ), - ); - }).pipe(Effect.provide(NodeServices.layer)), - ); - -/** Creates an isolated managed stack and destroys exactly that identity on disposal. */ -export const createTestStack = (options: CreateTestStackOptions = {}): Promise => - createTestStackWith(options); diff --git a/packages/stack/src/public/config-drift.integration.test.ts b/packages/stack/src/public/config-drift.integration.test.ts deleted file mode 100644 index f989fe4fac..0000000000 --- a/packages/stack/src/public/config-drift.integration.test.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, FileSystem, Option, Path, Redacted } from "effect"; -import { makePromiseApi } from "./PromiseStack.ts"; -import { createStack, inspectStack } from "./EffectStack.ts"; -import { - defaultRuntimeEnvironment, - StackRuntimeEnvironment, - type StackRuntimeEnvironmentValue, -} from "../supervisor/Launcher.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import type { StackConfig } from "./Config.ts"; -import { StackVersionUnsupportedError, InvalidStackConfigError } from "./Errors.ts"; - -const withRuntimeRoot = (effect: (project: string) => Effect.Effect) => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectory({ prefix: "supabase-config-drift-" }); - yield* Effect.addFinalizer(() => - fs.remove(root, { recursive: true, force: true }).pipe(Effect.ignore), - ); - const project = path.join(root, "project"); - yield* fs.makeDirectory(project); - const defaults = yield* defaultRuntimeEnvironment; - const runtime: StackRuntimeEnvironmentValue = { - ...defaults, - stateRoot: path.join(root, "managed", "stacks"), - tempRoot: "/tmp", - platform: "posix", - }; - return yield* effect(project).pipe(Effect.provideService(StackRuntimeEnvironment, runtime)); - }), - ).pipe(Effect.provide(NodeServices.layer)); - -const seedConfiguredStack = (projectRoot: string, config: StackConfig) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot, runtime: { kind: "native" } }); - const env = yield* StackRuntimeEnvironment; - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(stack.id); - if (state === undefined) return yield* Effect.die("stack state was not initialized"); - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: state.runtime, - config, - }); - const secrets = Object.fromEntries( - compiled.secrets.map((entry) => [ - entry.slot, - { - policy: entry.policy, - value: entry.value === undefined ? "generated" : String(Redacted.value(entry.value)), - }, - ]), - ); - yield* store.replace(stack.id, { ...state, definition: compiled.definition, secrets }); - return stack; - }); - -const baseConfig = (secret: string): StackConfig => ({ - capabilities: { - functions: { - settings: { - functions_root: "supabase/functions", - edge_runtime: { secrets: { TOKEN: Redacted.make(secret) } }, - }, - }, - }, - listeners: { api: { port: 55431 } }, -}); - -describe("inspectStack config drift", () => { - it.live( - "reports unchanged and changed settings, preparation, listeners, and secret paths without values", - () => - withRuntimeRoot((projectRoot) => - Effect.gen(function* () { - const stack = yield* seedConfiguredStack(projectRoot, baseConfig("old-secret")); - const unchanged = yield* inspectStack(stack.id, { config: baseConfig("old-secret") }); - expect(unchanged.configDrift).toEqual({ - status: "unchanged", - paths: [], - }); - - const changed = yield* inspectStack(stack.id, { - config: { - ...baseConfig("new-secret"), - preparation: "on-demand", - capabilities: { - functions: { - settings: { - functions_root: "supabase/functions", - edge_runtime: { - policy: "oneshot", - secrets: { TOKEN: Redacted.make("new-secret") }, - }, - }, - }, - }, - listeners: { api: { port: 55432 } }, - }, - }); - expect(changed.configDrift?.status).toBe("changed"); - expect(changed.configDrift?.paths).toEqual( - expect.arrayContaining([ - "definition.preparation", - "definition.capabilities.functions.settings.edge_runtime.policy", - "definition.listeners.api.port", - "secrets.secret:functions.settings.edge_runtime.secrets.TOKEN", - ]), - ); - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- JSON.stringify checks all fields for leaked secrets; schema encoding could omit unexpected fields. - expect(JSON.stringify(changed.configDrift)).not.toContain("old-secret"); - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- JSON.stringify checks all fields for leaked secrets; schema encoding could omit unexpected fields. - expect(JSON.stringify(changed.configDrift)).not.toContain("new-secret"); - }), - ), - ); - - it.live("marks an unconfigured stack", () => - withRuntimeRoot((projectRoot) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot, runtime: { kind: "native" } }); - const unconfigured = yield* inspectStack(stack.id, { config: {} }); - expect(unconfigured.configDrift).toEqual({ - status: "unconfigured", - paths: [], - }); - }), - ), - ); - - it.live( - "reuses omitted managed secrets and detects explicit changes or passthrough removal", - () => - withRuntimeRoot((projectRoot) => - Effect.gen(function* () { - const managed = (secret?: string): StackConfig => ({ - ...baseConfig("old-secret"), - capabilities: { - auth: { settings: secret === undefined ? {} : { jwt_secret: Redacted.make(secret) } }, - functions: baseConfig("old-secret").capabilities?.functions, - }, - }); - const stack = yield* seedConfiguredStack(projectRoot, managed("managed-secret")); - expect((yield* inspectStack(stack.id, { config: managed() })).configDrift).toEqual({ - status: "unchanged", - paths: [], - }); - const changed = yield* inspectStack(stack.id, { config: managed("new-managed-secret") }); - expect(changed.configDrift?.paths).toContain("secrets.secret:auth.settings.jwt_secret"); - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- JSON.stringify checks all fields for leaked secrets; schema encoding could omit unexpected fields. - expect(JSON.stringify(changed.configDrift)).not.toContain("managed-secret"); - const removed = yield* inspectStack(stack.id, { - config: { - ...baseConfig("old-secret"), - capabilities: { - functions: { settings: { functions_root: "supabase/functions", edge_runtime: {} } }, - }, - }, - }); - expect(removed.configDrift?.paths).toContain( - "secrets.secret:functions.settings.edge_runtime.secrets.TOKEN", - ); - }), - ), - ); - - it.live("rejects malformed candidate config with a typed config error", () => - withRuntimeRoot((projectRoot) => - Effect.gen(function* () { - const stack = yield* seedConfiguredStack(projectRoot, baseConfig("old-secret")); - const result = yield* inspectStack(stack.id, { - config: { capabilities: { database: { version: "unsupported" } } }, - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(StackVersionUnsupportedError); - expect(failure.value).not.toBeInstanceOf(InvalidStackConfigError); - } - } - }), - ), - ); - - it.live("decodes Promise facade config and returns the same redacted report", () => - withRuntimeRoot((projectRoot) => - Effect.gen(function* () { - const stack = yield* seedConfiguredStack(projectRoot, baseConfig("old-secret")); - const env = yield* StackRuntimeEnvironment; - const api = makePromiseApi(NodeServices.layer, env); - return yield* Effect.tryPromise(() => - api.inspectStack(stack.id, { config: { listeners: { api: { port: 55432 } } } }), - ); - }).pipe( - Effect.tap((inspection) => - Effect.sync(() => { - expect(inspection.configDrift?.status).toBe("changed"); - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- JSON.stringify checks all fields for leaked secrets; schema encoding could omit unexpected fields. - expect(JSON.stringify(inspection.configDrift)).not.toContain("old-secret"); - }), - ), - ), - ), - ); -}); diff --git a/packages/stack/src/public/effect-stack.integration.test.ts b/packages/stack/src/public/effect-stack.integration.test.ts deleted file mode 100644 index ad2a5b81ab..0000000000 --- a/packages/stack/src/public/effect-stack.integration.test.ts +++ /dev/null @@ -1,2616 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Crypto, - Deferred, - Effect, - Exit, - FileSystem, - Fiber, - Option, - Path, - Redacted, - Ref, - Schema, - Scope, - Stream, -} from "effect"; -import { ChildProcess } from "effect/unstable/process"; -import { - defaultRuntimeEnvironment, - ensureSupervisor, - type StackRuntimeEnvironmentValue, -} from "../supervisor/Launcher.ts"; -import { - StackRuntimeEnvironment, - acquireOwnership, - ownerLockExists, - publishOwnership, - readOwnerMetadata, -} from "../state/Ownership.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { resolveSecrets } from "../state/SecretStore.ts"; -import { deriveStackId, resolveStackIdentity } from "../identity/Identity.ts"; -import { toPersistedIdentity, type PersistedStackState } from "../state/StackState.ts"; -import { startControlServer } from "../control/ControlServer.ts"; -import { STACK_RPC_RELEASE, type StackRpcHandlers } from "../control/StackRpc.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { catalogEntryFor } from "../model/WorkloadCatalog.ts"; -import { - StackDestructionError, - StackCleanupError, - ContainerEngineError, - InvalidStackConfigError, - InvalidLogCursorError, - StackLifecycleConflictError, - StackOwnershipConflictError, - StackPreparationError, - StackRuntimeMismatchError, - StackStateInvalidError, - StackStateFormatUnsupportedError, - StackNotFoundError, - StackNotRunningError, - StackVersionUnsupportedError, - StackUpgradeRequiredError, - type StackError, -} from "./Errors.ts"; -import type { LogQuery, StackLogBatch, StackLogEntry } from "./Logs.ts"; -import { - createStack, - discoverStacks, - findStack, - inspectStack, - listStacks, - makeHandle, - openStack, - type EffectStack, - type HandleDependencies, - type PrepareStackResult, -} from "./EffectStack.ts"; -import { CAPABILITY_NAMES } from "./Capability.ts"; -import { StackIdSchema, type StackId } from "./StackId.ts"; -import type { ArtifactPreparationStatus, StackStatus } from "./Status.ts"; -import { makeDockerEngine } from "../runtime/DockerEngine.ts"; -import { - type ContainerCommandResult, - type ContainerCommandRunner, -} from "../runtime/ContainerEngine.ts"; -import { - ContainerEngineResolver, - defaultContainerEngineResolver, -} from "../runtime/ContainerEngineResolver.ts"; -import { runGit } from "../../tests/helpers/git.ts"; - -const defaultDatabaseVersion = catalogEntryFor("database:database").defaultVersion; -const defaultDatabaseMajor = defaultDatabaseVersion.split(".")[0]; -const defaultRestVersion = catalogEntryFor("rest:rest").defaultVersion; - -const stackId = StackIdSchema.make( - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", -); - -const expectPresent = (value: A | undefined, description: string): A => { - expect(value, description).toBeDefined(); - if (value === undefined) throw new Error(`Expected ${description}`); - return value; -}; - -const runningStatus: StackStatus = { - id: stackId, - lifecycle: "running", - desiredLifecycle: "running", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "functions" ? "lazy" : "eager", - state: name === "pooler" ? "disabled" : "ready", - })), - artifacts: [], -}; - -const credentials = { - database: { url: Redacted.make("postgres://localhost"), password: Redacted.make("secret") }, - api: { - publishableKey: "publishable", - secretKey: Redacted.make("secret"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service"), - }, -}; - -const stoppedState = (): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity: { - projectRoot: "/tmp/project", - branchContext: "branch", - stackName: "stack", - }, - runtime: { kind: "native" }, - desiredLifecycle: "stopped", - ports: [], - privatePorts: [], - secrets: {}, -}); - -const emptyLogs = () => - Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false } as const); - -const makeTestHandle = (id: StackId, overrides: Partial = {}) => - makeHandle(id, { - resolveOwner: () => Effect.succeed(Option.none()), - readOfflineState: Effect.succeed(Option.none()), - readPersistedState: Effect.succeed(Option.none()), - readLogs: emptyLogs, - waitForRelease: Effect.void, - prepare: () => - Effect.fail(new StackPreparationError({ message: "test preparation unavailable" })), - ...overrides, - }); - -const withRuntimeRoot = (effect: (project: string) => Effect.Effect) => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectory({ prefix: "supabase-effect-stack-owner-" }); - yield* Effect.addFinalizer(() => - fs.remove(root, { recursive: true, force: true }).pipe(Effect.ignore), - ); - const project = path.join(root, "project"); - yield* fs.makeDirectory(project); - const defaults = yield* defaultRuntimeEnvironment; - const runtime: StackRuntimeEnvironmentValue = { - ...defaults, - stateRoot: path.join(root, "managed", "stacks"), - tempRoot: "/tmp", - platform: "posix", - }; - return yield* effect(project).pipe( - Effect.provideService(StackRuntimeEnvironment, runtime), - Effect.provideService(ContainerEngineResolver, { - isInstalled: () => Effect.succeed(false), - resolve: (kind) => defaultContainerEngineResolver.resolve(kind), - }), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)); - -const lifecycleConfig = () => { - return { - capabilities: { - database: {}, - rest: {}, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - listeners: { - api: { enabled: false }, - database: { enabled: false }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - } as const; -}; - -describe("Effect stack lifecycle handoff", () => { - it.live("reclaims a stale owner after a failed maintenance connection", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-effect-stack-dead-owner-", - }); - const liveEndpoint = { kind: "unix" as const, path: path.join(root, "live.sock") }; - const deadEndpoint = { kind: "unix" as const, path: path.join(root, "dead.sock") }; - const liveOwnerSessionId = "live-owner-session"; - const deadOwner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint: deadEndpoint, - ownerSessionId: "dead-owner-session", - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - const liveOwner = { - ...deadOwner, - endpoint: liveEndpoint, - ownerSessionId: liveOwnerSessionId, - }; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const liveStopCalls = yield* Ref.make(0); - yield* startControlServer({ - endpoint: liveEndpoint, - stackId, - ownerSessionId: liveOwnerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId: liveOwnerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Ref.update(liveStopCalls, (calls) => calls + 1).pipe( - Effect.andThen(Effect.succeed({ ok: true, op: "stop" } as const)), - ), - }, - onShutdownReady: Scope.close(ownerScope, Exit.void), - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const launchCalls = yield* Ref.make(0); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: (launch) => - launch - ? Ref.update(launchCalls, (calls) => calls + 1).pipe( - Effect.as(Option.some({ owner: liveOwner, launched: true })), - ) - : Effect.succeed(Option.some({ owner: deadOwner, launched: false })), - }); - yield* stack.stop; - expect(yield* Ref.get(launchCalls)).toBe(1); - expect(yield* Ref.get(liveStopCalls)).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("preserves stop cleanup error identity through maintenance transport", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-effect-stack-stop-error-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "stop-error-session"; - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - const ownerScope = yield* Scope.make(); - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ - ok: false, - error: { - tag: "operation-failed", - message: "injected stop cleanup failure", - stackErrorTag: "StackCleanupError", - }, - }), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - const stopped = yield* stack.stop.pipe(Effect.exit); - expect(Exit.isFailure(stopped)).toBe(true); - if (Exit.isFailure(stopped)) - expect(Option.getOrUndefined(Cause.findErrorOption(stopped.cause))).toBeInstanceOf( - StackCleanupError, - ); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("hands off filtered logs through a real owner stop", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-logs-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "logs-session"; - const stopped = yield* Deferred.make(); - const followRead = yield* Deferred.make(); - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const initialAuth: StackLogEntry = { - cursor: { opaque: "v1_1" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "auth", - stream: "stdout", - message: "started", - }; - const unrelatedDatabase: StackLogEntry = { - cursor: { opaque: "v1_2" }, - timestamp: "2026-01-01T00:00:01.000Z", - source: "database", - stream: "stdout", - message: "ignored", - }; - const finalAuth: StackLogEntry = { - cursor: { opaque: "v1_3" }, - timestamp: "2026-01-01T00:00:02.000Z", - source: "auth", - stream: "stdout", - message: "stopped", - }; - const logEntries = yield* Ref.make>([ - initialAuth, - unrelatedDatabase, - ]); - const ownerRunning = yield* Ref.make(true); - const ownerAvailable = yield* Ref.make(true); - const readLogs = (query: LogQuery): Effect.Effect => - Effect.gen(function* () { - if (query.cursor !== undefined) - yield* Deferred.succeed(followRead, undefined).pipe(Effect.asVoid); - const allEntries = yield* Ref.get(logEntries); - const cursorIndex = - query.cursor === undefined - ? -1 - : allEntries.findIndex((entry) => entry.cursor.opaque === query.cursor?.opaque); - const capabilities = query.capabilities; - const matching = allEntries - .slice(cursorIndex + 1) - .filter( - (entry) => - capabilities === undefined || - (entry.source !== "gateway" && - entry.source !== "supervisor" && - capabilities.includes(entry.source)), - ); - const entries = - query.tail === undefined ? matching : matching.slice(-Math.floor(query.tail)); - const cursor = allEntries.at(-1)?.cursor ?? { opaque: "v1_0" }; - return { - entries, - cursor, - running: yield* Ref.get(ownerRunning), - }; - }); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: (query) => readLogs(query), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.gen(function* () { - yield* Ref.set(ownerRunning, false); - yield* Ref.update(logEntries, (entries) => [...entries, finalAuth]); - yield* Deferred.succeed(stopped, undefined); - return { ok: true, op: "stop" } as const; - }), - }, - onShutdownReady: Deferred.succeed(stopped, undefined).pipe(Effect.asVoid), - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => - Ref.get(ownerAvailable).pipe( - Effect.map((available) => - available ? Option.some({ owner, launched: false }) : Option.none(), - ), - ), - readOfflineState: Effect.succeed(Option.some(stoppedState())), - }); - expect((yield* stack.start()).lifecycle).toBe("running"); - const first = yield* stack.logs({ capabilities: ["auth"], tail: 1 }); - expect(first.entries).toEqual([initialAuth]); - expect(first.cursor).toEqual(unrelatedDatabase.cursor); - - const followedFiber = yield* Effect.forkChild( - stack - .followLogs({ capabilities: ["auth"], cursor: first.cursor }) - .pipe(Stream.runCollect), - { startImmediately: true }, - ); - yield* Deferred.await(followRead); - const stopFiber = yield* Effect.forkChild(stack.stop, { startImmediately: true }); - yield* Deferred.await(stopped); - const followed = yield* Fiber.join(followedFiber); - expect(Array.from(followed)).toEqual([finalAuth]); - yield* Ref.set(ownerAvailable, false); - yield* Scope.close(ownerScope, Exit.void); - yield* Fiber.join(stopFiber); - expect((yield* stack.status).lifecycle).toBe("stopped"); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("propagates malformed log cursors through the public control handle", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-cursor-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "cursor-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => - Effect.fail({ tag: "InvalidLogCursorError", message: "Log cursor is invalid" }), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - const result = yield* stack.logs({ cursor: { opaque: "not-a-cursor" } }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( - InvalidLogCursorError, - ); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("delivers final followed entries once and completes after stop", () => - withRuntimeRoot((_project) => - Effect.gen(function* () { - const calls = yield* Ref.make(0); - const entries: [StackLogEntry, StackLogEntry, StackLogEntry, StackLogEntry] = [ - { - cursor: { opaque: "v1_1" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "auth" as const, - stream: "stdout" as const, - message: "started", - }, - { - cursor: { opaque: "v1_2" }, - timestamp: "2026-01-01T00:00:01.000Z", - source: "auth" as const, - stream: "stdout" as const, - message: "stopped", - }, - { - cursor: { opaque: "v1_3" }, - timestamp: "2026-01-01T00:00:02.000Z", - source: "auth" as const, - stream: "stdout" as const, - message: "drained", - }, - { - cursor: { opaque: "v1_4" }, - timestamp: "2026-01-01T00:00:03.000Z", - source: "auth" as const, - stream: "stdout" as const, - message: "closed", - }, - ]; - const burst = entries.slice(1); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.none()), - readPersistedState: Effect.succeed(Option.some(stoppedState())), - readOfflineState: Effect.succeed(Option.none()), - readLogs: (query?: LogQuery) => - Ref.getAndUpdate(calls, (current) => current + 1).pipe( - Effect.map((index) => { - const batchEntries = index === 0 ? [entries[0]] : burst; - const visibleEntries = - query?.tail === undefined - ? batchEntries - : batchEntries.slice(-Math.floor(query.tail)); - return { - entries: visibleEntries, - cursor: entries.at(index === 0 ? 0 : 3)?.cursor ?? { opaque: "v1_0" }, - running: index === 0, - }; - }), - ), - }); - const followed = yield* stack - .followLogs({ capabilities: ["auth"], tail: 1 }) - .pipe(Stream.runCollect, Effect.exit); - expect(Exit.isSuccess(followed)).toBe(true); - if (Exit.isSuccess(followed)) expect(Array.from(followed.value)).toEqual(entries); - expect(yield* Ref.get(calls)).toBe(2); - }), - ), - ); - - it.live("creates a stopped stack without launching a Supervisor", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project }); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - const offline = yield* stack.status; - expect(offline.lifecycle).toBe("unconfigured"); - expect(offline.capabilities.every(({ state }) => state === "disabled")).toBe(true); - yield* openStack(stack.id); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - yield* stack.stop; - expect((yield* stack.status).lifecycle).toBe("unconfigured"); - expect((yield* stack.logs()).entries).toHaveLength(0); - }), - ), - ); - - it.live("isolates parallel stacks by project root, branch context, and name", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.dirname(project); - const monorepoA = path.join(project, "apps", "one"); - const monorepoB = path.join(project, "apps", "two"); - const detachedWorktreeA = path.join(root, "chat-worktree-a"); - const detachedWorktreeB = path.join(root, "chat-worktree-b"); - const plainProject = path.join(root, "plain-project"); - yield* fs.makeDirectory(monorepoA, { recursive: true }); - yield* fs.makeDirectory(monorepoB, { recursive: true }); - yield* fs.makeDirectory(plainProject); - yield* runGit(project, ["init", "-b", "main"]); - yield* runGit(project, ["config", "user.email", "stack-tests@example.test"]); - yield* runGit(project, ["config", "user.name", "Stack Tests"]); - yield* fs.writeFileString(path.join(project, "README.md"), "identity\n"); - yield* runGit(project, ["add", "README.md"]); - yield* runGit(project, ["commit", "-m", "initial"]); - yield* runGit(project, ["worktree", "add", "--detach", detachedWorktreeA, "HEAD"]); - yield* runGit(project, ["worktree", "add", "--detach", detachedWorktreeB, "HEAD"]); - - const [sameA, sameB, namedA, namedB, monoA, monoB, siblingA, siblingB, plainA, plainB] = - yield* Effect.all( - [ - createStack({ projectRoot: project }), - createStack({ projectRoot: project }), - createStack({ projectRoot: project, name: "preview-a" }), - createStack({ projectRoot: project, name: "preview-b" }), - createStack({ projectRoot: monorepoA }), - createStack({ projectRoot: monorepoB }), - createStack({ projectRoot: detachedWorktreeA }), - createStack({ projectRoot: detachedWorktreeB }), - createStack({ projectRoot: plainProject, name: "preview-a" }), - createStack({ projectRoot: plainProject, name: "preview-b" }), - ], - { concurrency: 10 }, - ); - yield* runGit(project, ["checkout", "-b", "feat-a"]); - const feature = yield* createStack({ projectRoot: project }); - yield* runGit(project, ["checkout", "main"]); - - expect(sameA.id).toBe(sameB.id); - expect( - new Set([ - sameA.id, - namedA.id, - namedB.id, - feature.id, - monoA.id, - monoB.id, - siblingA.id, - siblingB.id, - plainA.id, - plainB.id, - ]).size, - ).toBe(10); - const found = yield* findStack({ projectRoot: project }); - expect(Option.getOrUndefined(found)?.id).toBe(sameA.id); - expect((yield* listStacks({ projectRoot: project })).map(({ id }) => id).sort()).toEqual( - [sameA.id, namedA.id, namedB.id, feature.id].sort(), - ); - expect((yield* listStacks({ projectRoot: monorepoA })).map(({ id }) => id)).toEqual([ - monoA.id, - ]); - const detachedA = yield* listStacks({ projectRoot: detachedWorktreeA }); - const detachedB = yield* listStacks({ projectRoot: detachedWorktreeB }); - expect(detachedA.map(({ id }) => id)).toEqual([siblingA.id]); - expect(detachedB.map(({ id }) => id)).toEqual([siblingB.id]); - expect(detachedA[0]?.branchContext).toBe("detached"); - expect(detachedB[0]?.branchContext).toBe("detached"); - expect(siblingA.id).not.toBe(siblingB.id); - expect( - (yield* listStacks({ projectRoot: plainProject })).map(({ id }) => id).sort(), - ).toEqual([plainA.id, plainB.id].sort()); - }), - ), - ); - - it.live("does not relocate a stack when a linked Git worktree moves", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.dirname(project); - const linked = path.join(root, "linked-project"); - yield* runGit(project, ["init", "-b", "main"]); - yield* runGit(project, ["config", "user.email", "stack-tests@example.test"]); - yield* runGit(project, ["config", "user.name", "Stack Tests"]); - yield* fs.writeFileString(path.join(project, "README.md"), "identity\n"); - yield* runGit(project, ["add", "README.md"]); - yield* runGit(project, ["commit", "-m", "initial"]); - yield* runGit(project, ["worktree", "add", "--detach", linked, "HEAD"]); - const original = yield* createStack({ projectRoot: linked }); - const moved = path.join(root, "moved-project"); - yield* runGit(project, ["worktree", "move", linked, moved]); - - const movedBeforeCreate = yield* findStack({ projectRoot: moved }); - expect(Option.isNone(movedBeforeCreate)).toBe(true); - const replacement = yield* createStack({ projectRoot: moved }); - expect(replacement.id).not.toBe(original.id); - const movedCanonical = yield* fs.realPath(moved); - expect(yield* listStacks({ projectRoot: moved })).toEqual([ - expect.objectContaining({ id: replacement.id, projectRoot: movedCanonical }), - ]); - }), - ), - ); - - it.live("does not adopt state when a removed worktree basename is reused elsewhere", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.dirname(project); - const oldParent = path.join(root, "old-parent"); - const newParent = path.join(root, "new-parent"); - const oldWorktree = path.join(oldParent, "session"); - const newWorktree = path.join(newParent, "session"); - yield* fs.makeDirectory(oldParent); - yield* fs.makeDirectory(newParent); - yield* runGit(project, ["init", "-b", "main"]); - yield* runGit(project, ["config", "user.email", "stack-tests@example.test"]); - yield* runGit(project, ["config", "user.name", "Stack Tests"]); - yield* fs.writeFileString(path.join(project, "README.md"), "identity\n"); - yield* runGit(project, ["add", "README.md"]); - yield* runGit(project, ["commit", "-m", "initial"]); - yield* runGit(project, ["worktree", "add", "--detach", oldWorktree, "HEAD"]); - const original = yield* createStack({ projectRoot: oldWorktree }); - yield* runGit(project, ["worktree", "remove", "--force", oldWorktree]); - yield* runGit(project, ["worktree", "add", "--detach", newWorktree, "HEAD"]); - - const replacement = yield* createStack({ projectRoot: newWorktree }); - expect(replacement.id).not.toBe(original.id); - expect((yield* listStacks({ projectRoot: newWorktree })).map(({ id }) => id)).toEqual([ - replacement.id, - ]); - expect((yield* listStacks()).map(({ id }) => id).sort()).toEqual( - [original.id, replacement.id].sort(), - ); - }), - ), - ); - - it.live("releases a temporary Supervisor after a pre-commit start failure", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project, runtime: { kind: "native" } }); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(stack.id); - if (state === undefined) return yield* Effect.die("stack state was not initialized"); - yield* store.replace(stack.id, { ...state, desiredLifecycle: "stopped" }); - - const started = yield* stack - .start({ config: { capabilities: { database: { version: "unsupported" } } } }) - .pipe(Effect.exit); - - expect(Exit.isFailure(started)).toBe(true); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - expect((yield* stack.status).lifecycle).toBe("stopped"); - const reopened = yield* openStack(stack.id); - expect((yield* reopened.status).lifecycle).toBe("stopped"); - }), - ), - ); - - it.live("never projects offline state while metadata or the ownership lock remains", () => - Effect.gen(function* () { - for (const artifact of ["metadata", "lock"] as const) { - const ownership = new StackOwnershipConflictError({ - message: `owner ${artifact} is still present`, - }); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.none()), - readOfflineState: Effect.fail(ownership), - readLogs: () => Effect.fail(ownership), - }); - const status = yield* stack.status.pipe(Effect.exit); - expect(Exit.isFailure(status)).toBe(true); - const logs = yield* stack.logs().pipe(Effect.exit); - expect(Exit.isFailure(logs)).toBe(true); - } - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("preserves an unreachable owner error when offline status is still guarded", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const path = yield* Path.Path; - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint: { kind: "unix" as const, path: path.join(project, "missing-owner.sock") }, - ownerSessionId: "unreachable-owner", - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - let ownerReads = 0; - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => - Effect.sync(() => { - ownerReads += 1; - return ownerReads === 1 ? Option.some({ owner, launched: false }) : Option.none(); - }), - readOfflineState: Effect.fail( - new StackOwnershipConflictError({ message: "Owner metadata still exists" }), - ), - }); - const result = yield* stack.status.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(StackOwnershipConflictError); - expect(error?.message).toContain("Stack owner is unreachable"); - } - }), - ), - ); - - it.live("waits for owner teardown before falling back to offline logs", () => - withRuntimeRoot((_project) => - Effect.gen(function* () { - const attempts = yield* Ref.make(0); - const ownership = new StackOwnershipConflictError({ - message: "Supervisor is still shutting down", - }); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.none()), - readPersistedState: Effect.succeed(Option.some(stoppedState())), - readLogs: () => - Ref.getAndUpdate(attempts, (current) => current + 1).pipe( - Effect.flatMap((attempt) => - attempt === 0 - ? Effect.fail(ownership) - : Effect.succeed({ - entries: [], - cursor: { opaque: "v1_0" }, - running: false, - }), - ), - ), - }); - const batch = yield* stack.logs(); - expect(batch.entries).toEqual([]); - expect(yield* Ref.get(attempts)).toBe(2); - }), - ), - ); - - it.live("preserves an allowed error tag when reading offline logs", () => - Effect.gen(function* () { - const upgrade = new StackUpgradeRequiredError({ - message: "The retained log protocol is newer than this client", - expectedRelease: STACK_RPC_RELEASE, - actualRelease: "future", - }); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.none()), - readPersistedState: Effect.succeed(Option.some(stoppedState())), - readLogs: () => Effect.fail(upgrade), - }); - const result = yield* stack.logs().pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBe(upgrade); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live( - "guards offline operations against real lock-only and metadata ownership", - () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project }); - const lease = yield* acquireOwnership({ - stateRoot: env.stateRoot, - stackId: stack.id, - ownerSessionId: "offline-guard", - rpcRelease: STACK_RPC_RELEASE, - environment: env, - }); - const assertGuarded = (candidate: EffectStack) => - Effect.gen(function* () { - expect(Exit.isFailure(yield* candidate.status.pipe(Effect.exit))).toBe(true); - expect(Exit.isFailure(yield* candidate.logs().pipe(Effect.exit))).toBe(true); - }); - const lockOnly = yield* openStack(stack.id); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(true); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect((yield* inspectStack(stack.id)).owner).toBe("unreachable"); - yield* assertGuarded(lockOnly); - yield* lease.release; - - const metadataLease = yield* acquireOwnership({ - stateRoot: env.stateRoot, - stackId: stack.id, - ownerSessionId: "offline-metadata", - rpcRelease: STACK_RPC_RELEASE, - environment: env, - }); - yield* publishOwnership(metadataLease); - const metadataOnly = yield* openStack(stack.id); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeDefined(); - yield* assertGuarded(metadataOnly); - }), - ), - 60_000, - ); - - it.live("prepares cache-only without an owner or state mutation", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project }); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(stack.id); - if (state === undefined) return yield* Effect.die("stack state was not initialized"); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const before = yield* (yield* FileSystem.FileSystem).readFileString(paths.stateDocument); - const prepared = yield* stack.prepare({ capabilities: [] }); - expect(prepared.capabilities).toEqual([]); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - expect(yield* (yield* FileSystem.FileSystem).readFileString(paths.stateDocument)).toBe( - before, - ); - }), - ), - ); - - it.live("applies the configured artifact cache root during handle preparation", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-cache-" }); - const project = path.join(root, "project"); - yield* fs.makeDirectory(project); - const defaults = yield* defaultRuntimeEnvironment; - const stateRoot = path.join(root, "managed", "stacks"); - const artifactCacheRoot = path.join(root, "shared-artifacts"); - const configuredEnvironment: StackRuntimeEnvironmentValue = { - ...defaults, - stateRoot, - artifactCacheRoot, - }; - const stack = yield* createStack({ - projectRoot: project, - runtime: { kind: "native" }, - }).pipe(Effect.provideService(StackRuntimeEnvironment, configuredEnvironment)); - const prepared = yield* stack.prepare({ capabilities: [] }); - expect(prepared.capabilities).toEqual([]); - expect(yield* fs.exists(artifactCacheRoot)).toBe(true); - expect(yield* fs.exists(path.join(stateRoot, "artifacts"))).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("preserves input and version errors from prepare", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot: project, runtime: { kind: "native" } }); - const invalid = yield* stack - .prepare({ - config: yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Any))( - '{"capabilities":{"rest":{"settings":{"unknown":true}}}}', - ), - }) - .pipe(Effect.exit); - expect(Exit.isFailure(invalid)).toBe(true); - if (Exit.isFailure(invalid)) - expect(Option.getOrUndefined(Cause.findErrorOption(invalid.cause))).toBeInstanceOf( - InvalidStackConfigError, - ); - - const unsupported = yield* stack - .prepare({ config: { capabilities: { database: { version: "99" } } } }) - .pipe(Effect.exit); - expect(Exit.isFailure(unsupported)).toBe(true); - if (Exit.isFailure(unsupported)) - expect(Option.getOrUndefined(Cause.findErrorOption(unsupported.cause))).toBeInstanceOf( - StackVersionUnsupportedError, - ); - }), - ), - ); - - it.live("preserves unsupported persisted state format from prepare and start", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project }); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - yield* fs.writeFileString( - paths.stateDocument, - yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ - format: "supabase-stack-v0", - }), - ); - - const result = yield* stack.prepare().pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( - StackStateFormatUnsupportedError, - ); - - const started = yield* stack.start().pipe(Effect.exit); - expect(Exit.isFailure(started)).toBe(true); - if (Exit.isFailure(started)) - expect(Option.getOrUndefined(Cause.findErrorOption(started.cause))).toBeInstanceOf( - StackStateFormatUnsupportedError, - ); - }), - ), - ); - - it.live("reports a retained handle as not found after destroy", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot: project }); - yield* stack.destroy; - - const status = yield* stack.status.pipe(Effect.exit); - expect(Exit.isFailure(status)).toBe(true); - if (Exit.isFailure(status)) - expect(Option.getOrUndefined(Cause.findErrorOption(status.cause))).toBeInstanceOf( - StackNotFoundError, - ); - const logs = yield* stack.logs().pipe(Effect.exit); - expect(Exit.isFailure(logs)).toBe(true); - if (Exit.isFailure(logs)) - expect(Option.getOrUndefined(Cause.findErrorOption(logs.cause))).toBeInstanceOf( - StackNotFoundError, - ); - }), - ), - ); - - it.live("classifies credentials for offline and unreachable stacks", () => - Effect.gen(function* () { - const stopped = yield* makeTestHandle(stackId, { - readOfflineState: Effect.succeed(Option.some(stoppedState())), - }); - const stoppedResult = yield* stopped.credentials.pipe(Effect.exit); - expect(Exit.isFailure(stoppedResult)).toBe(true); - if (Exit.isFailure(stoppedResult)) { - const error = Cause.findErrorOption(stoppedResult.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(StackNotRunningError); - } - - const missing = yield* makeTestHandle(stackId); - const missingResult = yield* missing.credentials.pipe(Effect.exit); - expect(Exit.isFailure(missingResult)).toBe(true); - if (Exit.isFailure(missingResult)) { - const error = Cause.findErrorOption(missingResult.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(StackNotFoundError); - } - - const running = yield* makeTestHandle(stackId, { - readOfflineState: Effect.succeed( - Option.some({ ...stoppedState(), desiredLifecycle: "running" }), - ), - }); - const runningResult = yield* running.credentials.pipe(Effect.exit); - expect(Exit.isFailure(runningResult)).toBe(true); - if (Exit.isFailure(runningResult)) { - const error = Cause.findErrorOption(runningResult.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(StackOwnershipConflictError); - } - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("does not leave a runtime remnant when a retained handle starts after destroy", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project }); - yield* stack.destroy; - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - - const failed = yield* stack.start().pipe(Effect.exit); - - expect(Exit.isFailure(failed)).toBe(true); - expect(yield* fs.exists(paths.stackRoot)).toBe(false); - expect(yield* listStacks()).toEqual([]); - }), - ), - ); - - it.live("ignores a runtime-only missing-state sibling during list discovery", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const valid = yield* createStack({ projectRoot: project }); - const orphanId = StackIdSchema.make("a".repeat(64)); - const orphanPaths = yield* resolveStackPaths({ - stateRoot: env.stateRoot, - stackId: orphanId, - }); - yield* fs.makeDirectory(orphanPaths.runtime, { recursive: true }); - - const listed = yield* listStacks(); - - expect(listed).toHaveLength(1); - expect(listed[0]?.id).toBe(valid.id); - }), - ), - ); - - it.live("retains healthy stacks while reporting unreadable registry entries", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const env = yield* StackRuntimeEnvironment; - const healthy = yield* createStack({ projectRoot: project }); - const corruptProject = path.join(project, "corrupt"); - yield* fs.makeDirectory(corruptProject); - const corrupt = yield* createStack({ projectRoot: corruptProject }); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: corrupt.id }); - yield* fs.writeFileString(paths.stateDocument, "{ malformed"); - - const discovered = yield* discoverStacks(); - - expect(discovered.stacks.map(({ id }) => id)).toEqual([healthy.id]); - expect(discovered.errors).toHaveLength(1); - expect(discovered.errors[0]?.id).toBe(corrupt.id); - expect(discovered.errors[0]?.error).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); - - it.live("recovers a same-identity runtime-only missing-state remnant during create", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const identity = yield* resolveStackIdentity({ projectRoot: project }); - const id = yield* deriveStackId(identity); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: id }); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - - const recovered = yield* createStack({ projectRoot: project }); - - expect(recovered.id).toBe(id); - expect(yield* fs.exists(paths.stateDocument)).toBe(true); - }), - ), - ); - - it.live("keeps durable missing-state remnants fail-closed during list discovery", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - yield* createStack({ projectRoot: project }); - const orphanId = StackIdSchema.make("b".repeat(64)); - const orphanPaths = yield* resolveStackPaths({ - stateRoot: env.stateRoot, - stackId: orphanId, - }); - yield* fs.makeDirectory(orphanPaths.data, { recursive: true }); - - const listed = yield* listStacks().pipe(Effect.exit); - - expect(Exit.isFailure(listed)).toBe(true); - expect( - Exit.isFailure(listed) - ? Option.getOrUndefined(Cause.findErrorOption(listed.cause)) - : undefined, - ).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); - - it.live("treats an omitted container engine as Docker for runtime identity", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "podman" }, - }); - const result = yield* createStack({ - projectRoot: project, - runtime: { kind: "container" }, - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) - expect(failure.value).toBeInstanceOf(StackRuntimeMismatchError); - } - }), - ), - ); - - it.live("rejects unknown capabilities as a typed preparation error", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot: project }); - const malformedOptions = { capabilities: ["not-a-capability"] }; - const malformedPrepare = (): Effect.Effect => - Reflect.apply(stack.prepare, stack, [malformedOptions]); - const result = yield* malformedPrepare().pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) expect(failure.value).toBeInstanceOf(StackPreparationError); - } - }), - ), - ); - - it.live("rejects a directly disabled capability before preparing artifacts", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const fs = yield* FileSystem.FileSystem; - const stack = yield* createStack({ projectRoot: project }); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const before = yield* fs.readFileString(paths.stateDocument); - const progress: Array = []; - const result = yield* stack - .prepare({ - config: { capabilities: { pooler: { enabled: false } } }, - capabilities: ["pooler"], - onProgress: (status) => progress.push(status), - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(InvalidStackConfigError); - expect(failure.value).toMatchObject({ - stackId: stack.id, - capability: "pooler", - message: "Capability pooler is disabled", - }); - } - } - expect(progress).toEqual([]); - expect(yield* fs.readFileString(paths.stateDocument)).toBe(before); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - }), - ), - ); - - it.live( - "rejects an enabled capability with a disabled dependency before preparing artifacts", - () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const fs = yield* FileSystem.FileSystem; - const stack = yield* createStack({ projectRoot: project }); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const before = yield* fs.readFileString(paths.stateDocument); - const progress: Array = []; - const result = yield* stack - .prepare({ - config: { - capabilities: { - rest: { enabled: false }, - studio: { enabled: true }, - }, - }, - capabilities: ["studio"], - onProgress: (status) => progress.push(status), - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) - expect(failure.value).toBeInstanceOf(InvalidStackConfigError); - } - expect(progress).toEqual([]); - expect(yield* fs.readFileString(paths.stateDocument)).toBe(before); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - }), - ), - ); - - it.live("uses persisted pins and dependency closure for prospective preparation", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const calls: Array = []; - const runner: ContainerCommandRunner = { - run: (request) => { - calls.push(request.args.slice(0, 3).join(" ")); - if (request.args[0] === "version") - return Effect.succeed({ - stdout: '"1"\n', - stderr: "", - exitCode: 0, - }); - if (request.args[0] === "image" && request.args[1] === "ls") - return Effect.succeed({ - stdout: '"cached"\n', - stderr: "", - exitCode: 0, - }); - return Effect.succeed({ stdout: "", stderr: "", exitCode: 0 }); - }, - stream: () => Stream.empty, - }; - const engine = makeDockerEngine({ runner, platform: { os: "linux" } }); - const stack = yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "docker" }, - }).pipe( - Effect.provideService(ContainerEngineResolver, { - isInstalled: () => Effect.succeed(true), - resolve: () => Effect.succeed(engine), - }), - ); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(stack.id); - if (state === undefined) return yield* Effect.die("stack state was not initialized"); - const persisted = yield* compileStack({ - projectRoot: project, - runtime: { kind: "container", engine: "docker" }, - config: { capabilities: { database: { version: defaultDatabaseMajor } } }, - }).pipe( - Effect.provideService(Path.Path, path), - Effect.provideService(Crypto.Crypto, crypto), - ); - yield* store.replace(stack.id, { - ...state, - definition: persisted.definition, - }); - const before = yield* stack.status; - const progress: Array = []; - const prepared = yield* stack.prepare({ - config: { capabilities: { rest: { settings: { schemas: ["private"] } } } }, - capabilities: ["rest"], - onProgress: (status) => progress.push(status), - }); - expect(prepared.capabilities).toEqual([ - { capability: "database", version: defaultDatabaseVersion, outcome: "cached" }, - { capability: "rest", version: defaultRestVersion, outcome: "cached" }, - ]); - expect(calls.filter((call) => call.startsWith("image ls"))).toHaveLength(2); - expect(progress).toEqual( - expect.arrayContaining([ - { workloadId: "database:database", capability: "database", state: "queued" }, - { workloadId: "rest:rest", capability: "rest", state: "queued" }, - { workloadId: "database:database", capability: "database", state: "preparing" }, - { workloadId: "rest:rest", capability: "rest", state: "preparing" }, - { workloadId: "database:database", capability: "database", state: "ready" }, - { workloadId: "rest:rest", capability: "rest", state: "ready" }, - ]), - ); - expect(yield* stack.status).toEqual(before); - }), - ), - ); - - it.live( - "surfaces a container engine failure during public start", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-effect-stack-start-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "start-error-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => - Effect.fail({ - tag: "ContainerEngineError", - message: "Container engine command failed while starting database", - }), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - const result = yield* stack.start().pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(ContainerEngineError); - expect(failure.value.message).toContain("Container engine command failed"); - } - } - }), - ).pipe(Effect.provide(NodeServices.layer)), - 240_000, - ); - - it.live("surfaces concurrent lifecycle conflicts through the public start handle", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "ss-cstart-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "concurrent-start-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const startEntered = yield* Deferred.make(); - const releaseStart = yield* Deferred.make(); - let starts = 0; - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => - Effect.gen(function* () { - starts += 1; - if (starts === 1) { - yield* Deferred.succeed(startEntered, undefined); - yield* Deferred.await(releaseStart); - return runningStatus; - } - return yield* Effect.fail({ - tag: "StackLifecycleConflictError", - message: "A lifecycle transition is already in progress", - } as const); - }), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - const first = yield* Effect.forkChild(stack.start(), { startImmediately: true }); - yield* Deferred.await(startEntered); - const second = yield* stack.start().pipe(Effect.exit); - expect(Exit.isFailure(second)).toBe(true); - if (Exit.isFailure(second)) { - const failure = Cause.findErrorOption(second.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) - expect(failure.value).toBeInstanceOf(StackLifecycleConflictError); - } - yield* Deferred.succeed(releaseStart, undefined); - expect(Exit.isSuccess(yield* Fiber.join(first).pipe(Effect.exit))).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("cancels unfinished direct preparation while retaining completed artifacts", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const fs = yield* FileSystem.FileSystem; - const crypto = { - ...(yield* Effect.service(Crypto.Crypto)), - randomBytes: () => { - throw new Error("direct preparation must not generate managed secrets"); - }, - } satisfies Crypto.Crypto; - const firstPublished = yield* Deferred.make(); - const secondDownloadStarted = yield* Deferred.make(); - const secondDownloadCancelled = yield* Deferred.make(); - const secondDownloadRelease = yield* Deferred.make(); - const present = new Set(); - const runner: ContainerCommandRunner = { - run: (request) => { - const [command, subcommand, image] = request.args; - if (command === "version") - return Effect.succeed({ - stdout: '"1"\n', - stderr: "", - exitCode: 0, - }); - if (command === "image" && subcommand === "ls" && image !== undefined) { - return Effect.succeed({ - stdout: present.has(image) ? '"cached"\n' : "", - stderr: "", - exitCode: 0, - }); - } - if ( - command === "image" && - subcommand === "pull" && - image !== undefined && - image.includes("postgrest") - ) - return Deferred.succeed(secondDownloadStarted, undefined).pipe( - Effect.andThen(Deferred.await(secondDownloadRelease)), - Effect.onInterrupt(() => - Deferred.succeed(secondDownloadCancelled, undefined).pipe(Effect.asVoid), - ), - Effect.as({ stdout: "", stderr: "", exitCode: 0 }), - ); - if (command === "image" && subcommand === "pull" && image !== undefined) - return Effect.yieldNow.pipe( - Effect.andThen( - Effect.sync(() => { - present.add(image); - return { stdout: "", stderr: "", exitCode: 0 }; - }), - ), - Effect.andThen( - image.includes("postgres") - ? Deferred.succeed(firstPublished, undefined) - : Effect.void, - ), - Effect.as({ stdout: "", stderr: "", exitCode: 0 }), - ); - return Effect.succeed({ stdout: "", stderr: "", exitCode: 0 }); - }, - stream: () => Stream.empty, - }; - const engine = makeDockerEngine({ - runner, - platform: { os: "linux" }, - }); - const resolver = { - isInstalled: () => Effect.succeed(true), - resolve: () => Effect.succeed(engine), - }; - const stack = yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "docker" }, - }).pipe( - Effect.provideService(ContainerEngineResolver, resolver), - Effect.provideService(Crypto.Crypto, crypto), - ); - const stateStore = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const before = yield* fs.readFileString(paths.stateDocument); - const progress: Array = []; - const preparation = yield* Effect.forkChild( - stack.prepare({ - capabilities: ["database", "rest"], - onProgress: (status) => progress.push(status), - }), - { startImmediately: true }, - ); - yield* Deferred.await(firstPublished).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.await(secondDownloadStarted).pipe( - Effect.timeout("5 seconds"), - Effect.orDie, - ); - expect( - progress.some( - ({ workloadId, state }) => workloadId === "rest:rest" && state === "downloading", - ), - ).toBe(true); - yield* Fiber.interrupt(preparation); - yield* Deferred.await(secondDownloadCancelled).pipe( - Effect.timeout("5 seconds"), - Effect.orDie, - ); - const canceled = yield* Fiber.join(preparation).pipe(Effect.exit); - expect(Exit.isFailure(canceled)).toBe(true); - expect( - progress.some(({ workloadId, state }) => workloadId === "rest:rest" && state === "ready"), - ).toBe(false); - const retryProgress: Array = []; - const cached = yield* stack.prepare({ - capabilities: ["database"], - onProgress: (status) => retryProgress.push(status), - }); - expect(cached.capabilities).toEqual([ - { capability: "database", version: defaultDatabaseVersion, outcome: "cached" }, - ]); - expect(retryProgress.some(({ state }) => state === "downloading")).toBe(false); - expect(yield* fs.readFileString(paths.stateDocument)).toBe(before); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - expect(yield* stateStore.read(stack.id)).toBeDefined(); - }), - ), - ); - - it.live("prepares while running without consulting Supervisor ownership", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project }); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const state = yield* store.read(stack.id); - if (state === undefined) return yield* Effect.die("stack state was not initialized"); - yield* store.replace(stack.id, { ...state, desiredLifecycle: "running" }); - expect((yield* stack.prepare({ capabilities: [] })).capabilities).toEqual([]); - expect((yield* store.read(stack.id))?.desiredLifecycle).toBe("running"); - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - }), - ), - ); - - it.live("omits undefined optional RPC payload keys", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-rpc-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "session"; - const payloads: { start: Array } = { start: [] }; - const rpcHandlers: StackRpcHandlers = { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: (payload) => { - payloads.start.push(payload); - return Effect.succeed(runningStatus); - }, - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: emptyLogs, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - } as const), - stop: Effect.succeed({ ok: true, op: "stop" } as const), - }, - }); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - yield* stack.start(); - yield* stack.start({ config: {} }); - expect(payloads.start).toEqual([{}, { config: {} }]); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("launches a compatible Supervisor when start finds no live owner", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-start-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "start-session"; - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: emptyLogs, - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - }); - let launched = false; - const stack = yield* makeTestHandle(stackId, { - resolveOwner: (launch) => - Effect.sync(() => { - launched ||= launch; - return launched ? Option.some({ owner, launched: true }) : Option.none(); - }), - }); - expect((yield* stack.start()).lifecycle).toBe("running"); - expect(launched).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("completes destroy after the owner closes its control socket", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "session"; - const responseSent = yield* Deferred.make(); - const ownerScope = yield* Scope.make(); - const rpcHandlers: StackRpcHandlers = { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: emptyLogs, - }; - const maintenanceHandlers = { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - } as const), - stop: Effect.succeed({ ok: true, op: "stop" } as const), - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcHandlers, - maintenanceHandlers, - onShutdownReady: Deferred.succeed(responseSent, undefined).pipe(Effect.asVoid), - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => - Effect.succeed( - Option.some({ - owner: { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }, - launched: false, - }), - ), - readPersistedState: Effect.succeed(Option.some(stoppedState())), - }); - const destroyFiber = yield* Effect.forkChild(stack.destroy, { startImmediately: true }); - yield* Deferred.await(responseSent); - yield* Scope.close(ownerScope, Exit.void); - const destroyed = yield* Fiber.join(destroyFiber).pipe(Effect.exit); - expect(Exit.isSuccess(destroyed)).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("fails destroy when the owner control endpoint is unavailable", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-" }); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => - Effect.succeed( - Option.some({ - owner: { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint: { kind: "unix", path: path.join(root, "missing.sock") }, - ownerSessionId: "session", - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }, - launched: false, - }), - ), - readPersistedState: Effect.succeed(Option.some(stoppedState())), - }); - const result = yield* stack.destroy.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(StackDestructionError); - expect(error.value.message).toContain("control connection"); - } - } - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("fails destroy without launching an owner when persisted state is absent", () => - Effect.gen(function* () { - let launchCalls = 0; - const stack = yield* makeTestHandle(stackId, { - resolveOwner: (launch) => - Effect.sync(() => { - if (launch) launchCalls += 1; - return Option.none(); - }), - readPersistedState: Effect.succeed(Option.none()), - }); - const result = yield* stack.destroy.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) expect(failure.value).toBeInstanceOf(StackNotFoundError); - } - expect(launchCalls).toBe(0); - }), - ); - - it.live( - "stops an incompatible owner before starting a replacement", - () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project, runtime: { kind: "native" } }); - yield* Effect.addFinalizer(() => stack.destroy.pipe(Effect.ignore)); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - yield* ensureSupervisor({ - stackId: stack.id, - stateStore: store, - environment: env, - }); - const owner = yield* readOwnerMetadata(env.stateRoot, stack.id, env); - const originalOwner = expectPresent(owner, "owner metadata"); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const incompatibleOwner = yield* Schema.encodeEffect( - Schema.fromJsonString(Schema.Unknown), - )({ - ...originalOwner, - rpcRelease: "stack-rpc-v0@0.0.1", - }).pipe( - Effect.mapError( - (cause) => - new StackDestructionError({ message: "Unable to encode test owner", cause }), - ), - ); - yield* fs.writeFileString(paths.controlMetadata, incompatibleOwner); - - const oldOwner = yield* openStack(stack.id); - expect(Exit.isFailure(yield* oldOwner.status.pipe(Effect.exit))).toBe(true); - const ordinaryCreate = yield* createStack({ projectRoot: project }); - expect(ordinaryCreate.id).toBe(stack.id); - const restarted = yield* openStack(stack.id); - yield* Effect.addFinalizer(() => restarted.destroy.pipe(Effect.ignore)); - const directStart = yield* restarted - .start({ config: lifecycleConfig() }) - .pipe(Effect.exit); - expect(Exit.isFailure(directStart)).toBe(true); - if (Exit.isFailure(directStart)) { - const failure = Cause.findErrorOption(directStart.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) - expect(failure.value).toBeInstanceOf(StackUpgradeRequiredError); - } - yield* restarted.stop; - const status = yield* restarted.start({ config: lifecycleConfig() }); - const currentOwner = yield* readOwnerMetadata(env.stateRoot, stack.id, env); - expect(currentOwner?.rpcRelease).toBe(STACK_RPC_RELEASE); - expect(currentOwner?.ownerSessionId).not.toBe(originalOwner.ownerSessionId); - expect(status.id).toBe(stack.id); - expect(status.runtime).toEqual({ kind: "native" }); - expect((yield* restarted.status).lifecycle).toBe("running"); - yield* restarted.stop; - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - const startedAgain = yield* restarted.start({ config: lifecycleConfig() }); - expect(startedAgain.lifecycle).toBe("running"); - yield* restarted.destroy; - }), - ), - 240_000, - ); - - it.live( - "reclaims a dead incompatible owner through create and start", - () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const env = yield* StackRuntimeEnvironment; - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const makeDeadOwner = ( - projectRoot: string, - rpcRelease: string, - desiredLifecycle: "running" | "unconfigured" = "unconfigured", - ) => - Effect.gen(function* () { - const identity = yield* resolveStackIdentity({ projectRoot }); - const id = yield* deriveStackId(identity); - const persisted = - desiredLifecycle === "running" - ? yield* Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot, - runtime: { kind: "native" }, - config: lifecycleConfig(), - }); - const resolved = yield* resolveSecrets( - { - declarations: compiled.secrets.map( - ({ slot, policy, value, generator }) => ({ - slot, - policy, - ...(value === undefined ? {} : { value }), - ...(generator === undefined ? {} : { generator }), - }), - ), - }, - undefined, - "running", - ); - return { - definition: compiled.definition, - secrets: resolved.persisted, - }; - }) - : undefined; - const initialState: PersistedStackState = { - format: "supabase-stack-state-v1", - identity: toPersistedIdentity(identity), - runtime: { kind: "native" }, - desiredLifecycle, - ports: [], - privatePorts: [], - secrets: {}, - }; - yield* store.initialize( - id, - persisted === undefined ? initialState : { ...initialState, ...persisted }, - ); - const encodedEnvironment = yield* Schema.encodeEffect( - Schema.fromJsonString(Schema.Unknown), - )({ - stateRoot: env.stateRoot, - tempRoot: env.tempRoot, - platform: env.platform, - }); - // bare imports in a `node -e` script resolve from cwd, so run from the package root - const cwd = path.resolve(import.meta.dirname, "../.."); - const child = yield* ChildProcess.make( - process.execPath, - [ - "--input-type=module", - "-e", - ` - const { Effect } = await import("effect"); - const { NodeServices } = await import("@effect/platform-node"); - const { acquireOwnership, publishOwnership } = await import(process.env.OWNERSHIP_MODULE); - const environment = JSON.parse(process.env.OWNERSHIP_ENVIRONMENT); - await Effect.runPromise(Effect.scoped(Effect.gen(function* () { - const lease = yield* acquireOwnership({ - stateRoot: environment.stateRoot, - stackId: process.env.OWNERSHIP_STACK_ID, - ownerSessionId: "crashed-owner", - rpcRelease: process.env.OWNERSHIP_RPC_RELEASE, - environment, - }); - yield* publishOwnership(lease); - process.stdout.write("READY\\n"); - yield* Effect.never; - }).pipe(Effect.provide(NodeServices.layer)))); - `, - ], - { - cwd, - env: { - OWNERSHIP_MODULE: new URL("../state/Ownership.ts", import.meta.url).href, - OWNERSHIP_STACK_ID: id, - OWNERSHIP_RPC_RELEASE: rpcRelease, - OWNERSHIP_ENVIRONMENT: encodedEnvironment, - }, - extendEnv: true, - stdout: "pipe", - stderr: "pipe", - }, - ); - const ready = yield* Deferred.make(); - const output = yield* child.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - line === "READY" - ? Deferred.succeed(ready, undefined).pipe(Effect.asVoid) - : Effect.void, - ), - Effect.forkChild({ startImmediately: true }), - ); - const stderr = yield* child.stderr.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runDrain, - Effect.forkChild({ startImmediately: true }), - ); - try { - yield* Deferred.await(ready).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => - Effect.fail(new StackStateInvalidError({ message: "owner child not ready" })), - }), - ); - expect(yield* ownerLockExists(env.stateRoot, id)).toBe(true); - yield* child.kill({ killSignal: "SIGKILL" }); - yield* child.exitCode.pipe(Effect.ignore); - return id; - } finally { - yield* child.kill({ killSignal: "SIGKILL" }).pipe(Effect.ignore); - yield* Fiber.interrupt(output); - yield* Fiber.interrupt(stderr); - } - }); - - const createProject = path.join(project, "create"); - const openProject = path.join(project, "open"); - yield* fs.makeDirectory(createProject); - yield* fs.makeDirectory(openProject); - const created = yield* createStack({ projectRoot: createProject }); - expect(yield* readOwnerMetadata(env.stateRoot, created.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, created.id)).toBe(false); - - const openId = yield* makeDeadOwner(openProject, "stack-rpc-v0@0.0.1", "running"); - const replaced = yield* openStack(openId); - const cleanupRecovered = yield* Effect.cached( - Effect.uninterruptible( - Effect.gen(function* () { - yield* replaced.stop; - yield* replaced.destroy; - expect(yield* readOwnerMetadata(env.stateRoot, openId, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, openId)).toBe(false); - }), - ), - ); - // Cache the stop/destroy effect so the assertion below and the finalizer share one transition. - yield* Effect.addFinalizer(() => cleanupRecovered.pipe(Effect.ignore)); - yield* replaced.start(); - expect((yield* readOwnerMetadata(env.stateRoot, openId, env))?.rpcRelease).toBe( - STACK_RPC_RELEASE, - ); - expect((yield* readOwnerMetadata(env.stateRoot, openId, env))?.ownerSessionId).not.toBe( - "crashed-owner", - ); - expect(yield* ownerLockExists(env.stateRoot, openId)).toBe(true); - yield* cleanupRecovered; - }), - ), - 300_000, - ); - - it.live("accepts maintenance stop requests from an older RPC owner", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const stack = yield* createStack({ projectRoot: project, runtime: { kind: "native" } }); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - yield* ensureSupervisor({ - stackId: stack.id, - stateStore: store, - environment: env, - }); - const ownerHandle = yield* openStack(stack.id); - yield* Effect.addFinalizer(() => ownerHandle.stop.pipe(Effect.ignore)); - const owner = yield* readOwnerMetadata(env.stateRoot, stack.id, env); - const currentOwner = expectPresent(owner, "owner metadata"); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const incompatibleOwner = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))( - { - ...currentOwner, - rpcRelease: "stack-rpc-v0@0.0.1", - }, - ).pipe( - Effect.mapError( - (cause) => new StackDestructionError({ message: "Unable to encode test owner", cause }), - ), - ); - yield* fs.writeFileString(paths.controlMetadata, incompatibleOwner); - yield* ownerHandle.stop; - expect(yield* readOwnerMetadata(env.stateRoot, stack.id, env)).toBeUndefined(); - expect(yield* ownerLockExists(env.stateRoot, stack.id)).toBe(false); - }), - ), - ); - - it.live( - "rejects incompatible RPC calls before opening handlers while allowing maintenance stop", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-effect-stack-release-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "incompatible-rpc-session"; - const ownerRelease = "stack-rpc-v0@0.0.1"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const calls = yield* Ref.make(0); - const invoked = (value: A) => - Ref.update(calls, (count) => count + 1).pipe(Effect.andThen(Effect.succeed(value))); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: ownerRelease, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: ownerRelease, - rpcHandlers: { - status: () => invoked(runningStatus), - credentials: () => invoked(credentials), - start: () => invoked(runningStatus), - destroy: () => invoked(undefined), - resetDatabase: () => invoked(runningStatus), - logs: () => invoked({ entries: [], cursor: { opaque: "v1_0" }, running: false }), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: ownerRelease, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - onShutdownReady: Scope.close(ownerScope, Exit.void), - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - const status = yield* stack.status.pipe(Effect.exit); - const stackCredentials = yield* stack.credentials.pipe(Effect.exit); - const logs = yield* stack.logs().pipe(Effect.exit); - const assertUpgrade = (result: Exit.Exit) => { - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(StackUpgradeRequiredError); - expect(failure.value).toMatchObject({ - expectedRelease: STACK_RPC_RELEASE, - actualRelease: ownerRelease, - }); - } - } - }; - assertUpgrade(status); - assertUpgrade(stackCredentials); - assertUpgrade(logs); - expect(yield* Ref.get(calls)).toBe(0); - yield* stack.stop; - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("stops a freshly launched owner when the first start RPC is interrupted", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-launch-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "fresh-launch-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const startEntered = yield* Deferred.make(); - const stopCalls = yield* Ref.make(0); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => - Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Ref.update(stopCalls, (calls) => calls + 1).pipe( - Effect.andThen(Effect.succeed({ ok: true, op: "stop" } as const)), - ), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: true })), - }); - const start = yield* Effect.forkChild(stack.start(), { startImmediately: true }); - yield* Deferred.await(startEntered); - yield* Fiber.interrupt(start); - expect(yield* Ref.get(stopCalls)).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("stops a freshly launched owner when destroy is interrupted", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-effect-stack-destroy-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "fresh-destroy-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const destroyEntered = yield* Deferred.make(); - const stopCalls = yield* Ref.make(0); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => Effect.succeed(runningStatus), - destroy: () => - Deferred.succeed(destroyEntered, undefined).pipe(Effect.andThen(Effect.never)), - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Ref.update(stopCalls, (calls) => calls + 1).pipe( - Effect.andThen(Effect.succeed({ ok: true, op: "stop" } as const)), - ), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: true })), - readPersistedState: Effect.succeed(Option.some(stoppedState())), - }); - const destroy = yield* Effect.forkChild(stack.destroy, { startImmediately: true }); - yield* Deferred.await(destroyEntered); - yield* Fiber.interrupt(destroy); - const interrupted = yield* Fiber.join(destroy).pipe(Effect.exit); - expect(Exit.isFailure(interrupted)).toBe(true); - expect(yield* Ref.get(stopCalls)).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("reports fresh-owner rollback failures after an interrupted start", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-effect-stack-rollback-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "rollback-failure-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const startEntered = yield* Deferred.make(); - const stopCalls = yield* Ref.make(0); - const releaseCalls = yield* Ref.make(0); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => - Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Ref.update(stopCalls, (calls) => calls + 1).pipe( - Effect.andThen( - Effect.succeed({ - ok: false, - error: { tag: "operation-failed", message: "injected rollback failure" }, - } as const), - ), - ), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: true })), - waitForRelease: Ref.update(releaseCalls, (calls) => calls + 1).pipe( - Effect.andThen( - Effect.fail(new StackOwnershipConflictError({ message: "release failed" })), - ), - ), - }); - const start = yield* Effect.forkChild(stack.start(), { startImmediately: true }); - yield* Deferred.await(startEntered); - yield* Fiber.interrupt(start); - const interrupted = yield* Fiber.join(start).pipe(Effect.exit); - expect(Exit.isFailure(interrupted)).toBe(true); - if (Exit.isFailure(interrupted)) { - const failure = Cause.findErrorOption(interrupted.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(StackCleanupError); - expect(failure.value.message).toContain( - "Unable to clean up freshly launched Supervisor", - ); - expect(failure.value.cause).toBeDefined(); - } - } - expect(yield* Ref.get(stopCalls)).toBe(1); - expect(yield* Ref.get(releaseCalls)).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("preserves an interrupted start when fresh-owner cleanup is refused", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "ss-ar-", - }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "admitted-rollback-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const startEntered = yield* Deferred.make(); - const stopCalls = yield* Ref.make(0); - const releaseCalls = yield* Ref.make(0); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => - Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Ref.update(stopCalls, (calls) => calls + 1).pipe( - Effect.andThen( - Effect.succeed({ - ok: false, - error: { - tag: "operation-failed", - message: "a lifecycle transition is already in progress", - stackErrorTag: "StackLifecycleConflictError", - }, - } as const), - ), - ), - }, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: true })), - waitForRelease: Ref.update(releaseCalls, (calls) => calls + 1).pipe( - Effect.andThen( - Effect.fail(new StackOwnershipConflictError({ message: "release was unexpected" })), - ), - ), - }); - const start = yield* Effect.forkChild(stack.start(), { startImmediately: true }); - yield* Deferred.await(startEntered); - yield* Fiber.interrupt(start); - const interrupted = yield* Fiber.join(start).pipe(Effect.exit); - expect(Exit.isFailure(interrupted)).toBe(true); - if (Exit.isFailure(interrupted)) { - const failure = Cause.findErrorOption(interrupted.cause); - expect(Option.isNone(failure)).toBe(true); - expect(Cause.hasInterrupts(interrupted.cause)).toBe(true); - } - expect(yield* Ref.get(stopCalls)).toBe(1); - expect(yield* Ref.get(releaseCalls)).toBe(0); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("does not stop an owner joined through a concurrent launch", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-effect-stack-join-" }); - const endpoint = { kind: "unix" as const, path: path.join(root, "control.sock") }; - const ownerSessionId = "joined-launch-session"; - const ownerScope = yield* Scope.make(); - yield* Effect.addFinalizer(() => Scope.close(ownerScope, Exit.void)); - const startEntered = yield* Deferred.make(); - const stopCalls = yield* Ref.make(0); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId, - endpoint, - ownerSessionId, - leasePort: 45_001, - rpcRelease: STACK_RPC_RELEASE, - }; - yield* startControlServer({ - endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - rpcHandlers: { - status: () => Effect.succeed(runningStatus), - credentials: () => Effect.succeed(credentials), - start: () => - Deferred.succeed(startEntered, undefined).pipe(Effect.andThen(Effect.never)), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(runningStatus), - logs: () => emptyLogs(), - }, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Ref.update(stopCalls, (calls) => calls + 1).pipe( - Effect.andThen(Effect.succeed({ ok: true, op: "stop" } as const)), - ), - }, - onShutdownReady: Scope.close(ownerScope, Exit.void), - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* makeTestHandle(stackId, { - resolveOwner: () => Effect.succeed(Option.some({ owner, launched: false })), - }); - const start = yield* Effect.forkChild(stack.start(), { startImmediately: true }); - yield* Deferred.await(startEntered); - yield* Fiber.interrupt(start); - expect(yield* Ref.get(stopCalls)).toBe(0); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); -}); diff --git a/packages/stack/src/public/ephemeral-postgres.integration.test.ts b/packages/stack/src/public/ephemeral-postgres.integration.test.ts deleted file mode 100644 index 7b699a96f8..0000000000 --- a/packages/stack/src/public/ephemeral-postgres.integration.test.ts +++ /dev/null @@ -1,352 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { PgClient } from "@effect/sql-pg"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Duration, - Effect, - Exit, - Fiber, - FileSystem, - Layer, - Option, - Path, - Redacted, - Schedule, -} from "effect"; -import { ChildProcess } from "effect/unstable/process"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- docker availability probe for optional container cases. -import { spawnSync } from "node:child_process"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- test reserves a loopback port before fork. -import { createServer } from "node:net"; -import { tmpdir } from "node:os"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- isolated artifact cache path. -import { join } from "node:path"; -import { EphemeralPostgresError } from "./Errors.ts"; -import { createEphemeralPostgres } from "./EphemeralPostgres.ts"; -import { catalogEntryFor } from "../model/WorkloadCatalog.ts"; -import { schemaInit } from "./SchemaInit.ts"; -import { listStacks } from "./EffectStack.ts"; -import { defaultRuntimeEnvironment, StackRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { checkHostPort } from "../supervisor/HostListener.ts"; -import type { StackRuntimePreference } from "./Runtime.ts"; - -const NATIVE_TIMEOUT_MS = 180_000; -const PASSWORD = "ephemeral-test-password"; -const JWT_SECRET = "ephemeral-test-jwt-secret-value"; - -const dockerAvailable = (): boolean => - spawnSync("docker", ["info"], { encoding: "utf8" }).status === 0; - -const artifactCacheRoot = join(tmpdir(), "supabase-stack-test-artifacts"); - -const testEnvironment = (stateRoot: string) => - Layer.effect( - StackRuntimeEnvironment, - defaultRuntimeEnvironment.pipe( - Effect.map((env) => ({ - ...env, - stateRoot, - artifactCacheRoot, - })), - ), - ); - -const secrets = { - databasePassword: Redacted.make(PASSWORD), - jwtSecret: Redacted.make(JWT_SECRET), -}; - -const query = (url: Redacted.Redacted, statement: string) => - Effect.scoped( - Effect.gen(function* () { - const client = yield* PgClient.PgClient; - return yield* client.unsafe(statement); - }).pipe(Effect.provide(PgClient.layer({ url, connectTimeout: "10 seconds" }))), - ); - -const withIsolatedRoot = (effect: Effect.Effect) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectory({ prefix: "supabase-eph-" }); - yield* Effect.addFinalizer(() => fs.remove(root, { recursive: true }).pipe(Effect.ignore)); - const stateRoot = path.join(root, "managed", "stacks"); - yield* fs.makeDirectory(stateRoot, { recursive: true }); - return yield* effect.pipe(Effect.provide(testEnvironment(stateRoot))); - }); - -const writeForeignMarkerTar = (tarPath: string, marker: unknown) => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const staging = yield* fs.makeTempDirectoryScoped(); - const data = path.join(staging, "data"); - yield* fs.makeDirectory(data); - yield* fs.writeFileString( - path.join(data, ".supabase-ephemeral-runtime"), - // oxlint-disable-next-line effecttsgo/prefer-schema-over-json -- fixture marker bytes packed into a tar. - JSON.stringify(marker), - ); - const handle = yield* ChildProcess.make("tar", ["-C", staging, "-cf", tarPath, "data"], { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - }); - const code = yield* handle.exitCode; - expect(Number(code)).toBe(0); - }), - ); - -const reserveLoopbackPort = (): Effect.Effect => - Effect.callback((resume) => { - const server = createServer(); - let settled = false; - const finish = (effect: Effect.Effect) => { - if (settled) return; - settled = true; - resume(effect); - }; - server.once("error", (cause) => finish(Effect.die(cause))); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - const port = typeof address === "object" && address !== null ? address.port : 0; - server.close((error) => { - if (error !== undefined) { - finish(Effect.die(error)); - return; - } - finish(port > 0 ? Effect.succeed(port) : Effect.die("Unable to allocate a loopback port")); - }); - }); - return Effect.sync(() => { - if (settled) return; - settled = true; - try { - server.close(); - } catch { - // The listener never obtained a handle. - } - }); - }); - -describe("ephemeral Postgres", () => { - it.live("refuses a snapshot produced by a different runtime before starting Postgres", () => - withIsolatedRoot( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const tarPath = path.join(yield* fs.makeTempDirectoryScoped(), "foreign.tar"); - yield* writeForeignMarkerTar(tarPath, { kind: "container", engine: "docker" }); - const exit = yield* createEphemeralPostgres({ - runtime: { kind: "native" }, - restoreFrom: tarPath, - ...secrets, - }).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(EphemeralPostgresError); - if (!(error instanceof EphemeralPostgresError)) return; - expect(error.reason).toBe("restore-mismatch"); - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - - it.live("refuses a keyless snapshot marker when a snapshot key is expected", () => - withIsolatedRoot( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const tarPath = path.join(yield* fs.makeTempDirectoryScoped(), "keyless.tar"); - yield* writeForeignMarkerTar(tarPath, { kind: "native" }); - const exit = yield* createEphemeralPostgres({ - runtime: { kind: "native" }, - restoreFrom: tarPath, - snapshotKey: "expected-cache-key", - ...secrets, - }).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(EphemeralPostgresError); - if (!(error instanceof EphemeralPostgresError)) return; - expect(error.reason).toBe("restore-mismatch"); - expect(error.message).toContain("snapshot key"); - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - - it.live( - "starts a native cluster, snapshots, restores, and destroys without a stack identity", - () => - withIsolatedRoot( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const exportDir = yield* fs.makeTempDirectoryScoped(); - const tarPath = path.join(exportDir, "baseline.tar"); - const first = yield* createEphemeralPostgres({ runtime: { kind: "native" }, ...secrets }); - const rows = yield* query( - first.url, - "SELECT rolname FROM pg_roles WHERE rolname = 'supabase_admin'", - ); - expect(rows.length).toBeGreaterThan(0); - expect(first.runtime.kind).toBe("native"); - expect(first.artifactIdentity.startsWith("native:")).toBe(true); - const listedWhileRunning = yield* listStacks({}); - expect(listedWhileRunning).toEqual([]); - yield* first.stop; - yield* first.exportPgData(tarPath); - const exists = yield* fs.exists(tarPath); - expect(exists).toBe(true); - - const restored = yield* createEphemeralPostgres({ - runtime: { kind: "native" }, - restoreFrom: tarPath, - ...secrets, - }); - const restoredRows = yield* query(restored.url, "SELECT current_database() AS name"); - expect(restoredRows).toEqual([{ name: "postgres" }]); - expect(restored.port).not.toBe(first.port); - - const second = yield* createEphemeralPostgres({ - runtime: { kind: "native" }, - ...secrets, - }); - expect(second.port).not.toBe(first.port); - expect(second.port).not.toBe(restored.port); - yield* query(second.url, "SELECT 1"); - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - NATIVE_TIMEOUT_MS, - ); - - it.live( - "catalog native postgres can CREATE EXTENSION plpgsql_check on every database release", - () => - withIsolatedRoot( - Effect.gen(function* () { - const versions = Object.keys(catalogEntryFor("database:database").releases); - expect(versions.length).toBeGreaterThan(0); - for (const version of versions) { - const cluster = yield* createEphemeralPostgres({ - runtime: { kind: "native" }, - version, - ...secrets, - }); - expect(cluster.version).toBe(version); - yield* query(cluster.url, "CREATE EXTENSION IF NOT EXISTS plpgsql_check"); - const installed = yield* query( - cluster.url, - "SELECT extname FROM pg_extension WHERE extname = 'plpgsql_check'", - ); - expect(installed).toEqual([{ extname: "plpgsql_check" }]); - yield* query( - cluster.url, - "CREATE FUNCTION public.lint_probe() RETURNS void LANGUAGE plpgsql AS $$ BEGIN PERFORM id FROM lint_probe_missing; END $$", - ); - const reports = yield* query( - cluster.url, - "SELECT plpgsql_check_function('public.lint_probe()'::regprocedure, format := 'json')::text AS report", - ); - expect(reports).toEqual([{ report: expect.stringContaining("lint_probe_missing") }]); - } - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - NATIVE_TIMEOUT_MS * 2, - ); - - it.live( - "does not leave postgres listening after an interrupted native start", - () => - withIsolatedRoot( - Effect.gen(function* () { - const port = yield* reserveLoopbackPort(); - // Fiber-owned scope so interrupt always tears the cluster down, even after start returns. - const fiber = yield* Effect.forkChild( - Effect.scoped( - createEphemeralPostgres({ runtime: { kind: "native" }, port, ...secrets }).pipe( - Effect.andThen(Effect.never), - ), - ), - ); - const url = Redacted.make( - `postgresql://${encodeURIComponent("postgres")}:${encodeURIComponent(PASSWORD)}@127.0.0.1:${port}/postgres`, - ); - yield* Effect.raceFirst( - Effect.retry(query(url, "SELECT 1"), { - schedule: Schedule.spaced("100 millis"), - }).pipe(Effect.timeout(Duration.seconds(120))), - Fiber.join(fiber), - ); - yield* Fiber.interrupt(fiber); - yield* checkHostPort("127.0.0.1", port, "database"); - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - NATIVE_TIMEOUT_MS, - ); - - it.live.skipIf(!dockerAvailable())( - "starts a container cluster, snapshots, and restores", - () => - withIsolatedRoot( - Effect.gen(function* () { - const runtime: StackRuntimePreference = { kind: "container", engine: "docker" }; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const tarPath = path.join(yield* fs.makeTempDirectoryScoped(), "baseline.tar"); - const first = yield* createEphemeralPostgres({ runtime, ...secrets }); - yield* query(first.url, "SELECT 1"); - expect(first.runtime.kind).toBe("container"); - expect(first.networkId).toEqual(expect.any(String)); - expect(first.artifactIdentity.startsWith("container:docker:")).toBe(true); - yield* first.stop; - yield* first.exportPgData(tarPath); - const restored = yield* createEphemeralPostgres({ - runtime, - restoreFrom: tarPath, - ...secrets, - }); - yield* query(restored.url, "SELECT 1"); - expect(restored.port).not.toBe(first.port); - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - NATIVE_TIMEOUT_MS, - ); - - it.live.skipIf(process.platform !== "linux" || !dockerAvailable())( - "schema-init one-shots join the cluster network and reach Postgres", - () => - withIsolatedRoot( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-linux-" }); - const runtime: StackRuntimePreference = { kind: "container", engine: "docker" }; - const cluster = yield* createEphemeralPostgres({ runtime, ...secrets }); - expect(cluster.networkId).toEqual(expect.any(String)); - yield* schemaInit(["auth"], { - kind: "ephemeral", - projectRoot, - runtime: cluster.runtime, - config: { - capabilities: { - studio: { enabled: true }, - analytics: { enabled: false }, - }, - }, - databaseUrl: Redacted.value(cluster.url), - secrets, - ...(cluster.networkId === undefined ? {} : { networkId: cluster.networkId }), - }); - const rows = yield* query( - cluster.url, - "SELECT nspname FROM pg_namespace WHERE nspname = 'auth'", - ); - expect(rows.length).toBeGreaterThan(0); - }), - ).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - NATIVE_TIMEOUT_MS, - ); -}); diff --git a/packages/stack/src/public/ephemeral-postgres.unit.test.ts b/packages/stack/src/public/ephemeral-postgres.unit.test.ts deleted file mode 100644 index 528b9615ad..0000000000 --- a/packages/stack/src/public/ephemeral-postgres.unit.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Option } from "effect"; -import { DatabaseModule } from "../model/capabilities/database.ts"; -import { StackVersionUnsupportedError } from "./Errors.ts"; -import { resolveEphemeralPostgresRelease } from "./EphemeralPostgres.ts"; - -describe("resolveEphemeralPostgresRelease", () => { - it.effect("resolves the catalog default and a major selector", () => - Effect.gen(function* () { - const fallback = yield* resolveEphemeralPostgresRelease(); - expect(fallback.version).toBe(DatabaseModule.defaultVersion); - expect(fallback.image.length).toBeGreaterThan(0); - - const major = DatabaseModule.defaultVersion.split(".")[0]; - expect(major).toBeDefined(); - if (major === undefined) return; - const selected = yield* resolveEphemeralPostgresRelease(major); - expect(selected.version).toBe(fallback.version); - expect(selected.image).toBe(fallback.image); - }), - ); - - it.effect("maps a superseded exact pin to the current catalog release of that major", () => - Effect.gen(function* () { - const major = DatabaseModule.defaultVersion.split(".")[0]; - expect(major).toBeDefined(); - if (major === undefined) return; - const selected = yield* resolveEphemeralPostgresRelease(`${major}.0.0.1`); - expect(selected.version).toBe(DatabaseModule.defaultVersion); - }), - ); - - it.effect("fails for an unknown PostgreSQL version", () => - Effect.gen(function* () { - const exit = yield* resolveEphemeralPostgresRelease("99").pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(StackVersionUnsupportedError); - }), - ); -}); diff --git a/packages/stack/src/public/index.ts b/packages/stack/src/public/index.ts deleted file mode 100644 index 50839b488d..0000000000 --- a/packages/stack/src/public/index.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** Public stack model and APIs. */ -export * from "./StackId.ts"; -export * from "./Runtime.ts"; -export * from "./Capability.ts"; -export * from "./Status.ts"; -export * from "./Logs.ts"; -export * from "./Credentials.ts"; -export * from "./Errors.ts"; -export * from "./Config.ts"; -export { targetForPlatform } from "../model/WorkloadCatalog.ts"; -export { excludeStackCapabilities } from "../model/Exclusions.ts"; -export type { ExcludableCapabilityName } from "../model/Exclusions.ts"; -export { - selectDefaultRuntime, - selectDefaultRuntimeSelection, - ContainerEngineResolver, - DOCKER_DAEMON_FALLBACK_NOTICE, - NATIVE_ROOT_UNSUPPORTED_MESSAGE, -} from "../runtime/ContainerEngineResolver.ts"; -export { defaultRuntimeEnvironment } from "../supervisor/Launcher.ts"; -export { - createStack, - openStack, - findStack, - listStacks, - discoverStacks, - inspectStack, -} from "./EffectStack.ts"; -export type { - EffectStack, - InspectStackOptions, - StartStackOptions, - PrepareStackOptions, - CreateStackOptions, - FindStackOptions, - ListStacksOptions, - StackDiscoveryIssue, - StackDiscoveryResult, - PreparedCapability, - PrepareStackResult, -} from "./EffectStack.ts"; -export { databaseBootstrapIdentity } from "../model/DatabaseBootstrap.ts"; -export { createEphemeralPostgres, resolveEphemeralPostgresRelease } from "./EphemeralPostgres.ts"; -export type { - CreateEphemeralPostgresOptions, - EffectEphemeralPostgres, - EphemeralPostgresRelease, - EphemeralPostgresServices, - EphemeralPostgresSettings, -} from "./EphemeralPostgres.ts"; -export { runPostgresClient } from "./PostgresClient.ts"; -export type { - PostgresClientMount, - PostgresClientResult, - PostgresClientServices, - RunPostgresClientOptions, -} from "./PostgresClient.ts"; -export { - schemaInit, - SCHEMA_INIT_CAPABILITY_NAMES, - schemaInitArtifactIdentity, -} from "./SchemaInit.ts"; -export type { - SchemaInitCapabilityName, - SchemaInitEphemeralTarget, - SchemaInitLiveTarget, - SchemaInitOptions, - SchemaInitSecrets, - SchemaInitServices, - SchemaInitTarget, -} from "./SchemaInit.ts"; diff --git a/packages/stack/src/public/promise.integration.test.ts b/packages/stack/src/public/promise.integration.test.ts deleted file mode 100644 index afaa34cbef..0000000000 --- a/packages/stack/src/public/promise.integration.test.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { NodeServices } from "@effect/platform-node"; -import { CAPABILITY_NAMES, type CapabilityStatus } from "./Capability.ts"; -import { Data, Effect, FileSystem, Path, Redacted, Schema, Stream } from "effect"; -import type { EffectStack, PrepareStackOptions, StartStackOptions } from "./EffectStack.ts"; -import type { LogQuery, StackLogEntry } from "./Logs.ts"; -import { StackIdSchema } from "./StackId.ts"; -import type { ArtifactPreparationStatus, StackStatus } from "./Status.ts"; -import { InvalidStackConfigError, StackVersionUnsupportedError } from "./Errors.ts"; -import { adaptEffectStack, makePromiseApi, type PromiseStack } from "./PromiseStack.ts"; -import { compileStack } from "../model/Compiler.ts"; -class StreamFixtureError extends Data.TaggedError("StreamFixtureError")<{ - readonly cause: unknown; -}> {} -const malformedConfig = Schema.decodeSync(Schema.fromJsonString(Schema.Any))( - '{"capabilities":{"rest":{"settings":{"unknown":true}}}}', -); -const stackId = StackIdSchema.make( - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", -); -const capabilities: ReadonlyArray = CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "functions" ? "lazy" : "eager", - state: name === "functions" ? "ready" : "dormant", -})); -const status: StackStatus = { - id: stackId, - lifecycle: "running", - desiredLifecycle: "running", - runtime: { kind: "native" }, - endpoints: { - api: { - protocol: "http", - address: "127.0.0.1", - port: 54321, - url: "http://127.0.0.1:54321", - }, - }, - versions: {}, - capabilities, - artifacts: [], -}; -const effectStack = (): EffectStack => - ({ - id: stackId, - status: Effect.succeed(status), - credentials: Effect.succeed({ - database: { url: Redacted.make("postgres://secret"), password: Redacted.make("db-pass") }, - api: { - publishableKey: "publishable", - secretKey: Redacted.make("secret-key"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service-role"), - }, - storage: { - endpoint: "http://storage", - region: "local", - accessKeyId: "access", - secretAccessKey: Redacted.make("storage-secret"), - }, - }), - prepare: (_options?: PrepareStackOptions) => Effect.succeed({ capabilities: [] }), - start: () => Effect.succeed(status), - stop: Effect.void, - destroy: Effect.void, - resetDatabase: Effect.succeed(status), - logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), - followLogs: () => Stream.empty, - }) satisfies EffectStack; -describe("Promise stack facade", () => { - it.live("prepares a real stack without publishing owner metadata", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-promise-prepare-" }); - const project = path.join(root, "project"); - const stateRoot = path.join(root, "managed", "stacks"); - yield* fs.makeDirectory(project); - const api = makePromiseApi(NodeServices.layer, { - stateRoot, - tempRoot: "/tmp", - platform: "posix", - }); - const stack = yield* Effect.promise(() => - api.createStack({ projectRoot: project, runtime: { kind: "native" } }), - ); - const statePath = path.join(stateRoot, stack.id, "state.json"); - const before = yield* fs.readFileString(statePath); - yield* Effect.promise(() => - expect(stack.prepare({ capabilities: [] })).resolves.toEqual({ capabilities: [] }), - ); - expect(yield* fs.readFileString(statePath)).toBe(before); - expect(yield* fs.readDirectory(path.join(stateRoot, stack.id))).not.toContain("control.json"); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("preserves prepare validation tags", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-promise-prepare-errors-", - }); - const project = path.join(root, "project"); - yield* fs.makeDirectory(project); - const api = makePromiseApi(NodeServices.layer, { - stateRoot: path.join(root, "managed", "stacks"), - tempRoot: "/tmp", - platform: "posix", - }); - const stack = yield* Effect.promise(() => - api.createStack({ projectRoot: project, runtime: { kind: "native" } }), - ); - yield* Effect.promise(() => - expect( - stack.prepare({ - config: malformedConfig, - }), - ).rejects.toBeInstanceOf(InvalidStackConfigError), - ); - yield* Effect.promise(() => - expect( - stack.prepare({ config: { capabilities: { database: { version: "99" } } } }), - ).rejects.toBeInstanceOf(StackVersionUnsupportedError), - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("returns log batches and follows from their cursor", () => - Effect.gen(function* () { - const initial: StackLogEntry = { - cursor: { opaque: "v1_0" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "auth", - stream: "stdout", - message: "initial", - }; - const followedEntry: StackLogEntry = { - ...initial, - cursor: { opaque: "v1_1" }, - message: "followed", - }; - let logsQuery: LogQuery | undefined; - let followQuery: LogQuery | undefined; - const stack = adaptEffectStack({ - ...effectStack(), - logs: (query) => - Effect.sync(() => { - logsQuery = query; - return { entries: [initial], cursor: { opaque: "v1_0" }, running: true }; - }), - followLogs: (query) => { - followQuery = query; - return Stream.succeed(followedEntry); - }, - }); - const first = yield* Effect.promise(() => stack.logs({ capabilities: ["auth"], tail: 20 })); - expect(first.entries).toEqual([initial]); - expect(logsQuery).toEqual({ capabilities: ["auth"], tail: 20 }); - const followed = yield* Stream.fromAsyncIterable( - stack.followLogs({ capabilities: ["auth"], cursor: first.cursor }), - (cause) => new StreamFixtureError({ cause }), - ).pipe(Stream.runCollect); - expect(followed).toEqual([followedEntry]); - expect(followQuery).toEqual({ capabilities: ["auth"], cursor: { opaque: "v1_0" } }); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("unwraps every credential secret without a lifecycle close operation", () => - Effect.gen(function* () { - const stack: PromiseStack = adaptEffectStack(effectStack()); - yield* Effect.promise(() => - expect(stack.credentials()).resolves.toEqual({ - database: { url: "postgres://secret", password: "db-pass" }, - api: { - publishableKey: "publishable", - secretKey: "secret-key", - anonJwt: "anon", - serviceRoleJwt: "service-role", - }, - storage: { - endpoint: "http://storage", - region: "local", - accessKeyId: "access", - secretAccessKey: "storage-secret", - }, - }), - ); - expect(Symbol.asyncDispose in stack).toBe(false); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("cancels an active async stream and witnesses its finalizer", () => - Effect.gen(function* () { - let finalized = false; - const entry: StackLogEntry = { - cursor: { opaque: "v1_1" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "auth", - stream: "stdout", - message: "active", - }; - const source: EffectStack = { - ...effectStack(), - followLogs: () => - Stream.make(entry).pipe( - Stream.concat(Stream.never), - Stream.ensuring( - Effect.sync(() => { - finalized = true; - }), - ), - ), - }; - const stack = adaptEffectStack(source); - const iterator = stack.followLogs()[Symbol.asyncIterator](); - yield* Effect.promise(() => - expect(iterator.next()).resolves.toEqual({ done: false, value: entry }), - ); - yield* Effect.promise(() => - expect(iterator.return?.()).resolves.toMatchObject({ done: true }), - ); - expect(finalized).toBe(true); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("forwards prepare progress through the Promise facade", () => - Effect.gen(function* () { - const progress: Array = []; - const stack = adaptEffectStack({ - ...effectStack(), - prepare: (options?: PrepareStackOptions) => - Effect.sync(() => { - options?.onProgress?.({ - workloadId: "rest:rest", - capability: "rest", - state: "downloading", - }); - return { capabilities: [] }; - }), - }); - yield* Effect.promise(() => - expect(stack.prepare({ onProgress: (status) => progress.push(status) })).resolves.toEqual({ - capabilities: [], - }), - ); - expect(progress).toEqual([ - { workloadId: "rest:rest", capability: "rest", state: "downloading" }, - ]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("completes an empty follower immediately", () => - Effect.gen(function* () { - const stack = adaptEffectStack(effectStack()); - const logs = yield* Effect.promise(() => stack.logs()); - expect(logs.entries).toHaveLength(0); - const logsIterator = stack.followLogs()[Symbol.asyncIterator](); - yield* Effect.promise(() => - expect(logsIterator.next()).resolves.toMatchObject({ done: true }), - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("rejects malformed configs asynchronously with a tagged error", () => - Effect.gen(function* () { - const stack = adaptEffectStack(effectStack()); - yield* Effect.promise(() => - expect( - stack.start({ - config: malformedConfig, - }), - ).rejects.toBeInstanceOf(InvalidStackConfigError), - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("redacts nested config secrets for prepare and start", () => - Effect.gen(function* () { - let preparedConfig: StartStackOptions["config"] | undefined; - let startedConfig: StartStackOptions["config"] | undefined; - const source: EffectStack = { - ...effectStack(), - prepare: (options?: PrepareStackOptions) => - Effect.sync(() => { - preparedConfig = options?.config; - return { capabilities: [] }; - }), - start: (options?: StartStackOptions) => - Effect.sync(() => { - startedConfig = options?.config; - return status; - }), - }; - const stack = adaptEffectStack(source); - const config = { - capabilities: { - storage: { settings: { buckets: { assets: { public: false } } } }, - auth: { settings: { external: { github: { secret: "github-secret" } } } }, - functions: { - settings: { - edge_runtime: { secrets: { EDGE_TOKEN: "edge-secret" } }, - functions: { - hello: { env: { FUNCTION_TOKEN: "function-secret" }, static_files: ["index.html"] }, - }, - }, - }, - }, - }; - yield* Effect.promise(() => stack.prepare({ config })); - yield* Effect.promise(() => stack.start({ config })); - for (const value of [preparedConfig, startedConfig]) { - const auth = value?.capabilities?.auth; - const functions = value?.capabilities?.functions; - const authSettings = auth !== undefined && "settings" in auth ? auth.settings : undefined; - const functionSettings = - functions !== undefined && "settings" in functions ? functions.settings : undefined; - expect(Redacted.isRedacted(authSettings?.external?.github?.secret)).toBe(true); - expect(Redacted.isRedacted(functionSettings?.edge_runtime?.secrets?.EDGE_TOKEN)).toBe(true); - expect(Redacted.isRedacted(functionSettings?.functions?.hello?.env?.FUNCTION_TOKEN)).toBe( - true, - ); - } - if (startedConfig === undefined) - return yield* Effect.die("Expected Promise start to capture config"); - const compiled = yield* compileStack({ - projectRoot: "/tmp/promise-facade-project", - runtime: { kind: "native" }, - config: startedConfig, - }).pipe(Effect.provide(NodeServices.layer)); - expect(compiled.definition.capabilities.functions.settings.functions_root).toBe( - "/tmp/promise-facade-project/supabase/functions", - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/public/public-model.integration.test.ts b/packages/stack/src/public/public-model.integration.test.ts deleted file mode 100644 index a1cb70d6dd..0000000000 --- a/packages/stack/src/public/public-model.integration.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, Schema } from "effect"; -import { StackStatusSchema } from "./Status.ts"; - -const STATUS_CAPABILITIES = [ - "database", - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", -] as const; - -const STATUS_FIXTURE = { - id: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - lifecycle: "stopped", - desiredLifecycle: "stopped", - runtime: { kind: "native" }, - endpoints: { - api: { - protocol: "http", - address: "127.0.0.1", - port: 54321, - url: "http://127.0.0.1:54321", - }, - }, - versions: { - database: "17", - }, - capabilities: STATUS_CAPABILITIES.map((name) => ({ - name, - activation: "lazy", - state: name === "rest" ? "dormant" : "disabled", - })), - artifacts: [ - { - workloadId: "rest:rest", - capability: "rest", - state: "downloading", - }, - ], -}; - -describe("public stack model", () => { - it.effect("decodes a complete status snapshot", () => - Schema.decodeUnknownEffect(StackStatusSchema)(STATUS_FIXTURE).pipe( - Effect.map((status) => { - expect(status.capabilities.map(({ name }) => name)).toEqual([...STATUS_CAPABILITIES]); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("dormant"); - expect(status.artifacts).toEqual([ - { workloadId: "rest:rest", capability: "rest", state: "downloading" }, - ]); - }), - ), - ); - - it.effect("rejects status snapshots with missing capability entries", () => - Effect.gen(function* () { - const exit = yield* Schema.decodeUnknownEffect(StackStatusSchema)({ - ...STATUS_FIXTURE, - capabilities: STATUS_FIXTURE.capabilities.slice(1), - }).pipe(Effect.exit); - - expect(Exit.isFailure(exit)).toBe(true); - }), - ); - - it.effect("rejects status snapshots with duplicate capability entries", () => - Effect.gen(function* () { - const exit = yield* Schema.decodeUnknownEffect(StackStatusSchema)({ - ...STATUS_FIXTURE, - capabilities: STATUS_FIXTURE.capabilities.map((capability, index) => - index === STATUS_FIXTURE.capabilities.length - 1 - ? { ...capability, name: "database" } - : capability, - ), - }).pipe(Effect.exit); - - expect(Exit.isFailure(exit)).toBe(true); - }), - ); -}); diff --git a/packages/stack/src/public/reset-database.integration.test.ts b/packages/stack/src/public/reset-database.integration.test.ts deleted file mode 100644 index ae08a2b165..0000000000 --- a/packages/stack/src/public/reset-database.integration.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -// oxlint-disable effecttsgo/async-function -- Promise-facade live reset uses createTestStack. -// oxlint-disable-next-line effecttsgo/node-builtin-import -- docker availability probe for optional container cases. -import { execFile as execFileCallback, spawnSync } from "node:child_process"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- native storage marker path. -import { mkdir, readFile, writeFile } from "node:fs/promises"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- native storage marker path. -import { join } from "node:path"; -import { promisify } from "node:util"; -import { PgClient } from "@effect/sql-pg"; -import { Effect, Redacted } from "effect"; -import { describe, expect, it } from "vitest"; -import { createTestStack, type TestStack } from "../testing.ts"; -import type { StackRuntimePreference } from "./Runtime.ts"; - -const RESET_TIMEOUT_MS = 180_000; -const execFile = promisify(execFileCallback); -const MARKER_TABLE = "public.stack_reset_marker"; - -const dockerAvailable = (): boolean => - spawnSync("docker", ["info"], { encoding: "utf8" }).status === 0; - -const query = async (url: string, statement: string): Promise> => - Effect.runPromise( - Effect.scoped( - Effect.gen(function* () { - const client = yield* PgClient.PgClient; - return yield* client.unsafe(statement); - }).pipe( - Effect.provide(PgClient.layer({ url: Redacted.make(url), connectTimeout: "10 seconds" })), - ), - ), - ); - -const volumeWorkloadIds = async (stackId: string): Promise> => { - const listed = await execFile("docker", [ - "volume", - "ls", - "-q", - "--filter", - `label=com.supabase.stack.stackId=${stackId}`, - ]); - const ids = listed.stdout - .trim() - .split("\n") - .filter((value) => value.length > 0); - if (ids.length === 0) return []; - const inspected = await execFile("docker", [ - "inspect", - "--format", - '{{index .Labels "com.supabase.stack.workloadId"}}', - ...ids, - ]); - return inspected.stdout - .trim() - .split("\n") - .filter((value) => value.length > 0); -}; - -const resetAndAssert = async (stack: TestStack, runtime: StackRuntimePreference): Promise => { - const before = await stack.status(); - const credentials = await stack.credentials(); - await query(credentials.database.url, `CREATE TABLE ${MARKER_TABLE} (id integer PRIMARY KEY)`); - const storageMarker = join(stack.stateRoot, stack.id, "data", "storage", "keep.txt"); - if (runtime.kind === "native") { - await mkdir(join(stack.stateRoot, stack.id, "data", "storage"), { recursive: true }); - await writeFile(storageMarker, "keep"); - } - const volumesBefore = runtime.kind === "container" ? await volumeWorkloadIds(stack.id) : []; - - const after = await stack.resetDatabase(); - expect(after.id).toBe(stack.id); - expect(after.endpoints).toEqual(before.endpoints); - expect(after.lifecycle).toBe("running"); - const database = after.capabilities.find((capability) => capability.name === "database"); - expect(database?.state).toBe("ready"); - - const leftover = await query( - (await stack.credentials()).database.url, - `SELECT to_regclass('${MARKER_TABLE}') AS name`, - ); - expect(leftover).toEqual([{ name: null }]); - - if (runtime.kind === "native") { - expect(await readFile(storageMarker, "utf8")).toBe("keep"); - } else { - const volumesAfter = await volumeWorkloadIds(stack.id); - expect(volumesAfter.filter((id) => id !== "database:database")).toEqual( - volumesBefore.filter((id) => id !== "database:database"), - ); - } -}; - -describe("resetDatabase", () => { - it( - "wipes native Postgres while keeping identity, ports, and storage data", - async () => { - await using stack = await createTestStack({ - runtime: { kind: "native" }, - }); - await resetAndAssert(stack, { kind: "native" }); - }, - RESET_TIMEOUT_MS, - ); - - it.skipIf(!dockerAvailable())( - "wipes container Postgres while keeping identity, ports, and non-database volumes", - async () => { - await using stack = await createTestStack({ - runtime: { kind: "container", engine: "docker" }, - }); - await resetAndAssert(stack, { kind: "container", engine: "docker" }); - }, - RESET_TIMEOUT_MS, - ); -}); diff --git a/packages/stack/src/public/testing.integration.test.ts b/packages/stack/src/public/testing.integration.test.ts deleted file mode 100644 index 3d3c3bbf0f..0000000000 --- a/packages/stack/src/public/testing.integration.test.ts +++ /dev/null @@ -1,595 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, ConfigProvider, Data, Effect, Exit, Path, Stream } from "effect"; -import { homedir, tmpdir } from "node:os"; - -import type { PromiseStack } from "./PromiseStack.ts"; -import { createTestStackWith, type TestStackOperations } from "./Testing.ts"; -import { defaultRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { CAPABILITY_NAMES } from "./Capability.ts"; -import { StackIdSchema } from "./StackId.ts"; -import type { StackStatus } from "./Status.ts"; -class FixtureError extends Data.TaggedError("FixtureError")<{ readonly cause: unknown }> {} -const fixturePromise = (thunk: () => A): Promise => - Effect.runPromiseExit( - Effect.try({ try: thunk, catch: (cause) => new FixtureError({ cause }) }), - ).then((exit) => { - if (Exit.isSuccess(exit)) return exit.value; - const error = Cause.squash(exit.cause); - throw error instanceof FixtureError ? error.cause : error; - }); - -const stackId = StackIdSchema.make( - "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", -); -const status = ( - lifecycle: StackStatus["lifecycle"], - includeApi = true, - functionsState: "ready" | "dormant" | "stopping" | "stopped" = "dormant", - failedCapability?: string, -): StackStatus => ({ - id: stackId, - lifecycle, - desiredLifecycle: lifecycle === "starting" || lifecycle === "stopping" ? "running" : lifecycle, - runtime: { kind: "native" }, - endpoints: - lifecycle === "running" && includeApi - ? { - api: { - protocol: "http", - address: "127.0.0.1", - port: 54321, - url: "http://127.0.0.1:54321", - }, - } - : {}, - versions: {}, - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: name === "functions" ? "lazy" : "eager", - state: - lifecycle === "running" - ? name === "pooler" - ? "disabled" - : name === "functions" - ? functionsState - : "ready" - : "stopped", - ...(failedCapability === name ? { state: "failed", error: `${name} failed` } : {}), - })), - artifacts: [], -}); -const stream = (values: ReadonlyArray): AsyncIterable => - Stream.toAsyncIterable(Stream.fromIterable(values)); -type FakeStackOptions = { - readonly failStart?: boolean; - readonly reachesReadiness?: boolean; - readonly includeApi?: boolean; - readonly functionsState?: "ready" | "dormant" | "stopping" | "stopped"; - readonly failedCapability?: string; -}; -const fakeStack = (events: Array, options: FakeStackOptions = {}): PromiseStack => { - const { - failStart = false, - reachesReadiness = true, - includeApi = true, - functionsState = "dormant", - failedCapability, - } = options; - return { - id: stackId, - status: () => - fixturePromise(() => - status( - reachesReadiness ? "running" : "stopped", - includeApi, - functionsState, - failedCapability, - ), - ), - credentials: () => - fixturePromise(() => ({ - database: { url: "postgres://test", password: "test" }, - api: { - publishableKey: "publishable", - secretKey: "secret", - anonJwt: "anon", - serviceRoleJwt: "service", - }, - storage: { - endpoint: "http://storage", - region: "local", - accessKeyId: "access", - secretAccessKey: "storage", - }, - })), - prepare: () => fixturePromise(() => ({ capabilities: [] })), - start: () => - fixturePromise(() => { - events.push("start"); - if (failStart) throw new Error("startup failed"); - return status( - reachesReadiness ? "running" : "stopped", - includeApi, - functionsState, - failedCapability, - ); - }), - stop: () => fixturePromise(() => undefined), - destroy: () => - fixturePromise(() => { - events.push("destroy"); - if (failStart) throw new Error("destroy failed"); - }), - resetDatabase: () => - fixturePromise(() => - status( - reachesReadiness ? "running" : "stopped", - includeApi, - functionsState, - failedCapability, - ), - ), - logs: () => fixturePromise(() => ({ entries: [], cursor: { opaque: "v1_0" }, running: false })), - followLogs: () => stream([]), - }; -}; -const setupFixture = (root: string, stackOptions: FakeStackOptions = {}) => { - const events: Array = []; - const removed: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => root), - createStack: (options) => - fixturePromise(() => { - events.push(`create:${options.projectRoot}`); - return fakeStack(events, stackOptions); - }), - removeRoot: (removedRoot) => - fixturePromise(() => { - removed.push(removedRoot); - }), - }; - return { events, removed, operations }; -}; -describe("test stack resource", () => { - it.live("starts automatically and destroys only its owned identity", () => - Effect.gen(function* () { - const { events, removed, operations } = setupFixture("/tmp/stack-test-owned"); - const stack = yield* Effect.promise(() => createTestStackWith({}, operations)); - yield* Effect.promise(() => stack[Symbol.asyncDispose]()); - expect(events).toEqual(["create:/tmp/stack-test-owned", "start", "destroy"]); - expect(removed).toEqual(["/tmp/stack-test-owned"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("preserves startup failure while retaining the root when destroy fails", () => - Effect.gen(function* () { - const { events, removed, operations } = setupFixture("/tmp/stack-test-failed", { - failStart: true, - }); - yield* Effect.promise(() => - expect(createTestStackWith({}, operations)).rejects.toThrow( - /startup failed[\s\S]*retained test stack root \/tmp\/stack-test-failed/, - ), - ); - expect(events).toEqual(["create:/tmp/stack-test-failed", "start", "destroy"]); - expect(removed).toEqual([]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("includes bounded startup diagnostics before cleanup removes a failed stack", () => - Effect.gen(function* () { - const events: Array = []; - const removed: Array = []; - const logQueries: Array[0]> = []; - const entries = Array.from({ length: 51 }, (_, index) => ({ - cursor: { opaque: `v1_${(index + 1).toString(36)}` }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "pooler" as const, - stream: "stderr" as const, - message: index === 50 ? "pooler stderr" : `old-${index}`, - })); - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-diagnostics"), - createStack: () => - fixturePromise(() => ({ - ...fakeStack(events), - start: () => - fixturePromise(() => { - events.push("start"); - throw new Error("startup failed"); - }), - status: () => fixturePromise(() => status("starting")), - logs: (query) => - fixturePromise(() => { - logQueries.push(query); - return { entries, cursor: { opaque: "v1_1" }, running: false }; - }), - })), - removeRoot: (root) => - fixturePromise(() => { - removed.push(root); - }), - }; - const failure: unknown = yield* Effect.promise(() => - createTestStackWith({}, operations).then( - () => undefined, - (error: unknown) => error, - ), - ); - expect(failure).toBeInstanceOf(Error); - if (!(failure instanceof Error)) return yield* Effect.die("expected startup failure"); - expect(failure.message).toContain("startup failed"); - expect(failure.message).toContain("pooler stderr"); - expect(failure.message).not.toContain("old-0"); - expect(failure.cause).toEqual(expect.objectContaining({ message: "startup failed" })); - expect(logQueries).toEqual([{ tail: 50 }]); - expect(events).toEqual(["start", "destroy"]); - expect(removed).toEqual(["/tmp/stack-test-diagnostics"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("fails and cleans up when start returns before the stack is ready", () => - Effect.gen(function* () { - const events: Array = []; - const removed: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-unready"), - createStack: () => - fixturePromise(() => ({ - ...fakeStack(events), - start: () => - fixturePromise(() => { - events.push("start"); - return status("starting"); - }), - })), - removeRoot: (root) => - fixturePromise(() => { - removed.push(root); - }), - }; - yield* Effect.promise(() => - expect( - createTestStackWith({ config: { capabilities: { database: {} } } }, operations), - ).rejects.toThrow("Stack did not become ready after start (lifecycle starting)"), - ); - expect(events).toEqual(["start", "destroy"]); - expect(removed).toEqual(["/tmp/stack-test-unready"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("removes the exact root after disposal", () => - Effect.gen(function* () { - const events: Array = []; - const removed: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-close-failed"), - createStack: () => fixturePromise(() => fakeStack(events)), - removeRoot: (root) => - fixturePromise(() => { - removed.push(root); - }), - }; - const stack = yield* Effect.promise(() => createTestStackWith({}, operations)); - yield* Effect.promise(() => expect(stack[Symbol.asyncDispose]()).resolves.toBeUndefined()); - expect(events).toEqual(["start", "destroy"]); - expect(removed).toEqual(["/tmp/stack-test-close-failed"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("does not require disabled Functions or an unconfigured API listener", () => - Effect.gen(function* () { - const events: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-disabled-surfaces"), - createStack: () => - fixturePromise(() => fakeStack(events, { includeApi: false, functionsState: "stopped" })), - removeRoot: () => fixturePromise(() => undefined), - }; - const stack = yield* Effect.promise(() => - createTestStackWith( - { config: { capabilities: { functions: { enabled: false } } } }, - operations, - ), - ); - yield* Effect.promise(() => stack[Symbol.asyncDispose]()); - expect(events).toEqual(["start", "destroy"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("runs setupProject after creating the root and before creating the stack", () => - Effect.gen(function* () { - const events: Array = []; - const operations: TestStackOperations = { - createRoot: () => - fixturePromise(() => { - events.push("root"); - return "/tmp/stack-test-setup"; - }), - createStack: (options) => - fixturePromise(() => { - events.push(`create:${options.projectRoot}`); - return fakeStack(events); - }), - removeRoot: () => - fixturePromise(() => { - events.push("remove"); - }), - }; - const stack = yield* Effect.promise(() => - createTestStackWith( - { - setupProject: (root) => - fixturePromise(() => { - events.push(`setup:${root}`); - }), - }, - operations, - ), - ); - yield* Effect.promise(() => stack[Symbol.asyncDispose]()); - expect(events).toEqual([ - "root", - "setup:/tmp/stack-test-setup", - "create:/tmp/stack-test-setup", - "start", - "destroy", - "remove", - ]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("uses the managed runtime state root without mutating process environment", () => - Effect.gen(function* () { - const events: Array = []; - const originalEnvironment = yield* defaultRuntimeEnvironment; - let environment: Parameters>[1] | undefined; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-isolated-state"), - createStack: (_options, runtimeEnvironment) => - fixturePromise(() => { - environment = runtimeEnvironment; - events.push("create"); - return fakeStack(events); - }), - removeRoot: () => fixturePromise(() => undefined), - }; - const stack = yield* Effect.promise(() => createTestStackWith({}, operations)); - yield* Effect.promise(() => stack[Symbol.asyncDispose]()); - expect(environment?.stateRoot).toBe((yield* defaultRuntimeEnvironment).stateRoot); - expect(environment?.artifactCacheRoot).toBe( - (yield* Path.Path).join(tmpdir(), "supabase-stack-test-artifacts"), - ); - expect(environment?.artifactCacheRoot).not.toContain("stack-test-isolated-state"); - // Compare against the snapshot above to prove no global environment mutation occurred. - expect(yield* defaultRuntimeEnvironment).toEqual(originalEnvironment); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("falls back to the OS home when HOME is unavailable", () => - Effect.gen(function* () { - const environment = yield* defaultRuntimeEnvironment.pipe( - Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromUnknown({})), - ); - expect(environment.stateRoot).toBe(`${homedir()}/.supabase/managed/stacks`); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("removes the exact root when setupProject fails", () => - Effect.gen(function* () { - const events: Array = []; - let created = false; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-setup-failed"), - createStack: () => - fixturePromise(() => { - created = true; - return fakeStack(events); - }), - removeRoot: (root) => - fixturePromise(() => { - events.push(`remove:${root}`); - }), - }; - yield* Effect.promise(() => - expect( - createTestStackWith( - { - setupProject: () => - fixturePromise(() => { - throw new Error("project setup failed"); - }), - }, - operations, - ), - ).rejects.toThrow("project setup failed"), - ); - expect(created).toBe(false); - expect(events).toEqual(["remove:/tmp/stack-test-setup-failed"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("rejects readiness when a configured capability fails", () => - Effect.gen(function* () { - const events: Array = []; - const removed: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-capability-failed"), - createStack: () => fixturePromise(() => fakeStack(events, { failedCapability: "auth" })), - removeRoot: (root) => - fixturePromise(() => { - removed.push(root); - }), - }; - yield* Effect.promise(() => - expect(createTestStackWith({}, operations)).rejects.toThrow("auth failed"), - ); - expect(events).toEqual(["start", "destroy"]); - expect(removed).toEqual(["/tmp/stack-test-capability-failed"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("rejects readiness when the lifecycle stops before becoming ready", () => - Effect.gen(function* () { - const events: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-lifecycle-stopped"), - createStack: () => - fixturePromise(() => ({ - ...fakeStack(events), - start: () => - fixturePromise(() => { - events.push("start"); - return status("stopped"); - }), - })), - removeRoot: () => fixturePromise(() => undefined), - }; - yield* Effect.promise(() => - expect(createTestStackWith({}, operations)).rejects.toThrow("stopped"), - ); - expect(events).toEqual(["start", "destroy"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("rejects readiness when the lifecycle starts stopping before becoming ready", () => - Effect.gen(function* () { - const events: Array = []; - const operations: TestStackOperations = { - createRoot: () => fixturePromise(() => "/tmp/stack-test-lifecycle-stopping"), - createStack: () => - fixturePromise(() => ({ - ...fakeStack(events), - start: () => - fixturePromise(() => { - events.push("start"); - return status("stopping"); - }), - })), - removeRoot: () => fixturePromise(() => undefined), - }; - yield* Effect.promise(() => - expect(createTestStackWith({}, operations)).rejects.toThrow("stopping"), - ); - expect(events).toEqual(["start", "destroy"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("accepts a lazy capability that is stopping during a running stack", () => - Effect.gen(function* () { - const { events, operations } = setupFixture("/tmp/stack-test-lazy-stopping", { - functionsState: "stopping", - }); - const stack = yield* Effect.promise(() => - createTestStackWith({ config: { capabilities: { functions: {} } } }, operations), - ); - yield* Effect.promise(() => stack[Symbol.asyncDispose]()); - expect(events).toEqual(["create:/tmp/stack-test-lazy-stopping", "start", "destroy"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("uses the managed state root while test stacks overlap", () => - Effect.gen(function* () { - const roots = ["/tmp/stack-test-shared-a", "/tmp/stack-test-shared-b"]; - const environments: Array>[1]> = - []; - const removedRoots: Array = []; - const operations: TestStackOperations = { - createRoot: () => - fixturePromise(() => { - const root = roots.shift(); - if (root === undefined) throw new Error("No test root available"); - return root; - }), - createStack: (_options, environment) => - fixturePromise(() => { - environments.push(environment); - return fakeStack([]); - }), - removeRoot: (root) => - fixturePromise(() => { - removedRoots.push(root); - }), - }; - const [first, second] = yield* Effect.promise(() => - Promise.all([createTestStackWith({}, operations), createTestStackWith({}, operations)]), - ); - expect(environments.map((environment) => environment?.stateRoot)).toEqual([ - (yield* defaultRuntimeEnvironment).stateRoot, - (yield* defaultRuntimeEnvironment).stateRoot, - ]); - yield* Effect.promise(() => first[Symbol.asyncDispose]()); - yield* Effect.promise(() => second[Symbol.asyncDispose]()); - expect(removedRoots).toEqual(["/tmp/stack-test-shared-a", "/tmp/stack-test-shared-b"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("creates auto roots under the managed state root and cleans up the exact root", () => - Effect.gen(function* () { - let setupRoot: string | undefined; - let createdRoot: string | undefined; - let removedRoot: string | undefined; - const stack = yield* Effect.promise(() => - createTestStackWith( - { - setupProject: (root) => - fixturePromise(() => { - setupRoot = root; - }), - }, - { - createStack: (options) => - fixturePromise(() => { - createdRoot = options.projectRoot; - return fakeStack([]); - }), - removeRoot: (root) => - fixturePromise(() => { - removedRoot = root; - }), - }, - ), - ); - const managedRoot = (yield* defaultRuntimeEnvironment).stateRoot; - const { join, dirname, sep } = yield* Path.Path; - const projectsRoot = join(dirname(managedRoot), "test-projects"); - expect(setupRoot).toBe(createdRoot); - expect(createdRoot?.startsWith(`${projectsRoot}${sep}`)).toBe(true); - if (!managedRoot.startsWith(`${tmpdir()}${sep}`)) - expect(createdRoot?.startsWith(`${tmpdir()}${sep}`)).toBe(false); - yield* Effect.promise(() => stack[Symbol.asyncDispose]()); - expect(removedRoot).toBe(createdRoot); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.live("retains the project root and managed state when one stack destroy fails", () => - Effect.gen(function* () { - const roots = ["/tmp/stack-test-retained-a", "/tmp/stack-test-retained-b"]; - const removedRoots: Array = []; - const environments: Array>[1]> = - []; - const operations: TestStackOperations = { - createRoot: () => - fixturePromise(() => { - const root = roots.shift(); - if (root === undefined) throw new Error("No test root available"); - return root; - }), - createStack: (options, environment) => - fixturePromise(() => { - environments.push(environment); - return { - ...fakeStack([]), - destroy: () => - fixturePromise(() => { - if (options.projectRoot.endsWith("-a")) throw new Error("destroy a failed"); - }), - }; - }), - removeRoot: (root) => - fixturePromise(() => { - removedRoots.push(root); - }), - }; - const [first, second] = yield* Effect.promise(() => - Promise.all([createTestStackWith({}, operations), createTestStackWith({}, operations)]), - ); - yield* Effect.promise(() => - expect(first[Symbol.asyncDispose]()).rejects.toThrow( - "destroy a failed; retained test stack root /tmp/stack-test-retained-a", - ), - ); - yield* Effect.promise(() => second[Symbol.asyncDispose]()); - expect(removedRoots).toEqual(["/tmp/stack-test-retained-b"]); - expect(environments.map((environment) => environment?.stateRoot)).toEqual([ - (yield* defaultRuntimeEnvironment).stateRoot, - (yield* defaultRuntimeEnvironment).stateRoot, - ]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/public/whole-stack.e2e.test.ts b/packages/stack/src/public/whole-stack.e2e.test.ts deleted file mode 100644 index e0dada31b8..0000000000 --- a/packages/stack/src/public/whole-stack.e2e.test.ts +++ /dev/null @@ -1,2434 +0,0 @@ -import { PgClient } from "@effect/sql-pg"; -import { - Cause, - Clock, - Config, - Crypto, - Data, - DateTime, - Effect, - Exit, - FileSystem, - Layer, - ManagedRuntime, - Option, - Path, - PlatformError, - Predicate, - Redacted, - Schedule, - Schema, - Stream, - type Duration, -} from "effect"; -import { NodeServices } from "@effect/platform-node"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/http"; - -import { Socket } from "node:net"; -import { tmpdir } from "node:os"; - -import { WebSocket } from "ws"; -import { afterAll, describe, expect, test } from "vitest"; -import { createStack, listStacks, type PromiseStack } from "../index.ts"; -import { defaultRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { createTestStack, type TestStack } from "../testing.ts"; -import { catalogEntryFor } from "../model/WorkloadCatalog.ts"; -import { CAPABILITY_NAMES } from "./Capability.ts"; -import type { PromiseStackCredentials } from "./Credentials.ts"; -import type { StackLogEntry } from "./Logs.ts"; -import type { PromiseStackConfig } from "./PromiseStack.ts"; -import type { StackEndpoint, StackStatus } from "./Status.ts"; - -const E2E_TIMEOUT_MS = 15 * 60_000; - -const REQUEST_TIMEOUT_MS = 60_000; - -const hostRuntime = ManagedRuntime.make(Layer.mergeAll(NodeServices.layer, FetchHttpClient.layer)); -afterAll(() => hostRuntime.dispose()); - -const { join } = hostRuntime.runSync(Path.Path); - -const runNode = ( - program: Effect.Effect< - A, - E, - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | ChildProcessSpawner.ChildProcessSpawner - | HttpClient.HttpClient - >, -) => hostRuntime.runPromise(program); - -const execFileEffect = (command: string, args: ReadonlyArray) => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const child = yield* spawner.spawn(ChildProcess.make(command, args)); - const [stdout, stderr, code] = yield* Effect.all( - [ - child.stdout.pipe( - Stream.decodeText(), - Stream.runCollect, - Effect.map((chunks) => chunks.join("")), - ), - child.stderr.pipe( - Stream.decodeText(), - Stream.runCollect, - Effect.map((chunks) => chunks.join("")), - ), - child.exitCode, - ], - { concurrency: 3 }, - ); - if (code !== 0) - return yield* new E2ERequestError({ message: `${command} exited ${code}: ${stderr}` }); - return { stdout }; - }).pipe(Effect.scoped); - -const withFs = (operation: (fs: FileSystem.FileSystem) => Effect.Effect) => - Effect.flatMap(FileSystem.FileSystem, operation); - -const access = (path: string) => withFs((fs) => fs.access(path)); - -const mkdir = (path: string, options?: { recursive?: boolean }) => - withFs((fs) => fs.makeDirectory(path, options)); - -const readText = (path: string) => withFs((fs) => fs.readFileString(path)); - -const writeFile = (path: string, data: string) => withFs((fs) => fs.writeFileString(path, data)); - -const rm = (path: string, options: { recursive: boolean; force: boolean }) => - withFs((fs) => fs.remove(path, options)); - -const mkdtemp = (prefix: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - return yield* fs.makeTempDirectory({ - directory: path.dirname(prefix), - prefix: path.basename(prefix), - }); - }); - -const randomId = () => - hostRuntime.runSync(Effect.flatMap(Crypto.Crypto, (crypto) => crypto.randomUUIDv4)); - -const queryTimestamp = (offset: number) => - DateTime.formatIso(DateTime.makeUnsafe(hostRuntime.runSync(Clock.currentTimeMillis) + offset)); - -const ANALYTICS_QUERY_RETRY_SCHEDULE = Schedule.spaced("1 second").pipe( - Schedule.upTo({ duration: "3 minutes" }), -); - -const MAILPIT_DELIVERY_RETRY_SCHEDULE = Schedule.spaced("250 millis").pipe( - Schedule.upTo({ duration: "30 seconds" }), -); -// Studio's first lazy request can start four workloads sequentially (4 × 30s). -const LAZY_STUDIO_ACTIVATION_TIMEOUT_MS = 180_000; -// Pooler is activated by the first TCP connection and may take longer than the normal -// query deadline while its migration and tenant bootstrap processes run. -const LAZY_POOLER_ACTIVATION_TIMEOUT = "30 seconds"; - -const ALL_RUNTIME_CASES = [ - { name: "native", runtime: { kind: "native" as const } }, - { name: "Docker", runtime: { kind: "container" as const, engine: "docker" as const } }, -] as const; - -const SELECTED_RUNTIME = Option.getOrUndefined( - Effect.runSync(Config.option(Config.string("SUPABASE_STACK_E2E_RUNTIME"))), -); -if ( - SELECTED_RUNTIME !== undefined && - SELECTED_RUNTIME !== "native" && - SELECTED_RUNTIME !== "container" -) - throw new Error(`Unsupported stack E2E runtime: ${SELECTED_RUNTIME}`); - -const RUNTIME_CASES = ALL_RUNTIME_CASES.filter( - ({ runtime }) => SELECTED_RUNTIME === undefined || runtime.kind === SELECTED_RUNTIME, -); - -const databaseCatalog = catalogEntryFor("database:database"); - -const NON_DEFAULT_DATABASE_RELEASES = Object.keys(databaseCatalog.releases).filter( - (version) => version !== databaseCatalog.defaultVersion, -); - -const BASE_WORKLOAD_IDS = [ - "analytics:analytics", - "analytics:vector", - "auth:auth", - "database:database", - "functions:edge-runtime", - "mail:mail", - "pooler:pooler", - "realtime:realtime", - "rest:rest", - "storage:imgproxy", - "storage:storage", - "studio:pgmeta", - "studio:studio", -] as const; - -const OPTIONAL_STORAGE_ANALYTICS_WORKLOAD_IDS = [ - "database:database", - "analytics:vector", - "analytics:analytics", - "storage:imgproxy", - "storage:storage", -] as const; - -const ALL_EAGER_WORKLOAD_IDS = [ - "analytics:analytics", - "analytics:vector", - "auth:auth", - "database:database", - "functions:edge-runtime", - "mail:mail", - "pooler:pooler", - "realtime:realtime", - "rest:rest", - "storage:imgproxy", - "storage:storage", - "studio:pgmeta", - "studio:studio", -] as const; - -const dockerOwnedResourceCount = ( - stackId: string, - kind: "containers" | "networks" | "volumes", -): Effect.Effect< - number, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const args = - kind === "containers" - ? ["ps", "-aq", "--filter", `label=com.supabase.stack.stackId=${stackId}`] - : [ - kind === "networks" ? "network" : "volume", - "ls", - "-q", - "--filter", - `label=com.supabase.stack.stackId=${stackId}`, - ]; - const result = yield* execFileEffect("docker", args); - return result.stdout.trim().length === 0 ? 0 : result.stdout.trim().split("\n").length; - }); - -const dockerOwnedWorkloadIds = ( - stackId: string, -): Effect.Effect< - ReadonlyArray, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const listed = yield* execFileEffect("docker", [ - "ps", - "-aq", - "--filter", - `label=com.supabase.stack.stackId=${stackId}`, - ]); - const ids = listed.stdout - .trim() - .split("\n") - .filter((value) => value.length > 0); - if (ids.length === 0) return []; - const inspected = yield* execFileEffect("docker", [ - "inspect", - "--format", - '{{index .Config.Labels "com.supabase.stack.workloadId"}}', - ...ids, - ]); - return [ - ...new Set( - inspected.stdout - .trim() - .split("\n") - .filter((value) => value.length > 0), - ), - ].sort(); - }); -/** Native launchers carry the exact stack/workload marker in their command line. */ -const nativeWorkloadIds = ( - stackId: string, -): Effect.Effect< - ReadonlyArray, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const result = yield* execFileEffect("ps", ["-axo", "pid=,command="]); - const marker = `supabase-stack-id=${stackId}`; - return [ - ...new Set( - result.stdout.split("\n").flatMap((line) => { - if (!line.includes(marker)) return []; - const match = /supabase-workload-id=([^\s]+)/u.exec(line); - return match?.[1] === undefined ? [] : [match[1]]; - }), - ), - ].sort(); - }); - -const ownedWorkloadIds = ( - mode: (typeof RUNTIME_CASES)[number], - stackId: string, -): Effect.Effect< - ReadonlyArray, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - mode.runtime.kind === "container" ? dockerOwnedWorkloadIds(stackId) : nativeWorkloadIds(stackId); - -const expectOwnedWorkloads = ( - mode: (typeof RUNTIME_CASES)[number], - stackId: string, - expected: ReadonlyArray, -): Effect.Effect< - void, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - ownedWorkloadIds(mode, stackId).pipe( - Effect.tap((actual) => Effect.sync(() => expect(actual).toEqual([...expected].sort()))), - Effect.asVoid, - ); - -const supervisorPids = ( - stackId: string, -): Effect.Effect< - ReadonlyArray, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const result = yield* execFileEffect("ps", ["-axo", "pid=,command="]); - return result.stdout.split("\n").flatMap((line) => { - const match = /^\s*(\d+)\s+(.+)$/u.exec(line); - if (match === null) return []; - const pid = Number(match[1]); - const command = match[2] ?? ""; - return command.includes(stackId) && - (command.includes("supervisor-node.ts") || - command.includes("__supabase_stack_supervisor__")) - ? [pid] - : []; - }); - }); - -const supervisorPid = ( - stackId: string, -): Effect.Effect< - number, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const matches = yield* supervisorPids(stackId); - if (matches.length !== 1) - return yield* new E2ERequestError({ - message: `Expected one Supervisor process for ${stackId}, found ${matches.length}`, - }); - const pid = matches[0]; - if (pid === undefined) - return yield* new E2ERequestError({ - message: `Supervisor process for ${stackId} has no PID`, - }); - return pid; - }); - -const nativeDatabaseSharedMemoryId = ( - lockPath: string, -): Effect.Effect => - Effect.gen(function* () { - const lines = (yield* readText(lockPath)).split("\n"); - const sharedMemoryId = Number(lines[6]?.trim().split(/\s+/u)[1]); - if (!Number.isSafeInteger(sharedMemoryId) || sharedMemoryId < 0) - return yield* new E2ERequestError({ - message: "Native database lock did not contain a valid shared-memory ID", - }); - return sharedMemoryId; - }); - -const nativeDatabaseSharedMemoryExists = ( - sharedMemoryId: number, -): Effect.Effect< - boolean | undefined, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => { - if (process.platform !== "linux" && process.platform !== "darwin") - return Effect.void.pipe(Effect.as(undefined)); - return execFileEffect("ipcs", ["-m"]).pipe( - Effect.map((result) => - result.stdout - .split("\n") - .some((line) => line.trim().split(/\s+/u)[1] === String(sharedMemoryId)), - ), - Effect.catchIf( - (cause) => - Predicate.isTagged(cause, "PlatformError") && - "reason" in cause && - Predicate.isTagged(cause.reason, "NotFound"), - () => Effect.void.pipe(Effect.as(undefined)), - ), - ); -}; - -const waitForProcessExit = (pid: number): Effect.Effect => { - const exited = Effect.try({ - try: () => { - try { - process.kill(pid, 0); - } catch (cause) { - if ( - typeof cause === "object" && - cause !== null && - "code" in cause && - cause.code === "ESRCH" - ) - return; - throw cause; - } - throw new Error(`Process ${pid} is still running`); - }, - catch: (cause) => - new E2ERequestError({ - message: cause instanceof Error ? cause.message : String(cause), - cause, - }), - }); - return Effect.retry(exited, { - schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "15 seconds" })), - }); -}; -type JsonObject = Record; -class E2ERequestError extends Data.TaggedError("E2ERequestError")<{ - readonly message: string; - readonly cause?: unknown; -}> {} - -const isJsonObject = (value: unknown): value is JsonObject => - typeof value === "object" && value !== null && !Array.isArray(value); - -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -const websocketDataText = (data: WebSocket.RawData): string => { - if (typeof data === "string") return data; - if (Buffer.isBuffer(data)) return data.toString("utf8"); - if (data instanceof ArrayBuffer) return Buffer.from(data).toString("utf8"); - return Buffer.concat(data).toString("utf8"); -}; - -const endpoint = (status: StackStatus, name: keyof StackStatus["endpoints"]): StackEndpoint => { - const value = status.endpoints[name]; - if (value === undefined) throw new Error(`Expected ${name} listener in stack status`); - return value; -}; - -const capabilityState = (status: StackStatus, name: string): string | undefined => - status.capabilities.find((capability) => capability.name === name)?.state; - -const expectDefaultLazyState = (status: StackStatus): void => { - expect(status.lifecycle).toBe("running"); - expect(capabilityState(status, "database")).toBe("ready"); - expect(status.artifacts).toContainEqual( - expect.objectContaining({ - workloadId: "database:database", - capability: "database", - state: "ready", - }), - ); - for (const name of CAPABILITY_NAMES) { - if (name === "database") continue; - expect(capabilityState(status, name), `${name} should remain dormant`).toBe("dormant"); - } -}; -/** Wait for one capability transition while subscribing before sending traffic. */ -// oxlint-disable-next-line effecttsgo/async-function -- TestStack status and action use the public Promise contract. -const activate = async ( - stack: TestStack, - name: string, - action: () => Promise, -): Promise => { - const before = await stack.status(); - expect(capabilityState(before, name), `${name} should be dormant before activation`).toBe( - "dormant", - ); - const waitUntilReady = Effect.tryPromise(() => stack.status()).pipe( - Effect.flatMap((current) => - capabilityState(current, name) === "ready" - ? Effect.succeed(current) - : Effect.fail(new E2ERequestError({ message: `Capability ${name} is not ready yet` })), - ), - Effect.retry(Schedule.spaced("100 millis").pipe(Schedule.upTo({ duration: "3 minutes" }))), - ); - const result = await action(); - await Effect.runPromise(waitUntilReady); - return result; -}; - -const abortRequest = ( - signal: AbortSignal, - requestDescription: string, -): Effect.Effect => - Effect.callback((resume) => { - const onAbort = () => { - const reason = signal.reason; - resume( - Effect.fail( - new E2ERequestError({ - message: `${requestDescription} failed: ${reason instanceof Error ? reason.message : String(reason ?? "Request aborted")}`, - cause: reason, - }), - ), - ); - }; - if (signal.aborted) { - onAbort(); - return Effect.void; - } - signal.addEventListener("abort", onAbort, { once: true }); - return Effect.sync(() => signal.removeEventListener("abort", onAbort)); - }); - -const request = ( - base: string, - path: string, - init: RequestInit = {}, - options: Readonly<{ expectedStatus?: number }> = {}, -): Effect.Effect => { - const url = new URL(path, `${base.replace(/\/$/u, "")}/`); - const requestDescription = `${init.method ?? "GET"} ${url}`; - const program = Effect.gen(function* () { - const webRequest = new Request(url.href, init); - let outgoing = HttpClientRequest.fromWeb(webRequest); - if (webRequest.body !== null) { - const bytes = yield* Effect.tryPromise({ - try: () => webRequest.arrayBuffer(), - catch: (cause) => new E2ERequestError({ message: "Unable to read request body", cause }), - }); - outgoing = HttpClientRequest.bodyUint8Array( - outgoing, - new Uint8Array(bytes), - webRequest.headers.get("content-type") ?? undefined, - ); - } - const response = yield* HttpClient.execute(outgoing); - const body = yield* response.arrayBuffer; - const statusMatches = - options.expectedStatus === undefined - ? response.status >= 200 && response.status < 300 - : response.status === options.expectedStatus; - if (!statusMatches) - return yield* new E2ERequestError({ - message: `${init.method ?? "GET"} ${url} returned ${response.status}: ${new TextDecoder().decode(body)}`, - }); - return new Response(response.status === 204 || response.status === 205 ? null : body, { - status: response.status, - headers: response.headers, - }); - }); - const requestProgram = - init.signal == null - ? program.pipe(Effect.timeout(REQUEST_TIMEOUT_MS)) - : Effect.raceFirst(abortRequest(init.signal, requestDescription), program); - return requestProgram.pipe( - Effect.mapError((cause) => { - if (cause instanceof E2ERequestError) return cause; - const reason = cause instanceof Error ? cause.message : String(cause); - return new E2ERequestError({ - message: `${requestDescription} failed: ${reason}`, - cause, - }); - }), - ); -}; - -const connect = (endpoint: StackEndpoint): Effect.Effect => - Effect.callback((resume) => { - const socket = new Socket(); - let settled = false; - const finish = (result: Effect.Effect) => { - if (settled) return; - settled = true; - socket.off("connect", onConnect); - socket.off("error", onError); - socket.destroy(); - resume(result); - }; - const onConnect = () => finish(Effect.void); - const onError = (cause: Error) => - finish(Effect.fail(new E2ERequestError({ message: cause.message, cause }))); - socket.once("connect", onConnect); - socket.once("error", onError); - socket.connect(endpoint.port, endpoint.address); - return Effect.sync(() => { - settled = true; - socket.off("connect", onConnect); - socket.off("error", onError); - socket.destroy(); - }); - }).pipe( - Effect.timeout("2 seconds"), - Effect.mapError( - (cause) => - new E2ERequestError({ - message: cause instanceof Error ? cause.message : String(cause), - cause, - }), - ), - ); - -const openTcpSocket = (endpoint: StackEndpoint): Effect.Effect => - Effect.callback((resume) => { - const socket = new Socket(); - let settled = false; - const cleanup = () => { - socket.off("connect", onConnect); - socket.off("error", onError); - }; - const onConnect = () => { - if (settled) return; - settled = true; - cleanup(); - resume(Effect.succeed(socket)); - }; - const onError = (cause: Error) => { - if (settled) return; - settled = true; - cleanup(); - socket.destroy(); - resume(Effect.fail(new E2ERequestError({ message: cause.message, cause }))); - }; - socket.once("connect", onConnect); - socket.once("error", onError); - socket.connect(endpoint.port, endpoint.address); - return Effect.sync(() => { - if (settled) return; - settled = true; - cleanup(); - socket.destroy(); - }); - }).pipe( - Effect.timeout(REQUEST_TIMEOUT_MS), - Effect.mapError((cause) => - cause instanceof E2ERequestError - ? cause - : new E2ERequestError({ message: "TCP connection failed", cause }), - ), - ); - -const openHttpTransport = ( - endpoint: StackEndpoint, - credentials: PromiseStackCredentials, - path: string, -): Effect.Effect => - Effect.gen(function* () { - const socket = yield* openTcpSocket(endpoint); - yield* Effect.callback((resume) => { - let settled = false; - let received = Buffer.alloc(0); - const finish = (result: Effect.Effect) => { - if (settled) return; - settled = true; - socket.off("data", onData); - socket.off("error", onError); - socket.off("close", onClose); - resume(result); - }; - const onData = (chunk: Buffer) => { - received = Buffer.concat([received, chunk]); - const headerEnd = received.indexOf("\r\n\r\n"); - if (headerEnd === -1) return; - if (!received.subarray(0, headerEnd).toString("utf8").startsWith("HTTP/1.1 100")) { - finish( - Effect.fail( - new E2ERequestError({ - message: "HTTP transport did not acknowledge Expect: 100-continue", - }), - ), - ); - return; - } - socket.write("{"); - finish(Effect.void); - }; - const onError = (cause: Error) => - finish(Effect.fail(new E2ERequestError({ message: cause.message, cause }))); - const onClose = () => - finish( - Effect.fail( - new E2ERequestError({ message: "HTTP transport closed before 100-continue" }), - ), - ); - socket.on("data", onData); - socket.once("error", onError); - socket.once("close", onClose); - socket.write( - [ - `POST ${path} HTTP/1.1`, - `Host: ${endpoint.address}:${endpoint.port}`, - `Content-Length: 1000000`, - `Content-Type: application/json`, - `Expect: 100-continue`, - `apikey: ${apiCredentials(credentials).publishableKey}`, - `Authorization: Bearer ${apiCredentials(credentials).anonJwt}`, - `Connection: keep-alive`, - ``, - ``, - ].join("\r\n"), - ); - return Effect.sync(() => { - settled = true; - socket.off("data", onData); - socket.off("error", onError); - socket.off("close", onClose); - }); - }).pipe( - Effect.timeout(REQUEST_TIMEOUT_MS), - Effect.onExit((result) => - Exit.isFailure(result) ? Effect.sync(() => socket.destroy()) : Effect.void, - ), - Effect.mapError((cause) => - cause instanceof E2ERequestError - ? cause - : new E2ERequestError({ message: "HTTP transport failed", cause }), - ), - ); - return socket; - }); - -const waitForSocketClose = (socket: Socket | WebSocket): Effect.Effect => - Effect.callback((resume) => { - if (socket instanceof WebSocket ? socket.readyState === WebSocket.CLOSED : socket.destroyed) { - resume(Effect.void); - return Effect.void; - } - let settled = false; - const finish = (result: Effect.Effect) => { - if (settled) return; - settled = true; - socket.off("close", onClose); - socket.off("error", onError); - resume(result); - }; - const onClose = () => finish(Effect.void); - const onError = (cause: Error) => - finish(Effect.fail(new E2ERequestError({ message: cause.message, cause }))); - socket.once("close", onClose); - socket.once("error", onError); - return Effect.sync(() => { - settled = true; - socket.off("close", onClose); - socket.off("error", onError); - }); - }).pipe( - Effect.timeout(REQUEST_TIMEOUT_MS), - Effect.mapError((cause) => - cause instanceof E2ERequestError - ? cause - : new E2ERequestError({ message: "Socket close observation failed", cause }), - ), - ); - -const waitForLogEntry = ( - stack: Pick, - iterator: AsyncIterator, - predicate: (entry: StackLogEntry) => boolean, -): Effect.Effect => { - const observation = Effect.gen(function* () { - while (true) { - const next = yield* Effect.tryPromise({ - try: () => iterator.next(), - catch: (cause) => - new E2ERequestError({ - message: cause instanceof Error ? cause.message : String(cause), - cause, - }), - }); - if (next.done) - return yield* new E2ERequestError({ message: "Stack log stream ended before idle stop" }); - if (predicate(next.value)) return next.value; - } - }); - return observation.pipe( - Effect.timeoutOrElse({ - duration: REQUEST_TIMEOUT_MS, - orElse: () => - Effect.fail(new E2ERequestError({ message: "Timed out waiting for idle stop log" })), - }), - Effect.catchCause((cause) => - Effect.tryPromise({ - // oxlint-disable-next-line effecttsgo/async-function -- Diagnostics read the public TestStack Promise API. - try: async () => { - const [status, logs] = await Promise.all([stack.status(), stack.logs({ tail: 20 })]); - const rest = status.capabilities.find(({ name }) => name === "rest"); - const recent = logs.entries - .map((entry) => `${entry.source}/${entry.stream}: ${entry.message}`) - .join("\n"); - return `lifecycle=${status.lifecycle}; rest=${rest?.state ?? "unavailable"}; recent logs:\n${recent}`; - }, - catch: (cause) => - new E2ERequestError({ - message: cause instanceof Error ? cause.message : String(cause), - cause, - }), - }).pipe( - Effect.catchCause(() => Effect.succeed("unavailable")), - Effect.flatMap((diagnostics) => - Effect.fail( - new E2ERequestError({ - message: `Idle stop observation failed: ${diagnostics}`, - cause: Cause.squash(cause), - }), - ), - ), - ), - ), - ); -}; - -const expectEndpointsRefused = (endpoints: ReadonlyArray): Effect.Effect => - Effect.forEach( - endpoints, - (listener) => - connect(listener).pipe( - Effect.matchEffect({ - onFailure: () => Effect.void, - onSuccess: () => - Effect.sync(() => - expect(false, `${listener.protocol} ${listener.url} should be closed`).toBe(true), - ), - }), - ), - { discard: true }, - ); - -const expectRuntimeInputsAbsent = ( - stack: Pick, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const pathApi = yield* Path.Path; - const stackRoot = pathApi.join(stack.stateRoot, stack.id); - yield* fs.access(stackRoot); - for (const relativePath of ["runtime/env", "runtime/inputs", "runtime/functions"]) { - const path = pathApi.join(stackRoot, relativePath); - yield* fs.readDirectory(path).pipe( - Effect.tap((entries) => - Effect.sync(() => - expect(entries, `${path} should be empty after cleanup`).toHaveLength(0), - ), - ), - Effect.catchIf( - (cause) => - Predicate.isTagged(cause, "PlatformError") && - "reason" in cause && - Predicate.isTagged(cause.reason, "NotFound"), - () => Effect.succeed([]), - ), - ); - } - }); - -// oxlint-disable-next-line effecttsgo/async-function -- diagnostics consume the public TestStack Promise API. -const throwCapabilityDiagnostics = async ( - stack: TestStack, - capabilityName: string, - operation: string, - cause: unknown, -): Promise => { - let status: StackStatus | undefined; - try { - status = await stack.status(); - } catch { - // Preserve the original request failure when status is unavailable. - } - const capability = status?.capabilities.find(({ name }) => name === capabilityName); - const recentLogs: StackLogEntry[] = []; - try { - for (const entry of (await stack.logs()).entries) { - if (entry.source !== capabilityName && entry.source !== "gateway") continue; - recentLogs.push(entry); - if (recentLogs.length > 50) recentLogs.shift(); - } - } catch { - // Preserve the original request failure when logs are unavailable. - } - const reason = cause instanceof Error ? cause.message : String(cause); - const state = capability === undefined ? "unavailable" : capability.state; - const error = capability?.error === undefined ? "none" : capability.error; - const logs = - recentLogs.length === 0 - ? "none" - : recentLogs.map((entry) => `${entry.source}/${entry.stream}: ${entry.message}`).join("\n"); - throw new Error( - `${operation} failed: ${reason}; ${capabilityName} state=${state}; error=${error}; recent logs:\n${logs}`, - { cause }, - ); -}; - -const jsonValue = (response: Response): Effect.Effect => - Effect.tryPromise({ - try: () => response.json(), - catch: (cause) => new E2ERequestError({ message: "Unable to decode JSON response", cause }), - }); - -const jsonObject = (response: Response): Effect.Effect => - Effect.gen(function* () { - const value = yield* jsonValue(response); - if (!isJsonObject(value)) - return yield* new E2ERequestError({ message: "Expected a JSON object response" }); - return Object.fromEntries(Object.entries(value)); - }); - -const databaseQuery = ( - url: string, - statement: string, - parameters: ReadonlyArray = [], - options: Readonly<{ - readonly connectTimeout?: Duration.Input; - }> = {}, -): Effect.Effect, E2ERequestError> => { - let target = ""; - try { - const parsed = new URL(url); - target = `${parsed.protocol}//${parsed.hostname}:${parsed.port || "default"}${parsed.pathname}`; - } catch { - // Keep diagnostics safe even when the connection URL is malformed. - } - return Effect.scoped( - Effect.gen(function* () { - const client = yield* PgClient.PgClient; - return yield* client.unsafe(statement, parameters); - }).pipe( - Effect.provide( - PgClient.layer({ - url: Redacted.make(url), - connectTimeout: options.connectTimeout ?? "10 seconds", - }), - ), - ), - ).pipe( - Effect.mapError( - (cause) => - new E2ERequestError({ - message: `databaseQuery failed for ${target}: ${statement}`, - cause, - }), - ), - ); -}; - -const apiCredentials = (credentials: PromiseStackCredentials) => { - if (credentials.api === undefined) throw new Error("API credentials are required"); - return credentials.api; -}; - -const apiHeaders = ( - credentials: PromiseStackCredentials, - token: string = apiCredentials(credentials).anonJwt, -): Record => ({ - apikey: apiCredentials(credentials).publishableKey, - Authorization: `Bearer ${token}`, -}); - -const serviceHeaders = (credentials: PromiseStackCredentials): Record => - apiHeaders(credentials, apiCredentials(credentials).serviceRoleJwt); - -const functionSource = (table: string, marker: string): string => ` -Deno.serve(async (request) => { - if (request.headers.get("x-reject-before-body") === "true") { - return new Response("rejected", { status: 400 }); - } - console.log("${marker}"); - let publishableKey: unknown; - try { - publishableKey = JSON.parse( - Deno.env.get("SUPABASE_PUBLISHABLE_KEYS") ?? "{}", - ).default; - } catch { - publishableKey = undefined; - } - if (typeof publishableKey !== "string" || publishableKey.length === 0) { - return new Response("Missing SUPABASE_PUBLISHABLE_KEYS.default", { status: 500 }); - } - const response = await fetch( - \`${'${Deno.env.get("SUPABASE_URL")}'}/rest/v1/${table}?select=id,payload\`, - { - headers: { - apikey: publishableKey, - Authorization: \`Bearer \${publishableKey}\`, - }, - }, - ); - const body = await response.text(); - return new Response(JSON.stringify({ - marker: "${marker}", - functionSlug: Deno.env.get("SUPABASE_FUNCTION_SLUG"), - rows: JSON.parse(body), - }), { - status: response.status, - headers: { "content-type": "application/json" }, - }); -}); -`; - -const waitForSocket = ( - socket: WebSocket, - predicate: (value: JsonObject) => boolean, -): Effect.Effect => - Effect.callback((resume) => { - let settled = false; - const finish = (result: Effect.Effect) => { - if (settled) return; - settled = true; - socket.off("message", onMessage); - socket.off("error", onError); - socket.off("close", onClose); - resume(result); - }; - const onMessage = (data: WebSocket.RawData) => { - try { - const value: unknown = JSON.parse(websocketDataText(data)); - if (isJsonObject(value) && predicate(value)) - finish(Effect.succeed(Object.fromEntries(Object.entries(value)))); - } catch { - // Realtime can send non-JSON protocol frames. - } - }; - const onError = () => - finish( - Effect.fail(new E2ERequestError({ message: "Realtime WebSocket failed while waiting" })), - ); - const onClose = () => - finish( - Effect.fail(new E2ERequestError({ message: "Realtime WebSocket closed while waiting" })), - ); - socket.on("message", onMessage); - socket.on("error", onError); - socket.on("close", onClose); - return Effect.sync(() => { - settled = true; - socket.off("message", onMessage); - socket.off("error", onError); - socket.off("close", onClose); - }); - }).pipe( - Effect.timeout(REQUEST_TIMEOUT_MS), - Effect.mapError( - (cause) => - new E2ERequestError({ - message: cause instanceof Error ? cause.message : String(cause), - cause, - }), - ), - ); - -const openSocket = (url: string): Effect.Effect => - Effect.callback((resume) => { - const socket = new WebSocket(url, { - handshakeTimeout: REQUEST_TIMEOUT_MS, - perMessageDeflate: false, - }); - let settled = false; - const cleanup = () => { - socket.off("open", onOpen); - socket.off("error", onError); - socket.off("close", cleanup); - }; - const terminate = () => { - socket.off("open", onOpen); - // Terminating an unfinished handshake emits an error before close. - socket.once("close", cleanup); - socket.terminate(); - }; - const onOpen = () => { - if (settled) return; - settled = true; - cleanup(); - resume(Effect.succeed(socket)); - }; - const onError = (cause: Error) => { - if (settled) return; - settled = true; - terminate(); - resume(Effect.fail(new E2ERequestError({ message: cause.message, cause }))); - }; - socket.once("open", onOpen); - socket.once("error", onError); - return Effect.sync(() => { - if (settled) return; - settled = true; - terminate(); - }); - }); - -const makeRealtimeUrl = (api: StackEndpoint, apikey: string): string => { - const url = new URL(api.url); - url.protocol = api.protocol === "http" ? "ws:" : "wss:"; - url.pathname = "/realtime/v1/websocket"; - url.search = new URLSearchParams({ apikey, vsn: "1.0.0" }).toString(); - return url.toString(); -}; - -const onePixelPng = Uint8Array.from( - atob( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", - ), - (character) => character.charCodeAt(0), -); - -const optionalWorkloadConfig = ( - functionSlug: string, - analyticsApiKey: string, -): PromiseStackConfig => ({ - capabilities: { - storage: { settings: { image_transformation: { enabled: true } } }, - functions: { settings: { functions: { [functionSlug]: { verify_jwt: false } } } }, - analytics: { settings: { api_key: analyticsApiKey } }, - }, - listeners: { smtp: { enabled: true } }, -}); - -const allEagerConfig = (analyticsApiKey: string): PromiseStackConfig => ({ - capabilities: { - rest: { activation: "eager" }, - auth: { activation: "eager" }, - realtime: { activation: "eager" }, - storage: { - activation: "eager", - settings: { image_transformation: { enabled: true } }, - }, - functions: { activation: "eager" }, - studio: { activation: "eager" }, - mail: { activation: "eager" }, - analytics: { - activation: "eager", - settings: { api_key: analyticsApiKey }, - }, - pooler: { activation: "eager" }, - }, - listeners: { smtp: { enabled: true } }, -}); - -const queryAnalyticsMarker = ( - api: StackEndpoint, - analyticsApiKey: string, - marker: string, -): Effect.Effect => { - const query = new URLSearchParams({ - project: "default", - iso_timestamp_start: queryTimestamp(-3_600_000), - iso_timestamp_end: queryTimestamp(3_600_000), - sql: - "select count(*) as c from postgres_logs where regexp_contains(event_message, '" + - marker + - "')", - }); - const attempt = Effect.gen(function* () { - const response = yield* request( - api.url, - "/analytics/v1/api/endpoints/query/logs.all?" + query, - { headers: { "x-api-key": analyticsApiKey } }, - ); - const value = yield* jsonValue(response); - if (!isJsonObject(value)) - return yield* new E2ERequestError({ message: "Expected a JSON object response" }); - const responseObject = value; - const rows = responseObject.result; - const count = Array.isArray(rows) && isJsonObject(rows[0]) ? Number(rows[0].c) : 0; - if (count <= 0) - return yield* new E2ERequestError({ message: `Analytics query pending (count=${count})` }); - return count; - }); - return Effect.retry(attempt, { schedule: ANALYTICS_QUERY_RETRY_SCHEDULE }); -}; -type RuntimeCase = (typeof RUNTIME_CASES)[number]; -type WholeStackMarkers = Readonly<{ - first: string; - second: string; - live: string; -}>; -type WholeStackScenario = Readonly<{ - mode: RuntimeCase; - stack: TestStack; - identity: string; - projectRoot: string; - table: string; - bucket: string; - functionSlug: string; - email: string; - password: string; - markers: WholeStackMarkers; - credentials: PromiseStackCredentials; - api: StackEndpoint; - pooler: StackEndpoint; - studio: StackEndpoint; - mailUi: StackEndpoint; -}>; - -const arrangeWholeStackDatabase = ( - scenario: WholeStackScenario, -): Effect.Effect => { - const { credentials, markers, table } = scenario; - return Effect.gen(function* () { - yield* databaseQuery( - credentials.database.url, - `CREATE TABLE public."${table}" (id integer PRIMARY KEY, payload text NOT NULL)`, - ); - yield* databaseQuery( - credentials.database.url, - `GRANT SELECT, INSERT, UPDATE, DELETE ON public."${table}" TO anon, authenticated, service_role`, - ); - yield* databaseQuery( - credentials.database.url, - `ALTER PUBLICATION supabase_realtime ADD TABLE public."${table}"`, - ); - const directRows = yield* databaseQuery( - credentials.database.url, - `INSERT INTO public."${table}" (id, payload) VALUES (1, $1) RETURNING id, payload`, - [markers.first], - ); - yield* Effect.sync(() => expect(directRows).toEqual([{ id: 1, payload: markers.first }])); - }); -}; - -// oxlint-disable-next-line effecttsgo/async-function -- this verifies the public async log iterator contract. -const verifyWholeStackDatabaseLogs = async (stack: TestStack): Promise => { - expect((await stack.logs({ capabilities: ["database"], tail: 1 })).entries).not.toHaveLength(0); - const logIterator = stack - .followLogs({ capabilities: ["database"], tail: 1 }) - [Symbol.asyncIterator](); - try { - const logEntry = await logIterator.next(); - expect(logEntry.done).toBe(false); - if (!logEntry.done) expect(logEntry.value.source).toBe("database"); - } finally { - await logIterator.return?.(); - } -}; - -// oxlint-disable-next-line effecttsgo/async-function -- this scenario consumes the public TestStack Promise contract. -const exerciseWholeStackRestAndAuth = async ( - scenario: WholeStackScenario, -): Promise<{ - readonly restPath: string; - readonly accessToken: string; -}> => { - const { api, credentials, email, identity, password, markers, stack, table } = scenario; - const restPath = `/rest/v1/${table}?select=id,payload&order=id`; - const restRows = await activate(stack, "rest", () => - runNode( - request(api.url, restPath, { - headers: { ...apiHeaders(credentials), Accept: "application/json" }, - }).pipe(Effect.flatMap(jsonValue)), - ), - ); - expect(restRows).toEqual(expect.arrayContaining([{ id: 1, payload: markers.first }])); - const signup = await activate(stack, "auth", () => - runNode( - request(api.url, "/auth/v1/signup", { - method: "POST", - headers: { ...apiHeaders(credentials), "content-type": "application/json" }, - body: JSON.stringify({ email, password }), - }).pipe(Effect.flatMap(jsonObject)), - ), - ); - const accessToken = signup.access_token; - if (typeof accessToken !== "string") - throw new Error("Auth signup did not return an access token"); - const authenticatedInsert = await runNode( - request(api.url, `/rest/v1/${table}`, { - method: "POST", - headers: { - ...apiHeaders(credentials, accessToken), - "content-type": "application/json", - Prefer: "return=representation", - }, - body: JSON.stringify({ id: 2, payload: `auth-${identity}` }), - }).pipe(Effect.flatMap(jsonValue)), - ); - expect(authenticatedInsert).toEqual( - expect.arrayContaining([expect.objectContaining({ id: 2, payload: `auth-${identity}` })]), - ); - return { restPath, accessToken }; -}; - -// oxlint-disable-next-line effecttsgo/async-function -- this scenario consumes the public TestStack Promise contract. -const exerciseWholeStackRealtime = async ( - scenario: WholeStackScenario, - accessToken: string, -): Promise => { - const { api, credentials, identity, stack, table } = scenario; - let openedSocket: WebSocket | undefined; - let socket: WebSocket; - try { - socket = await activate(stack, "realtime", () => - runNode( - openSocket(makeRealtimeUrl(api, apiCredentials(credentials).publishableKey)).pipe( - Effect.tap((candidate) => Effect.sync(() => void (openedSocket = candidate))), - ), - ), - ); - } catch (cause) { - openedSocket?.close(); - throw cause; - } - const socketWaiters: Array> = []; - try { - const joined = runNode( - waitForSocket(socket, (value) => { - const payload = value.payload; - return value.event === "phx_reply" && typeof payload === "object" && payload !== null; - }), - ); - socketWaiters.push(joined); - const subscribed = runNode( - waitForSocket(socket, (value) => { - const payload = value.payload; - return ( - value.event === "system" && - isJsonObject(payload) && - payload.status === "ok" && - payload.extension === "postgres_changes" - ); - }), - ); - socketWaiters.push(subscribed); - const change = runNode(waitForSocket(socket, (value) => value.event === "postgres_changes")); - socketWaiters.push(change); - socket.send( - JSON.stringify({ - topic: `realtime:public:${table}`, - event: "phx_join", - payload: { - config: { - broadcast: { ack: false, self: false }, - presence: { key: "" }, - postgres_changes: [{ event: "INSERT", schema: "public", table }], - }, - access_token: accessToken, - }, - ref: "1", - }), - ); - await joined; - await subscribed; - await runNode( - request(api.url, `/rest/v1/${table}`, { - method: "POST", - headers: { - ...apiHeaders(credentials, accessToken), - "content-type": "application/json", - Prefer: "return=minimal", - }, - body: JSON.stringify({ id: 3, payload: `realtime-${identity}` }), - }), - ); - const realtimeChange = await change; - expect(JSON.stringify(realtimeChange)).toContain(`realtime-${identity}`); - } finally { - socket.close(); - await Promise.allSettled(socketWaiters); - } -}; - -// oxlint-disable-next-line effecttsgo/async-function -- this scenario consumes the public TestStack Promise contract. -const exerciseWholeStackStorage = async (scenario: WholeStackScenario): Promise => { - const { api, bucket, credentials, stack } = scenario; - await activate(stack, "storage", () => - runNode( - Effect.gen(function* () { - yield* request(api.url, "/storage/v1/bucket", { - method: "POST", - headers: { ...serviceHeaders(credentials), "content-type": "application/json" }, - body: encodeJson({ id: bucket, name: bucket, public: true }), - }); - yield* request(api.url, `/storage/v1/object/${bucket}/pixel.png`, { - method: "POST", - headers: { ...serviceHeaders(credentials), "content-type": "image/png" }, - body: new Blob([onePixelPng], { type: "image/png" }), - }); - const downloaded = yield* request( - api.url, - `/storage/v1/object/public/${bucket}/pixel.png`, - { headers: serviceHeaders(credentials) }, - ); - const bytes = yield* Effect.tryPromise(() => downloaded.arrayBuffer()); - yield* Effect.sync(() => expect(bytes.byteLength).toBeGreaterThan(0)); - }), - ), - ); -}; - -// oxlint-disable-next-line effecttsgo/async-function -- this scenario consumes public TestStack Promise and async log iterator contracts. -const exerciseWholeStackFunctions = async (scenario: WholeStackScenario): Promise => { - const { api, credentials, functionSlug, markers, projectRoot, stack, table } = scenario; - const functionPath = `/functions/v1/${functionSlug}`; - try { - let firstFunction: JsonObject = {}; - await activate(stack, "functions", () => - runNode( - request(api.url, functionPath, { headers: apiHeaders(credentials) }).pipe( - Effect.flatMap(jsonObject), - Effect.tap((value) => Effect.sync(() => void (firstFunction = value))), - ), - ), - ); - expect(firstFunction).toEqual( - expect.objectContaining({ - marker: markers.first, - functionSlug, - rows: expect.arrayContaining([{ id: 1, payload: markers.first }]), - }), - ); - const earlyResponse = await runNode( - request( - api.url, - functionPath, - { - method: "POST", - headers: { ...apiHeaders(credentials), "x-reject-before-body": "true" }, - body: new Uint8Array(128 * 1024), - signal: AbortSignal.timeout(5_000), - }, - { expectedStatus: 400 }, - ), - ); - expect(earlyResponse.status).toBe(400); - expect(await earlyResponse.text()).toBe("rejected"); - - await runNode( - writeFile( - join(projectRoot, "supabase", "functions", functionSlug, "index.ts"), - functionSource(table, markers.second), - ), - ); - const secondFunction = await runNode( - request(api.url, functionPath, { headers: apiHeaders(credentials) }).pipe( - Effect.flatMap(jsonObject), - ), - ); - expect(secondFunction).toEqual( - expect.objectContaining({ - marker: markers.second, - functionSlug, - rows: expect.arrayContaining([{ id: 1, payload: markers.first }]), - }), - ); - const beforeLiveLogs = await stack.logs({ capabilities: ["functions"] }); - const liveIterator = stack - .followLogs({ capabilities: ["functions"], cursor: beforeLiveLogs.cursor }) - [Symbol.asyncIterator](); - try { - const liveNext = liveIterator.next(); - await runNode( - writeFile( - join(projectRoot, "supabase", "functions", functionSlug, "index.ts"), - functionSource(table, markers.live), - ), - ); - await runNode( - request(api.url, functionPath, { headers: apiHeaders(credentials) }).pipe( - Effect.flatMap(jsonObject), - ), - ); - let liveEntry = await liveNext; - for (let attempt = 0; attempt < 100; attempt += 1) { - if (liveEntry.done) break; - if ( - liveEntry.value.source === "functions" && - liveEntry.value.message.includes(markers.live) - ) - break; - liveEntry = await liveIterator.next(); - } - expect(liveEntry.done).toBe(false); - if (!liveEntry.done) { - expect(liveEntry.value.source).toBe("functions"); - expect(liveEntry.value.message).toContain(markers.live); - } - } finally { - await liveIterator.return?.(); - } - expect((await liveIterator.next()).done).toBe(true); - } catch (cause) { - await throwCapabilityDiagnostics(stack, "functions", "Functions flow", cause); - } -}; - -// oxlint-disable-next-line effecttsgo/async-function -- this scenario consumes the public TestStack Promise contract. -const exerciseWholeStackAuxiliary = async (scenario: WholeStackScenario): Promise => { - const { api, credentials, mailUi, pooler, stack, studio } = scenario; - await activate(stack, "mail", () => runNode(request(mailUi.url, "/api/v1/messages?limit=100"))); - await activate(stack, "analytics", () => runNode(request(api.url, "/analytics/v1/health"))); - try { - await activate(stack, "studio", () => - runNode( - request(studio.url, "/api/platform/profile", { - headers: serviceHeaders(credentials), - signal: AbortSignal.timeout(LAZY_STUDIO_ACTIVATION_TIMEOUT_MS), - }), - ), - ); - } catch (cause) { - await throwCapabilityDiagnostics(stack, "studio", "Studio profile request", cause); - } - const poolerUrl = new URL(credentials.database.url); - poolerUrl.port = String(pooler.port); - poolerUrl.username = "postgres.pooler-dev"; - const poolerRows = await activate(stack, "pooler", () => - runNode( - databaseQuery(poolerUrl.toString(), "SELECT 42 AS answer", [], { - connectTimeout: LAZY_POOLER_ACTIVATION_TIMEOUT, - }), - ), - ); - expect(poolerRows).toEqual([{ answer: 42 }]); - return poolerUrl; -}; - -const assertWholeStackReady = ( - scenario: WholeStackScenario, - status: StackStatus, -): Effect.Effect< - void, - E2ERequestError | PlatformError.PlatformError, - ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - yield* Effect.sync(() => { - expect(status.capabilities.map(({ name, state }) => ({ name, state }))).toEqual( - CAPABILITY_NAMES.map((name) => ({ name, state: "ready" })), - ); - for (const workloadId of ["studio:pgmeta", "studio:studio"] as const) { - expect(status.artifacts).toContainEqual( - expect.objectContaining({ workloadId, capability: "studio", state: "ready" }), - ); - } - }); - yield* expectOwnedWorkloads(scenario.mode, scenario.stack.id, BASE_WORKLOAD_IDS); - }); - -// oxlint-disable-next-line effecttsgo/async-function -- this helper consumes the public TestStack Promise contract. -const reactivateWholeStackCapabilities = async ( - scenario: WholeStackScenario, - restPath: string, - functionPath: string, - poolerUrl: URL, -): Promise => { - const { api, credentials, mailUi, stack, studio } = scenario; - await activate(stack, "rest", () => - runNode(request(api.url, restPath, { headers: apiHeaders(credentials) })), - ); - await activate(stack, "auth", () => - runNode(request(api.url, "/auth/v1/settings", { headers: apiHeaders(credentials) })), - ); - await activate(stack, "realtime", () => - runNode( - openSocket(makeRealtimeUrl(api, apiCredentials(credentials).publishableKey)).pipe( - Effect.tap((probe) => Effect.sync(() => probe.close())), - Effect.asVoid, - ), - ), - ); - await activate(stack, "storage", () => - runNode(request(api.url, "/storage/v1/bucket", { headers: serviceHeaders(credentials) })), - ); - await activate(stack, "functions", () => - runNode(request(api.url, functionPath, { headers: apiHeaders(credentials) })), - ); - await activate(stack, "mail", () => runNode(request(mailUi.url, "/api/v1/messages?limit=1"))); - await activate(stack, "analytics", () => runNode(request(api.url, "/analytics/v1/health"))); - await activate(stack, "studio", () => - runNode( - request(studio.url, "/api/platform/profile", { - headers: serviceHeaders(credentials), - signal: AbortSignal.timeout(LAZY_STUDIO_ACTIVATION_TIMEOUT_MS), - }), - ), - ); - await activate(stack, "pooler", () => - runNode( - databaseQuery(poolerUrl.toString(), "SELECT 42 AS answer", [], { - connectTimeout: LAZY_POOLER_ACTIVATION_TIMEOUT, - }), - ), - ); -}; - -// oxlint-disable-next-line effecttsgo/async-function -- restart flow consumes the public TestStack Promise API. -const restartWholeStackFromPersistedData = async ( - scenario: WholeStackScenario, - endpointSnapshot: Readonly>, - restPath: string, - functionPath: string, - poolerUrl: URL, -): Promise => { - const { credentials, markers, mode, stack, table } = scenario; - const restarted = await stack.start(); - expectDefaultLazyState(restarted); - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database"])); - expect( - Object.fromEntries( - Object.entries(restarted.endpoints).map(([name, value]) => [name, value?.port]), - ), - ).toEqual(endpointSnapshot); - expect( - await runNode( - databaseQuery(credentials.database.url, `SELECT payload FROM public."${table}" WHERE id = 1`), - ), - ).toEqual([{ payload: markers.first }]); - await reactivateWholeStackCapabilities(scenario, restPath, functionPath, poolerUrl); -}; - -// oxlint-disable-next-line effecttsgo/async-function -- whole-stack flow consumes PromiseStack and AsyncDisposable contracts. -const runWholeStackScenario = async (mode: (typeof RUNTIME_CASES)[number]): Promise => { - const identity = randomId().replaceAll("-", "").slice(0, 20).toLowerCase(); - const table = `stack_e2e_${identity}`; - const bucket = `stack-e2e-${identity}`; - const functionSlug = `cross_service_${identity}`; - const email = `${identity}@example.test`; - const password = "SupabaseStackE2e!123"; - const markers = { - first: `first-${identity}`, - second: `second-${identity}`, - live: `live-${identity}`, - }; - let projectRoot = ""; - await using stack: TestStack = await createTestStack({ - name: `stack-e2e-${identity}`, - runtime: mode.runtime, - config: { - capabilities: { - rest: { idleTimeoutSeconds: false }, - auth: { idleTimeoutSeconds: false }, - realtime: { idleTimeoutSeconds: false }, - studio: { idleTimeoutSeconds: false }, - pooler: { idleTimeoutSeconds: false }, - }, - }, - setupProject: (root) => { - projectRoot = root; - const directory = join(root, "supabase", "functions", functionSlug); - return runNode( - Effect.gen(function* () { - yield* mkdir(directory, { recursive: true }); - yield* writeFile(join(directory, "index.ts"), functionSource(table, markers.first)); - }), - ); - }, - }); - const initialRunning = await stack.status(); - expect(initialRunning.runtime).toEqual(mode.runtime); - expect(projectRoot.length).toBeGreaterThan(0); - expectDefaultLazyState(initialRunning); - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database"])); - // Preparation is an explicit cache-only operation. It runs after the helper's - // initial session is stopped, so the test proves it creates no owner, listener, - // or workload. - const initialSupervisorPid = await runNode(supervisorPid(stack.id)); - await stack.stop(); - const warmed = await stack.prepare({ capabilities: ["rest"] }); - expect(warmed.capabilities).toEqual( - expect.arrayContaining([expect.objectContaining({ capability: "rest" })]), - ); - expect((await stack.status()).lifecycle).toBe("stopped"); - await runNode( - expectEndpointsRefused( - Object.values(initialRunning.endpoints).filter( - (value): value is StackEndpoint => value !== undefined, - ), - ), - ); - // Process exit follows lease finalization for a detached supervisor. Observe the exact - // owner PID before scanning for leftovers, without masking a genuine process leak. - await runNode(waitForProcessExit(initialSupervisorPid)); - expect(await runNode(supervisorPids(stack.id))).toHaveLength(0); - const initial = await stack.start(); - expectDefaultLazyState(initial); - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database"])); - const credentials = await stack.credentials(); - const api = endpoint(initial, "api"); - const pooler = endpoint(initial, "pooler"); - const studio = endpoint(initial, "studio"); - const mailUi = endpoint(initial, "mailUi"); - const initialEndpoints = Object.values(initial.endpoints).filter( - (value): value is StackEndpoint => value !== undefined, - ); - // The warmed capability remains lazy: only its artifact is cached, not its - // workload. The first request below still performs the normal activation. - const scenario: WholeStackScenario = { - mode, - stack, - identity, - projectRoot, - table, - bucket, - functionSlug, - email, - password, - markers, - credentials, - api, - pooler, - studio, - mailUi, - }; - await runNode(arrangeWholeStackDatabase(scenario)); - await verifyWholeStackDatabaseLogs(stack); - const { restPath, accessToken } = await exerciseWholeStackRestAndAuth(scenario); - await exerciseWholeStackRealtime(scenario, accessToken); - await exerciseWholeStackStorage(scenario); - // Functions are request-time discovered and call REST through SUPABASE_URL. - const functionPath = `/functions/v1/${functionSlug}`; - await exerciseWholeStackFunctions(scenario); - const poolerUrl = await exerciseWholeStackAuxiliary(scenario); - const ready = await stack.status(); - await runNode(assertWholeStackReady(scenario, ready)); - const idempotentStart = await stack.start(); - await runNode(assertWholeStackReady(scenario, idempotentStart)); - const endpointSnapshot = Object.fromEntries( - Object.entries(ready.endpoints).map(([name, value]) => [name, value?.port]), - ); - const persistedMarker = await runNode( - databaseQuery(credentials.database.url, `SELECT payload FROM public."${table}" WHERE id = 1`), - ); - expect(persistedMarker).toEqual([{ payload: markers.first }]); - const volumesBeforeStop = - mode.runtime.kind === "container" - ? await runNode(dockerOwnedResourceCount(initial.id, "volumes")) - : undefined; - try { - await stack.stop(); - } catch (cause) { - let diagnostic: StackStatus | undefined; - try { - diagnostic = await stack.status(); - } catch { - // Preserve the original stop failure if the owner is already gone. - } - const reason = cause instanceof Error ? cause.message : String(cause); - const states = - diagnostic === undefined - ? "unavailable" - : diagnostic.capabilities - .map( - ({ name, state, error }) => - `${name}=${state}${error === undefined ? "" : ` (${error})`}`, - ) - .join(", "); - throw new Error( - `Stack stop failed: ${reason}; lifecycle=${diagnostic?.lifecycle ?? "unknown"}; ${states}`, - { - cause, - }, - ); - } - const stopped = await stack.status(); - expect(stopped.lifecycle).toBe("stopped"); - expect(stopped.capabilities.every(({ state }) => state === "stopped")).toBe(true); - expect(stopped.artifacts).toEqual([]); - await runNode(expectOwnedWorkloads(mode, stack.id, [])); - const retainedLogs: StackLogEntry[] = (await stack.logs()).entries.slice(); - expect(retainedLogs.length).toBeGreaterThan(0); - await runNode(expectRuntimeInputsAbsent(stack)); - await runNode(expectEndpointsRefused(initialEndpoints)); - if (mode.runtime.kind === "container") { - expect(await runNode(dockerOwnedResourceCount(initial.id, "containers"))).toBe(0); - expect(await runNode(dockerOwnedResourceCount(initial.id, "networks"))).toBe(0); - expect(await runNode(dockerOwnedResourceCount(initial.id, "volumes"))).toBe(volumesBeforeStop); - } - await restartWholeStackFromPersistedData( - scenario, - endpointSnapshot, - restPath, - functionPath, - poolerUrl, - ); - await stack.stop(); - await restartWholeStackFromPersistedData( - scenario, - endpointSnapshot, - restPath, - functionPath, - poolerUrl, - ); - const final = await stack.status(); - await runNode(assertWholeStackReady(scenario, final)); - await stack.stop(); - await runNode(expectOwnedWorkloads(mode, stack.id, [])); - const reconfigured = await stack.start({ - config: { - capabilities: { studio: { enabled: false } }, - listeners: { - api: { port: studio.port }, - studio: { enabled: false }, - }, - }, - }); - expect(capabilityState(reconfigured, "database")).toBe("ready"); - expect(capabilityState(reconfigured, "studio")).toBe("disabled"); - expect(endpoint(reconfigured, "api").port).toBe(studio.port); - expect(endpoint(reconfigured, "api").port).not.toBe(api.port); - expect(reconfigured.endpoints.studio).toBeUndefined(); -}; -describe("managed Supabase stack whole-stack E2E", () => { - test.skipIf(SELECTED_RUNTIME === "container")( - "recovers the native database after abrupt Supervisor termination", - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- Vitest callback consumes the public TestStack Promise contract. - async () => { - await using stack: TestStack = await createTestStack({ - name: `stack-crash-recovery-${randomId().replaceAll("-", "").slice(0, 16)}`, - runtime: { kind: "native" }, - }); - const before = await stack.status(); - const database = endpoint(before, "database"); - const lockPath = join(stack.stateRoot, stack.id, "data", "database", "postmaster.pid"); - const databasePid = Number((await runNode(readText(lockPath))).split("\n", 1)[0]); - if (!Number.isSafeInteger(databasePid) || databasePid <= 0) - throw new Error("Native database lock did not contain a valid PID"); - const sharedMemoryId = await runNode(nativeDatabaseSharedMemoryId(lockPath)); - const sharedMemoryAvailable = await runNode(nativeDatabaseSharedMemoryExists(sharedMemoryId)); - if (sharedMemoryAvailable !== undefined) expect(sharedMemoryAvailable).toBe(true); - process.kill(await runNode(supervisorPid(stack.id)), "SIGKILL"); - await runNode(waitForProcessExit(databasePid)); - if (sharedMemoryAvailable !== undefined) - expect(await runNode(nativeDatabaseSharedMemoryExists(sharedMemoryId))).toBe(false); - await expect(runNode(connect(database))).rejects.toThrow(); - await expect(runNode(access(lockPath))).rejects.toThrow(); - const recovered = await stack.start(); - expectDefaultLazyState(recovered); - expect( - await runNode(databaseQuery((await stack.credentials()).database.url, "SELECT 1 AS value")), - ).toEqual([{ value: 1 }]); - }, - ); - for (const mode of RUNTIME_CASES) { - for (const databaseVersion of NON_DEFAULT_DATABASE_RELEASES) { - const major = databaseVersion.split(".")[0]; - test( - `starts PostgreSQL ${major} in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- Vitest callback consumes the public TestStack Promise and AsyncDisposable contracts. - async () => { - await using stack: TestStack = await createTestStack({ - name: `stack-postgres-${major}-${randomId().replaceAll("-", "").slice(0, 16)}`, - runtime: mode.runtime, - config: { capabilities: { database: { version: major } } }, - }); - const rows = await runNode( - databaseQuery( - (await stack.credentials()).database.url, - "SELECT current_setting('server_version') AS version", - ), - ); - expect(rows).toEqual([{ version: expect.stringMatching(new RegExp(`^${major}\\.`)) }]); - }, - ); - } - test( - `coordinates concurrent isolated stacks in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- concurrent test consumes PromiseStack and AsyncDisposable contracts. - async () => { - const identity = randomId().replaceAll("-", "").slice(0, 16); - const creations = await Promise.allSettled([ - createTestStack({ - name: `stack-concurrent-a-${identity}`, - runtime: mode.runtime, - }), - createTestStack({ - name: `stack-concurrent-b-${identity}`, - runtime: mode.runtime, - }), - ]); - const firstResult = creations[0]; - const secondResult = creations[1]; - if (firstResult?.status !== "fulfilled" || secondResult?.status !== "fulfilled") { - await Promise.allSettled( - creations.flatMap((result) => - result.status === "fulfilled" ? [result.value[Symbol.asyncDispose]()] : [], - ), - ); - const firstRejection = creations.find( - (result): result is PromiseRejectedResult => result.status === "rejected", - ); - if (firstRejection !== undefined) throw firstRejection.reason; - throw new Error("Concurrent test stack creation returned an incomplete result"); - } - const firstStack = firstResult.value; - const secondStack = secondResult.value; - await using first: TestStack = firstStack; - await using second: TestStack = secondStack; - const [firstStatus, secondStatus] = await Promise.all([first.status(), second.status()]); - expect(firstStatus.lifecycle).toBe("running"); - expect(secondStatus.lifecycle).toBe("running"); - const firstApi = endpoint(firstStatus, "api"); - const secondApi = endpoint(secondStatus, "api"); - expect(firstApi.port).not.toBe(secondApi.port); - const [firstCredentials, secondCredentials] = await Promise.all([ - first.credentials(), - second.credentials(), - ]); - const firstTable = `concurrent_${identity}_a`; - const secondTable = `concurrent_${identity}_b`; - const firstMarker = `marker-a-${identity}`; - const secondMarker = `marker-b-${identity}`; - await Promise.all([ - runNode( - databaseQuery( - firstCredentials.database.url, - `CREATE TABLE public."${firstTable}" (payload text NOT NULL)`, - ), - ), - runNode( - databaseQuery( - secondCredentials.database.url, - `CREATE TABLE public."${secondTable}" (payload text NOT NULL)`, - ), - ), - ]); - await Promise.all([ - runNode( - databaseQuery( - firstCredentials.database.url, - `INSERT INTO public."${firstTable}" (payload) VALUES ($1)`, - [firstMarker], - ), - ), - runNode( - databaseQuery( - secondCredentials.database.url, - `INSERT INTO public."${secondTable}" (payload) VALUES ($1)`, - [secondMarker], - ), - ), - ]); - const [firstRows, secondRows] = await Promise.all([ - runNode( - databaseQuery( - firstCredentials.database.url, - `SELECT payload FROM public."${firstTable}"`, - ), - ), - runNode( - databaseQuery( - secondCredentials.database.url, - `SELECT payload FROM public."${secondTable}"`, - ), - ), - ]); - expect(firstRows).toEqual([{ payload: firstMarker }]); - expect(secondRows).toEqual([{ payload: secondMarker }]); - }, - ); - test( - `coordinates with an ordinary package stack in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- Vitest callback consumes public PromiseStack contracts. - async () => { - const identity = randomId().replaceAll("-", "").slice(0, 16); - const ordinaryRoot = await runNode(mkdtemp(join(tmpdir(), "supabase-stack-cli-consumer-"))); - let ordinary: PromiseStack | undefined; - let helper: TestStack | undefined; - let primary: unknown; - try { - ordinary = await createStack({ - projectRoot: ordinaryRoot, - name: `stack-cli-consumer-${identity}`, - runtime: mode.runtime, - }); - const ordinaryStack = ordinary; - await ordinaryStack.start(); - helper = await createTestStack({ - name: `stack-helper-consumer-${identity}`, - runtime: mode.runtime, - }); - const ordinaryStatus = await ordinaryStack.status(); - const helperStatus = await helper.status(); - expect(ordinaryStatus.lifecycle).toBe("running"); - expect(helperStatus.lifecycle).toBe("running"); - expect(helper.stateRoot).toBe( - (await Effect.runPromise(defaultRuntimeEnvironment)).stateRoot, - ); - expect(endpoint(ordinaryStatus, "api").port).not.toBe(endpoint(helperStatus, "api").port); - const scoped = await listStacks({ projectRoot: ordinaryRoot }); - expect(scoped.map(({ id }) => id)).toContain(ordinaryStack.id); - expect(scoped.map(({ id }) => id)).not.toContain(helper.id); - const all = await listStacks(); - expect(all.map(({ id }) => id)).toEqual(expect.arrayContaining([ordinary.id, helper.id])); - } catch (error) { - primary = error; - } - const [helperResult, ordinaryResult] = await Promise.allSettled([ - helper === undefined ? Promise.resolve() : helper[Symbol.asyncDispose](), - ordinary === undefined ? Promise.resolve() : ordinary.destroy(), - ]); - let cleanupFailure: unknown; - for (const result of [helperResult, ordinaryResult]) { - if (result.status === "rejected") { - cleanupFailure ??= result.reason; - } - } - if (ordinaryResult.status === "fulfilled") { - try { - await runNode(rm(ordinaryRoot, { recursive: true, force: true })); - } catch (error) { - cleanupFailure ??= error; - } - } - if (primary !== undefined) throw primary; - if (cleanupFailure !== undefined) throw cleanupFailure; - }, - ); - test( - `stops and wakes idle REST in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- E2E scenario consumes the public TestStack Promise contract. - async () => { - const identity = randomId().replaceAll("-", "").slice(0, 16).toLowerCase(); - const table = `idle_${identity}`; - const marker = `idle-${identity}`; - await using stack: TestStack = await createTestStack({ - name: `stack-idle-rest-${identity}`, - runtime: mode.runtime, - config: { - capabilities: { - database: {}, - rest: { activation: "lazy", idleTimeoutSeconds: 5 }, - // API gateway credentials are materialized by auth even while its workload stays lazy. - auth: {}, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const initial = await stack.status(); - expect(initial.lifecycle).toBe("running"); - const api = endpoint(initial, "api"); - const initialSupervisorPid = await runNode(supervisorPid(stack.id)); - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database"])); - - const credentials = await stack.credentials(); - await runNode( - databaseQuery( - credentials.database.url, - `CREATE TABLE public."${table}" (id integer PRIMARY KEY, payload text NOT NULL)`, - ), - ); - await runNode( - databaseQuery( - credentials.database.url, - `GRANT SELECT ON public."${table}" TO anon, authenticated, service_role`, - ), - ); - expect( - await runNode( - databaseQuery( - credentials.database.url, - `INSERT INTO public."${table}" (id, payload) VALUES (1, $1) RETURNING id, payload`, - [marker], - ), - ), - ).toEqual([{ id: 1, payload: marker }]); - - const restPath = `/rest/v1/${table}?select=id,payload`; - const beforeLogs = await stack.logs(); - const logIterator = stack.followLogs({ cursor: beforeLogs.cursor })[Symbol.asyncIterator](); - const idleStopLog = waitForLogEntry( - stack, - logIterator, - (entry) => - entry.source === "supervisor" && - entry.stream === "internal" && - entry.message === "Stopped rest after inactivity", - ); - try { - const firstRows = await runNode( - request(api.url, restPath, { - headers: { ...apiHeaders(credentials), Accept: "application/json" }, - }).pipe(Effect.flatMap(jsonValue)), - ); - expect(firstRows).toEqual([{ id: 1, payload: marker }]); - await runNode(idleStopLog); - } finally { - await logIterator.return?.(); - } - - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database"])); - const stoppedRest = await stack.status(); - expect(capabilityState(stoppedRest, "rest")).toBe("dormant"); - expect(endpoint(stoppedRest, "api").port).toBe(api.port); - expect(await runNode(supervisorPid(stack.id))).toBe(initialSupervisorPid); - expect( - await runNode( - databaseQuery( - credentials.database.url, - `SELECT payload FROM public."${table}" WHERE id = 1`, - ), - ), - ).toEqual([{ payload: marker }]); - - const secondRows = await runNode( - request(api.url, restPath, { - headers: { ...apiHeaders(credentials), Accept: "application/json" }, - }).pipe(Effect.flatMap(jsonValue)), - ); - expect(secondRows).toEqual([{ id: 1, payload: marker }]); - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database", "rest:rest"])); - const restartedRest = await stack.status(); - expect(capabilityState(restartedRest, "rest")).toBe("ready"); - expect(endpoint(restartedRest, "api").port).toBe(api.port); - expect(await runNode(supervisorPid(stack.id))).toBe(initialSupervisorPid); - }, - ); - test( - `settles live HTTP, WebSocket, and TCP transports on stop in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- E2E scenario consumes the public TestStack Promise contract. - async () => { - const identity = randomId().replaceAll("-", "").slice(0, 16).toLowerCase(); - const table = `transport_stop_${identity}`; - await using stack: TestStack = await createTestStack({ - name: `stack-transport-stop-${identity}`, - runtime: mode.runtime, - config: { - capabilities: { - database: {}, - rest: {}, - realtime: {}, - pooler: {}, - auth: {}, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - }, - }, - }); - const running = await stack.status(); - expect(running.lifecycle).toBe("running"); - const api = endpoint(running, "api"); - const pooler = endpoint(running, "pooler"); - const credentials = await stack.credentials(); - await runNode( - databaseQuery( - credentials.database.url, - `CREATE TABLE public."${table}" (id integer PRIMARY KEY)`, - ), - ); - await runNode( - databaseQuery( - credentials.database.url, - `GRANT INSERT ON public."${table}" TO anon, authenticated, service_role`, - ), - ); - - const http = await activate(stack, "rest", () => - runNode(openHttpTransport(api, credentials, `/rest/v1/${table}?select=id`)), - ); - let websocket: WebSocket | undefined; - let tcp: Socket | undefined; - try { - websocket = await activate(stack, "realtime", () => - runNode(openSocket(makeRealtimeUrl(api, apiCredentials(credentials).publishableKey))), - ); - await activate(stack, "pooler", () => { - const poolerUrl = new URL(credentials.database.url); - poolerUrl.port = String(pooler.port); - poolerUrl.username = "postgres.pooler-dev"; - return runNode( - databaseQuery(poolerUrl.toString(), "SELECT 42 AS answer", [], { - connectTimeout: LAZY_POOLER_ACTIVATION_TIMEOUT, - }), - ); - }); - tcp = await runNode(openTcpSocket(pooler)); - expect(http.destroyed).toBe(false); - expect(websocket.readyState).toBe(WebSocket.OPEN); - expect(tcp.destroyed).toBe(false); - const closed = runNode( - Effect.all( - [waitForSocketClose(http), waitForSocketClose(websocket), waitForSocketClose(tcp)], - { discard: true, concurrency: "unbounded" }, - ), - ); - - await Promise.all([stack.stop(), closed]); - const stopped = await stack.status(); - expect(stopped.lifecycle).toBe("stopped"); - expect( - stopped.capabilities - .filter(({ name }) => ["pooler", "realtime", "rest"].includes(name)) - .every(({ state }) => state === "stopped"), - ).toBe(true); - expect(stopped.artifacts).toEqual([]); - await runNode(expectOwnedWorkloads(mode, stack.id, [])); - await runNode(expectRuntimeInputsAbsent(stack)); - await runNode( - expectEndpointsRefused( - Object.values(running.endpoints).filter( - (value): value is StackEndpoint => value !== undefined, - ), - ), - ); - } finally { - http.destroy(); - websocket?.terminate(); - tcp?.destroy(); - } - }, - ); - - test(`supports the complete user flow in ${mode.name} mode`, { timeout: E2E_TIMEOUT_MS }, () => - runWholeStackScenario(mode), - ); - test( - `starts every capability eagerly in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- Vitest callback consumes public PromiseStack contracts. - async () => { - const analyticsApiKey = `analytics-key-${randomId()}`; - await using stack: TestStack = await createTestStack({ - name: `stack-eager-${randomId().replaceAll("-", "").slice(0, 16)}`, - runtime: mode.runtime, - config: allEagerConfig(analyticsApiKey), - setupProject: (root) => - runNode(mkdir(join(root, "supabase", "functions"), { recursive: true })), - }); - const status = await stack.status(); - expect(status.lifecycle).toBe("running"); - expect(status.endpoints.smtp).toBeDefined(); - expect( - status.capabilities.map(({ name, state, activation }) => ({ name, state, activation })), - ).toEqual(CAPABILITY_NAMES.map((name) => ({ name, state: "ready", activation: "eager" }))); - await runNode(expectOwnedWorkloads(mode, stack.id, ALL_EAGER_WORKLOAD_IDS)); - const credentials = await stack.credentials(); - expect(await runNode(databaseQuery(credentials.database.url, "SELECT 1 AS value"))).toEqual( - [{ value: 1 }], - ); - await stack.stop(); - const stopped = await stack.status(); - expect(stopped.lifecycle).toBe("stopped"); - expect(stopped.capabilities.every(({ state }) => state === "stopped")).toBe(true); - await runNode(expectOwnedWorkloads(mode, stack.id, [])); - }, - ); - test( - `supports optional image and vector workloads in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- Vitest callback consumes public PromiseStack contracts. - async () => { - const identity = randomId().replaceAll("-", "").slice(0, 20).toLowerCase(); - const bucket = `stack-optional-${identity}`; - const functionSlug = `optional_${identity}`; - const analyticsApiKey = `analytics-key-${identity}`; - const email = `${identity}@example.test`; - const password = "SupabaseStackE2e!123"; - await using stack: TestStack = await createTestStack({ - name: `stack-optional-${identity}`, - runtime: mode.runtime, - config: optionalWorkloadConfig(functionSlug, analyticsApiKey), - setupProject: (root) => - runNode( - Effect.gen(function* () { - yield* mkdir(join(root, "supabase", "functions", functionSlug), { - recursive: true, - }); - yield* writeFile( - join(root, "supabase", "functions", functionSlug, "index.ts"), - "Deno.serve(() => new Response('ok'))", - ); - }), - ), - }); - const credentials = await stack.credentials(); - const status = await stack.status(); - const api = endpoint(status, "api"); - const mailUi = endpoint(status, "mailUi"); - await activate(stack, "storage", () => - runNode( - Effect.gen(function* () { - yield* request(api.url, "/storage/v1/bucket", { - method: "POST", - headers: { ...serviceHeaders(credentials), "content-type": "application/json" }, - body: encodeJson({ id: bucket, name: bucket, public: true }), - }); - yield* request(api.url, `/storage/v1/object/${bucket}/pixel.png`, { - method: "POST", - headers: { ...serviceHeaders(credentials), "content-type": "image/png" }, - body: new Blob([onePixelPng], { type: "image/png" }), - }); - const transformed = yield* request( - api.url, - `/storage/v1/render/image/public/${bucket}/pixel.png?width=1&height=1`, - { headers: serviceHeaders(credentials) }, - ); - const bytes = yield* Effect.tryPromise(() => transformed.arrayBuffer()); - yield* Effect.sync(() => expect(bytes.byteLength).toBeGreaterThan(0)); - }), - ), - ); - await activate(stack, "analytics", () => - runNode( - Effect.gen(function* () { - yield* request(api.url, "/analytics/v1/health", { - headers: { "x-api-key": analyticsApiKey }, - }); - const marker = `analytics-${identity}`; - yield* request(api.url, "/analytics/v1/logs?source_name=postgres.logs", { - method: "POST", - headers: { "x-api-key": analyticsApiKey, "content-type": "application/json" }, - body: encodeJson({ - event_message: marker, - project: "default", - metadata: { source: "stack-e2e" }, - }), - }); - const count = yield* queryAnalyticsMarker(api, analyticsApiKey, marker); - const vectorCount = yield* queryAnalyticsMarker( - api, - analyticsApiKey, - "supabase-stack-vector", - ); - yield* Effect.sync(() => { - expect(count).toBeGreaterThan(0); - expect(vectorCount).toBeGreaterThan(0); - }); - yield* expectOwnedWorkloads(mode, stack.id, OPTIONAL_STORAGE_ANALYTICS_WORKLOAD_IDS); - }), - ), - ); - await activate(stack, "mail", () => - runNode(request(mailUi.url, "/api/v1/messages?limit=100")), - ); - let signup: JsonObject = {}; - await activate(stack, "auth", () => - runNode( - request(api.url, "/auth/v1/signup", { - method: "POST", - headers: { ...apiHeaders(credentials), "content-type": "application/json" }, - body: JSON.stringify({ email, password }), - }).pipe( - Effect.flatMap(jsonObject), - Effect.tap((value) => Effect.sync(() => void (signup = value))), - ), - ), - ); - expect(typeof signup.access_token).toBe("string"); - await runNode( - request(api.url, "/auth/v1/recover", { - method: "POST", - headers: { ...apiHeaders(credentials), "content-type": "application/json" }, - body: JSON.stringify({ email }), - }), - ); - const mailAttempt = Effect.gen(function* () { - const response = yield* request(mailUi.url, "/api/v1/messages?limit=100"); - const body = yield* Effect.tryPromise({ - try: () => response.text(), - catch: (cause) => - new E2ERequestError({ message: "Unable to read Mailpit response", cause }), - }); - if (!body.includes(email)) - return yield* new E2ERequestError({ message: `Mailpit delivery pending for ${email}` }); - return body; - }); - await runNode(Effect.retry(mailAttempt, { schedule: MAILPIT_DELIVERY_RETRY_SCHEDULE })); - }, - ); - } - for (const mode of RUNTIME_CASES) { - test( - `releases owned resources across repeated stop/start cycles in ${mode.name} mode`, - { timeout: E2E_TIMEOUT_MS }, - // oxlint-disable-next-line effecttsgo/async-function -- repeated lifecycle test consumes PromiseStack and AsyncDisposable contracts. - async () => { - let stackId: string | undefined; - let projectRoot = ""; - const snapshots: Array< - Readonly<{ - containers: number; - networks: number; - volumes: number; - }> - > = []; - let endpointSnapshot: ReadonlyArray = []; - { - await using stack: TestStack = await createTestStack({ - name: `stack-resource-audit-${randomId().replaceAll("-", "").slice(0, 16)}`, - runtime: mode.runtime, - setupProject: (root) => { - projectRoot = root; - return Promise.resolve(); - }, - }); - stackId = stack.id; - const initial = await stack.status(); - expectDefaultLazyState(initial); - endpointSnapshot = Object.values(initial.endpoints).filter( - (value): value is StackEndpoint => value !== undefined, - ); - for (let cycle = 0; cycle < 3; cycle += 1) { - const stoppedSupervisorPid = await runNode(supervisorPid(stack.id)); - await stack.stop(); - const stopped = await stack.status(); - expect(stopped.lifecycle).toBe("stopped"); - expect(stopped.capabilities.every(({ state }) => state === "stopped")).toBe(true); - await runNode(expectOwnedWorkloads(mode, stack.id, [])); - await runNode(expectRuntimeInputsAbsent(stack)); - await runNode(expectEndpointsRefused(endpointSnapshot)); - // The stop contract waits for lease/resource release. Detached supervisor exit can - // lag that handoff, so observe the exact owner PID before scanning for leftovers. - await runNode(waitForProcessExit(stoppedSupervisorPid)); - expect(await runNode(supervisorPids(stack.id))).toHaveLength(0); - if (mode.runtime.kind === "container") { - if (stackId === undefined) throw new Error("Stack id was not assigned"); - snapshots.push({ - containers: await runNode(dockerOwnedResourceCount(stackId, "containers")), - networks: await runNode(dockerOwnedResourceCount(stackId, "networks")), - volumes: await runNode(dockerOwnedResourceCount(stackId, "volumes")), - }); - expect(snapshots.at(-1)).toEqual({ containers: 0, networks: 0, volumes: 1 }); - } - await stack.start(); - const restarted = await stack.status(); - expectDefaultLazyState(restarted); - await runNode(expectOwnedWorkloads(mode, stack.id, ["database:database"])); - expect( - Object.values(restarted.endpoints).filter( - (value): value is StackEndpoint => value !== undefined, - ), - ).toEqual(endpointSnapshot); - } - if (mode.runtime.kind === "container") - expect(new Set(snapshots.map((snapshot) => JSON.stringify(snapshot))).size).toBe(1); - } - expect(projectRoot.length).toBeGreaterThan(0); - await expect(runNode(access(projectRoot))).rejects.toThrow(); - if (mode.runtime.kind === "container") { - if (stackId === undefined) throw new Error("Stack id was not assigned"); - expect(await runNode(dockerOwnedResourceCount(stackId, "containers"))).toBe(0); - expect(await runNode(dockerOwnedResourceCount(stackId, "networks"))).toBe(0); - expect(await runNode(dockerOwnedResourceCount(stackId, "volumes"))).toBe(0); - } - if (stackId === undefined) throw new Error("Stack id was not assigned"); - await runNode(expectOwnedWorkloads(mode, stackId, [])); - }, - ); - } -}); diff --git a/packages/stack/src/runtime/Container.integration.test.ts b/packages/stack/src/runtime/Container.integration.test.ts new file mode 100644 index 0000000000..a2e8bdbaf0 --- /dev/null +++ b/packages/stack/src/runtime/Container.integration.test.ts @@ -0,0 +1,222 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Data, Deferred, Effect, Exit, Fiber, Option, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { HttpClient } from "effect/unstable/http"; +import { ContainerLaunchError, makeContainerRuntime, type ContainerProcess } from "./Container.ts"; + +const image = "oven/bun:1.4.1-slim"; + +class ContainerTestError extends Data.TaggedError("ContainerTestError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +describe("container process adapter", () => { + it.live("prepares, launches, streams, waits, and removes one exact container", () => + Effect.gen(function* () { + const id = yield* Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker" }); + yield* runtime.prepare(image); + const process = yield* runtime.launch({ + image, + stackId: "a".repeat(64), + instanceId: "fixture-output", + env: { FIXTURE: "true" }, + args: [ + "-e", + "console.log('stdout-marker'); console.error('stderr-marker'); process.exit(7)", + ], + }); + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + process.stdout.pipe(Stream.decodeText, Stream.mkString), + process.stderr.pipe(Stream.decodeText, Stream.mkString), + process.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(stdout).toContain("stdout-marker"); + expect(stderr).toContain("stderr-marker"); + expect(exitCode).toBe(7); + return process.id; + }), + ); + expect(yield* exists(id)).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("streams tool input and output and preserves a nonzero exit", () => + Effect.gen(function* () { + const id = yield* Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker" }); + yield* runtime.prepare(image); + const process = yield* runtime.launchTool({ + image, + stackId: "e".repeat(64), + instanceId: "tool-input", + env: {}, + args: [ + "-e", + "const input = await Bun.stdin.text(); console.log(input); console.error('tool-error'); process.exit(7)", + ], + }); + const [, stdout, stderr, exitCode] = yield* Effect.all( + [ + Stream.make(new TextEncoder().encode("attached-input")).pipe( + Stream.run(process.stdin), + ), + process.stdout.pipe(Stream.decodeText, Stream.mkString), + process.stderr.pipe(Stream.decodeText, Stream.mkString), + process.exitCode, + ], + { concurrency: "unbounded" }, + ); + expect(stdout).toContain("attached-input"); + expect(stderr).toContain("tool-error"); + expect(exitCode).toBe(7); + return process.id; + }), + ); + expect(yield* exists(id)).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "publishes two private ports and keeps the second service alive after the first stops", + () => + Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker" }); + yield* runtime.prepare(image); + const launch = (instanceId: string, marker: string) => + runtime.launch({ + image, + stackId: "b".repeat(64), + instanceId, + env: {}, + ports: [8080], + args: [ + "-e", + `process.on('SIGTERM', () => process.exit(0)); Bun.serve({ hostname: '0.0.0.0', port: 8080, fetch() { return new Response(${JSON.stringify(marker)}) } }); console.log('ready')`, + ], + }); + const first = yield* launch("http-one", "one"); + const firstReady = yield* ready(first); + const second = yield* launch("http-two", "two"); + const secondReady = yield* ready(second); + expect(first.ports[8080]).toBeDefined(); + expect(second.ports[8080]).toBeDefined(); + expect(first.ports[8080]).not.toBe(second.ports[8080]); + expect(yield* get(first.ports[8080])).toBe("one"); + expect(yield* get(second.ports[8080])).toBe("two"); + yield* first.stop; + yield* first.remove; + expect(yield* get(second.ports[8080])).toBe("two"); + yield* second.stop; + yield* second.remove; + yield* Fiber.interrupt(firstReady); + yield* Fiber.interrupt(secondReady); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("returns cleanup authority when container start fails", () => + Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker" }); + yield* runtime.prepare(image); + const result = yield* runtime + .launch({ + image, + stackId: "c".repeat(64), + instanceId: "invalid-entrypoint", + env: {}, + entrypoint: "/definitely/missing/entrypoint", + }) + .pipe(Effect.exit); + expect(Exit.isFailure(result)).toBe(true); + if (Exit.isSuccess(result)) + return yield* Effect.die("invalid entrypoint unexpectedly started"); + const failure = Cause.findErrorOption(result.cause); + expect(Option.isSome(failure) && failure.value instanceof ContainerLaunchError).toBe(true); + if (Option.isNone(failure) || !(failure.value instanceof ContainerLaunchError)) + return yield* Effect.die("launch failure did not retain cleanup authority"); + const id = failure.value.process.id; + yield* failure.value.process.remove; + expect(yield* exists(id)).toBe(false); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("keeps exit observation shared after a caller cancels its wait", () => + Effect.scoped( + Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ engine: "docker" }); + yield* runtime.prepare(image); + const process = yield* runtime.launch({ + image, + stackId: "d".repeat(64), + instanceId: "cancelled-waiter", + env: {}, + args: [ + "-e", + "process.on('SIGTERM', () => process.exit(23)); setInterval(() => {}, 1000); console.log('ready')", + ], + }); + const logs = yield* ready(process); + const waiter = yield* process.exitCode.pipe(Effect.forkChild({ startImmediately: true })); + yield* Fiber.interrupt(waiter); + expect(Exit.hasInterrupts(yield* Fiber.await(waiter))).toBe(true); + yield* process.stop; + expect(yield* process.exitCode).toBe(23); + yield* Fiber.interrupt(logs); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +const ready = (process: ContainerProcess) => + Effect.gen(function* () { + const signal = yield* Deferred.make(); + const observer = yield* process.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach((chunk) => + chunk === "ready" ? Deferred.succeed(signal, undefined) : Effect.void, + ), + Effect.forkChild, + ); + yield* Deferred.await(signal); + return observer; + }); + +const exists = (id: string) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make("docker", ["inspect", id], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }), + ); + return Number(yield* child.exitCode) === 0; + }); + +const get = (port: number | undefined, path = "/") => { + if (port === undefined) + return Effect.fail(new ContainerTestError({ message: "container port was not assigned" })); + return Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const response = yield* client.get(`http://127.0.0.1:${port}${path}`); + return yield* response.text; + }).pipe( + Effect.mapError( + (cause) => new ContainerTestError({ message: "container request failed", cause }), + ), + Effect.provide(NodeHttpClient.layerNodeHttp), + ); +}; diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts new file mode 100644 index 0000000000..82bed8b63a --- /dev/null +++ b/packages/stack/src/runtime/Container.ts @@ -0,0 +1,347 @@ +import { + Cause, + Crypto, + Data, + type Duration, + Effect, + Exit, + FileSystem, + Fiber, + Option, + Path, + Ref, + Schema, + Scope, + Sink, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; + +export class ContainerError extends Data.TaggedError("ContainerError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +interface ContainerSpec { + readonly image: string; + readonly stackId: string; + readonly instanceId: string; + readonly env: Readonly>; + readonly args?: ReadonlyArray; + readonly entrypoint?: string; + readonly mounts?: ReadonlyArray<{ + readonly source: string; + readonly target: string; + readonly readOnly: boolean; + }>; + readonly workingDir?: string; + readonly ports?: ReadonlyArray; +} + +export interface ContainerProcess { + readonly id: string; + readonly ports: Readonly>; + /** A single-consumer stream; callers that need fanout should publish observations. */ + readonly stdout: Stream.Stream; + /** A single-consumer stream; callers that need fanout should publish observations. */ + readonly stderr: Stream.Stream; + readonly exitCode: Effect.Effect; + readonly stdin: Sink.Sink; + readonly stop: Effect.Effect; + readonly remove: Effect.Effect; +} + +export class ContainerLaunchError extends Data.TaggedError("ContainerLaunchError")<{ + readonly failure: ContainerError; + readonly process: ContainerProcess; +}> {} + +export interface ContainerRuntime { + readonly prepare: (image: string) => Effect.Effect; + readonly launch: ( + spec: ContainerSpec, + ) => Effect.Effect; + readonly launchTool: ( + spec: Omit, + ) => Effect.Effect; +} + +const errorFor = (operation: string, cause: unknown) => + new ContainerError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const PublishedPorts = Schema.Record( + Schema.String, + Schema.NullOr( + Schema.Array( + Schema.Struct({ + HostIp: Schema.String, + HostPort: Schema.String, + }), + ), + ), +); + +const mountField = (key: string, value: string) => { + const field = `${key}=${value}`; + return /[,"\n\r]/u.test(field) ? `"${field.replaceAll('"', '""')}"` : field; +}; + +/** Captures the selected local engine; each launch owns one exact container. */ +export const makeContainerRuntime = (options: { + readonly engine: "docker" | "podman"; +}): Effect.Effect< + ContainerRuntime, + never, + ChildProcessSpawner.ChildProcessSpawner | FileSystem.FileSystem | Path.Path | Crypto.Crypto +> => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + + const run = Effect.fn("Container.command")(function* ( + args: ReadonlyArray, + commandOptions: { readonly timeout?: Duration.Input } = { timeout: "30 seconds" }, + ) { + return yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make(options.engine, args, { stdin: "ignore" }), + ); + const tail = (stream: typeof child.stdout) => + stream.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (text, chunk) => (text + chunk).slice(-65536), + ), + ); + const [stdout, stderr, code] = yield* Effect.all( + [tail(child.stdout), tail(child.stderr), child.exitCode], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor( + args[0] ?? "command", + stderr.trim() || `Engine exited with ${code}`, + ); + return stdout.trim(); + }), + ).pipe( + (effect) => + commandOptions.timeout === undefined + ? effect + : effect.pipe(Effect.timeout(commandOptions.timeout)), + Effect.mapError((cause) => errorFor(args[0] ?? "command", cause)), + ); + }); + + const prepare = Effect.fn("Container.prepare")(function* (image: string) { + const present = yield* run(["image", "ls", "--quiet", "--no-trunc", image]); + if (present.length === 0) yield* run(["pull", image], { timeout: "5 minutes" }); + }); + + const launch = Effect.fn("Container.launch")(function* ( + spec: ContainerSpec, + interactive = false, + ) { + const owner = yield* Scope.Scope; + for (const [key, value] of Object.entries(spec.env)) { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key) || /[\0\r\n]/u.test(value)) { + return yield* errorFor( + "environment", + "Container environment contains an invalid key or value", + ); + } + } + for (const port of spec.ports ?? []) { + if (!Number.isInteger(port) || port < 1 || port > 65535) + return yield* errorFor("ports", "Invalid container port"); + } + const directory = yield* fs + .makeTempDirectoryScoped({ prefix: "supabase-container-" }) + .pipe(Effect.mapError((cause) => errorFor("environment", cause))); + const envPath = path.join(directory, "environment"); + const cidPath = path.join(directory, "container-id"); + yield* fs + .writeFileString( + envPath, + Object.entries(spec.env) + .map(([key, value]) => `${key}=${value}`) + .join("\n"), + { mode: 0o600 }, + ) + .pipe(Effect.mapError((cause) => errorFor("environment", cause))); + const token = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("identity", cause)), + ); + const args = [ + "create", + "--pull", + "never", + "--cidfile", + cidPath, + ...(interactive ? ["--interactive", "--init"] : []), + ...(options.engine === "docker" && process.platform === "linux" + ? ["--add-host", "host.docker.internal:host-gateway"] + : []), + "--name", + `supabase-${token}`, + "--label", + `com.supabase.stack=${spec.stackId}`, + "--label", + `com.supabase.instance=${spec.instanceId}`, + "--env-file", + envPath, + ...(spec.mounts ?? []).flatMap((mount) => [ + "--mount", + `type=bind,${mountField("src", mount.source)},${mountField("dst", mount.target)}${mount.readOnly ? ",ro" : ""}`, + ]), + ...(spec.workingDir === undefined ? [] : ["--workdir", spec.workingDir]), + ...(spec.ports ?? []).flatMap((port) => ["--publish", `127.0.0.1::${port}`]), + ...(spec.entrypoint === undefined ? [] : ["--entrypoint", spec.entrypoint]), + spec.image, + ...(spec.args ?? []), + ]; + + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const creation = yield* restore(run(args)).pipe(Effect.exit); + const recorded = yield* fs.readFileString(cidPath).pipe(Effect.option); + const fromFile = + Option.isSome(recorded) && recorded.value.trim().length > 0 + ? Option.some(recorded.value.trim()) + : Option.none(); + const id = Option.isSome(fromFile) + ? fromFile.value + : Exit.isSuccess(creation) + ? creation.value + : undefined; + if (id === undefined) + return yield* creation.pipe( + Effect.andThen( + Effect.fail(errorFor("create", "Engine returned no container identity")), + ), + ); + if (!/^[a-f0-9]{12,64}$/u.test(id)) + return yield* errorFor("create", "Engine returned an invalid container identity"); + const stopped = yield* Ref.make(false); + const removed = yield* Ref.make(false); + const stop = Effect.gen(function* () { + if ((yield* Ref.get(removed)) || (yield* Ref.get(stopped))) return; + yield* run(["stop", "--time", "10", id]); + yield* Ref.set(stopped, true); + }); + const remove = Effect.gen(function* () { + if (yield* Ref.get(removed)) return; + yield* run(["rm", id]); + yield* Ref.set(removed, true); + }); + yield* Scope.addFinalizer( + owner, + stop.pipe( + Effect.andThen(remove), + Effect.catch((error) => Effect.logError(error)), + ), + ); + const partial: ContainerProcess = { + id, + ports: {}, + stdout: Stream.empty, + stderr: Stream.empty, + exitCode: Effect.fail(errorFor("wait", "Container did not start")), + stdin: Sink.fail(errorFor("stdin", "Container did not start")), + stop, + remove, + }; + let owned = partial; + const wrapFailure = (failure: ContainerError) => + new ContainerLaunchError({ failure, process: owned }); + if (Exit.isFailure(creation)) { + const failure = Cause.findErrorOption(creation.cause); + return yield* Option.isSome(failure) + ? Effect.fail(wrapFailure(failure.value)) + : Effect.failCause(creation.cause); + } + return yield* restore( + Effect.gen(function* () { + const attached = interactive + ? yield* spawner + .spawn( + ChildProcess.make( + options.engine, + ["start", "--attach", "--interactive", id], + { stdin: "pipe", forceKillAfter: "5 seconds" }, + ), + ) + .pipe(Effect.mapError((cause) => errorFor("start", cause))) + : undefined; + if (attached === undefined) yield* run(["start", id]); + const wait: Effect.Effect = + attached === undefined + ? run(["wait", id], {}).pipe( + Effect.flatMap((output) => { + const code = Number(output); + return /^\d+$/u.test(output) && Number.isSafeInteger(code) + ? Effect.succeed(code) + : Effect.fail(errorFor("wait", "Invalid container exit code")); + }), + ) + : attached.exitCode.pipe( + Effect.map(Number), + Effect.mapError((cause) => errorFor("wait", cause)), + ); + const getWaiter = yield* Effect.cached(Effect.forkIn(wait, owner)); + const exitCode: Effect.Effect = Effect.uninterruptible( + getWaiter, + ).pipe(Effect.flatMap((fiber) => Fiber.join(fiber))); + owned = { ...partial, exitCode }; + const text = yield* run([ + "inspect", + "--format", + "{{json .NetworkSettings.Ports}}", + id, + ]); + const bindings = yield* Schema.decodeEffect(Schema.fromJsonString(PublishedPorts))( + text, + ).pipe(Effect.mapError((cause) => errorFor("inspect", cause))); + const ports: Record = {}; + for (const port of spec.ports ?? []) { + const value = bindings[`${port}/tcp`]?.find( + (binding) => binding.HostIp === "127.0.0.1", + )?.HostPort; + const actual = Number(value); + if (!Number.isInteger(actual) || actual < 1 || actual > 65535) + return yield* errorFor("inspect", `No loopback publication for ${port}`); + ports[port] = actual; + } + const logProcess = + attached ?? + (yield* spawner + .spawn( + ChildProcess.make(options.engine, ["logs", "--follow", id], { + stdin: "ignore", + }), + ) + .pipe(Effect.mapError((cause) => errorFor("logs", cause)))); + return { + ...partial, + ports, + exitCode, + stdin: logProcess.stdin.pipe(Sink.mapError((cause) => errorFor("stdin", cause))), + stdout: logProcess.stdout.pipe(Stream.mapError((cause) => errorFor("logs", cause))), + stderr: logProcess.stderr.pipe(Stream.mapError((cause) => errorFor("logs", cause))), + }; + }), + ).pipe(Effect.mapError(wrapFailure)); + }), + ); + }); + return { prepare, launch, launchTool: (spec) => launch(spec, true) }; + }); diff --git a/packages/stack/src/runtime/ContainerEngine.ts b/packages/stack/src/runtime/ContainerEngine.ts deleted file mode 100644 index 4d68544251..0000000000 --- a/packages/stack/src/runtime/ContainerEngine.ts +++ /dev/null @@ -1,893 +0,0 @@ -import { Data, Effect, Predicate, Schema, Stream } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import type * as ChildProcessSpawnerService from "effect/unstable/process/ChildProcessSpawner"; -import type { ContainerArtifact } from "../model/CapabilityModule.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; -import { NetworkPortSchema } from "../public/Status.ts"; - -export type ContainerEngineKind = "docker" | "podman"; -export interface ContainerPlatform { - readonly os: "linux" | "darwin" | "windows"; - readonly desktop?: boolean; - readonly rootless?: boolean; - readonly remote?: boolean; -} -export interface ContainerHostRoute { - readonly host: string; - /** Host address where the in-process gateway also binds for rootful Linux containers. */ - readonly bindAddress?: string; -} -class ContainerExecutableNotFoundError extends Data.TaggedError( - "ContainerExecutableNotFoundError", -)<{ readonly executable: string; readonly message: string }> {} -class ContainerRoutingUnsupportedError extends Data.TaggedError( - "ContainerRoutingUnsupportedError", -)<{ readonly engine: ContainerEngineKind; readonly message: string }> {} -export class ContainerEngineProtocolError extends Data.TaggedError("ContainerEngineProtocolError")<{ - readonly operation: string; - readonly message: string; - readonly cause?: unknown; -}> {} -export class ContainerCommandError extends Data.TaggedError("ContainerCommandError")<{ - readonly operation: string; - readonly message: string; -}> {} -export type ContainerEngineFailure = - | ContainerExecutableNotFoundError - | ContainerRoutingUnsupportedError - | ContainerEngineProtocolError - | ContainerCommandError; - -export type ContainerResourceRole = "network" | "volume" | "workload"; -interface ContainerIdentityLabels { - readonly stackId: StackId; - readonly ownerSessionId: string; -} -export interface ContainerNetworkLabels extends ContainerIdentityLabels { - readonly role: "network"; -} -export interface ContainerWorkloadLabels extends ContainerIdentityLabels { - readonly workloadId: string; - readonly startup?: boolean; - readonly role: "workload"; -} -export interface ContainerVolumeLabels { - readonly stackId: StackId; - readonly workloadId: string; - readonly role: "volume"; -} -export type ContainerLabels = - | ContainerNetworkLabels - | ContainerWorkloadLabels - | ContainerVolumeLabels; - -const CONTAINER_LABEL_PREFIX = "com.supabase.stack"; - -/** Encodes one complete key=value field in Docker and Podman's mount CSV syntax. */ -const mountField = (key: string, value: string): string => { - const field = `${key}=${value}`; - return /[,"\n\r]/.test(field) ? `"${field.replaceAll('"', '""')}"` : field; -}; -export const CONTAINER_LABEL_KEYS = { - stackId: `${CONTAINER_LABEL_PREFIX}.stackId`, - ownerSessionId: `${CONTAINER_LABEL_PREFIX}.ownerSessionId`, - workloadId: `${CONTAINER_LABEL_PREFIX}.workloadId`, - startup: `${CONTAINER_LABEL_PREFIX}.startup`, - role: `${CONTAINER_LABEL_PREFIX}.role`, -}; - -/** Shapes byte-identical label argv for Docker and Podman. */ -const containerLabels = (value: ContainerLabels): ReadonlyArray => { - const pairs: ReadonlyArray = - value.role === "network" - ? [ - ["stackId", value.stackId], - ["ownerSessionId", value.ownerSessionId], - ["role", value.role], - ] - : value.role === "volume" - ? [ - ["stackId", value.stackId], - ["workloadId", value.workloadId], - ["role", value.role], - ] - : [ - ["stackId", value.stackId], - ["ownerSessionId", value.ownerSessionId], - ["workloadId", value.workloadId], - ["startup", value.startup === true ? "true" : "false"], - ["role", value.role], - ]; - return pairs.flatMap(([key, label]) => ["--label", `${CONTAINER_LABEL_PREFIX}.${key}=${label}`]); -}; -export interface ContainerResource { - readonly id: string; - readonly name: string; - readonly kind: ContainerResourceRole; - readonly labels: ContainerLabels; - readonly state?: "created" | "running" | "stopped"; -} -export interface ContainerMount { - readonly source: string; - readonly target: string; - readonly readOnly: boolean; -} -export interface ContainerVolumeMount { - readonly volume: string; - readonly target: string; - readonly readOnly: boolean; -} -export interface ContainerStartupProcess { - readonly entrypoint: string; - readonly command: ReadonlyArray; -} -interface ContainerPortPublication { - readonly address: "127.0.0.1"; - readonly hostPort: number; - readonly containerPort: number; -} -export interface ContainerNetworkSpec { - readonly name: string; - readonly labels: ContainerNetworkLabels; -} -export interface ContainerVolumeSpec { - readonly name: string; - readonly labels: ContainerVolumeLabels; -} -export interface ContainerContainerSpec { - readonly name: string; - readonly image: ContainerArtifact["image"]; - readonly labels: ContainerWorkloadLabels; - readonly network: string; - readonly mounts: ReadonlyArray; - readonly volumeMounts: ReadonlyArray; - readonly publications: ReadonlyArray; - readonly role: "workload"; - /** Optional image entrypoint override used by service-owned init processes. */ - readonly entrypoint?: string; - readonly command?: ReadonlyArray; - /** Path to an owned 0600 env file. Secret values must never be argv. */ - readonly envFile?: string; - readonly networkAliases?: ReadonlyArray; - /** Linux Engine needs `host.docker.internal:host-gateway` to reach published loopback. */ - readonly extraHosts?: ReadonlyArray; -} - -export type ContainerCommand = - | { readonly operation: "probe" } - | { readonly operation: "inspect-image"; readonly image: string } - | { readonly operation: "pull-image"; readonly image: string } - | { readonly operation: "inspect-containers"; readonly stackId: StackId } - | { readonly operation: "inspect-networks"; readonly stackId: StackId } - | { readonly operation: "inspect-network-gateway"; readonly id: string } - | { readonly operation: "inspect-volumes"; readonly stackId: StackId } - | { readonly operation: "create-network"; readonly spec: ContainerNetworkSpec } - | { readonly operation: "remove-network"; readonly id: string } - | { readonly operation: "create-volume"; readonly spec: ContainerVolumeSpec } - | { readonly operation: "remove-volume"; readonly id: string } - | { readonly operation: "create-container"; readonly spec: ContainerContainerSpec } - | { - readonly operation: "copy-container"; - readonly id: string; - readonly source: string; - readonly destination: string; - } - | { readonly operation: "start-container"; readonly id: string } - | { readonly operation: "wait-container"; readonly id: string } - | { readonly operation: "stop-container"; readonly id: string } - | { readonly operation: "remove-container"; readonly id: string }; - -type CommonContainerCommand = Extract< - ContainerCommand, - | { readonly operation: "create-network" } - | { readonly operation: "remove-network" } - | { readonly operation: "create-volume" } - | { readonly operation: "remove-volume" } - | { readonly operation: "create-container" } - | { readonly operation: "copy-container" } - | { readonly operation: "start-container" } - | { readonly operation: "wait-container" } - | { readonly operation: "stop-container" } - | { readonly operation: "remove-container" } ->; - -export const serializeCommonContainerCommand = ( - command: CommonContainerCommand, -): ContainerProcessRequest => { - switch (command.operation) { - case "create-network": - return { - args: ["network", "create", ...containerLabels(command.spec.labels), command.spec.name], - }; - case "remove-network": - return { args: ["network", "rm", command.id] }; - case "create-volume": - return { - args: ["volume", "create", ...containerLabels(command.spec.labels), command.spec.name], - }; - case "remove-volume": - return { args: ["volume", "rm", command.id] }; - case "create-container": { - const bindMounts = command.spec.mounts.flatMap((mount) => [ - "--mount", - `type=bind,${mountField("src", mount.source)},${mountField("dst", mount.target)}${mount.readOnly ? ",ro" : ""}`, - ]); - const volumeMounts = command.spec.volumeMounts.flatMap((mount) => [ - "--mount", - `type=volume,${mountField("src", mount.volume)},${mountField("dst", mount.target)}${mount.readOnly ? ",ro" : ""}`, - ]); - const publications = command.spec.publications.flatMap((port) => [ - "--publish", - `${port.address}:${port.hostPort}:${port.containerPort}`, - ]); - const environment = - command.spec.envFile === undefined ? [] : ["--env-file", command.spec.envFile]; - const networkAliases = - command.spec.networkAliases === undefined - ? [] - : command.spec.networkAliases.flatMap((alias) => ["--network-alias", alias]); - const extraHosts = (command.spec.extraHosts ?? []).flatMap((host) => ["--add-host", host]); - const entrypoint = - command.spec.entrypoint === undefined ? [] : ["--entrypoint", command.spec.entrypoint]; - return { - args: [ - "create", - "--name", - command.spec.name, - "--network", - command.spec.network, - ...networkAliases, - ...extraHosts, - ...containerLabels(command.spec.labels), - ...bindMounts, - ...volumeMounts, - ...publications, - ...environment, - ...entrypoint, - command.spec.image, - ...(command.spec.command ?? []), - ], - }; - } - case "copy-container": - return { args: ["cp", command.source, `${command.id}:${command.destination}`] }; - case "start-container": - return { args: ["start", command.id] }; - case "wait-container": - return { args: ["wait", command.id] }; - case "stop-container": - return { args: ["stop", command.id] }; - case "remove-container": - return { args: ["rm", "--force", command.id] }; - } -}; - -export interface ContainerCommandResult { - readonly stdout: string; - readonly stderr: string; - readonly exitCode: number; -} -export interface ContainerProcessRequest { - readonly args: ReadonlyArray; - readonly stdin?: string; - readonly env?: Readonly>; -} -interface ContainerProcessOutputChunk { - readonly stream: "stdout" | "stderr"; - readonly bytes: Uint8Array; -} -export interface ContainerLogLine { - readonly stream: "stdout" | "stderr"; - readonly message: string; -} -export interface ContainerLogOptions { - /** Number of historical lines to replay before following. `all` is the default. */ - readonly tail?: "all" | 0; -} -export interface ContainerCommandRunner { - readonly run: ( - request: ContainerProcessRequest, - ) => Effect.Effect; - /** Follows one exact process's stdout/stderr until it exits. */ - readonly stream: ( - request: ContainerProcessRequest, - ) => Stream.Stream; -} -export interface ProcessCommandRunnerOptions { - readonly executable: string; - readonly baseArgs?: ReadonlyArray; - readonly maxOutputBytes?: number; -} -const boundedOutput = ( - stream: Stream.Stream, - maxBytes: number, -): Effect.Effect => - Stream.runFoldEffect( - stream.pipe( - Stream.mapError( - (cause) => - new ContainerEngineProtocolError({ - operation: "process-output", - message: "Container engine output stream failed", - cause, - }), - ), - ), - () => new Uint8Array(), - (current: Uint8Array, chunk: Uint8Array) => { - const remaining = Math.max(0, maxBytes - current.length); - if (chunk.length > remaining) - return Effect.fail( - new ContainerEngineProtocolError({ - operation: "process-output", - message: "Container engine output exceeded the configured limit", - }), - ); - const result = new Uint8Array(current.length + chunk.length); - result.set(current); - result.set(chunk, current.length); - return Effect.succeed(result); - }, - ).pipe(Effect.map((bytes) => new TextDecoder().decode(bytes))); -export const makeProcessCommandRunner = ( - options: ProcessCommandRunnerOptions, -): Effect.Effect< - ContainerCommandRunner, - ContainerEngineFailure, - ChildProcessSpawnerService.ChildProcessSpawner -> => - Effect.gen(function* () { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const maxOutputBytes = options.maxOutputBytes ?? 1024 * 1024; - const run = ( - request: ContainerProcessRequest, - ): Effect.Effect => { - const stdin = - request.stdin === undefined - ? "ignore" - : { - stream: Stream.succeed(new TextEncoder().encode(request.stdin)), - endOnDone: true, - }; - return Effect.scoped( - Effect.acquireUseRelease( - ChildProcess.make(options.executable, [...(options.baseArgs ?? []), ...request.args], { - stdin, - stdout: "pipe", - stderr: "pipe", - ...(request.env === undefined ? {} : { env: request.env }), - extendEnv: true, - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)), - (handle) => - Effect.gen(function* () { - const [stdout, stderr, exitCode] = yield* Effect.all( - [ - boundedOutput(handle.stdout, maxOutputBytes), - boundedOutput(handle.stderr, maxOutputBytes), - handle.exitCode, - ], - { concurrency: "unbounded" }, - ); - return { stdout, stderr, exitCode: Number(exitCode) }; - }), - (handle) => { - const terminate = (signal: "SIGTERM" | "SIGKILL") => - Effect.gen(function* () { - if (!(yield* handle.isRunning)) return; - yield* handle.kill({ killSignal: signal }); - }); - return Effect.timeoutOrElse(terminate("SIGTERM").pipe(Effect.interruptible), { - duration: "2 seconds", - orElse: () => terminate("SIGKILL"), - }); - }, - ), - ).pipe( - Effect.mapError((error) => - typeof error === "object" && - error !== null && - "reason" in error && - Predicate.isTagged(error.reason, "NotFound") - ? new ContainerExecutableNotFoundError({ - executable: options.executable, - message: `${options.executable} executable was not found`, - }) - : new ContainerEngineProtocolError({ - operation: request.args[0] ?? "command", - message: "Container engine process failed", - cause: error, - }), - ), - ); - }; - const stream = ( - request: ContainerProcessRequest, - ): Stream.Stream => - Stream.scoped( - Stream.unwrap( - ChildProcess.make(options.executable, [...(options.baseArgs ?? []), ...request.args], { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - ...(request.env === undefined ? {} : { env: request.env }), - extendEnv: true, - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.mapError( - () => - new ContainerEngineProtocolError({ - operation: request.args[0] ?? "stream", - message: "Container engine log follower failed to start", - }), - ), - Effect.map((handle) => { - const mapOutput = ( - output: Stream.Stream, - streamName: "stdout" | "stderr", - ): Stream.Stream => - output.pipe( - Stream.map((bytes) => ({ stream: streamName, bytes })), - Stream.mapError( - () => - new ContainerEngineProtocolError({ - operation: request.args[0] ?? "stream", - message: "Container engine log follower output failed", - }), - ), - ); - const output = Stream.merge( - mapOutput(handle.stdout, "stdout"), - mapOutput(handle.stderr, "stderr"), - ); - const verifyExit = Stream.fromEffect( - handle.exitCode.pipe( - Effect.mapError( - () => - new ContainerEngineProtocolError({ - operation: request.args[0] ?? "stream", - message: "Container engine log follower exit status failed", - }), - ), - Effect.flatMap((code) => - Number(code) === 0 - ? Effect.void - : Effect.fail( - new ContainerCommandError({ - operation: request.args[0] ?? "stream", - message: `Container engine log follower exited (${String(code)})`, - }), - ), - ), - ), - ).pipe(Stream.flatMap(() => Stream.empty)); - return output.pipe(Stream.concat(verifyExit)); - }), - ), - ), - ); - return { run, stream }; - }); - -export interface ContainerEngineCodecs { - readonly serialize: (command: ContainerCommand) => ContainerProcessRequest; - readonly decodeProbe: ( - result: ContainerCommandResult, - ) => Effect.Effect; - readonly decodeImage: ( - result: ContainerCommandResult, - ) => Effect.Effect, ContainerEngineFailure>; - readonly decodeContainers: ( - result: ContainerCommandResult, - ) => Effect.Effect, ContainerEngineFailure>; - readonly decodeNetworks: ( - result: ContainerCommandResult, - ) => Effect.Effect, ContainerEngineFailure>; - readonly decodeNetworkGateway: ( - result: ContainerCommandResult, - ) => Effect.Effect; - readonly decodeVolumes: ( - result: ContainerCommandResult, - ) => Effect.Effect, ContainerEngineFailure>; - readonly decodeCreate: ( - operation: string, - result: ContainerCommandResult, - spec: { readonly name: string; readonly labels: ContainerLabels }, - kind: R, - ) => Effect.Effect; - readonly decodeWait: ( - result: ContainerCommandResult, - ) => Effect.Effect; -} - -export const makeContainerEngineCodecs = (options: { - readonly engineName: "Docker" | "Podman"; - readonly scalarFormat: "json" | "raw"; - readonly serialize: ContainerEngineCodecs["serialize"]; -}): ContainerEngineCodecs => { - const protocol = (operation: string, cause?: unknown): ContainerEngineProtocolError => - new ContainerEngineProtocolError({ - operation, - message: `${operation} returned an invalid ${options.engineName} response`, - ...(cause === undefined ? {} : { cause }), - }); - const lines = (raw: string): ReadonlyArray => - raw - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line.length > 0); - const scalar = ( - operation: string, - raw: string, - ): Effect.Effect => { - if (options.scalarFormat === "raw") { - const value = raw.trim(); - return value.length > 0 && !/\\t|\t/.test(value) - ? Effect.succeed(value) - : Effect.fail(protocol(operation)); - } - return Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(raw).pipe( - Effect.mapError((cause) => protocol(operation, cause)), - Effect.flatMap((value) => - typeof value === "string" && value.length > 0 - ? Effect.succeed(value) - : Effect.fail(protocol(operation)), - ), - ); - }; - const fields = (operation: string, line: string, count: number) => { - const values = line.split(/\\t|\t/); - if (values.length !== count) return Effect.fail(protocol(operation)); - return options.scalarFormat === "json" - ? Effect.forEach(values, (value) => scalar(operation, value)) - : Effect.succeed(values); - }; - const decodeIdentity = (operation: string, stack: string | undefined) => - stack === undefined - ? Effect.fail(protocol(operation)) - : Schema.decodeEffect(StackIdSchema)(stack).pipe( - Effect.mapError((error) => protocol(operation, error)), - ); - const networkLabels = (operation: string, values: ReadonlyArray) => { - const [stack, owner, role] = values; - if (owner === undefined || role !== "network") return Effect.fail(protocol(operation)); - return decodeIdentity(operation, stack).pipe( - Effect.map((stackId) => ({ stackId, ownerSessionId: owner, role: "network" as const })), - ); - }; - const workloadLabels = (operation: string, values: ReadonlyArray) => { - const [stack, owner, workload, startup, role] = values; - if (owner === undefined || workload === undefined || role !== "workload") - return Effect.fail(protocol(operation)); - return decodeIdentity(operation, stack).pipe( - Effect.map((stackId) => ({ - stackId, - ownerSessionId: owner, - workloadId: workload, - ...(startup === "true" ? { startup: true } : {}), - role: "workload" as const, - })), - ); - }; - const decodeRows = ( - operation: string, - raw: string, - count: number, - decode: (values: ReadonlyArray) => Effect.Effect, - ) => - Effect.forEach(lines(raw), (line) => - fields(operation, line, count).pipe(Effect.flatMap(decode)), - ); - const decodeContainers: ContainerEngineCodecs["decodeContainers"] = (result) => - decodeRows("inspect-containers", result.stdout, 8, (values) => { - const [id, name, stack, owner, workload, startup, role, state] = values; - if ( - id === undefined || - name === undefined || - stack === undefined || - owner === undefined || - workload === undefined || - role !== "workload" || - state === undefined - ) - return Effect.fail(protocol("inspect-containers")); - return workloadLabels("inspect-containers", [ - stack, - owner, - workload, - startup ?? "", - role, - ]).pipe( - Effect.map((labels): ContainerResource => ({ - id, - name, - kind: "workload", - labels, - state: state.includes("running") ? "running" : "stopped", - })), - ); - }); - const decodeNetworks: ContainerEngineCodecs["decodeNetworks"] = (result) => - decodeRows("inspect-networks", result.stdout, 5, (values) => { - const [id, name, stack, owner, role] = values; - if (id === undefined || name === undefined || role === undefined) - return Effect.fail(protocol("inspect-networks")); - return networkLabels("inspect-networks", [stack ?? "", owner ?? "", role]).pipe( - Effect.map((labels): ContainerResource => ({ id, name, kind: "network", labels })), - ); - }); - const decodeNetworkGateway: ContainerEngineCodecs["decodeNetworkGateway"] = (result) => - Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(result.stdout).pipe( - Effect.mapError((cause) => protocol("inspect-network-gateway", cause)), - Effect.flatMap((value) => { - const entries = Array.isArray(value) ? value : [value]; - const gateway = entries.find( - (entry): entry is { readonly Gateway: string } => - typeof entry === "object" && - entry !== null && - "Gateway" in entry && - typeof entry.Gateway === "string" && - /^(?:\d{1,3}\.){3}\d{1,3}$/u.test(entry.Gateway) && - entry.Gateway.split(".").every((part: string) => Number(part) <= 255), - )?.Gateway; - return gateway === undefined - ? Effect.fail(protocol("inspect-network-gateway")) - : Effect.succeed(gateway); - }), - ); - const decodeVolumes: ContainerEngineCodecs["decodeVolumes"] = (result) => - decodeRows("inspect-volumes", result.stdout, 4, (values) => { - const [name, stack, workload, role] = values; - if (name === undefined || stack === undefined || workload === undefined || role !== "volume") - return Effect.fail(protocol("inspect-volumes")); - return decodeIdentity("inspect-volumes", stack).pipe( - Effect.map((stackId): ContainerResource => ({ - id: name, - name, - kind: "volume", - labels: { stackId, workloadId: workload, role: "volume" }, - })), - ); - }); - const decodeCreate: ContainerEngineCodecs["decodeCreate"] = (operation, result, spec, kind) => { - const id = result.stdout.trim(); - return id.length > 0 && !/[\r\n]/.test(id) - ? Effect.succeed({ id, name: spec.name, kind, labels: spec.labels, state: "created" }) - : Effect.fail(protocol(operation)); - }; - const decodeWait: ContainerEngineCodecs["decodeWait"] = (result) => { - const values = lines(result.stdout); - const value = values[0]; - if (values.length !== 1 || value === undefined || !/^\d+$/.test(value)) - return Effect.fail(protocol("wait-container")); - const exitCode = Number(value); - return Number.isSafeInteger(exitCode) - ? Effect.succeed(exitCode) - : Effect.fail(protocol("wait-container")); - }; - return { - serialize: options.serialize, - decodeProbe: (result) => scalar("probe", result.stdout).pipe(Effect.asVoid), - decodeImage: (result) => - Effect.forEach(lines(result.stdout), (line) => scalar("inspect-image", line)).pipe( - Effect.map((values) => ({ present: values.length > 0 })), - ), - decodeContainers, - decodeNetworks, - decodeNetworkGateway, - decodeVolumes, - decodeCreate, - decodeWait, - }; -}; -export interface ContainerEngineOptions { - readonly kind: ContainerEngineKind; - readonly runner: ContainerCommandRunner; - readonly platform: ContainerPlatform; - readonly codecs: ContainerEngineCodecs; -} -export interface ContainerEngine { - readonly kind: ContainerEngineKind; - readonly preflight: Effect.Effect; - readonly probe: Effect.Effect; - readonly inspectImage: ( - image: string, - ) => Effect.Effect, ContainerEngineFailure>; - readonly pullImage: (image: string) => Effect.Effect; - readonly listResources: ( - stackId: StackId, - ) => Effect.Effect, ContainerEngineFailure>; - readonly createNetwork: ( - spec: ContainerNetworkSpec, - ) => Effect.Effect; - /** Resolves a rootful Docker network's IPv4 bridge gateway. */ - readonly resolveNetworkGateway?: (id: string) => Effect.Effect; - readonly removeNetwork: (id: string) => Effect.Effect; - readonly createVolume: ( - spec: ContainerVolumeSpec, - ) => Effect.Effect; - readonly removeVolume: (id: string) => Effect.Effect; - readonly createContainer: ( - spec: ContainerContainerSpec, - ) => Effect.Effect; - /** Copies an owner-created bootstrap path into a newly-created container. */ - readonly copyToContainer: ( - id: string, - source: string, - destination: string, - ) => Effect.Effect; - readonly startContainer: (id: string) => Effect.Effect; - /** Waits for one exact container and returns its process exit code. */ - readonly waitContainer: (id: string) => Effect.Effect; - readonly stopContainer: (id: string) => Effect.Effect; - readonly removeContainer: (id: string) => Effect.Effect; - /** Follows one exact container and emits complete stdout/stderr lines. */ - readonly streamLogs: ( - id: string, - options?: ContainerLogOptions, - ) => Stream.Stream; -} -export const makeContainerEngineCore = (options: ContainerEngineOptions): ContainerEngine => { - const run = (command: ContainerCommand) => options.runner.run(options.codecs.serialize(command)); - const check = (operation: string, command: ContainerCommand) => - run(command).pipe( - Effect.flatMap((result) => - result.exitCode === 0 - ? Effect.succeed(result) - : Effect.fail( - new ContainerCommandError({ - operation, - message: - result.stderr.trim().length > 0 - ? `Container engine command failed (${result.exitCode}): ${result.stderr.trim()}` - : `Container engine command failed (${result.exitCode})`, - }), - ), - ), - ); - const noResult = (operation: string, command: ContainerCommand) => - check(operation, command).pipe(Effect.asVoid); - const streamLogs = ( - id: string, - logOptions?: ContainerLogOptions, - ): Stream.Stream => { - const source = options.runner.stream; - interface LineState { - readonly stdout: { readonly decoder: TextDecoder; remainder: string }; - readonly stderr: { readonly decoder: TextDecoder; remainder: string }; - } - const stateFor = (): LineState => ({ - stdout: { decoder: new TextDecoder(), remainder: "" }, - stderr: { decoder: new TextDecoder(), remainder: "" }, - }); - const split = ( - state: LineState, - chunk: ContainerProcessOutputChunk, - ): ReadonlyArray => { - const accumulator = state[chunk.stream]; - accumulator.remainder += accumulator.decoder.decode(chunk.bytes, { stream: true }); - const lines = accumulator.remainder.split(/\r?\n/); - accumulator.remainder = lines.pop() ?? ""; - return lines.map((message) => ({ stream: chunk.stream, message })); - }; - const flush = (state: LineState): ReadonlyArray => - (["stdout", "stderr"] as const).flatMap((streamName) => { - const accumulator = state[streamName]; - accumulator.remainder += accumulator.decoder.decode(); - if (accumulator.remainder.length === 0) return []; - const message = accumulator.remainder; - accumulator.remainder = ""; - return [{ stream: streamName, message }]; - }); - return source({ - args: [ - "logs", - "--follow", - "--tail", - logOptions?.tail === undefined ? "all" : String(logOptions.tail), - id, - ], - }).pipe( - Stream.mapAccum(stateFor, (state, chunk) => [state, split(state, chunk)] as const, { - onHalt: flush, - }), - ); - }; - const preflight = - options.kind === "docker" - ? options.platform.remote === true - ? Effect.fail( - new ContainerRoutingUnsupportedError({ - engine: options.kind, - message: "Docker remote daemon has no verified host route", - }), - ) - : Effect.succeed({ host: "host.docker.internal" }) - : options.platform.remote === true || options.platform.os !== "linux" - ? Effect.fail( - new ContainerRoutingUnsupportedError({ - engine: options.kind, - message: "Podman host route is unsupported on this platform", - }), - ) - : Effect.succeed({ host: "host.containers.internal" }); - return { - kind: options.kind, - preflight, - probe: check("probe", { operation: "probe" }).pipe(Effect.flatMap(options.codecs.decodeProbe)), - inspectImage: (image) => - check("inspect-image", { operation: "inspect-image", image }).pipe( - Effect.flatMap(options.codecs.decodeImage), - ), - pullImage: (image) => noResult("pull-image", { operation: "pull-image", image }), - listResources: (stackId) => - Effect.all([ - check("inspect-containers", { operation: "inspect-containers", stackId }).pipe( - Effect.flatMap(options.codecs.decodeContainers), - ), - check("inspect-networks", { operation: "inspect-networks", stackId }).pipe( - Effect.flatMap(options.codecs.decodeNetworks), - ), - check("inspect-volumes", { operation: "inspect-volumes", stackId }).pipe( - Effect.flatMap(options.codecs.decodeVolumes), - ), - ]).pipe( - Effect.map(([containers, networks, volumes]) => [...containers, ...networks, ...volumes]), - ), - createNetwork: (spec) => - check("create-network", { operation: "create-network", spec }).pipe( - Effect.flatMap((result) => - options.codecs.decodeCreate("create-network", result, spec, "network"), - ), - ), - resolveNetworkGateway: - options.kind === "docker" && - options.platform.os === "linux" && - options.platform.desktop !== true && - options.platform.rootless !== true && - options.platform.remote !== true - ? (id) => - check("inspect-network-gateway", { - operation: "inspect-network-gateway", - id, - }).pipe(Effect.flatMap(options.codecs.decodeNetworkGateway)) - : undefined, - removeNetwork: (id) => noResult("remove-network", { operation: "remove-network", id }), - createVolume: (spec) => - check("create-volume", { operation: "create-volume", spec }).pipe( - Effect.flatMap((result) => - options.codecs.decodeCreate("create-volume", result, spec, "volume"), - ), - ), - removeVolume: (id) => noResult("remove-volume", { operation: "remove-volume", id }), - createContainer: (spec) => { - const invalidPublication = spec.publications.some( - (publication) => - publication.address !== "127.0.0.1" || - !Schema.is(NetworkPortSchema)(publication.hostPort) || - !Schema.is(NetworkPortSchema)(publication.containerPort), - ); - return invalidPublication - ? Effect.fail( - new ContainerEngineProtocolError({ - operation: "create-container", - message: "Container publications must use valid loopback host ports", - }), - ) - : check("create-container", { operation: "create-container", spec }).pipe( - Effect.flatMap((result) => - options.codecs.decodeCreate("create-container", result, spec, spec.role), - ), - ); - }, - copyToContainer: (id, source, destination) => - noResult("copy-container", { operation: "copy-container", id, source, destination }), - startContainer: (id) => noResult("start-container", { operation: "start-container", id }), - waitContainer: (id) => - check("wait-container", { operation: "wait-container", id }).pipe( - Effect.flatMap(options.codecs.decodeWait), - ), - stopContainer: (id) => noResult("stop-container", { operation: "stop-container", id }), - removeContainer: (id) => noResult("remove-container", { operation: "remove-container", id }), - streamLogs, - }; -}; diff --git a/packages/stack/src/runtime/ContainerEngineResolver.ts b/packages/stack/src/runtime/ContainerEngineResolver.ts deleted file mode 100644 index ce073edb0d..0000000000 --- a/packages/stack/src/runtime/ContainerEngineResolver.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { Context, Effect } from "effect"; -import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; -import { ContainerEngineError } from "../public/Errors.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import { - makeProcessCommandRunner, - type ContainerEngine, - ContainerCommandError, - type ContainerEngineFailure, - type ContainerEngineKind, - type ContainerPlatform, -} from "./ContainerEngine.ts"; -import { makeDockerEngine } from "./DockerEngine.ts"; -import { makePodmanEngine } from "./PodmanEngine.ts"; - -/** - * Host-composition seam for selecting one concrete container engine, kept narrow so - * createStack can be tested without a local daemon while production uses the real Docker/Podman - * adapters. - */ -export interface ContainerEngineResolverShape { - /** Checks for an installed client without requiring a running daemon. */ - readonly isInstalled: ( - kind: ContainerEngineKind, - ) => Effect.Effect; - /** - * Checks daemon liveness. Omitted on test doubles: auto-select then treats an installed client - * as reachable. - */ - readonly isDaemonReachable?: ( - kind: ContainerEngineKind, - ) => Effect.Effect; - readonly resolve: ( - preference: ContainerEngineKind, - ) => Effect.Effect; -} - -/** @effect-expect-leaking ChildProcessSpawner */ -export class ContainerEngineResolver extends Context.Service< - ContainerEngineResolver, - ContainerEngineResolverShape ->()("@supabase/stack/ContainerEngineResolver") {} - -const hostContainerPlatform = (): ContainerPlatform => { - if (process.platform === "darwin") return { os: "darwin", desktop: true }; - if (process.platform === "win32") return { os: "windows", desktop: true }; - return { os: "linux", desktop: false }; -}; - -const DOCKER_PROBE_TIMEOUT = "2 seconds"; - -const processUid = (): number | undefined => - typeof process.getuid === "function" ? process.getuid() : undefined; - -export const nativeRuntimeBlockedForUid = (uid = processUid()): boolean => uid === 0; - -export const NATIVE_ROOT_UNSUPPORTED_MESSAGE = - "Native Postgres cannot run as uid 0 because initdb refuses root. Use --runtime docker."; - -export const DOCKER_DAEMON_FALLBACK_NOTICE = - "Docker daemon is not reachable; using native Postgres. This runtime is persisted for this stack. To use Docker later, run stack destroy and start again, or choose a new --stack name."; - -export interface DefaultRuntimeSelection { - readonly runtime: StackRuntime; - readonly dockerFallbackNotice?: string; -} - -export interface SelectDefaultRuntimeOptions { - readonly uid?: number; -} - -export const defaultContainerEngineResolver: ContainerEngineResolverShape = { - isInstalled: (kind) => - Effect.gen(function* () { - const runner = yield* makeProcessCommandRunner({ executable: kind }); - const result = yield* runner.run({ args: ["--version"] }); - if (result.exitCode !== 0) - return yield* new ContainerCommandError({ - operation: "version", - message: `${kind} --version exited (${String(result.exitCode)})`, - }); - return true; - }).pipe(Effect.catchTag("ContainerExecutableNotFoundError", () => Effect.succeed(false))), - isDaemonReachable: (kind) => - Effect.gen(function* () { - const engine = yield* defaultContainerEngineResolver.resolve(kind); - return yield* engine.probe.pipe( - Effect.as(true), - Effect.timeoutOrElse({ - duration: DOCKER_PROBE_TIMEOUT, - orElse: () => Effect.succeed(false), - }), - Effect.orElseSucceed(() => false), - ); - }), - resolve: (kind) => - Effect.gen(function* () { - const runner = yield* makeProcessCommandRunner({ executable: kind }); - const platform = hostContainerPlatform(); - return kind === "docker" - ? makeDockerEngine({ runner, platform }) - : makePodmanEngine({ runner, platform }); - }), -}; - -export const resolveContainerEngine = ( - kind: ContainerEngineKind, - resolver?: ContainerEngineResolverShape, -): Effect.Effect => - (resolver ?? defaultContainerEngineResolver).resolve(kind); - -const mapSelectError = (error: ContainerEngineFailure): ContainerEngineError => - new ContainerEngineError({ - engine: "docker", - message: `Unable to determine whether Docker is installed: ${error.message}`, - cause: error, - }); - -/** Docker when the daemon is reachable, otherwise native (unless uid 0). */ -export const selectDefaultRuntimeSelection = ( - resolver?: ContainerEngineResolverShape, - options?: SelectDefaultRuntimeOptions, -): Effect.Effect => { - const selected = resolver ?? defaultContainerEngineResolver; - return Effect.gen(function* () { - const installed = yield* selected.isInstalled("docker").pipe(Effect.mapError(mapSelectError)); - const reachable = !installed - ? false - : selected.isDaemonReachable === undefined - ? true - : yield* selected.isDaemonReachable("docker").pipe( - Effect.mapError(mapSelectError), - Effect.orElseSucceed(() => false), - ); - if (reachable) - return { runtime: { kind: "container", engine: "docker" } satisfies StackRuntime }; - if (nativeRuntimeBlockedForUid(options?.uid ?? processUid())) - return yield* new ContainerEngineError({ - engine: "docker", - message: NATIVE_ROOT_UNSUPPORTED_MESSAGE, - }); - return { - runtime: { kind: "native" } satisfies StackRuntime, - ...(installed ? { dockerFallbackNotice: DOCKER_DAEMON_FALLBACK_NOTICE } : {}), - }; - }); -}; - -export const selectDefaultRuntime = ( - resolver?: ContainerEngineResolverShape, - options?: SelectDefaultRuntimeOptions, -): Effect.Effect => - selectDefaultRuntimeSelection(resolver, options).pipe(Effect.map((selected) => selected.runtime)); diff --git a/packages/stack/src/runtime/ContainerEngineResolver.unit.test.ts b/packages/stack/src/runtime/ContainerEngineResolver.unit.test.ts deleted file mode 100644 index be322bde36..0000000000 --- a/packages/stack/src/runtime/ContainerEngineResolver.unit.test.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit } from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import { - NATIVE_ROOT_UNSUPPORTED_MESSAGE, - selectDefaultRuntime, - selectDefaultRuntimeSelection, - type ContainerEngineResolverShape, -} from "./ContainerEngineResolver.ts"; - -const unusedSpawner = ChildProcessSpawner.make(() => Effect.die("unused")); - -const resolver = (opts: { - readonly installed: boolean; - readonly reachable?: boolean; -}): ContainerEngineResolverShape => { - const reachable = opts.reachable; - return { - isInstalled: () => Effect.succeed(opts.installed), - ...(reachable === undefined ? {} : { isDaemonReachable: () => Effect.succeed(reachable) }), - resolve: () => Effect.die("unused"), - }; -}; - -describe("selectDefaultRuntime", () => { - it.effect("selects Docker when the client is installed", () => - Effect.gen(function* () { - expect(yield* selectDefaultRuntime(resolver({ installed: true }))).toEqual({ - kind: "container", - engine: "docker", - }); - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), - ); - - it.effect("selects native when Docker is not installed", () => - Effect.gen(function* () { - expect(yield* selectDefaultRuntime(resolver({ installed: false }))).toEqual({ - kind: "native", - }); - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), - ); - - it.effect("selects native when the Docker client is installed but the daemon is down", () => - Effect.gen(function* () { - const selected = yield* selectDefaultRuntimeSelection( - resolver({ installed: true, reachable: false }), - ); - expect(selected.runtime).toEqual({ kind: "native" }); - expect(selected.dockerFallbackNotice).toContain("Docker daemon is not reachable"); - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), - ); - - it.effect("refuses native auto-select as uid 0", () => - Effect.gen(function* () { - const failed = yield* selectDefaultRuntime(resolver({ installed: false }), { uid: 0 }).pipe( - Effect.exit, - ); - expect(Exit.isFailure(failed)).toBe(true); - if (Exit.isFailure(failed)) { - const error = failed.cause; - expect(String(error)).toContain(NATIVE_ROOT_UNSUPPORTED_MESSAGE); - } - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), - ); - - it.effect("still selects Docker as uid 0 when the daemon is reachable", () => - Effect.gen(function* () { - expect(yield* selectDefaultRuntime(resolver({ installed: true }), { uid: 0 })).toEqual({ - kind: "container", - engine: "docker", - }); - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, unusedSpawner)), - ); -}); diff --git a/packages/stack/src/runtime/ContainerRuntime.ts b/packages/stack/src/runtime/ContainerRuntime.ts deleted file mode 100644 index c108756589..0000000000 --- a/packages/stack/src/runtime/ContainerRuntime.ts +++ /dev/null @@ -1,1110 +0,0 @@ -import { - Cause, - Deferred, - Effect, - Exit, - Fiber, - Option, - Ref, - Scope, - Semaphore, - Stream, - type Duration, -} from "effect"; -import type { ContainerArtifact } from "../model/CapabilityModule.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { StackId } from "../public/StackId.ts"; -import type { LogStore } from "../supervisor/LogStore.ts"; -import { - type ContainerContainerSpec, - type ContainerEngine, - type ContainerEngineFailure, - type ContainerLabels, - type ContainerMount, - type ContainerStartupProcess, - type ContainerVolumeMount, - type ContainerVolumeLabels, - type ContainerVolumeSpec, - type ContainerResource, - type ContainerResourceRole, - type ContainerNetworkLabels, - type ContainerWorkloadLabels, - type ContainerLogOptions, -} from "./ContainerEngine.ts"; -import { - RuntimeDriverError, - type RuntimeCleanupRequest, - type ObservedWorkload, - type RuntimeDriver, - type RuntimeWorkloadKey, -} from "./RuntimeDriver.ts"; -import { ContainerEngineError } from "../public/Errors.ts"; - -export interface ContainerRuntimeOptions { - readonly engine: ContainerEngine; - readonly ownerSessionId: string; - /** Maximum time allowed for each service-owned startup process. */ - readonly startupProcessTimeout?: Duration.Input; - readonly resolveWorkload?: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - ) => Effect.Effect; - /** Runs the one-shot initial database bootstrap after readiness. */ - readonly bootstrapDatabase?: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - resource: ContainerResource, - ) => Effect.Effect; - /** Probes a newly-started workload before reporting it ready. */ - readonly waitForReadiness?: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - resource: ContainerResource, - ) => Effect.Effect; - /** Updates container-to-host routing after the exact stack network is ready. */ - readonly onNetworkReady?: ( - network: ContainerResource, - ) => Effect.Effect; - /** Persists exact container stdout/stderr lines while a workload is running. */ - readonly logStore?: LogStore; -} - -export interface ContainerWorkloadResolution { - /** Optional main-container entrypoint override (startup processes are explicit). */ - readonly entrypoint?: string; - readonly mounts?: ReadonlyArray; - readonly volume?: ContainerVolumeRequest; - /** Path to an owned env file; secret bytes are kept out of engine argv. */ - readonly envFile?: string; - readonly networkAliases?: ReadonlyArray; - /** Private host-loopback publications used by the in-process gateway. */ - readonly publications?: ReadonlyArray; - /** Service-owned one-shot processes run before a newly-created main container. */ - readonly startup?: ReadonlyArray; - readonly command?: ReadonlyArray; - /** Owner-created bootstrap copied into a newly-created container before start. */ - readonly bootstrap?: Readonly<{ readonly source: string; readonly destination: string }>; - readonly waitForReadiness?: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - resource: ContainerResource, - ) => Effect.Effect; -} - -/** Untrusted host-publication input validated before any engine operation. */ -interface ContainerPortPublicationInput { - readonly address: string; - readonly hostPort: number; - readonly containerPort: number; -} - -interface ContainerVolumeRequest { - readonly target: string; - readonly readOnly: boolean; - /** Stable logical workload that owns the persistent volume. Defaults to the mounting workload. */ - readonly ownerWorkloadId?: string; -} - -interface ContainerRuntimeResource { - readonly container: string; - state: "starting" | "running" | "stopped" | "failed"; - readonly error?: string; - readonly key: RuntimeWorkloadKey; - readonly workload: PlannedWorkload; - readonly failure: Deferred.Deferred; - logFiber?: Fiber.Fiber; - watchFiber?: Fiber.Fiber; - stopRequested: boolean; -} - -const resourceKey = (key: RuntimeWorkloadKey): string => `${key.stackId}:${key.workloadId}`; - -const nameFor = (key: RuntimeWorkloadKey, role: ContainerResourceRole): string => - role === "network" - ? `supabase-${key.stackId.slice(0, 16)}-network` - : `supabase-${key.stackId.slice(0, 16)}-${key.workloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-${role}`; - -/** Distinct from {@link nameFor}(..., "workload") so one-shots cannot collide with an eager main container. */ -export const schemaInitContainerName = (key: RuntimeWorkloadKey): string => - `supabase-${key.stackId.slice(0, 16)}-${key.workloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-schema-init`; - -const networkLabelsFor = ( - key: RuntimeWorkloadKey, - ownerSessionId: string, -): ContainerNetworkLabels => ({ - stackId: key.stackId, - ownerSessionId, - role: "network", -}); -const workloadLabelsFor = ( - key: RuntimeWorkloadKey, - ownerSessionId: string, -): ContainerWorkloadLabels => ({ - stackId: key.stackId, - ownerSessionId, - workloadId: key.workloadId, - role: "workload", -}); -const startupLabelsFor = ( - key: RuntimeWorkloadKey, - ownerSessionId: string, -): ContainerWorkloadLabels => ({ - ...workloadLabelsFor(key, ownerSessionId), - startup: true, -}); -const volumeOwnerFor = (key: RuntimeWorkloadKey, request: ContainerVolumeRequest): string => - request.ownerWorkloadId ?? key.workloadId; -const volumeLabelsFor = ( - key: RuntimeWorkloadKey, - request: ContainerVolumeRequest, -): ContainerVolumeLabels => ({ - stackId: key.stackId, - workloadId: volumeOwnerFor(key, request), - role: "volume", -}); -const volumeNameFor = (key: RuntimeWorkloadKey, ownerWorkloadId: string): string => - `supabase-${key.stackId}-${ownerWorkloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-volume`; -const volumeSpecFor = ( - key: RuntimeWorkloadKey, - request: ContainerVolumeRequest, -): ContainerVolumeSpec => ({ - name: volumeNameFor(key, volumeOwnerFor(key, request)), - labels: volumeLabelsFor(key, request), -}); -const volumeMountFor = ( - key: RuntimeWorkloadKey, - request: ContainerVolumeRequest, -): ContainerVolumeMount => ({ - volume: volumeNameFor(key, volumeOwnerFor(key, request)), - target: request.target, - readOnly: request.readOnly, -}); - -const sameLabels = (left: ContainerLabels, right: ContainerLabels): boolean => - left.role === right.role && - (left.role === "volume" && right.role === "volume" - ? left.stackId === right.stackId && left.workloadId === right.workloadId - : left.stackId === right.stackId && - "ownerSessionId" in left && - "ownerSessionId" in right && - left.ownerSessionId === right.ownerSessionId && - (left.role === "network" || - ("workloadId" in left && "workloadId" in right && left.workloadId === right.workloadId))); - -/** Workload identity excludes ownerSessionId; stack ownership is fenced externally. */ -const sameWorkloadIdentity = (left: ContainerLabels, right: ContainerWorkloadLabels): boolean => - left.role === "workload" && - left.stackId === right.stackId && - left.workloadId === right.workloadId && - left.startup !== true; - -/** Networks are identified by stack identity. */ -const sameNetworkIdentity = (left: ContainerLabels, right: ContainerNetworkLabels): boolean => - left.role === "network" && - left.stackId === right.stackId && - "ownerSessionId" in left && - left.ownerSessionId === right.ownerSessionId; - -const toDriverError = ( - key: Pick, - error: unknown, -): RuntimeDriverError => - new RuntimeDriverError({ - message: error instanceof Error ? error.message : String(error), - stackId: key.stackId, - workloadId: key.workloadId, - cause: error, - }); - -const toContainerEngineError = ( - engine: ContainerEngine["kind"], - error: unknown, -): ContainerEngineError => - new ContainerEngineError({ - engine, - message: error instanceof Error ? error.message : String(error), - cause: error, - }); - -const withEngine = ( - engine: ContainerEngine, - key: Pick, - effect: Effect.Effect, -): Effect.Effect => - effect.pipe( - Effect.mapError((error) => toDriverError(key, toContainerEngineError(engine.kind, error))), - ); - -/** Runs one service-owned one-shot container: create, start, wait for exit 0, remove. */ -export const runContainerStartupProcess = (input: { - readonly engine: ContainerEngine; - readonly key: RuntimeWorkloadKey; - readonly specification: ContainerContainerSpec; - readonly timeout: Duration.Input; - readonly logStore?: LogStore; - readonly capability?: PlannedWorkload["capability"]; - readonly runtimeScope?: Scope.Scope; -}): Effect.Effect => { - let logFiber: Fiber.Fiber | undefined; - const acquire = withEngine( - input.engine, - input.key, - input.engine.createContainer(input.specification), - ); - const use = (container: ContainerResource): Effect.Effect => - Effect.gen(function* () { - yield* withEngine(input.engine, input.key, input.engine.startContainer(container.id)); - const logStore = input.logStore; - const runtimeScope = input.runtimeScope; - const capability = input.capability; - if (logStore !== undefined && runtimeScope !== undefined && capability !== undefined) { - const consume = input.engine.streamLogs(container.id, { tail: "all" }).pipe( - Stream.runForEach((line) => - logStore - .append({ - source: capability, - stream: line.stream, - message: line.message, - }) - .pipe(Effect.asVoid), - ), - Effect.mapError((error) => - toDriverError(input.key, toContainerEngineError(input.engine.kind, error)), - ), - ); - logFiber = yield* Effect.forkIn(consume, runtimeScope); - } - const exitCode = yield* withEngine( - input.engine, - input.key, - input.engine.waitContainer(container.id), - ); - const logs = logFiber === undefined ? Exit.succeed(undefined) : yield* Fiber.await(logFiber); - const logFailure = - Exit.isFailure(logs) && !Cause.hasInterruptsOnly(logs.cause) ? logs.cause : undefined; - const exitFailure = - exitCode === 0 - ? undefined - : toDriverError( - input.key, - new Error( - `Container startup process exited with code ${String(exitCode)} for ${input.key.workloadId}`, - ), - ); - if (exitFailure !== undefined) { - const exitCause = Cause.fail(exitFailure); - return yield* logFailure !== undefined - ? Effect.failCause(Cause.combine(exitCause, logFailure)) - : Effect.failCause(exitCause); - } - if (logFailure !== undefined) return yield* Effect.failCause(logFailure); - }); - const release = ( - container: ContainerResource, - useExit: Exit.Exit, - ): Effect.Effect => - Effect.gen(function* () { - if (logFiber !== undefined) yield* Fiber.interrupt(logFiber); - const removed = yield* Effect.exit( - withEngine(input.engine, input.key, input.engine.removeContainer(container.id)), - ); - if (Exit.isFailure(removed)) - return yield* Effect.failCause( - Exit.isFailure(useExit) ? Cause.combine(useExit.cause, removed.cause) : removed.cause, - ); - }); - return Effect.acquireUseRelease(acquire, use, release).pipe( - Effect.timeoutOrElse({ - duration: input.timeout, - orElse: () => - Effect.fail( - toDriverError( - input.key, - new Error(`Container startup process timed out for ${input.key.workloadId}`), - ), - ), - }), - ); -}; - -const containerArtifact = (workload: PlannedWorkload): ContainerArtifact | undefined => - workload.selected.kind === "container" ? workload.selected : undefined; - -const isWorkloadResource = ( - entry: ContainerResource, -): entry is ContainerResource & { readonly labels: ContainerWorkloadLabels } => - entry.kind === "workload" && entry.labels.role === "workload"; - -/** Startup containers are marked by the startup label. */ -const isStartupContainer = (entry: ContainerResource): boolean => - isWorkloadResource(entry) && entry.labels.startup === true; - -const isNetworkResource = ( - entry: ContainerResource, -): entry is ContainerResource & { readonly labels: ContainerNetworkLabels } => - entry.kind === "network" && entry.labels.role === "network"; - -const isValidPublication = ( - publication: ContainerPortPublicationInput, -): publication is ContainerPortPublicationInput & { readonly address: "127.0.0.1" } => - publication.address === "127.0.0.1" && - Number.isInteger(publication.hostPort) && - Number.isInteger(publication.containerPort) && - publication.hostPort >= 1 && - publication.hostPort <= 65_535 && - publication.containerPort >= 1 && - publication.containerPort <= 65_535; - -/** RuntimeDriver implementation backed by one exact, injected container engine. */ -export const makeContainerRuntime = ( - options: ContainerRuntimeOptions, -): Effect.Effect => - Effect.gen(function* () { - const startupProcessTimeout = - options.startupProcessTimeout ?? ("5 minutes" satisfies Duration.Input); - const parentScope = yield* Scope.Scope; - const runtimeScope = yield* Scope.fork(parentScope, "parallel"); - const registration = yield* Semaphore.make(1); - // Serialize only exact shared-network (and volume identity) establishment. Resolution, - // image pulls, startup migrations, container creation, and readiness run outside this gate. - const setup = yield* Semaphore.make(1); - const resources = new Map(); - const startGuards = new Map>(); - const startFibers = new Map>(); - - const withEngine = ( - key: Pick, - effect: Effect.Effect, - ): Effect.Effect => - effect.pipe( - Effect.mapError((error) => - toDriverError(key, toContainerEngineError(options.engine.kind, error)), - ), - ); - - const stopLogs = (resource: ContainerRuntimeResource): Effect.Effect => - resource.logFiber === undefined ? Effect.void : Fiber.interrupt(resource.logFiber); - - const stopExitWatcher = (resource: ContainerRuntimeResource): Effect.Effect => - resource.watchFiber === undefined ? Effect.void : Fiber.interrupt(resource.watchFiber); - - const cleanupStartedContainer = ( - key: RuntimeWorkloadKey, - containerId: string, - ): Effect.Effect => - registration.withPermit( - Effect.uninterruptible( - Effect.gen(function* () { - const exact = (yield* withEngine(key, options.engine.listResources(key.stackId))).find( - (entry) => entry.kind === "workload" && entry.id === containerId, - ); - if (exact === undefined) return; - if (exact.state === "running") - yield* withEngine(key, options.engine.stopContainer(containerId)); - yield* withEngine(key, options.engine.removeContainer(containerId)); - }), - ), - ); - - const cleanupRegisteredResource = ( - resource: ContainerRuntimeResource, - ): Effect.Effect => - Effect.gen(function* () { - resource.stopRequested = true; - yield* stopLogs(resource); - yield* stopExitWatcher(resource); - const cleanup = yield* cleanupStartedContainer(resource.key, resource.container).pipe( - Effect.exit, - ); - const id = resourceKey(resource.key); - if (resources.get(id) === resource) resources.delete(id); - if (Exit.isFailure(cleanup)) return yield* Effect.failCause(cleanup.cause); - }); - - const reportFailure = ( - resource: ContainerRuntimeResource, - error: unknown, - message = `Container workload failed for ${resource.key.workloadId}`, - ): Effect.Effect => { - const current = resources.get(resourceKey(resource.key)); - if (resource.stopRequested || current !== resource || resource.state === "failed") - return Effect.void; - const failure = - error instanceof RuntimeDriverError - ? error - : new RuntimeDriverError({ - message, - stackId: resource.key.stackId, - workloadId: resource.key.workloadId, - cause: error, - }); - resources.set(resourceKey(resource.key), { - ...resource, - state: "failed", - error: failure.message, - }); - return Deferred.fail(resource.failure, failure).pipe(Effect.asVoid); - }; - - const attachLogs = ( - resource: ContainerRuntimeResource, - tail: ContainerLogOptions["tail"], - ): Effect.Effect => { - const logStore = options.logStore; - if (logStore === undefined) return Effect.void; - const stream = options.engine.streamLogs(resource.container, { tail }); - const consume = stream.pipe( - Stream.runForEach((line) => - logStore - .append({ - source: resource.workload.capability, - stream: line.stream, - message: line.message, - }) - .pipe(Effect.asVoid), - ), - Effect.mapError((error) => - toDriverError(resource.key, toContainerEngineError(options.engine.kind, error)), - ), - Effect.catch((error) => reportFailure(resource, error)), - ); - return Effect.gen(function* () { - resource.logFiber = yield* Effect.forkIn(consume, runtimeScope); - // Give the follower one scheduling turn so process acquisition begins before readiness. - yield* Effect.yieldNow; - }); - }; - - const attachExitWatcher = (resource: ContainerRuntimeResource): Effect.Effect => { - if (resource.watchFiber !== undefined) return Effect.void; - const wait = Effect.exit( - withEngine(resource.key, options.engine.waitContainer(resource.container)), - ).pipe( - Effect.flatMap((result) => - resource.stopRequested - ? Effect.void - : reportFailure( - resource, - Exit.isFailure(result) - ? Option.getOrElse( - Cause.findErrorOption(result.cause), - () => new Error("Container wait failed"), - ) - : new Error(`Container exited with code ${String(result.value)}`), - ), - ), - Effect.ignore, - ); - return Effect.forkIn(wait, runtimeScope).pipe( - Effect.tap((fiber) => - Effect.sync(() => { - resource.watchFiber = fiber; - }), - ), - Effect.asVoid, - ); - }; - - const observe = ( - stackId: StackId, - ): Effect.Effect, RuntimeDriverError> => - withEngine({ stackId, workloadId: "" }, options.engine.listResources(stackId)).pipe( - Effect.map((entries) => - entries - .filter(isWorkloadResource) - .filter((entry) => entry.labels.stackId === stackId) - .filter((entry) => !isStartupContainer(entry)) - .map((entry) => { - const key: RuntimeWorkloadKey = { - stackId: entry.labels.stackId, - workloadId: entry.labels.workloadId, - }; - const local = resources.get(resourceKey(key)); - return { - ...key, - state: - local?.state === "failed" - ? "failed" - : local?.state === "starting" - ? "starting" - : entry.state === "running" - ? "ready" - : entry.state === "stopped" - ? "stopped" - : "starting", - ...(local?.error === undefined ? {} : { error: local.error }), - }; - }), - ), - ); - - const runStartupProcess = ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - startupProcess: ContainerStartupProcess, - context: Readonly<{ - readonly artifact: ContainerArtifact; - readonly network: ContainerResource; - readonly resolution: ContainerWorkloadResolution; - readonly volumeRequest?: ContainerVolumeRequest; - }>, - ): Effect.Effect => { - const labels = startupLabelsFor(key, options.ownerSessionId); - return runContainerStartupProcess({ - engine: options.engine, - key, - timeout: startupProcessTimeout, - specification: { - // Reuse the main name so crash-orphaned init containers are exact collisions to clean up. - name: nameFor(key, "workload"), - image: context.artifact.image, - labels, - network: context.network.id, - mounts: context.resolution.mounts ?? [], - volumeMounts: - context.volumeRequest === undefined ? [] : [volumeMountFor(key, context.volumeRequest)], - publications: [], - role: "workload", - entrypoint: startupProcess.entrypoint, - command: startupProcess.command, - ...(context.resolution.envFile === undefined - ? {} - : { envFile: context.resolution.envFile }), - }, - ...(options.logStore === undefined - ? {} - : { - logStore: options.logStore, - capability: workload.capability, - runtimeScope, - }), - }); - }; - - const start = ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - ): Effect.Effect => { - const artifact = containerArtifact(workload); - if (artifact === undefined) - return Effect.fail( - new RuntimeDriverError({ - message: "Container runtime cannot start a native artifact", - stackId: key.stackId, - workloadId: key.workloadId, - }), - ); - - const labels = workloadLabelsFor(key, options.ownerSessionId); - const networkLabels = networkLabelsFor(key, options.ownerSessionId); - const networkName = nameFor(key, "network"); - const resolved = - options.resolveWorkload === undefined - ? Effect.succeed({}) - : options.resolveWorkload(key, workload); - const id = resourceKey(key); - const guard = Effect.gen(function* () { - const current = startGuards.get(id); - if (current === undefined) - return yield* toDriverError( - key, - new Error("Container workload start was not registered"), - ); - const stopped = yield* Ref.get(current); - if (stopped) - return yield* toDriverError(key, new Error("Container workload start was stopped")); - }); - const startEffect = Effect.gen(function* () { - yield* guard; - const existing = resources.get(resourceKey(key)); - if (existing !== undefined && existing.state === "running") - return { ...key, state: "ready" } satisfies ObservedWorkload; - const setupResult = yield* Effect.gen(function* () { - if (existing !== undefined && existing.state !== "running") { - existing.stopRequested = true; - yield* stopLogs(existing); - yield* stopExitWatcher(existing); - if (existing.state === "failed") - yield* withEngine(key, options.engine.stopContainer(existing.container)); - resources.delete(resourceKey(key)); - } - const initialResolution = yield* resolved; - yield* guard; - const initialPublications = initialResolution.publications ?? []; - if (initialPublications.some((publication) => !isValidPublication(publication))) - return yield* toDriverError( - key, - new Error("Container publications must use valid loopback host ports"), - ); - const initialVolumeRequest = initialResolution.volume; - if (initialVolumeRequest !== undefined && initialVolumeRequest.target.length === 0) - return yield* toDriverError(key, new Error("Container volume mapping is invalid")); - yield* withEngine(key, options.engine.preflight); - const entries = yield* withEngine(key, options.engine.listResources(key.stackId)); - const existingExact = entries.find( - (entry) => entry.kind === "workload" && sameWorkloadIdentity(entry.labels, labels), - ); - const namedCollision = entries.find( - (entry) => entry.kind === "workload" && entry.name === nameFor(key, "workload"), - ); - const collision = - namedCollision !== undefined && isWorkloadResource(namedCollision) - ? namedCollision - : undefined; - if ( - existingExact === undefined && - namedCollision !== undefined && - (collision === undefined || - collision.labels.stackId !== key.stackId || - collision.labels.workloadId !== key.workloadId || - collision.labels.role !== "workload") - ) - return yield* toDriverError( - key, - new Error("Container name is owned by another workload"), - ); - - // A new owner never adopts a durable container. Remove any exact identity first; - // lifecycle ownership has already fenced the stack before this activation. - if (existingExact !== undefined) { - if (existingExact.state === "running") - yield* withEngine(key, options.engine.stopContainer(existingExact.id)); - yield* withEngine(key, options.engine.removeContainer(existingExact.id)); - } - yield* withEngine(key, options.engine.inspectImage(artifact.image)).pipe( - Effect.flatMap((inspected) => - inspected.present - ? Effect.void - : withEngine(key, options.engine.pullImage(artifact.image)), - ), - ); - yield* guard; - - const networkResource = yield* setup.withPermit( - Effect.gen(function* () { - const networkEntries = yield* withEngine( - key, - options.engine.listResources(key.stackId), - ); - const exactNetwork = networkEntries - .filter(isNetworkResource) - .find((entry) => sameNetworkIdentity(entry.labels, networkLabels)); - const namedNetworkCollision = networkEntries.find( - (entry) => entry.kind === "network" && entry.name === networkName, - ); - const networkCollision = - namedNetworkCollision !== undefined && isNetworkResource(namedNetworkCollision) - ? namedNetworkCollision - : undefined; - if ( - exactNetwork === undefined && - namedNetworkCollision !== undefined && - (networkCollision === undefined || networkCollision.labels.stackId !== key.stackId) - ) - return yield* toDriverError( - key, - new Error("Container network name is owned by another stack"), - ); - if (exactNetwork === undefined && networkCollision !== undefined) - yield* withEngine(key, options.engine.removeNetwork(networkCollision.id)); - return ( - exactNetwork ?? - (yield* withEngine( - key, - options.engine.createNetwork({ name: networkName, labels: networkLabels }), - )) - ); - }), - ); - - if (options.onNetworkReady !== undefined) yield* options.onNetworkReady(networkResource); - // Resolve after the shared network is established so every concurrent caller observes - // the final host route, including callers whose initial preflight raced this setup. - const resolution = - options.resolveWorkload === undefined - ? yield* Effect.succeed({}) - : yield* options.resolveWorkload(key, workload); - yield* guard; - const requestedPublications = resolution.publications ?? []; - if (requestedPublications.some((publication) => !isValidPublication(publication))) - return yield* toDriverError( - key, - new Error("Container publications must use valid loopback host ports"), - ); - const publications = requestedPublications.filter(isValidPublication); - const volumeRequest = resolution.volume; - if (volumeRequest !== undefined && volumeRequest.target.length === 0) - return yield* toDriverError(key, new Error("Container volume mapping is invalid")); - const volumeSpec = - volumeRequest === undefined ? undefined : volumeSpecFor(key, volumeRequest); - if (volumeSpec !== undefined) { - const volume = volumeSpec; - yield* setup.withPermit( - Effect.gen(function* () { - const volumeEntries = yield* withEngine( - key, - options.engine.listResources(key.stackId), - ); - const exactVolume = volumeEntries.find( - (entry) => - entry.kind === "volume" && - entry.name === volume.name && - sameLabels(entry.labels, volume.labels), - ); - if (exactVolume !== undefined) return; - const sameLabel = volumeEntries.find( - (entry) => entry.kind === "volume" && sameLabels(entry.labels, volume.labels), - ); - const sameName = volumeEntries.find( - (entry) => entry.kind === "volume" && entry.name === volume.name, - ); - if (sameLabel !== undefined || sameName !== undefined) - return yield* toDriverError( - key, - new Error("Container volume identity collision"), - ); - yield* withEngine(key, options.engine.createVolume(volume)); - }), - ); - } - - if (collision !== undefined && collision.id !== existingExact?.id) { - if (collision.state === "running") - yield* withEngine(key, options.engine.stopContainer(collision.id)); - yield* withEngine(key, options.engine.removeContainer(collision.id)); - } - - for (const startup of resolution.startup ?? []) - yield* runStartupProcess(key, workload, startup, { - artifact, - network: networkResource, - resolution, - ...(volumeRequest === undefined ? {} : { volumeRequest }), - }); - yield* guard; - - const container = yield* withEngine( - key, - options.engine.createContainer({ - name: nameFor(key, "workload"), - image: artifact.image, - labels, - network: networkResource.id, - mounts: resolution.mounts ?? [], - publications, - volumeMounts: volumeRequest === undefined ? [] : [volumeMountFor(key, volumeRequest)], - role: "workload", - ...(resolution.envFile === undefined ? {} : { envFile: resolution.envFile }), - ...(resolution.networkAliases === undefined - ? {} - : { networkAliases: resolution.networkAliases }), - ...(resolution.entrypoint === undefined ? {} : { entrypoint: resolution.entrypoint }), - ...(resolution.command === undefined ? {} : { command: resolution.command }), - } satisfies ContainerContainerSpec), - ); - if (resolution.bootstrap !== undefined) { - const copied = yield* Effect.exit( - withEngine( - key, - options.engine.copyToContainer( - container.id, - resolution.bootstrap.source, - resolution.bootstrap.destination, - ), - ), - ); - if (Exit.isFailure(copied)) { - const removed = yield* Effect.exit( - withEngine(key, options.engine.removeContainer(container.id)), - ); - const cleanupCause = Exit.isFailure(removed) ? removed.cause : Cause.empty; - return yield* Effect.failCause( - cleanupCause.reasons.length === 0 - ? copied.cause - : Cause.combine(copied.cause, cleanupCause), - ); - } - } - yield* guard; - yield* withEngine(key, options.engine.startContainer(container.id)); - const registered = yield* Effect.uninterruptibleMask((restore) => - restore( - Effect.gen(function* () { - yield* guard; - const failure = yield* Deferred.make(); - yield* guard; - const createdResource: ContainerRuntimeResource = { - key, - workload, - container: container.id, - state: "starting", - failure, - stopRequested: false, - }; - resources.set(resourceKey(key), createdResource); - return createdResource; - }), - ).pipe(Effect.exit), - ); - if (Exit.isFailure(registered)) { - const cleanup = yield* cleanupStartedContainer(key, container.id).pipe(Effect.exit); - return yield* Effect.failCause( - Exit.isFailure(cleanup) - ? Cause.combine(registered.cause, cleanup.cause) - : registered.cause, - ); - } - return { resource: registered.value, resolution, container }; - }); - const { resource, resolution, container } = setupResult; - const postRegistration = Effect.gen(function* () { - yield* attachLogs(resource, "all"); - yield* attachExitWatcher(resource); - const readiness = - resolution.waitForReadiness === undefined - ? options.waitForReadiness === undefined - ? Effect.void - : options.waitForReadiness(key, workload, container) - : resolution.waitForReadiness(key, workload, container); - yield* Effect.raceFirst(readiness, Deferred.await(resource.failure)); - if (workload.bootstrap === "database") { - if (options.bootstrapDatabase === undefined) - return yield* toDriverError( - key, - new Error("Database bootstrap resolver is not configured"), - ); - yield* options.bootstrapDatabase(key, workload, container); - } - resource.state = "running"; - return { ...key, state: "ready" } satisfies ObservedWorkload; - }); - // The mask lets Exit observe ordinary failures from the restored work; interruptions stay - // pending and re-raise at mask exit, where the outer wrapper performs cleanup. - const completed = yield* Effect.uninterruptibleMask((restore) => - restore(postRegistration).pipe(Effect.exit), - ); - if (Exit.isFailure(completed)) { - const cleanup = yield* cleanupRegisteredResource(resource).pipe(Effect.exit); - return yield* Effect.failCause( - Exit.isFailure(cleanup) - ? Cause.combine(completed.cause, cleanup.cause) - : completed.cause, - ); - } - return completed.value; - }); - return Effect.gen(function* () { - const inFlight = startFibers.get(id); - if (inFlight !== undefined) return yield* Fiber.join(inFlight); - const guardRef = yield* Ref.make(false); - startGuards.set(id, guardRef); - // Track the whole registration wait, not only the body after permit acquisition. Cleanup - // must be able to interrupt starts queued behind another activation as well. The permit - // only covers forking the body; readiness and log following continue after it is released. - const run = Effect.gen(function* () { - const body = yield* registration.withPermit( - Effect.forkChild(startEffect, { startImmediately: true }), - ); - return yield* Fiber.join(body); - }); - const fiber = yield* Effect.forkChild(run, { startImmediately: true }); - startFibers.set(id, fiber); - return yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const joined = yield* restore(Fiber.join(fiber)).pipe(Effect.exit); - if (Exit.isFailure(joined)) { - yield* Fiber.interrupt(fiber); - const resource = resources.get(id); - if (resource !== undefined) { - const cleanup = yield* cleanupRegisteredResource(resource).pipe(Effect.exit); - if (Exit.isFailure(cleanup)) - return yield* Effect.failCause(Cause.combine(joined.cause, cleanup.cause)); - } - return yield* Effect.failCause(joined.cause); - } - return joined.value; - }).pipe( - Effect.ensuring( - Effect.sync(() => { - if (startGuards.get(id) === guardRef) startGuards.delete(id); - if (startFibers.get(id) === fiber) startFibers.delete(id); - }), - ), - ), - ); - }); - }; - - const stopInPermit = (key: RuntimeWorkloadKey): Effect.Effect => { - const found = resources.get(resourceKey(key)); - const guard = startGuards.get(resourceKey(key)); - return found !== undefined - ? found.state === "stopped" - ? Effect.void - : Effect.gen(function* () { - found.stopRequested = true; - if (guard !== undefined) yield* Ref.set(guard, true); - yield* Deferred.fail( - found.failure, - new RuntimeDriverError({ - message: "Container workload was stopped while starting", - stackId: key.stackId, - workloadId: key.workloadId, - }), - ); - yield* stopLogs(found); - yield* stopExitWatcher(found); - yield* withEngine(key, options.engine.stopContainer(found.container)); - const id = resourceKey(key); - if (resources.get(id) === found) resources.set(id, { ...found, state: "stopped" }); - }) - : (guard === undefined ? Effect.void : Ref.set(guard, true)).pipe( - Effect.andThen(withEngine(key, options.engine.listResources(key.stackId))), - Effect.flatMap((entries) => { - const exact = entries - .filter(isWorkloadResource) - .find((entry) => - sameWorkloadIdentity( - entry.labels, - workloadLabelsFor(key, options.ownerSessionId), - ), - ); - return exact === undefined - ? Effect.void - : exact.state === "running" - ? withEngine(key, options.engine.stopContainer(exact.id)) - : Effect.void; - }), - ); - }; - - const stop = (key: RuntimeWorkloadKey): Effect.Effect => - registration.withPermit(stopInPermit(key)); - - const remove = (key: RuntimeWorkloadKey): Effect.Effect => - registration.withPermit( - Effect.gen(function* () { - const found = resources.get(resourceKey(key)); - if (found !== undefined) { - found.stopRequested = true; - const guard = startGuards.get(resourceKey(key)); - if (guard !== undefined) yield* Ref.set(guard, true); - yield* Deferred.fail( - found.failure, - new RuntimeDriverError({ - message: "Container workload was removed while starting", - stackId: key.stackId, - workloadId: key.workloadId, - }), - ); - yield* stopLogs(found); - yield* stopExitWatcher(found); - if (found.state !== "stopped") - yield* withEngine(key, options.engine.stopContainer(found.container)); - yield* withEngine(key, options.engine.removeContainer(found.container)); - resources.delete(resourceKey(key)); - return; - } - const guard = startGuards.get(resourceKey(key)); - if (guard !== undefined) yield* Ref.set(guard, true); - const entries = yield* withEngine(key, options.engine.listResources(key.stackId)); - const exact = entries - .filter(isWorkloadResource) - .find((entry) => - sameWorkloadIdentity(entry.labels, workloadLabelsFor(key, options.ownerSessionId)), - ); - if (exact === undefined) return; - if (exact.state === "running") - yield* withEngine(key, options.engine.stopContainer(exact.id)); - yield* withEngine(key, options.engine.removeContainer(exact.id)); - }), - ); - - const cleanup = (request: RuntimeCleanupRequest): Effect.Effect => - registration.withPermit( - Effect.gen(function* () { - // Stop and interrupt starts that have not registered a resource yet. Without this - // fence, cleanup can observe an empty resource list while an activation is still inside - // createContainer, then the activation creates a new orphan after cleanup returns. - const pendingStartIds = new Set( - [...startGuards.keys(), ...startFibers.keys()].filter((id) => - id.startsWith(`${request.stackId}:`), - ), - ); - for (const id of pendingStartIds) { - const guard = startGuards.get(id); - if (guard !== undefined) yield* Ref.set(guard, true); - } - for (const id of pendingStartIds) { - const fiber = startFibers.get(id); - if (fiber !== undefined) yield* Fiber.interrupt(fiber); - } - - const stackKey = { - stackId: request.stackId, - workloadId: "", - } satisfies RuntimeWorkloadKey; - const entries = yield* withEngine( - stackKey, - options.engine.listResources(request.stackId), - ); - const owned = entries - .filter((entry) => entry.labels.stackId === request.stackId) - .sort((left, right) => left.id.localeCompare(right.id)); - let cleanupCause: Cause.Cause = Cause.empty; - const attempt = (effect: Effect.Effect) => - Effect.gen(function* () { - const result = yield* Effect.exit(effect); - if (Exit.isFailure(result)) cleanupCause = Cause.combine(cleanupCause, result.cause); - }); - - for (const resource of resources.values()) - if (resource.key.stackId === request.stackId) { - resource.stopRequested = true; - yield* stopLogs(resource); - yield* stopExitWatcher(resource); - } - - // Stop and remove every stack workload before touching its network. A failed - // stop does not prevent the remove attempt; all failures are returned together. - for (const entry of owned.filter(isWorkloadResource)) { - if (entry.state === "running") - yield* attempt(withEngine(stackKey, options.engine.stopContainer(entry.id))); - yield* attempt(withEngine(stackKey, options.engine.removeContainer(entry.id))); - } - for (const entry of owned.filter(isNetworkResource)) - yield* attempt(withEngine(stackKey, options.engine.removeNetwork(entry.id))); - if (request.destroy) - for (const entry of owned.filter((candidate) => candidate.kind === "volume")) - yield* attempt(withEngine(stackKey, options.engine.removeVolume(entry.id))); - - for (const id of resources.keys()) - if (id.startsWith(`${request.stackId}:`)) resources.delete(id); - if (cleanupCause.reasons.length > 0) return yield* Effect.failCause(cleanupCause); - }), - ); - - const wipePersistentData = (key: RuntimeWorkloadKey): Effect.Effect => - registration.withPermit( - Effect.gen(function* () { - const entries = yield* withEngine(key, options.engine.listResources(key.stackId)); - const volumes = entries.filter( - (entry) => - entry.kind === "volume" && - entry.labels.role === "volume" && - entry.labels.workloadId === key.workloadId, - ); - for (const volume of volumes) - yield* withEngine(key, options.engine.removeVolume(volume.id)); - }), - ); - - return { - observe, - start, - stop, - remove, - cleanup, - wipePersistentData, - } satisfies RuntimeDriver; - }); diff --git a/packages/stack/src/model/DatabaseBootstrap.ts b/packages/stack/src/runtime/DatabaseBootstrap.ts similarity index 91% rename from packages/stack/src/model/DatabaseBootstrap.ts rename to packages/stack/src/runtime/DatabaseBootstrap.ts index 0a32a89ddf..9b7f765894 100644 --- a/packages/stack/src/model/DatabaseBootstrap.ts +++ b/packages/stack/src/runtime/DatabaseBootstrap.ts @@ -5,6 +5,7 @@ export type DatabaseSqlValue = string | number | boolean | null; /** Login roles provisioned by the managed database template. */ const DATABASE_BOOTSTRAP_ROLES = [ + "supabase_admin", "postgres", "authenticator", "pgbouncer", @@ -82,17 +83,6 @@ export const INTERNAL_DATABASE = "_supabase"; /** Service-owned schemas created in the private database. */ export const INTERNAL_SCHEMAS = ["_analytics", "_supavisor"] as const; -/** Cache/key material for the managed bootstrap. */ -export const databaseBootstrapIdentity = [ - ...DATABASE_BOOTSTRAP_ROLES, - ADVISORY_LOCK_STATEMENT, - REALTIME_SCHEMA_STATEMENT, - JWT_SECRET_SETTING, - JWT_EXP_SETTING, - INTERNAL_DATABASE, - ...INTERNAL_SCHEMAS, -].join("\n"); - const statementError = (error: DatabaseBootstrapError, statement: string) => new DatabaseBootstrapError({ message: error.message, @@ -102,11 +92,11 @@ const statementError = (error: DatabaseBootstrapError, statement: string) => }); /** Reconciles the runtime-owned schema, roles, and settings in one transaction. */ -export const runDatabaseBootstrap = ( +export const runDatabaseBootstrap = Effect.fn("DatabaseBootstrap.run")(function* ( session: DatabaseSession, options: DatabaseBootstrapOptions, -): Effect.Effect => - session.transaction((transaction) => +) { + return yield* session.transaction((transaction) => Effect.gen(function* () { yield* transaction .execute(ADVISORY_LOCK_STATEMENT) @@ -114,6 +104,10 @@ export const runDatabaseBootstrap = ( yield* transaction .execute(REALTIME_SCHEMA_STATEMENT) .pipe(Effect.mapError((error) => statementError(error, REALTIME_SCHEMA_STATEMENT))); + yield* transaction.execute("SET LOCAL log_statement = 'none'"); + yield* transaction.execute("SET LOCAL log_min_error_statement = 'panic'"); + yield* transaction.execute("SET LOCAL log_min_duration_statement = -1"); + yield* transaction.execute("SET LOCAL log_min_duration_sample = -1"); yield* transaction.setRolePasswords(DATABASE_BOOTSTRAP_ROLES, options.databasePassword).pipe( Effect.mapError( (error) => @@ -139,3 +133,4 @@ export const runDatabaseBootstrap = ( ); }), ); +}); diff --git a/packages/stack/src/runtime/DatabaseBootstrapCatalog.ts b/packages/stack/src/runtime/DatabaseBootstrapCatalog.ts deleted file mode 100644 index cb15e3357f..0000000000 --- a/packages/stack/src/runtime/DatabaseBootstrapCatalog.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Effect, Redacted } from "effect"; -import type { DatabaseBootstrapOptions } from "../model/DatabaseBootstrap.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { StackPreparationError } from "../public/Errors.ts"; -import { AUTH_JWT_SECRET_SLOT, DATABASE_INTERNAL_PASSWORD_SLOT } from "../state/SecretStore.ts"; - -const missingMaterial = (message: string) => new StackPreparationError({ message }); - -const secretValue = (state: PersistedStackState, slot: string): string | undefined => { - const value = state.secrets[slot]?.value; - return typeof value === "string" && value.length > 0 ? value : undefined; -}; - -/** - * Builds the initial database bootstrap from fully materialized state. - * - * Returns only the managed material the fixed runtime bootstrap reconciliation needs; it does - * not read artifact SQL files or perform caller-driven reset/migration/seed work. - */ -export const databaseBootstrapPlan = ( - state: PersistedStackState, -): Effect.Effect => - Effect.gen(function* () { - if (state.definition === undefined) - return yield* missingMaterial( - "A materialized stack definition is required for database bootstrap", - ); - - const databasePassword = secretValue(state, DATABASE_INTERNAL_PASSWORD_SLOT); - if (databasePassword === undefined) - return yield* missingMaterial("Managed database password is unavailable for bootstrap"); - - const jwtSecret = secretValue(state, AUTH_JWT_SECRET_SLOT); - if (jwtSecret === undefined) - return yield* missingMaterial("Managed JWT secret is unavailable for database bootstrap"); - - const jwtExpiry = state.definition.capabilities.auth.settings.jwt_expiry; - if ( - typeof jwtExpiry !== "number" || - !Number.isFinite(jwtExpiry) || - !Number.isInteger(jwtExpiry) || - jwtExpiry <= 0 - ) - return yield* missingMaterial("Auth JWT expiry must be a finite positive integer"); - - return { - databasePassword: Redacted.make(databasePassword), - jwtSecret: Redacted.make(jwtSecret), - jwtExpiry, - } satisfies DatabaseBootstrapOptions; - }); diff --git a/packages/stack/src/runtime/Diagnostics.ts b/packages/stack/src/runtime/Diagnostics.ts deleted file mode 100644 index 8f2d110b16..0000000000 --- a/packages/stack/src/runtime/Diagnostics.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { redactKnownSecrets } from "../state/SecretStore.ts"; - -const DIAGNOSTIC_TAIL_LINES = 40; - -const LEFTOVER_PERSISTENT_DATA = - /exists but is not empty|directory not empty|not empty directory|PGDATA.*exist|initdb:.*exist|volume is in use|already exists.*volume/i; - -/** Wipe guidance when a first-create retry cannot launch over leftover PGDATA/volume. */ -const LEFTOVER_PERSISTENT_DATA_GUIDANCE = - "Leftover files from a failed first start remain. stack destroy wipes them."; - -export const looksLikeLeftoverPersistentData = (message: string): boolean => - LEFTOVER_PERSISTENT_DATA.test(message); - -export const withLeftoverPersistentDataGuidance = (message: string): string => { - if (!looksLikeLeftoverPersistentData(message) || message.includes("stack destroy")) - return message; - return `${message}. ${LEFTOVER_PERSISTENT_DATA_GUIDANCE}`; -}; - -export const formatStopTimeoutMessage = (running: ReadonlyArray): string => { - const listed = running.length > 0 ? ` Still running: ${running.join(", ")}.` : ""; - return `Stop timed out after 60s.${listed} The owner stop continues in the background.`; -}; - -export interface ProcessOutputTail { - readonly pushBytes: (stream: "stdout" | "stderr", bytes: Uint8Array) => void; - readonly finish: (knownSecrets?: Iterable) => string; -} - -/** Bounded, secret-redacted last lines from a one-shot or dying native process. */ -export const makeProcessOutputTail = (): ProcessOutputTail => { - const lines: Array = []; - const accumulators = { - stdout: { decoder: new TextDecoder(), remainder: "" }, - stderr: { decoder: new TextDecoder(), remainder: "" }, - }; - const pushLine = (line: string): void => { - if (line.length === 0) return; - lines.push(line); - if (lines.length > DIAGNOSTIC_TAIL_LINES) lines.shift(); - }; - return { - pushBytes: (stream, bytes) => { - const accumulator = accumulators[stream]; - accumulator.remainder += accumulator.decoder.decode(bytes, { stream: true }); - const parts = accumulator.remainder.split(/\r?\n/); - accumulator.remainder = parts.pop() ?? ""; - for (const line of parts) pushLine(`${stream}: ${line}`); - }, - finish: (knownSecrets = []) => { - for (const stream of ["stdout", "stderr"] as const) { - const accumulator = accumulators[stream]; - accumulator.remainder += accumulator.decoder.decode(); - if (accumulator.remainder.length > 0) pushLine(`${stream}: ${accumulator.remainder}`); - accumulator.remainder = ""; - } - if (lines.length === 0) return ""; - return redactKnownSecrets(lines.join("\n"), knownSecrets); - }, - }; -}; diff --git a/packages/stack/src/runtime/Diagnostics.unit.test.ts b/packages/stack/src/runtime/Diagnostics.unit.test.ts deleted file mode 100644 index 69f3a4ea40..0000000000 --- a/packages/stack/src/runtime/Diagnostics.unit.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { - formatStopTimeoutMessage, - looksLikeLeftoverPersistentData, - makeProcessOutputTail, - withLeftoverPersistentDataGuidance, -} from "./Diagnostics.ts"; - -describe("leftover persistent data guidance", () => { - it("appends wipe guidance to initdb leftover errors", () => { - const message = "initdb: directory exists but is not empty"; - expect(looksLikeLeftoverPersistentData(message)).toBe(true); - const guided = withLeftoverPersistentDataGuidance(message); - expect(guided).toBe( - "initdb: directory exists but is not empty. Leftover files from a failed first start remain. stack destroy wipes them.", - ); - expect(guided).not.toContain("db reset --local"); - expect(withLeftoverPersistentDataGuidance(guided)).toBe(guided); - }); - - it("leaves unrelated launch errors unchanged", () => { - expect(withLeftoverPersistentDataGuidance("Native startup process exited with code 1")).toBe( - "Native startup process exited with code 1", - ); - }); -}); - -describe("stop timeout message", () => { - it("names still-running capabilities and that stop continues", () => { - expect(formatStopTimeoutMessage(["database", "auth"])).toBe( - "Stop timed out after 60s. Still running: database, auth. The owner stop continues in the background.", - ); - }); -}); - -describe("process output tail", () => { - it("redacts known secrets from captured lines", () => { - const tail = makeProcessOutputTail(); - tail.pushBytes("stderr", new TextEncoder().encode("password=s3cret\n")); - expect(tail.finish(["s3cret"])).toBe("stderr: password=[REDACTED]"); - }); -}); diff --git a/packages/stack/src/runtime/DockerEngine.ts b/packages/stack/src/runtime/DockerEngine.ts deleted file mode 100644 index f5842b8ae4..0000000000 --- a/packages/stack/src/runtime/DockerEngine.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { - makeContainerEngineCodecs, - makeContainerEngineCore, - CONTAINER_LABEL_KEYS, - serializeCommonContainerCommand, - type ContainerCommand, - type ContainerEngine, - type ContainerEngineOptions, - type ContainerProcessRequest, -} from "./ContainerEngine.ts"; - -const jsonLabel = (label: string): string => `{{json (.Label "${label}")}}`; - -// Emit a closed set of JSON scalar fields. In particular, do not decode the -// daemon's raw `.Labels` map: unknown labels must never become runtime state. -const containerFormat = [ - "{{json .ID}}", - "{{json .Names}}", - jsonLabel(CONTAINER_LABEL_KEYS.stackId), - jsonLabel(CONTAINER_LABEL_KEYS.ownerSessionId), - jsonLabel(CONTAINER_LABEL_KEYS.workloadId), - jsonLabel(CONTAINER_LABEL_KEYS.startup), - jsonLabel(CONTAINER_LABEL_KEYS.role), - "{{json .State}}", -].join("\\t"); -const networkFormat = [ - "{{json .ID}}", - "{{json .Name}}", - jsonLabel(CONTAINER_LABEL_KEYS.stackId), - jsonLabel(CONTAINER_LABEL_KEYS.ownerSessionId), - jsonLabel(CONTAINER_LABEL_KEYS.role), -].join("\\t"); -const volumeFormat = [ - "{{json .Name}}", - jsonLabel(CONTAINER_LABEL_KEYS.stackId), - jsonLabel(CONTAINER_LABEL_KEYS.workloadId), - jsonLabel(CONTAINER_LABEL_KEYS.role), -].join("\\t"); - -export const serializeDockerCommand = (command: ContainerCommand): ContainerProcessRequest => { - switch (command.operation) { - case "probe": - return { args: ["version", "--format", "{{json .Server.Version}}"] }; - case "inspect-image": - return { - args: ["image", "ls", command.image, "--format", "{{json .ID}}"], - }; - case "pull-image": - return { args: ["image", "pull", command.image] }; - case "inspect-containers": - return { - args: [ - "ps", - "--all", - "--filter", - `label=${CONTAINER_LABEL_KEYS.stackId}=${command.stackId}`, - "--format", - containerFormat, - ], - }; - case "inspect-networks": - return { - args: [ - "network", - "ls", - "--filter", - `label=${CONTAINER_LABEL_KEYS.stackId}=${command.stackId}`, - "--format", - networkFormat, - ], - }; - case "inspect-network-gateway": - return { args: ["network", "inspect", command.id, "--format", "{{json .IPAM.Config}}"] }; - case "inspect-volumes": - return { - args: [ - "volume", - "ls", - "--filter", - `label=${CONTAINER_LABEL_KEYS.stackId}=${command.stackId}`, - "--format", - volumeFormat, - ], - }; - default: - return serializeCommonContainerCommand(command); - } -}; - -export const makeDockerEngine = ( - options: Omit, -): ContainerEngine => - makeContainerEngineCore({ - ...options, - kind: "docker", - codecs: makeContainerEngineCodecs({ - engineName: "Docker", - scalarFormat: "json", - serialize: serializeDockerCommand, - }), - }); diff --git a/packages/stack/src/runtime/EphemeralPostgres.ts b/packages/stack/src/runtime/EphemeralPostgres.ts deleted file mode 100644 index ac94f6fab7..0000000000 --- a/packages/stack/src/runtime/EphemeralPostgres.ts +++ /dev/null @@ -1,1161 +0,0 @@ -import { PgClient } from "@effect/sql-pg"; -import { - Crypto, - Duration, - Effect, - Exit, - FileSystem, - Option, - Path, - Redacted, - Schedule, - Schema, - Scope, - Semaphore, - Stream, -} from "effect"; -import { ChildProcess } from "effect/unstable/process"; -import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- loopback bind is the port allocator. -import { createServer } from "node:net"; -import { DatabaseBootstrapError } from "../model/DatabaseBootstrap.ts"; -import { - DEFAULT_DATABASE_HEALTH_TIMEOUT, - parseGoDuration, -} from "../model/capabilities/database.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { - ContainerEngineError, - EphemeralPostgresError, - PortUnavailableError, - StackPreparationError, - StackRuntimeError, - type EphemeralPostgresCreateError, -} from "../public/Errors.ts"; -import { - resolveEphemeralPostgresRelease, - type CreateEphemeralPostgresOptions, - type EffectEphemeralPostgres, -} from "../public/EphemeralPostgres.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; -import { defaultRuntimeEnvironment, StackRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { checkHostPort } from "../supervisor/HostListener.ts"; -import { probeReadiness } from "./ReadinessProbe.ts"; -import { - defaultNativeProcessLauncher, - spawnNativeProcess, - type NativeProcess, -} from "./NativeProcess.ts"; -import { bootstrapManagedPostgres } from "./PostgresDatabaseSession.ts"; -import { makeProductionRuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; -import { - resolveContainerEngine, - ContainerEngineResolver, - selectDefaultRuntimeSelection, - nativeRuntimeBlockedForUid, - NATIVE_ROOT_UNSUPPORTED_MESSAGE, - type ContainerEngineResolverShape, - type DefaultRuntimeSelection, -} from "./ContainerEngineResolver.ts"; -import type { ContainerEngine } from "./ContainerEngine.ts"; -import { encodeRuntimeEnvFile } from "./RuntimeEnvFile.ts"; -import { containerAliasFor } from "../model/WorkloadCatalog.ts"; - -const DATABASE_WORKLOAD_ID = "database:database"; -const PGDATA_DIR_NAME = "data"; -const CONTAINER_PGDATA_PARENT = "/var/lib/postgresql"; -const SNAPSHOT_MOUNT = "/snapshot"; -const BUSYBOX = "/usr/bin/busybox"; -const RUNTIME_MARKER = ".supabase-ephemeral-runtime"; -const DEFAULT_JWT_EXPIRY = 3600; - -const RuntimeMarkerSchema = Schema.Struct({ - kind: Schema.Literals(["native", "container"] as const), - engine: Schema.optionalKey(Schema.Literals(["docker", "podman"] as const)), - snapshotKey: Schema.optionalKey(Schema.String), -}); -type RuntimeMarker = Schema.Schema.Type; -const TAR_EXTRACT_FLAGS = ["--no-same-owner"] as const; - -const ephemeralError = ( - message: string, - fields: Omit[0], "message"> = {}, -) => new EphemeralPostgresError({ message, ...fields }); - -const resolvedRuntime = ( - preference: CreateEphemeralPostgresOptions["runtime"] | undefined, - resolver: ContainerEngineResolverShape | undefined, -): Effect.Effect< - DefaultRuntimeSelection, - ContainerEngineError | StackRuntimeError, - ChildProcessSpawnerService -> => { - if (preference !== undefined) { - const runtime = - preference.kind === "container" - ? ({ kind: "container", engine: preference.engine ?? "docker" } satisfies StackRuntime) - : ({ kind: "native" } satisfies StackRuntime); - return runtime.kind === "native" && nativeRuntimeBlockedForUid() - ? Effect.fail(new StackRuntimeError({ message: NATIVE_ROOT_UNSUPPORTED_MESSAGE })) - : Effect.succeed({ runtime }); - } - return selectDefaultRuntimeSelection(resolver).pipe( - Effect.flatMap((selected) => - selected.runtime.kind === "native" && nativeRuntimeBlockedForUid() - ? Effect.fail(new StackRuntimeError({ message: NATIVE_ROOT_UNSUPPORTED_MESSAGE })) - : Effect.succeed(selected), - ), - ); -}; - -const plannedWorkload = ( - version: string, - image: string, - runtime: StackRuntime, -): PlannedWorkload => ({ - id: DATABASE_WORKLOAD_ID, - capability: "database", - bootstrap: "database", - dependencies: [], - readiness: { portField: "database" }, - artifacts: { - native: { kind: "native", release: version }, - container: { kind: "container", image }, - }, - selected: - runtime.kind === "native" ? { kind: "native", release: version } : { kind: "container", image }, -}); - -const postgresArgs = ( - port: number, - runtime: StackRuntime, - settings: CreateEphemeralPostgresOptions["postgresSettings"], -): ReadonlyArray => { - const tuned = Object.entries(settings ?? {}).flatMap(([key, value]) => { - if (value === undefined) return []; - const rendered = String(value); - return rendered.length === 0 ? [] : ["-c", `${key}=${rendered}`]; - }); - return [ - "-p", - String(port), - "-c", - runtime.kind === "container" ? "listen_addresses=*" : "listen_addresses=127.0.0.1", - ...tuned, - ]; -}; - -const postgresEnv = (input: { - readonly port: number; - readonly dataPath: string; - readonly password: string; -}): Record => ({ - SUPABASE_STACK_WORKLOAD: DATABASE_WORKLOAD_ID, - SUPABASE_STACK_PRIVATE_PORT: String(input.port), - PGDATA: input.dataPath, - POSTGRES_USER: "supabase_admin", - POSTGRES_DB: "postgres", - POSTGRES_PASSWORD: input.password, - TZDIR: "/var/db/timezone/zoneinfo", -}); - -const databaseUrl = (port: number, password: string): string => - `postgresql://${encodeURIComponent("postgres")}:${encodeURIComponent(password)}@127.0.0.1:${port}/postgres`; - -const markerFor = (runtime: StackRuntime, snapshotKey?: string): RuntimeMarker => ({ - ...(runtime.kind === "native" - ? { kind: "native" as const } - : { kind: "container" as const, engine: runtime.engine }), - ...(snapshotKey === undefined ? {} : { snapshotKey }), -}); - -const sameSnapshotKey = (marker: RuntimeMarker, expected: string | undefined): boolean => - expected === undefined ? marker.snapshotKey === undefined : marker.snapshotKey === expected; - -const encodeMarker = (marker: RuntimeMarker): string => JSON.stringify(marker); - -const decodeMarker = (text: string): Effect.Effect => - Schema.decodeEffect(Schema.fromJsonString(RuntimeMarkerSchema))(text).pipe( - Effect.mapError(() => - ephemeralError("Ephemeral Postgres snapshot marker is invalid", { reason: "snapshot" }), - ), - ); - -const sameRuntime = (left: RuntimeMarker, right: StackRuntime): boolean => - left.kind === right.kind && (right.kind === "native" || left.engine === right.engine); - -const allocateLoopbackPort = ( - requested: number | undefined, -): Effect.Effect => { - if (requested !== undefined) - return checkHostPort("127.0.0.1", requested, "database").pipe(Effect.as(requested)); - return Effect.callback((resume) => { - const server = createServer(); - let settled = false; - const finish = (effect: Effect.Effect) => { - if (settled) return; - settled = true; - resume(effect); - }; - server.once("error", (cause) => - finish(Effect.fail(ephemeralError("Unable to allocate a loopback port", { cause }))), - ); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - const port = typeof address === "object" && address !== null ? address.port : 0; - server.close(() => - finish( - port > 0 - ? Effect.succeed(port) - : Effect.fail(ephemeralError("Unable to allocate a loopback port")), - ), - ); - }); - return Effect.sync(() => { - if (settled) return; - settled = true; - try { - server.close(); - } catch { - // The listener never obtained a handle. - } - }); - }); -}; - -const runTar = ( - args: ReadonlyArray, -): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const handle = yield* ChildProcess.make("tar", args, { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - }).pipe(Effect.mapError((cause) => ephemeralError("Unable to start tar", { cause }))); - const drain = Effect.all([Stream.runDrain(handle.stdout), Stream.runDrain(handle.stderr)], { - concurrency: "unbounded", - discard: true, - }).pipe(Effect.ignore); - const [code] = yield* Effect.all([handle.exitCode, drain], { concurrency: "unbounded" }).pipe( - Effect.mapError((cause) => ephemeralError("tar failed", { cause })), - ); - if (Number(code) !== 0) - return yield* ephemeralError(`tar failed (${String(code)})`, { reason: "snapshot" }); - }), - ); - -const writeEnvFile = ( - filePath: string, - values: Readonly>, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const text = yield* encodeRuntimeEnvFile(values).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to write Postgres environment file", { cause, path: filePath }), - ), - ); - yield* fs - .writeFileString(filePath, text, { mode: 0o600 }) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to write Postgres environment file", { cause, path: filePath }), - ), - ); - yield* fs - .chmod(filePath, 0o600) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to restrict Postgres environment file", { cause, path: filePath }), - ), - ); - return filePath; - }); - -const waitForPostgres = ( - port: number, - healthTimeout: string, -): Effect.Effect => - Effect.try({ - try: () => parseGoDuration(healthTimeout), - catch: (cause) => ephemeralError("Invalid database health timeout", { cause }), - }).pipe( - Effect.flatMap((deadline) => - probeReadiness( - { mode: "tcp", host: "127.0.0.1", port }, - { deadline: Duration.isZero(deadline) ? Duration.seconds(1) : deadline }, - ).pipe( - Effect.mapError((cause) => - ephemeralError("Ephemeral Postgres did not become ready", { cause }), - ), - ), - ), - ); - -const pingAdvertised = ( - port: number, - password: string, -): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const client = yield* PgClient.PgClient; - yield* client.unsafe("SELECT 1"); - }).pipe( - Effect.provide( - PgClient.layer({ - url: Redacted.make(databaseUrl(port, password)), - connectTimeout: "2 seconds", - }), - ), - ), - ).pipe( - Effect.mapError((cause) => - ephemeralError("Ephemeral Postgres did not accept a connection", { cause }), - ), - ); - -const waitForAdvertised = ( - port: number, - password: string, - healthTimeout: string, -): Effect.Effect => - Effect.try({ - try: () => parseGoDuration(healthTimeout), - catch: (cause) => ephemeralError("Invalid database health timeout", { cause }), - }).pipe( - Effect.flatMap((deadline) => - Effect.timeout( - Effect.retry(pingAdvertised(port, password), { - schedule: Schedule.spaced("100 millis"), - }), - Duration.isZero(deadline) ? Duration.seconds(1) : deadline, - ).pipe( - Effect.mapError((cause) => - ephemeralError("Ephemeral Postgres did not become ready", { cause }), - ), - ), - ), - ); - -const bootstrap = ( - port: number, - options: CreateEphemeralPostgresOptions, - healthTimeout: string, -): Effect.Effect => - Effect.try({ - try: () => parseGoDuration(healthTimeout), - catch: (cause) => ephemeralError("Invalid database health timeout", { cause }), - }).pipe( - Effect.flatMap((deadline) => - Effect.timeout( - Effect.retry( - bootstrapManagedPostgres({ - host: "127.0.0.1", - port, - databasePassword: options.databasePassword, - jwtSecret: options.jwtSecret, - jwtExpiry: options.jwtExpiry ?? DEFAULT_JWT_EXPIRY, - }), - { - schedule: Schedule.spaced("100 millis"), - while: (error) => error instanceof DatabaseBootstrapError && error.retryable === true, - }, - ), - Duration.isZero(deadline) ? Duration.seconds(1) : deadline, - ).pipe( - Effect.mapError((cause) => - ephemeralError("Ephemeral Postgres bootstrap failed", { reason: "bootstrap", cause }), - ), - ), - ), - ); - -interface NativeResources { - readonly kind: "native"; - process?: NativeProcess; - processScope?: Scope.Closeable; -} - -interface ContainerResources { - readonly kind: "container"; - readonly engine: ContainerEngine; - networkName?: string; - volumeName?: string; - containerName?: string; - networkId?: string; - volumeId?: string; - containerId?: string; -} - -type RuntimeResources = NativeResources | ContainerResources; - -interface Cluster { - readonly identity: StackId; - readonly root: string; - readonly dataPath: string; - readonly host: "127.0.0.1"; - readonly port: number; - readonly version: string; - readonly runtime: StackRuntime; - readonly artifactIdentity: string; - readonly executable?: string; - readonly artifactRoot?: string; - readonly image?: string; - readonly lifecycle: Semaphore.Semaphore; - running: boolean; - bootstrapped: boolean; - snapshotKey?: string; - resources: RuntimeResources; -} - -const resourceName = (identity: StackId, role: string): string => - `supabase-eph-${identity.slice(0, 16)}-${role}`; - -const createIdentity = (crypto: Crypto.Crypto): Effect.Effect => - Effect.gen(function* () { - const first = yield* crypto.randomUUIDv4; - const second = yield* crypto.randomUUIDv4; - return yield* Schema.decodeEffect(StackIdSchema)(`${first}${second}`.replaceAll("-", "")); - }).pipe( - Effect.mapError((cause) => ephemeralError("Unable to allocate ephemeral identity", { cause })), - ); - -const writeRuntimeMarker = ( - cluster: Cluster, - snapshotKey?: string, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const markerPath = `${cluster.dataPath}/${RUNTIME_MARKER}`; - const encoded = encodeMarker(markerFor(cluster.runtime, snapshotKey ?? cluster.snapshotKey)); - if (cluster.resources.kind === "native") { - yield* fs.writeFileString(markerPath, encoded).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to write snapshot runtime marker", { - cause, - path: markerPath, - reason: "snapshot", - }), - ), - ); - return; - } - const containerId = cluster.resources.containerId; - if (containerId === undefined) - return yield* ephemeralError("Ephemeral Postgres container is missing", { - reason: "snapshot", - }); - const tempPath = `${cluster.root}/${RUNTIME_MARKER}`; - yield* fs.writeFileString(tempPath, encoded).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to write snapshot runtime marker", { - cause, - path: tempPath, - reason: "snapshot", - }), - ), - ); - yield* cluster.resources.engine - .copyToContainer( - containerId, - tempPath, - `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}/${RUNTIME_MARKER}`, - ) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to copy snapshot runtime marker", { cause, reason: "snapshot" }), - ), - ); - }); - -const snapshotFileName = ( - tarPath: string, - path: Path.Path, -): Effect.Effect => { - const name = path.basename(tarPath); - if (name.length === 0 || name === "." || name === "..") - return Effect.fail( - ephemeralError("Ephemeral Postgres snapshot path is invalid", { - reason: "snapshot", - path: tarPath, - }), - ); - return Effect.succeed(name); -}; - -/** Catalog image has no tar on PATH; busybox tar archives the volume in place. */ -const runVolumeTar = ( - cluster: Cluster, - tarPath: string, - mode: "create" | "extract", -): Effect.Effect => - Effect.gen(function* () { - if (cluster.resources.kind !== "container") return; - const { engine, networkId, volumeId } = cluster.resources; - if (networkId === undefined || volumeId === undefined) - return yield* ephemeralError("Ephemeral Postgres volume is unavailable", { - reason: "snapshot", - }); - const image = cluster.image; - if (image === undefined) - return yield* ephemeralError("Ephemeral Postgres image is unavailable", { - reason: "snapshot", - }); - const path = yield* Path.Path; - const fs = yield* FileSystem.FileSystem; - const fileName = yield* snapshotFileName(tarPath, path); - const parent = path.dirname(tarPath); - yield* fs.makeDirectory(parent, { recursive: true }).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to create snapshot directory", { - cause, - path: parent, - reason: "snapshot", - }), - ), - ); - const snapshotPath = `${SNAPSHOT_MOUNT}/${fileName}`; - const command = - mode === "create" - ? ["tar", "-C", CONTAINER_PGDATA_PARENT, "-cf", snapshotPath, PGDATA_DIR_NAME] - : ["tar", "-C", CONTAINER_PGDATA_PARENT, "-xf", snapshotPath]; - yield* Effect.acquireUseRelease( - engine - .createContainer({ - name: resourceName(cluster.identity, "snapshot"), - image, - labels: { - stackId: cluster.identity, - ownerSessionId: cluster.identity.slice(0, 32), - workloadId: `${DATABASE_WORKLOAD_ID}:snapshot`, - role: "workload", - }, - network: networkId, - mounts: [{ source: parent, target: SNAPSHOT_MOUNT, readOnly: mode === "extract" }], - volumeMounts: [ - { - volume: volumeId, - target: `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}`, - readOnly: false, - }, - ], - publications: [], - role: "workload", - entrypoint: BUSYBOX, - command, - }) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to create snapshot helper", { cause, reason: "snapshot" }), - ), - ), - (created) => - Effect.gen(function* () { - yield* engine.startContainer(created.id).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to start snapshot helper", { - cause, - reason: "snapshot", - }), - ), - ); - const code = yield* engine.waitContainer(created.id).pipe( - Effect.mapError((cause) => - ephemeralError("Snapshot helper did not finish", { - cause, - reason: "snapshot", - }), - ), - ); - if (code !== 0) - return yield* ephemeralError(`Snapshot helper failed (${String(code)})`, { - reason: "snapshot", - }); - }), - (created) => engine.removeContainer(created.id).pipe(Effect.ignore), - ); - }); - -const verifyRestoredMarker = ( - cluster: Cluster, - restoreFrom: string, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const markerPath = `${cluster.dataPath}/${RUNTIME_MARKER}`; - const exists = yield* fs.exists(markerPath).pipe(Effect.orElseSucceed(() => false)); - if (!exists) - return yield* ephemeralError("Ephemeral Postgres snapshot is missing a runtime marker", { - reason: "restore-mismatch", - path: restoreFrom, - }); - const marker = yield* fs.readFileString(markerPath).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to read snapshot runtime marker", { - cause, - path: markerPath, - reason: "snapshot", - }), - ), - Effect.flatMap(decodeMarker), - ); - if (!sameRuntime(marker, cluster.runtime) || !sameSnapshotKey(marker, cluster.snapshotKey)) - return yield* ephemeralError( - "Ephemeral Postgres snapshot was produced by a different runtime", - { - reason: "restore-mismatch", - path: restoreFrom, - }, - ); - }); - -const stopNative = (cluster: Cluster): Effect.Effect => - Effect.gen(function* () { - if (cluster.resources.kind !== "native") return; - const process = cluster.resources.process; - if (process !== undefined) { - const running = yield* process.isRunning.pipe(Effect.orElseSucceed(() => false)); - if (running) - yield* process.kill.pipe( - Effect.mapError((cause) => - ephemeralError("Unable to stop ephemeral Postgres", { cause }), - ), - ); - } - const scope = cluster.resources.processScope; - if (scope !== undefined) yield* Scope.close(scope, Exit.void); - cluster.resources.process = undefined; - cluster.resources.processScope = undefined; - cluster.running = false; - }); - -const stopContainer = (cluster: Cluster): Effect.Effect => - Effect.gen(function* () { - if (cluster.resources.kind !== "container") return; - const containerId = cluster.resources.containerId; - if (containerId !== undefined) - yield* cluster.resources.engine - .stopContainer(containerId) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to stop ephemeral Postgres", { cause }), - ), - ); - cluster.running = false; - }); - -const startNative = ( - cluster: Cluster, - options: CreateEphemeralPostgresOptions, - healthTimeout: string, - password: string, -): Effect.Effect => - Effect.gen(function* () { - const executable = cluster.executable; - if (executable === undefined) - return yield* ephemeralError("Native Postgres executable is unavailable"); - const parentScope = yield* Scope.Scope; - const processScope = yield* Scope.fork(parentScope, "parallel"); - yield* Effect.uninterruptibleMask((restore) => - restore( - spawnNativeProcess( - { - executable, - args: postgresArgs(cluster.port, cluster.runtime, options.postgresSettings), - env: postgresEnv({ - port: cluster.port, - dataPath: cluster.dataPath, - password, - }), - cwd: cluster.root, - gracefulStopSignal: "SIGINT", - gracefulStopTimeout: "15 seconds", - }, - defaultNativeProcessLauncher(), - { stackId: cluster.identity, workloadId: DATABASE_WORKLOAD_ID }, - ).pipe(Scope.provide(processScope)), - ).pipe( - Effect.mapError((cause) => ephemeralError("Unable to start native Postgres", { cause })), - Effect.tap((process) => - Effect.gen(function* () { - if (cluster.resources.kind === "native") { - cluster.resources.process = process; - cluster.resources.processScope = processScope; - } - yield* Effect.forkIn( - Effect.all([Stream.runDrain(process.stdout), Stream.runDrain(process.stderr)], { - concurrency: "unbounded", - discard: true, - }).pipe(Effect.ignore), - processScope, - ); - }), - ), - Effect.onExit((exit) => - Exit.isSuccess(exit) - ? Effect.void - : Scope.close(processScope, Exit.void).pipe(Effect.asVoid), - ), - ), - ); - yield* Effect.gen(function* () { - yield* waitForPostgres(cluster.port, healthTimeout); - if (!cluster.bootstrapped) { - yield* bootstrap(cluster.port, options, healthTimeout); - cluster.bootstrapped = true; - } - yield* waitForAdvertised(cluster.port, password, healthTimeout); - cluster.running = true; - }).pipe( - Effect.onExit((exit) => - Exit.isSuccess(exit) ? Effect.void : stopNative(cluster).pipe(Effect.ignore), - ), - ); - }); - -const startContainer = ( - cluster: Cluster, - options: CreateEphemeralPostgresOptions, - healthTimeout: string, - password: string, -): Effect.Effect => - Effect.gen(function* () { - if (cluster.resources.kind !== "container") return; - const resources = cluster.resources; - const image = cluster.image; - if (image === undefined) - return yield* ephemeralError("Ephemeral Postgres image is unavailable"); - const networkId = resources.networkId; - const volumeId = resources.volumeId; - if (networkId === undefined || volumeId === undefined) - return yield* ephemeralError("Ephemeral Postgres volume is unavailable"); - yield* Effect.gen(function* () { - if (resources.containerId !== undefined) { - yield* resources.engine - .startContainer(resources.containerId) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to start ephemeral Postgres", { cause }), - ), - ); - } else { - const path = yield* Path.Path; - const envFile = yield* writeEnvFile( - path.join(cluster.root, "postgres.env"), - postgresEnv({ - port: 5432, - dataPath: `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}`, - password, - }), - ); - const containerName = resources.containerName ?? resourceName(cluster.identity, "database"); - resources.containerName = containerName; - const created = yield* Effect.uninterruptibleMask((restore) => - restore( - resources.engine.createContainer({ - name: containerName, - image, - labels: { - stackId: cluster.identity, - ownerSessionId: cluster.identity.slice(0, 32), - workloadId: DATABASE_WORKLOAD_ID, - role: "workload", - }, - network: networkId, - mounts: [], - volumeMounts: [ - { - volume: volumeId, - target: `${CONTAINER_PGDATA_PARENT}/${PGDATA_DIR_NAME}`, - readOnly: false, - }, - ], - publications: [{ address: "127.0.0.1", hostPort: cluster.port, containerPort: 5432 }], - role: "workload", - command: postgresArgs(5432, cluster.runtime, options.postgresSettings), - envFile, - networkAliases: [containerAliasFor("database:database")], - }), - ).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to create ephemeral Postgres", { cause }), - ), - Effect.tap((created) => - Effect.sync(() => { - resources.containerId = created.id; - }), - ), - ), - ); - yield* resources.engine - .startContainer(created.id) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to start ephemeral Postgres", { cause }), - ), - ); - } - yield* waitForPostgres(cluster.port, healthTimeout); - if (!cluster.bootstrapped) { - yield* bootstrap(cluster.port, options, healthTimeout); - cluster.bootstrapped = true; - } - yield* waitForAdvertised(cluster.port, password, healthTimeout); - cluster.running = true; - }).pipe( - Effect.onExit((exit) => - Exit.isSuccess(exit) ? Effect.void : stopContainer(cluster).pipe(Effect.ignore), - ), - ); - }); - -const exportNative = ( - cluster: Cluster, - tarPath: string, -): Effect.Effect< - void, - EphemeralPostgresError, - ChildProcessSpawnerService | Scope.Scope | FileSystem.FileSystem -> => - Effect.gen(function* () { - yield* writeRuntimeMarker(cluster, cluster.snapshotKey); - yield* runTar(["-C", cluster.root, "-cf", tarPath, PGDATA_DIR_NAME]); - }); - -const exportContainer = ( - cluster: Cluster, - tarPath: string, -): Effect.Effect => - Effect.gen(function* () { - yield* writeRuntimeMarker(cluster, cluster.snapshotKey); - yield* runVolumeTar(cluster, tarPath, "create"); - }); - -const restoreNative = ( - cluster: Cluster, - restoreFrom: string, -): Effect.Effect< - void, - EphemeralPostgresError, - ChildProcessSpawnerService | Scope.Scope | FileSystem.FileSystem -> => - Effect.gen(function* () { - yield* runTar(["-C", cluster.root, "-xf", restoreFrom, ...TAR_EXTRACT_FLAGS]); - yield* verifyRestoredMarker(cluster, restoreFrom); - }); - -const restoreContainer = ( - cluster: Cluster, - restoreFrom: string, -): Effect.Effect => - runVolumeTar(cluster, restoreFrom, "extract"); - -const peekSnapshotRuntime = ( - restoreFrom: string, - runtime: StackRuntime, - peekRoot: string, - snapshotKey?: string, -): Effect.Effect< - void, - EphemeralPostgresError, - FileSystem.FileSystem | ChildProcessSpawnerService | Scope.Scope -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(peekRoot, { recursive: true }).pipe( - Effect.mapError((cause) => - ephemeralError("Unable to inspect snapshot", { - cause, - path: peekRoot, - reason: "snapshot", - }), - ), - ); - yield* runTar([ - "-xf", - restoreFrom, - "-C", - peekRoot, - ...TAR_EXTRACT_FLAGS, - `${PGDATA_DIR_NAME}/${RUNTIME_MARKER}`, - ]).pipe( - Effect.mapError(() => - ephemeralError("Ephemeral Postgres snapshot is missing a runtime marker", { - reason: "restore-mismatch", - path: restoreFrom, - }), - ), - ); - const marker = yield* fs - .readFileString(`${peekRoot}/${PGDATA_DIR_NAME}/${RUNTIME_MARKER}`) - .pipe( - Effect.mapError((cause) => - ephemeralError("Unable to read snapshot runtime marker", { cause, reason: "snapshot" }), - ), - Effect.flatMap(decodeMarker), - ); - if (!sameRuntime(marker, runtime)) - return yield* ephemeralError( - "Ephemeral Postgres snapshot was produced by a different runtime", - { - reason: "restore-mismatch", - path: restoreFrom, - }, - ); - if (!sameSnapshotKey(marker, snapshotKey)) - return yield* ephemeralError("Ephemeral Postgres snapshot key does not match", { - reason: "restore-mismatch", - path: restoreFrom, - }); - yield* fs.remove(peekRoot, { recursive: true }).pipe(Effect.ignore); - }); - -const destroyCluster = (cluster: Cluster): Effect.Effect => - Effect.gen(function* () { - if (cluster.resources.kind === "native") yield* stopNative(cluster).pipe(Effect.ignore); - else { - yield* stopContainer(cluster).pipe(Effect.ignore); - const containerRef = cluster.resources.containerId ?? cluster.resources.containerName; - if (containerRef !== undefined) - yield* cluster.resources.engine.removeContainer(containerRef).pipe(Effect.ignore); - const volumeRef = cluster.resources.volumeId ?? cluster.resources.volumeName; - if (volumeRef !== undefined) - yield* cluster.resources.engine.removeVolume(volumeRef).pipe(Effect.ignore); - const networkRef = cluster.resources.networkId ?? cluster.resources.networkName; - if (networkRef !== undefined) - yield* cluster.resources.engine.removeNetwork(networkRef).pipe(Effect.ignore); - } - const fs = yield* FileSystem.FileSystem; - yield* fs.remove(cluster.root, { recursive: true }).pipe(Effect.ignore); - }); - -const clusterHandle = ( - cluster: Cluster, - options: CreateEphemeralPostgresOptions, - healthTimeout: string, - password: string, -): EffectEphemeralPostgres => { - const requireStopped = (): Effect.Effect => - cluster.running - ? Effect.fail( - ephemeralError("Ephemeral Postgres must be stopped before exporting PGDATA", { - reason: "not-stopped", - }), - ) - : Effect.void; - return { - host: cluster.host, - port: cluster.port, - version: cluster.version, - runtime: cluster.runtime, - artifactIdentity: cluster.artifactIdentity, - ...(cluster.artifactRoot === undefined ? {} : { nativeArtifactRoot: cluster.artifactRoot }), - url: Redacted.make(databaseUrl(cluster.port, password)), - ...(cluster.resources.kind === "container" && cluster.resources.networkId !== undefined - ? { networkId: cluster.resources.networkId } - : {}), - start: Effect.suspend(() => - cluster.lifecycle.withPermit( - Effect.gen(function* () { - if (cluster.running) { - if (cluster.runtime.kind === "native" && cluster.resources.kind === "native") { - const process = cluster.resources.process; - const stillRunning = - process === undefined - ? false - : yield* process.isRunning.pipe(Effect.orElseSucceed(() => false)); - if (stillRunning) return; - yield* stopNative(cluster).pipe(Effect.ignore); - } else { - const probe = yield* waitForPostgres(cluster.port, "1s").pipe(Effect.exit); - if (Exit.isSuccess(probe)) return; - cluster.running = false; - } - } - if (cluster.runtime.kind === "native") - yield* startNative(cluster, options, healthTimeout, password); - else yield* startContainer(cluster, options, healthTimeout, password); - }), - ), - ), - stop: Effect.suspend(() => - cluster.lifecycle.withPermit( - cluster.runtime.kind === "native" ? stopNative(cluster) : stopContainer(cluster), - ), - ), - exportPgData: (tarPath, snapshotKey) => - cluster.lifecycle.withPermit( - Effect.gen(function* () { - yield* requireStopped(); - if (snapshotKey !== undefined) cluster.snapshotKey = snapshotKey; - if (cluster.runtime.kind === "native") yield* exportNative(cluster, tarPath); - else yield* exportContainer(cluster, tarPath); - }), - ), - }; -}; - -export const createEphemeralPostgresCluster = ( - options: CreateEphemeralPostgresOptions, -): Effect.Effect< - EffectEphemeralPostgres, - EphemeralPostgresCreateError, - Scope.Scope | FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => - Effect.gen(function* () { - const resolver = yield* Effect.serviceOption(ContainerEngineResolver).pipe( - Effect.map(Option.getOrUndefined), - ); - const selection = yield* resolvedRuntime(options.runtime, resolver); - const runtime = selection.runtime; - const release = yield* resolveEphemeralPostgresRelease(options.version); - const envOption = yield* Effect.serviceOption(StackRuntimeEnvironment); - const env = Option.isSome(envOption) ? envOption.value : yield* defaultRuntimeEnvironment; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const identity = yield* createIdentity(crypto); - const root = path.join(path.dirname(env.stateRoot), "ephemeral-postgres", identity); - const dataPath = path.join(root, PGDATA_DIR_NAME); - yield* fs.makeDirectory(dataPath, { recursive: true, mode: 0o700 }).pipe( - Effect.mapError( - (cause) => - new StackPreparationError({ - message: "Unable to create ephemeral Postgres data directory", - path: dataPath, - cause, - }), - ), - ); - yield* Effect.addFinalizer(() => fs.remove(root, { recursive: true }).pipe(Effect.ignore)); - if (options.restoreFrom !== undefined) - yield* peekSnapshotRuntime( - options.restoreFrom, - runtime, - path.join(root, "peek"), - options.snapshotKey, - ); - const port = yield* allocateLoopbackPort(options.port); - const healthTimeout = options.healthTimeout ?? DEFAULT_DATABASE_HEALTH_TIMEOUT; - const password = Redacted.value(options.databasePassword); - const workload = plannedWorkload(release.version, release.image, runtime); - const preparer = yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: env.stateRoot, - ...(env.artifactCacheRoot === undefined ? {} : { artifactCacheRoot: env.artifactCacheRoot }), - runtime, - }); - const prepared = yield* preparer.prepare(runtime, workload); - let resources: RuntimeResources; - if (runtime.kind === "native") { - resources = { kind: "native" }; - } else { - const engine = yield* resolveContainerEngine(runtime.engine, resolver).pipe( - Effect.mapError( - (cause) => - new ContainerEngineError({ - message: `Unable to configure ${runtime.engine} for ephemeral Postgres`, - engine: runtime.engine, - cause, - }), - ), - ); - resources = { kind: "container", engine }; - } - const cluster: Cluster = { - identity, - root, - dataPath, - host: "127.0.0.1", - port, - version: release.version, - runtime, - artifactIdentity: - runtime.kind === "native" - ? `native:${release.version}` - : `container:${runtime.engine}:${release.image}`, - ...(prepared.executablePath === undefined || prepared.artifactRoot === undefined - ? {} - : { - executable: prepared.artifactRoot.endsWith("/") - ? `${prepared.artifactRoot}${prepared.executablePath}` - : `${prepared.artifactRoot}/${prepared.executablePath}`, - artifactRoot: prepared.artifactRoot, - }), - ...(prepared.image === undefined ? {} : { image: prepared.image }), - lifecycle: Semaphore.makeUnsafe(1), - running: false, - bootstrapped: options.restoreFrom !== undefined, - ...(options.snapshotKey === undefined ? {} : { snapshotKey: options.snapshotKey }), - resources, - }; - yield* Effect.addFinalizer(() => destroyCluster(cluster)); - if (cluster.resources.kind === "container") { - const resources = cluster.resources; - const engine = resources.engine; - const engineKind = cluster.runtime.kind === "container" ? cluster.runtime.engine : "docker"; - const networkName = resourceName(identity, "network"); - const volumeName = resourceName(identity, "database-volume"); - const containerName = resourceName(identity, "database"); - resources.networkName = networkName; - resources.volumeName = volumeName; - resources.containerName = containerName; - yield* Effect.uninterruptibleMask((restore) => - restore( - engine.createNetwork({ - name: networkName, - labels: { stackId: identity, ownerSessionId: identity.slice(0, 32), role: "network" }, - }), - ).pipe( - Effect.mapError( - (cause) => - new ContainerEngineError({ - message: "Unable to create ephemeral Postgres network", - engine: engineKind, - cause, - }), - ), - Effect.tap((created) => - Effect.sync(() => { - resources.networkId = created.id; - }), - ), - ), - ); - yield* Effect.uninterruptibleMask((restore) => - restore( - engine.createVolume({ - name: volumeName, - labels: { stackId: identity, workloadId: DATABASE_WORKLOAD_ID, role: "volume" }, - }), - ).pipe( - Effect.mapError( - (cause) => - new ContainerEngineError({ - message: "Unable to create ephemeral Postgres volume", - engine: engineKind, - cause, - }), - ), - Effect.tap((created) => - Effect.sync(() => { - resources.volumeId = created.id; - }), - ), - ), - ); - } - if (options.restoreFrom !== undefined) { - if (runtime.kind === "native") yield* restoreNative(cluster, options.restoreFrom); - else yield* restoreContainer(cluster, options.restoreFrom); - } - if (runtime.kind === "native") yield* startNative(cluster, options, healthTimeout, password); - else yield* startContainer(cluster, options, healthTimeout, password); - return { - ...clusterHandle(cluster, options, healthTimeout, password), - ...(selection.dockerFallbackNotice === undefined - ? {} - : { dockerFallbackNotice: selection.dockerFallbackNotice }), - }; - }); diff --git a/packages/stack/src/runtime/NativeProcess.ts b/packages/stack/src/runtime/NativeProcess.ts index d69b23ed8b..382e4a1c2f 100644 --- a/packages/stack/src/runtime/NativeProcess.ts +++ b/packages/stack/src/runtime/NativeProcess.ts @@ -1,7 +1,8 @@ -import { Data, Duration, Effect, Option, Scope, Stream } from "effect"; +import { Data, Duration, Effect, Fiber, Option, Scope, Stream } from "effect"; import { fileURLToPath } from "node:url"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { PlatformError } from "effect/PlatformError"; +import { isBunVirtualPath } from "../internal/dispatch-markers.ts"; import type { ChildProcessHandle, ExitCode, @@ -52,12 +53,9 @@ export interface NativeProcess { readonly kill: Effect.Effect; } -export { NATIVE_PROCESS_DISPATCH_SENTINEL } from "../internal/dispatch-markers.ts"; import { NATIVE_PROCESS_DISPATCH_SENTINEL } from "../internal/dispatch-markers.ts"; -const isBunVirtualPath = (value: string): boolean => /(?:^|[\\/])\$bunfs(?:[\\/]|$)/.test(value); - -export const nativeLauncherEntrypointFor = (moduleUrl: string): string => { +const nativeLauncherEntrypointFor = (moduleUrl: string): string => { if (isBunVirtualPath(moduleUrl)) return NATIVE_PROCESS_DISPATCH_SENTINEL; const sourceEntrypoint = fileURLToPath(new URL("./native-launcher.ts", moduleUrl)); return isBunVirtualPath(sourceEntrypoint) ? NATIVE_PROCESS_DISPATCH_SENTINEL : sourceEntrypoint; @@ -106,16 +104,12 @@ const mapProcessError = (error: unknown, spec: NativeProcessSpec): NativeProcess * that pipe and terminates the exact process group. Normal process-tree * termination remains owned by Effect's ChildProcessSpawner. */ -export const spawnNativeProcess = ( +export const spawnNativeProcess = Effect.fn("NativeProcess.spawn")(function* ( spec: NativeProcessSpec, launcher: NativeProcessLauncher = defaultNativeProcessLauncher(), identity?: NativeProcessIdentity, -): Effect.Effect< - NativeProcess, - NativeProcessError, - import("effect/unstable/process/ChildProcessSpawner").ChildProcessSpawner | Scope.Scope -> => - Effect.gen(function* () { +) { + return yield* Effect.gen(function* () { // Shutdown must precede the spawner's finalizer even when the caller closes in parallel. const processScope = yield* Scope.fork(yield* Scope.Scope, "sequential"); const launcherArgs = @@ -147,71 +141,76 @@ export const spawnNativeProcess = ( ): Stream.Stream => stream.pipe(Stream.mapError((error) => mapProcessError(error, spec))); const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - // Darwin can report EPERM after a process group has become zombie-only. - const inspectProcessGroup = Effect.scoped( - Effect.gen(function* () { - const inspection = yield* ChildProcess.make("/bin/ps", ["-axo", "pgid=,stat="], { - killSignal: "SIGKILL", - stdin: "ignore", - stdout: "pipe", - stderr: "ignore", - }); - const [output, exitCode] = yield* Effect.all( - [inspection.stdout.pipe(Stream.decodeText, Stream.mkString), inspection.exitCode], - { concurrency: 2 }, - ); - if (Number(exitCode) !== 0 || output.trim().length === 0) return false; - const targetGroup = Number(handle.pid); - for (const line of output.split("\n")) { - if (line.trim() === "") continue; - const match = /^(\d+)\s+(\S+)$/.exec(line.trim()); - if (match === null) return false; - const group = Number(match[1]); - const state = match[2]; - if (state === undefined) return false; - if (!Number.isSafeInteger(group)) return false; - if (group === targetGroup && !state.startsWith("Z")) return false; - } - return true; - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)), - ); - const cleanupProcessGroup = Effect.try({ - try: () => { - if (globalThis.process.platform === "win32") return; - try { - globalThis.process.kill(-Number(handle.pid), "SIGKILL"); - } catch (error) { - if ( - typeof error === "object" && - error !== null && - "code" in error && - error.code === "ESRCH" - ) - return; - throw error; - } - }, - catch: (error) => mapProcessError(error, spec), - }).pipe( - Effect.catch((error) => { - const cause = error.cause; - if ( - globalThis.process.platform === "darwin" && - typeof cause === "object" && - cause !== null && - "code" in cause && - cause.code === "EPERM" - ) { - return inspectProcessGroup.pipe( - Effect.timeout("2 seconds"), - Effect.catchDefect(() => Effect.fail(error)), - Effect.mapError(() => error), - Effect.flatMap((noLiveMembers) => (noLiveMembers ? Effect.void : Effect.fail(error))), + // Darwin can report EPERM after a process group has become zombie-only or is exiting. + const inspectProcessGroup = Effect.fn("NativeProcess.inspectProcessGroup")(function* () { + return yield* Effect.scoped( + Effect.gen(function* () { + const inspection = yield* ChildProcess.make("/bin/ps", ["-axo", "pgid=,stat="], { + killSignal: "SIGKILL", + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + const [output, exitCode] = yield* Effect.all( + [inspection.stdout.pipe(Stream.decodeText, Stream.mkString), inspection.exitCode], + { concurrency: 2 }, ); - } - return Effect.fail(error); - }), - ); + if (Number(exitCode) !== 0 || output.trim().length === 0) return false; + const targetGroup = Number(handle.pid); + for (const line of output.split("\n")) { + if (line.trim() === "") continue; + const match = /^(\d+)\s+(\S+)$/.exec(line.trim()); + if (match === null) return false; + const group = Number(match[1]); + const state = match[2]; + if (state === undefined) return false; + if (!Number.isSafeInteger(group)) return false; + if (group === targetGroup && !(state.startsWith("Z") || state.includes("E"))) + return false; + } + return true; + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)), + ); + }); + const cleanupProcessGroup = Effect.fn("NativeProcess.cleanupProcessGroup")(function* () { + return yield* Effect.try({ + try: () => { + if (globalThis.process.platform === "win32") return; + try { + globalThis.process.kill(-Number(handle.pid), "SIGKILL"); + } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ESRCH" + ) + return; + throw error; + } + }, + catch: (error) => mapProcessError(error, spec), + }).pipe( + Effect.catch((error) => { + const cause = error.cause; + if ( + globalThis.process.platform === "darwin" && + typeof cause === "object" && + cause !== null && + "code" in cause && + cause.code === "EPERM" + ) { + return inspectProcessGroup().pipe( + Effect.timeout("2 seconds"), + Effect.catchDefect(() => Effect.fail(error)), + Effect.mapError(() => error), + Effect.flatMap((noLiveMembers) => (noLiveMembers ? Effect.void : Effect.fail(error))), + ); + } + return Effect.fail(error); + }), + ); + }); const signalLauncher = (signal: NodeJS.Signals): Effect.Effect => Effect.try({ try: () => { @@ -229,11 +228,11 @@ export const spawnNativeProcess = ( : Effect.fail(error); }), ); - const killProcess = Effect.gen(function* () { + const killProcess = Effect.fn("NativeProcess.kill")(function* () { const running = yield* handle.isRunning.pipe( Effect.mapError((error) => mapProcessError(error, spec)), ); - if (!running) return yield* cleanupProcessGroup; + if (!running) return yield* cleanupProcessGroup(); // Record the stop in the launcher before it forwards the signal to the workload. const graceful = globalThis.process.platform === "win32" @@ -256,15 +255,17 @@ export const spawnNativeProcess = ( .kill({ killSignal: "SIGKILL" }) .pipe(Effect.mapError((error) => mapProcessError(error, spec))); } - if (globalThis.process.platform !== "win32") yield* cleanupProcessGroup; + if (globalThis.process.platform !== "win32") yield* cleanupProcessGroup(); }); yield* Scope.addFinalizer( processScope, - killProcess.pipe(Effect.catch((error) => Effect.logError(error.message))), + killProcess().pipe(Effect.catch((error) => Effect.logError(error.message))), ); yield* Stream.run(Stream.succeed(encodeSpec(spec)), handle.getInputFd(4)); - const exitCode = yield* Effect.cached( - mapError(handle.exitCode).pipe(Effect.tap(() => cleanupProcessGroup)), + const waitForExit = mapError(handle.exitCode).pipe(Effect.tap(() => cleanupProcessGroup())); + const getWaiter = yield* Effect.cached(Effect.forkIn(waitForExit, processScope)); + const exitCode = Effect.uninterruptible(getWaiter).pipe( + Effect.flatMap((fiber) => Fiber.join(fiber)), ); return { pid: handle.pid, @@ -272,6 +273,7 @@ export const spawnNativeProcess = ( stderr: mapStreamError(handle.stderr), exitCode, isRunning: mapError(handle.isRunning), - kill: killProcess, + kill: killProcess(), } satisfies NativeProcess; }).pipe(Effect.mapError((error) => mapProcessError(error, spec))); +}); diff --git a/packages/stack/src/runtime/NativeRuntime.ts b/packages/stack/src/runtime/NativeRuntime.ts deleted file mode 100644 index e4a5a7c7a8..0000000000 --- a/packages/stack/src/runtime/NativeRuntime.ts +++ /dev/null @@ -1,645 +0,0 @@ -import { Cause, Deferred, Effect, Exit, Fiber, Ref, Scope, Semaphore, Stream } from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import type { ExitCode } from "effect/unstable/process/ChildProcessSpawner"; -import type * as ChildProcessSpawnerService from "effect/unstable/process/ChildProcessSpawner"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { StackId } from "../public/StackId.ts"; -import type { LogStore } from "../supervisor/LogStore.ts"; -import { LogStoreError } from "../supervisor/LogStore.ts"; -import { - RuntimeDriverError, - type RuntimeCleanupRequest, - type ObservedWorkload, - type RuntimeDriver, - type RuntimeWorkloadKey, -} from "./RuntimeDriver.ts"; -import { - NativeProcessError, - spawnNativeProcess, - type NativeProcess, - type NativeProcessLauncher, - type NativeProcessSpec, -} from "./NativeProcess.ts"; -import { - makeProcessOutputTail, - withLeftoverPersistentDataGuidance, - type ProcessOutputTail, -} from "./Diagnostics.ts"; -import { - nativeRuntimeBlockedForUid, - NATIVE_ROOT_UNSUPPORTED_MESSAGE, -} from "./ContainerEngineResolver.ts"; - -export interface NativeRuntimeOptions { - /** - * Resolves the process-ownership launcher. Development defaults use the - * bundled native-launcher entrypoint. - */ - readonly resolveLauncher?: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - ) => Effect.Effect; - /** Resolves the complete native process plan for one workload. */ - readonly resolveProcess: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - ) => Effect.Effect; - /** Private readiness is resolved by the owning Supervisor/gateway seam. */ - readonly waitForReadiness: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - process: NativeProcess, - ) => Effect.Effect; - /** Runs the one-shot initial database bootstrap after readiness. */ - readonly bootstrapDatabase?: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - process?: NativeProcess, - ) => Effect.Effect; - readonly logStore?: LogStore; - /** Wipes native PGDATA after the database workload has been stopped and removed. */ - readonly wipeDatabaseData?: Effect.Effect; - readonly knownSecrets?: Effect.Effect>; -} - -/** One-shot startup processes followed by the long-lived workload process. */ -interface NativeProcessPlan { - readonly startup: ReadonlyArray; - readonly main: NativeProcessSpec; -} - -interface OutputAccumulator { - readonly decoder: TextDecoder; - remainder: string; -} - -interface Resource { - readonly key: RuntimeWorkloadKey; - readonly workload: PlannedWorkload; - readonly scope: Scope.Closeable; - readonly state: Ref.Ref; - readonly output: Readonly<{ stdout: OutputAccumulator; stderr: OutputAccumulator }>; - readonly tail: ProcessOutputTail; - readonly result: Deferred.Deferred; - readonly failure: Deferred.Deferred; - stopRequested: boolean; - process?: NativeProcess; - startFiber?: Fiber.Fiber; -} - -const resourceKey = (key: RuntimeWorkloadKey): string => - JSON.stringify([key.stackId, key.workloadId]); - -const sameKey = (left: RuntimeWorkloadKey, right: RuntimeWorkloadKey): boolean => - left.stackId === right.stackId && left.workloadId === right.workloadId; - -const driverError = ( - key: Pick, - message: string, - cause?: unknown, -): RuntimeDriverError => - new RuntimeDriverError({ - message: withLeftoverPersistentDataGuidance(message), - stackId: key.stackId, - workloadId: key.workloadId, - ...(cause === undefined ? {} : { cause }), - }); - -const errorMessage = (error: unknown): string => - error instanceof Error ? error.message : String(error); - -const processError = ( - error: unknown, - key: RuntimeWorkloadKey, - operation: string, -): RuntimeDriverError => - driverError( - key, - `Native ${operation} failed for ${key.workloadId}: ${errorMessage(error)}`, - error, - ); - -const appendLines = ( - logStore: LogStore, - resource: Resource, - stream: "stdout" | "stderr", - bytes: Uint8Array, -): Effect.Effect => { - const accumulator = resource.output[stream]; - accumulator.remainder += accumulator.decoder.decode(bytes, { stream: true }); - const lines = accumulator.remainder.split(/\r?\n/); - accumulator.remainder = lines.pop() ?? ""; - return Effect.forEach( - lines, - (message) => logStore.append({ source: resource.workload.capability, stream, message }), - { discard: true }, - ); -}; - -const flushLines = ( - logStore: LogStore, - resource: Resource, - stream: "stdout" | "stderr", -): Effect.Effect => { - const accumulator = resource.output[stream]; - accumulator.remainder += accumulator.decoder.decode(); - if (accumulator.remainder.length === 0) return Effect.void; - const message = accumulator.remainder; - accumulator.remainder = ""; - return logStore - .append({ source: resource.workload.capability, stream, message }) - .pipe(Effect.asVoid); -}; - -/** Creates a Supervisor-owned native runtime with exact process identity fencing. */ -export const makeNativeRuntime = ( - options: NativeRuntimeOptions, -): Effect.Effect< - RuntimeDriver, - RuntimeDriverError, - ChildProcessSpawnerService.ChildProcessSpawner | Scope.Scope -> => - Effect.gen(function* () { - const childSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; - const parentScope = yield* Scope.Scope; - const runtimeScope = yield* Scope.fork(parentScope, "parallel"); - const registration = yield* Semaphore.make(1); - const resources = new Map(); - - const knownSecrets = options.knownSecrets ?? Effect.succeed>([]); - const diagnosticMessage = (resource: Resource, message: string) => - knownSecrets.pipe( - Effect.map((secrets) => { - const tail = resource.tail.finish(secrets); - return tail.length === 0 ? message : `${message}\n${tail}`; - }), - ); - - const consumeProcessOutput = ( - resource: Resource, - process: NativeProcess, - logStore: LogStore | undefined, - ) => - Effect.all( - (["stdout", "stderr"] as const).map((stream) => - Stream.runForEach(process[stream], (bytes) => - Effect.gen(function* () { - resource.tail.pushBytes(stream, bytes); - if (logStore !== undefined) yield* appendLines(logStore, resource, stream, bytes); - }), - ).pipe( - Effect.andThen( - logStore === undefined ? Effect.void : flushLines(logStore, resource, stream), - ), - ), - ), - { concurrency: "unbounded", discard: true }, - ); - - const cleanup = (resource: Resource): Effect.Effect => - Scope.close(resource.scope, Exit.void).pipe( - Effect.asVoid, - Effect.andThen( - Effect.sync(() => { - const id = resourceKey(resource.key); - if (resources.get(id) === resource) resources.delete(id); - }), - ), - ); - - const watchProcess = ( - resource: Resource, - process: NativeProcess, - exitCode: Effect.Effect, - ) => - Effect.gen(function* () { - const result = yield* exitCode.pipe(Effect.exit); - const next: ObservedWorkload = Exit.isSuccess(result) - ? { - ...resource.key, - state: resource.stopRequested ? "stopped" : "failed", - ...(resource.stopRequested - ? {} - : { - error: `Native workload exited before an explicit stop (code ${String(result.value)})`, - }), - } - : { - ...resource.key, - state: resource.stopRequested ? "stopped" : "failed", - ...(resource.stopRequested ? {} : { error: Cause.pretty(result.cause) }), - }; - yield* Ref.update(resource.state, (current): ObservedWorkload => - current.state === "failed" ? current : next, - ); - if (!resource.stopRequested && Exit.isFailure(result)) { - const message = yield* diagnosticMessage( - resource, - "Native workload exited before readiness", - ); - yield* Deferred.fail(resource.failure, driverError(resource.key, message, result.cause)); - } - }).pipe(Effect.ignore); - - const reportLogFailure = ( - resource: Resource, - error: LogStoreError | NativeProcessError, - ): Effect.Effect => { - if (resource.stopRequested) return Effect.void; - const failure = driverError( - resource.key, - `Native log stream failed for ${resource.key.workloadId}: ${error.message}`, - error, - ); - return Ref.modify(resource.state, (current) => { - if (current.state === "stopped" || current.state === "failed") - return [false, current] satisfies readonly [boolean, ObservedWorkload]; - const next: ObservedWorkload = { ...current, state: "failed", error: failure.message }; - return [true, next] satisfies readonly [boolean, ObservedWorkload]; - }).pipe( - Effect.flatMap((changed) => - changed ? Deferred.fail(resource.failure, failure).pipe(Effect.asVoid) : Effect.void, - ), - ); - }; - - const attachLogs = (resource: Resource, process: NativeProcess) => { - const logStore = options.logStore; - const runOutput = consumeProcessOutput(resource, process, logStore).pipe( - Effect.catch((error) => reportLogFailure(resource, error)), - ); - return runOutput.pipe(Effect.forkIn(resource.scope)); - }; - - /** Runs one short-lived, service-owned startup process in its own scope. */ - const runStartupProcess = ( - resource: Resource, - spec: NativeProcessSpec, - launcher: NativeProcessLauncher | undefined, - phase: "startup", - ): Effect.Effect => - Effect.gen(function* () { - const startupScope = yield* Scope.fork(resource.scope, "parallel"); - let started: NativeProcess | undefined; - const run = Effect.gen(function* () { - const process = yield* spawnNativeProcess(spec, launcher, resource.key).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childSpawner), - Scope.provide(startupScope), - Effect.mapError((error) => processError(error, resource.key, `${phase} spawn`)), - ); - started = process; - const exitCode = yield* Effect.cached( - process.exitCode.pipe( - Effect.mapError((error) => processError(error, resource.key, `${phase} wait`)), - ), - ); - const consume = consumeProcessOutput(resource, process, options.logStore); - const consumed = consume.pipe( - Effect.mapError((error) => - driverError( - resource.key, - `Native ${phase} log stream failed for ${resource.key.workloadId}: ${errorMessage(error)}`, - error, - ), - ), - ); - const outputFiber = yield* Effect.forkIn(consumed, startupScope); - // A one-shot process finishes only once both its exit and output streams complete: - // Effect.all keeps draining after the child exits, and a drain failure interrupts the - // exit waiter while startup cleanup kills the exact child. - const completed = Effect.all([exitCode.pipe(Effect.asVoid), Fiber.join(outputFiber)], { - concurrency: "unbounded", - discard: true, - }); - yield* spec.timeout === undefined - ? completed - : completed.pipe( - Effect.timeoutOrElse({ - duration: spec.timeout, - orElse: () => - Effect.fail( - driverError( - resource.key, - `Native ${phase} process timed out for ${resource.key.workloadId}`, - ), - ), - }), - ); - const result = yield* exitCode.pipe(Effect.exit); - if (Exit.isFailure(result)) - return yield* driverError( - resource.key, - yield* diagnosticMessage( - resource, - `Native ${phase} process failed for ${resource.key.workloadId}`, - ), - result.cause, - ); - if (result.value !== 0) - return yield* driverError( - resource.key, - yield* diagnosticMessage( - resource, - `Native ${phase} process exited with code ${String(result.value)} for ${resource.key.workloadId}`, - ), - ); - }); - yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const result = yield* restore(run).pipe(Effect.exit); - const terminated = - started === undefined - ? Exit.succeed(undefined) - : yield* Effect.suspend(() => { - const child = started; - if (child === undefined) return Effect.void; - return child.isRunning.pipe( - Effect.mapError((error) => - processError(error, resource.key, `${phase} probe`), - ), - Effect.flatMap((running) => - running - ? child.kill.pipe( - Effect.mapError((error) => - processError(error, resource.key, `${phase} stop`), - ), - ) - : Effect.void, - ), - ); - }).pipe(Effect.exit); - const closed = yield* Scope.close(startupScope, Exit.void).pipe(Effect.exit); - let failureCause: Cause.Cause | undefined; - for (const candidate of [result, terminated, closed]) { - if (!Exit.isFailure(candidate)) continue; - failureCause = - failureCause === undefined - ? candidate.cause - : Cause.combine(failureCause, candidate.cause); - } - if (failureCause !== undefined) return yield* Effect.failCause(failureCause); - }), - ); - }); - - const runStart = (resource: Resource): Effect.Effect => - Effect.gen(function* () { - const { key, workload } = resource; - const ensureNotStopped = (): Effect.Effect => - resource.stopRequested - ? Effect.fail(driverError(key, "Native workload was stopped while starting")) - : Effect.void; - const operation = Effect.gen(function* () { - const launcher = - options.resolveLauncher === undefined - ? undefined - : yield* options.resolveLauncher(key, workload); - const resolved = yield* options.resolveProcess(key, workload); - for (const startup of resolved.startup) - yield* runStartupProcess(resource, startup, launcher, "startup"); - const process = yield* spawnNativeProcess(resolved.main, launcher, key).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childSpawner), - Scope.provide(resource.scope), - Effect.mapError((error) => processError(error, key, "spawn")), - ); - resource.process = process; - // The readiness race's waiter fibers can be interrupted, so exit observation is forked - // into the resource scope rather than shared via Effect.cached, which would let that - // interruption poison the long-lived workload watcher's view. - const exitFiber = yield* Effect.forkIn(process.exitCode, resource.scope); - const exitCode = Fiber.join(exitFiber); - yield* Effect.forkIn(watchProcess(resource, process, exitCode), resource.scope); - const outputFiber = yield* attachLogs(resource, process); - const readiness = options.waitForReadiness; - const mainExit = exitCode.pipe( - Effect.flatMap((code) => - Fiber.join(outputFiber).pipe( - Effect.ignore, - Effect.timeoutOrElse({ - duration: "1 second", - orElse: () => Effect.void, - }), - Effect.andThen( - diagnosticMessage( - resource, - `Native workload ${key.workloadId} exited before readiness (${String(code)})`, - ).pipe(Effect.flatMap((message) => Effect.fail(driverError(key, message)))), - ), - ), - ), - ); - const preReady = Effect.gen(function* () { - yield* readiness(key, workload, process); - yield* ensureNotStopped(); - if (workload.bootstrap === "database") { - if (options.bootstrapDatabase === undefined) - return yield* driverError(key, "Database bootstrap resolver is not configured"); - yield* options - .bootstrapDatabase(key, workload, process) - .pipe(Effect.mapError((error) => driverError(key, error.message, error))); - } - yield* ensureNotStopped(); - if (yield* Deferred.isDone(resource.failure)) - return yield* Deferred.await(resource.failure); - const ready = yield* Ref.modify(resource.state, (current) => { - if (current.state !== "starting") return [undefined, current]; - const next: ObservedWorkload = { ...current, state: "ready" }; - return [next, next]; - }); - if (ready === undefined) - return yield* driverError(key, "Native workload exited or stopped before readiness"); - return ready; - }); - return yield* Effect.raceFirst( - Effect.raceFirst(preReady, Deferred.await(resource.failure)), - mainExit, - ); - }); - const result = yield* operation.pipe( - Effect.mapError((error) => - error instanceof RuntimeDriverError - ? error - : driverError(key, `Native workload ${key.workloadId} failed`, error), - ), - Effect.exit, - ); - if (Exit.isFailure(result)) { - const failed: ObservedWorkload = { - ...resource.key, - state: resource.stopRequested ? "stopped" : "failed", - ...(resource.stopRequested ? {} : { error: Cause.pretty(result.cause) }), - }; - yield* Ref.set(resource.state, failed); - yield* cleanup(resource); - } - yield* Deferred.done(resource.result, result); - }); - - const start = ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - ): Effect.Effect => { - if (nativeRuntimeBlockedForUid()) - return Effect.fail(driverError(key, NATIVE_ROOT_UNSUPPORTED_MESSAGE)); - if (workload.selected.kind === "container") - return Effect.fail( - new RuntimeDriverError({ - message: "Native runtime cannot start a container artifact", - stackId: key.stackId, - workloadId: key.workloadId, - }), - ); - return Effect.flatMap( - // The permit serializes registration only; process readiness and log following continue - // after it is released, while stop/remove/cleanup hold it for their full operation. - registration.withPermit( - Effect.gen(function* () { - const id = resourceKey(key); - const existing = resources.get(id); - if (existing !== undefined) { - const current = yield* Ref.get(existing.state); - if (current.state === "starting" || current.state === "ready") return existing; - yield* cleanup(existing); - } - const processScope = yield* Scope.fork(runtimeScope, "parallel"); - const state = yield* Ref.make({ ...key, state: "starting" }); - const result = yield* Deferred.make(); - const failure = yield* Deferred.make(); - const resource: Resource = { - key, - workload, - scope: processScope, - state, - result, - failure, - output: { - stdout: { decoder: new TextDecoder(), remainder: "" }, - stderr: { decoder: new TextDecoder(), remainder: "" }, - }, - tail: makeProcessOutputTail(), - stopRequested: false, - }; - resources.set(id, resource); - resource.startFiber = yield* Effect.forkIn(runStart(resource), runtimeScope); - return resource; - }), - ), - (resource) => Deferred.await(resource.result), - ); - }; - - const observe = ( - stackId: StackId, - ): Effect.Effect, RuntimeDriverError> => - Effect.forEach( - [...resources.values()].filter((resource) => resource.key.stackId === stackId), - (resource) => Ref.get(resource.state), - ); - - const stopResource = (resource: Resource): Effect.Effect => - Effect.gen(function* () { - const { key } = resource; - resource.stopRequested = true; - if (resource.startFiber !== undefined) { - yield* Deferred.fail( - resource.result, - driverError(key, "Native workload was stopped while starting"), - ); - } - const nativeProcess = resource.process; - const killResult = - nativeProcess === undefined - ? Exit.succeed(undefined) - : yield* nativeProcess.isRunning.pipe( - Effect.mapError((error) => processError(error, key, "probe")), - Effect.flatMap((running) => - running - ? nativeProcess.kill.pipe( - Effect.mapError((error) => processError(error, key, "stop")), - ) - : Effect.void, - ), - Effect.exit, - ); - if (resource.startFiber !== undefined) yield* Fiber.interrupt(resource.startFiber); - if (Exit.isFailure(killResult)) { - yield* Ref.update(resource.state, (current): ObservedWorkload => ({ - ...current, - state: "failed", - error: Cause.pretty(killResult.cause), - })); - return yield* Effect.failCause(killResult.cause); - } - yield* Ref.update(resource.state, (current): ObservedWorkload => ({ - ...current, - state: "stopped", - })); - }); - - const removeResource = (resource: Resource): Effect.Effect => - Effect.gen(function* () { - resource.stopRequested = true; - if (resource.startFiber !== undefined) yield* Fiber.interrupt(resource.startFiber); - yield* cleanup(resource); - }); - - const stop = (key: RuntimeWorkloadKey): Effect.Effect => - registration.withPermit( - Effect.gen(function* () { - const resource = resources.get(resourceKey(key)); - if (resource === undefined) return; - if (!sameKey(resource.key, key)) - return yield* driverError(key, "Native workload identity mismatch"); - yield* stopResource(resource); - }), - ); - - const remove = (key: RuntimeWorkloadKey): Effect.Effect => - registration.withPermit( - Effect.gen(function* () { - const resource = resources.get(resourceKey(key)); - if (resource === undefined) return; - if (!sameKey(resource.key, key)) - return yield* driverError(key, "Native workload identity mismatch"); - yield* removeResource(resource); - }), - ); - - const cleanupRuntime = ( - request: RuntimeCleanupRequest, - ): Effect.Effect => - registration.withPermit( - Effect.gen(function* () { - let cleanupCause: Cause.Cause = Cause.empty; - const attempt = (effect: Effect.Effect) => - Effect.gen(function* () { - const result = yield* Effect.exit(effect); - if (Exit.isFailure(result)) cleanupCause = Cause.combine(cleanupCause, result.cause); - }); - const owned = [...resources.values()].filter( - (resource) => resource.key.stackId === request.stackId, - ); - for (const resource of owned) { - yield* attempt(stopResource(resource)); - yield* attempt(removeResource(resource)); - } - if (cleanupCause.reasons.length > 0) return yield* Effect.failCause(cleanupCause); - }), - ); - - const wipePersistentData = ( - key: RuntimeWorkloadKey, - ): Effect.Effect => - key.workloadId === "database:database" && options.wipeDatabaseData !== undefined - ? options.wipeDatabaseData - : Effect.void; - - return { - observe, - start, - stop, - remove, - cleanup: cleanupRuntime, - wipePersistentData, - } satisfies RuntimeDriver; - }); diff --git a/packages/stack/src/runtime/PodmanEngine.ts b/packages/stack/src/runtime/PodmanEngine.ts deleted file mode 100644 index 55e03591cd..0000000000 --- a/packages/stack/src/runtime/PodmanEngine.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { - makeContainerEngineCodecs, - makeContainerEngineCore, - CONTAINER_LABEL_KEYS, - serializeCommonContainerCommand, - type ContainerCommand, - type ContainerEngine, - type ContainerEngineOptions, - type ContainerProcessRequest, -} from "./ContainerEngine.ts"; - -const templateLabel = (label: string): string => `{{.Label "${label}"}}`; -const templateMapLabel = (label: string): string => `{{index .Labels "${label}"}}`; - -// Podman has a separate documented Go-template contract. Delimited scalar -// fields avoid relying on its version-specific JSON row shape. -const containerFormat = [ - "{{.ID}}", - "{{.Names}}", - templateLabel(CONTAINER_LABEL_KEYS.stackId), - templateLabel(CONTAINER_LABEL_KEYS.ownerSessionId), - templateLabel(CONTAINER_LABEL_KEYS.workloadId), - templateLabel(CONTAINER_LABEL_KEYS.startup), - templateLabel(CONTAINER_LABEL_KEYS.role), - "{{.State}}", -].join("\\t"); -const networkFormat = [ - "{{.ID}}", - "{{.Name}}", - templateMapLabel(CONTAINER_LABEL_KEYS.stackId), - templateMapLabel(CONTAINER_LABEL_KEYS.ownerSessionId), - templateMapLabel(CONTAINER_LABEL_KEYS.role), -].join("\\t"); -const volumeFormat = [ - "{{.Name}}", - templateMapLabel(CONTAINER_LABEL_KEYS.stackId), - templateMapLabel(CONTAINER_LABEL_KEYS.workloadId), - templateMapLabel(CONTAINER_LABEL_KEYS.role), -].join("\\t"); - -export const serializePodmanCommand = (command: ContainerCommand): ContainerProcessRequest => { - switch (command.operation) { - case "probe": - return { args: ["version", "--format", "{{.Version}}"] }; - case "inspect-image": - return { - args: ["image", "ls", command.image, "--format", "{{.ID}}"], - }; - case "pull-image": - return { args: ["pull", command.image] }; - case "inspect-containers": - return { - args: [ - "ps", - "--all", - "--filter", - `label=${CONTAINER_LABEL_KEYS.stackId}=${command.stackId}`, - "--format", - containerFormat, - ], - }; - case "inspect-networks": - return { - args: [ - "network", - "ls", - "--filter", - `label=${CONTAINER_LABEL_KEYS.stackId}=${command.stackId}`, - "--format", - networkFormat, - ], - }; - case "inspect-network-gateway": - return { args: ["network", "inspect", command.id, "--format", "{{json .Subnets}}"] }; - case "inspect-volumes": - return { - args: [ - "volume", - "ls", - "--filter", - `label=${CONTAINER_LABEL_KEYS.stackId}=${command.stackId}`, - "--format", - volumeFormat, - ], - }; - default: - return serializeCommonContainerCommand(command); - } -}; - -export const makePodmanEngine = ( - options: Omit, -): ContainerEngine => - makeContainerEngineCore({ - ...options, - kind: "podman", - codecs: makeContainerEngineCodecs({ - engineName: "Podman", - scalarFormat: "raw", - serialize: serializePodmanCommand, - }), - }); diff --git a/packages/stack/src/runtime/PostgresDatabaseSession.ts b/packages/stack/src/runtime/PostgresDatabaseSession.ts index c5cf9a4a68..62222d3398 100644 --- a/packages/stack/src/runtime/PostgresDatabaseSession.ts +++ b/packages/stack/src/runtime/PostgresDatabaseSession.ts @@ -1,28 +1,14 @@ -import { PgClient } from "@effect/sql-pg"; -import { Context, Duration, Effect, Layer, Predicate, Redacted, Schema, Scope } from "effect"; +import { Effect, Predicate, Redacted, Schema, Scope } from "effect"; import { isSqlError, type SqlError } from "effect/unstable/sql/SqlError"; import { DatabaseBootstrapError, INTERNAL_DATABASE, INTERNAL_SCHEMAS, JWT_SECRET_SETTING, - type DatabaseBootstrapOptions, type DatabaseSession, type DatabaseSqlValue, type DatabaseTransaction, - runDatabaseBootstrap, -} from "../model/DatabaseBootstrap.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { StackPreparationError } from "../public/Errors.ts"; -import { databaseBootstrapPlan } from "./DatabaseBootstrapCatalog.ts"; - -interface PostgresDatabaseSessionOptions { - readonly host: string; - readonly port: number; - readonly password: Redacted.Redacted; - readonly database?: string; - readonly connectTimeout?: Duration.Input; -} +} from "./DatabaseBootstrap.ts"; export interface DatabaseSqlClient { readonly unsafe: ( @@ -67,23 +53,27 @@ export interface PostgresDatabaseSession extends DatabaseSession { export const makeDatabaseSessionFromSqlClient = ( client: DatabaseSqlClient, ): PostgresDatabaseSession => { - const executeWith = ( + const executeWith = Effect.fn("PostgresDatabaseSession.execute")(function* ( sql: string, parameters: ReadonlyArray = [], - ): Effect.Effect => - client.unsafe(sql, parameters).pipe(Effect.asVoid, Effect.mapError(mapSqlError("statement"))); + ) { + return yield* client + .unsafe(sql, parameters) + .pipe(Effect.asVoid, Effect.mapError(mapSqlError("statement"))); + }); - const queryWith = ( + const queryWith = Effect.fn("PostgresDatabaseSession.query")(function* ( sql: string, parameters: ReadonlyArray = [], - ): Effect.Effect>, DatabaseBootstrapError> => - client.unsafe(sql, parameters).pipe(Effect.mapError(mapSqlError("query"))); + ) { + return yield* client.unsafe(sql, parameters).pipe(Effect.mapError(mapSqlError("query"))); + }); - const generated = ( + const generated = Effect.fn("PostgresDatabaseSession.generated")(function* ( format: string, parameters: ReadonlyArray, - ): Effect.Effect => - Effect.gen(function* () { + ) { + yield* Effect.gen(function* () { const rows = yield* client .unsafe(format, parameters) .pipe(Effect.mapError(mapSqlError("statement preparation"))); @@ -93,6 +83,7 @@ export const makeDatabaseSessionFromSqlClient = ( ); yield* executeWith(decoded.statement); }); + }); const transaction: DatabaseTransaction = { execute: executeWith, @@ -127,118 +118,37 @@ export const makeDatabaseSessionFromSqlClient = ( }; /** Acquires a SQL client in the caller's scope and keeps it live with the session. */ -export const makeDatabaseSessionFromAcquisition = ( - acquire: Effect.Effect, -): Effect.Effect => - acquire.pipe( +export const makeDatabaseSessionFromAcquisition = Effect.fn( + "PostgresDatabaseSession.fromAcquisition", +)(function* (acquire: Effect.Effect) { + return yield* acquire.pipe( Effect.map(makeDatabaseSessionFromSqlClient), Effect.mapError(mapSqlError("connection")), ); - -/** Creates a scoped production session backed by the Effect PostgreSQL pool. */ -const makePostgresDatabaseSession = ( - options: PostgresDatabaseSessionOptions, -): Effect.Effect => - makeDatabaseSessionFromAcquisition( - Effect.gen(function* () { - const services = yield* Layer.build( - PgClient.layer({ - host: options.host, - port: options.port, - database: options.database ?? "postgres", - // The canonical Supabase migration demotes the bootstrap-created - // `postgres` role. Keep the session on the init superuser so role - // and database-setting reconciliation remains authorized. - username: "supabase_admin", - password: options.password, - connectTimeout: options.connectTimeout ?? "5 seconds", - }), - ); - return Context.get(services, PgClient.PgClient); - }), - ); +}); /** * Ensures the private database and service-owned schemas exist before any * dependent workload is started. Database creation happens outside a transaction because * PostgreSQL does not support CREATE DATABASE there. */ -export const ensureInternalDatabase = ( - postgres: PostgresDatabaseSession, - openInternal: Effect.Effect, -): Effect.Effect => - Effect.gen(function* () { - const databases = yield* postgres.query("SELECT 1 FROM pg_database WHERE datname = $1", [ - INTERNAL_DATABASE, - ]); - if (databases.length === 0) - yield* postgres.execute(`CREATE DATABASE ${INTERNAL_DATABASE} WITH OWNER postgres`); - - const internal = yield* openInternal; - for (const schema of INTERNAL_SCHEMAS) { - yield* internal.execute(`CREATE SCHEMA IF NOT EXISTS ${schema} AUTHORIZATION postgres`); - yield* internal.execute(`ALTER SCHEMA ${schema} OWNER TO postgres`); - } - }); - -/** Runs the initial bootstrap through the durable loopback database endpoint. */ -export const bootstrapDatabaseAt = ( - state: PersistedStackState, -): Effect.Effect => - Effect.gen(function* () { - const plan = yield* databaseBootstrapPlan(state); - const port = state.privatePorts.find( - (assignment) => - assignment.workloadId === "database:database" && assignment.binding === "primary", - )?.port; - if (port === undefined) - return yield* new StackPreparationError({ - message: "A persisted database private port is required for bootstrap", - }); - yield* Effect.scoped( - Effect.gen(function* () { - const session = yield* makePostgresDatabaseSession({ - host: "127.0.0.1", - port, - password: plan.databasePassword, - }); - yield* ensureInternalDatabase( - session, - makePostgresDatabaseSession({ - host: "127.0.0.1", - port, - database: INTERNAL_DATABASE, - password: plan.databasePassword, - }), - ); - yield* runDatabaseBootstrap(session, plan); - }), - ); - }); - -/** Reconciles roles, JWT settings, and `_supabase` against an already-ready Postgres. */ -export const bootstrapManagedPostgres = ( - options: DatabaseBootstrapOptions & { - readonly host: string; - readonly port: number; +export const ensureInternalDatabase = Effect.fn("PostgresDatabaseSession.ensureInternalDatabase")( + function* ( + postgres: PostgresDatabaseSession, + openInternal: Effect.Effect, + ) { + return yield* Effect.gen(function* () { + const databases = yield* postgres.query("SELECT 1 FROM pg_database WHERE datname = $1", [ + INTERNAL_DATABASE, + ]); + if (databases.length === 0) + yield* postgres.execute(`CREATE DATABASE ${INTERNAL_DATABASE} WITH OWNER postgres`); + + const internal = yield* openInternal; + for (const schema of INTERNAL_SCHEMAS) { + yield* internal.execute(`CREATE SCHEMA IF NOT EXISTS ${schema} AUTHORIZATION postgres`); + yield* internal.execute(`ALTER SCHEMA ${schema} OWNER TO postgres`); + } + }); }, -): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const session = yield* makePostgresDatabaseSession({ - host: options.host, - port: options.port, - password: options.databasePassword, - }); - yield* ensureInternalDatabase( - session, - makePostgresDatabaseSession({ - host: options.host, - port: options.port, - database: INTERNAL_DATABASE, - password: options.databasePassword, - }), - ); - yield* runDatabaseBootstrap(session, options); - }), - ); +); diff --git a/packages/stack/src/runtime/ProductionRuntime.ts b/packages/stack/src/runtime/ProductionRuntime.ts deleted file mode 100644 index a1c4551fdb..0000000000 --- a/packages/stack/src/runtime/ProductionRuntime.ts +++ /dev/null @@ -1,1131 +0,0 @@ -import { NodeHttpClient } from "@effect/platform-node"; -import { HttpClient } from "effect/unstable/http"; -import { - Cause, - Context, - Crypto, - Duration, - Effect, - Exit, - FileSystem, - Fiber, - Path, - Ref, - Result, - Scope, - Schedule, - Semaphore, -} from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import { type PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { rebuildExecutionPlan, type StackDefinition } from "../model/Compiler.ts"; -import { - makeFunctionsBootstrapOwner, - type FunctionsBootstrapOwner, -} from "../functions/FunctionsBootstrap.ts"; -import type { StackStateStore } from "../state/StackStateStore.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { redactKnownSecrets } from "../state/SecretStore.ts"; -import { privateBindingKey, type PersistedStackState } from "../state/StackState.ts"; -import type { PersistedSecretValues } from "../state/StackState.ts"; -import type { StackId } from "../public/StackId.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import type { ArtifactPreparationStatus } from "../public/Status.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import { - GatewayActivationError, - StackPreparationError, - ContainerEngineError, - PortUnavailableError, - StackRuntimeMismatchError, - StackStateInvalidError, - type StackError, -} from "../public/Errors.ts"; -import { makeSupervisorIngress, type SupervisorIngress } from "../supervisor/Ingress.ts"; -import { checkHostPort } from "../supervisor/HostListener.ts"; -import { - LogStoreError, - makeLogStore, - type LogStore, - type LogRecord, -} from "../supervisor/LogStore.ts"; -import type { LifecycleInput } from "../supervisor/Lifecycle.ts"; -import type { SupervisorRuntime } from "../supervisor/Supervisor.ts"; -import { - makeProductionRuntimeArtifactPreparer, - type RuntimeArtifactPreparationProgress, - type PreparedWorkloadArtifact, - type RuntimeArtifactPreparer, -} from "../preparation/RuntimeArtifacts.ts"; -import { makeRuntimeEnvFileOwner, type RuntimeEnvFileOwner } from "./RuntimeEnvFile.ts"; -import { - makeRuntimeInputOwner, - type RuntimeInputOwner, - type RuntimeJsonFetcher, -} from "./RuntimeInputOwner.ts"; -import type { NativeProcessSpec } from "./NativeProcess.ts"; -import { - resolveContainerResolutionFor, - privateBindingIntentsFor, - runtimeSpecFor, - validatePrivateAssignments, - validateWorkloadRuntimeInputs, - type WorkloadRuntimeInputs, -} from "./WorkloadRuntimeSpec.ts"; -import { makeNativeRuntime } from "./NativeRuntime.ts"; -import { makeContainerRuntime, type ContainerWorkloadResolution } from "./ContainerRuntime.ts"; -import { DEFAULT_READINESS_DEADLINE, probeReadiness } from "./ReadinessProbe.ts"; -import { parseGoDuration } from "../model/capabilities/database.ts"; -import type { - ContainerEngine, - ContainerEngineKind, - ContainerHostRoute, -} from "./ContainerEngine.ts"; -import { resolveContainerEngine } from "./ContainerEngineResolver.ts"; -import { bootstrapDatabaseAt } from "./PostgresDatabaseSession.ts"; -import { DatabaseBootstrapError } from "../model/DatabaseBootstrap.ts"; -import { validateMaterializedSecrets } from "../state/MaterializedSettings.ts"; -import { - RuntimeDriverError, - type RuntimeDriver, - type RuntimeWorkloadKey, -} from "./RuntimeDriver.ts"; - -type RuntimeContext = FileSystem.FileSystem | Path.Path | Crypto.Crypto; - -export interface ProductionRuntimeOptions { - readonly stateRoot: string; - readonly artifactCacheRoot?: string; - readonly stackId: StackId; - readonly ownerSessionId: string; - readonly stateStore: StackStateStore; - readonly context: Context.Context; - readonly artifactPreparer?: RuntimeArtifactPreparer; - readonly containerEngine?: ContainerEngine; - readonly ingress?: SupervisorIngress; - readonly logStore?: LogStore; - readonly envFileOwner?: RuntimeEnvFileOwner; - readonly functionsBootstrapOwner?: FunctionsBootstrapOwner; - readonly fetchJson?: RuntimeJsonFetcher; - readonly bootstrapDatabase?: ( - state: PersistedStackState, - ) => Effect.Effect; -} - -const preparationError = (message: string, cause?: unknown): StackPreparationError => - new StackPreparationError({ message, ...(cause === undefined ? {} : { cause }) }); - -const unavailableLogStore = (error: LogStoreError, path: string): LogStore => ({ - path, - append: () => Effect.fail(error), - read: () => Effect.fail(error), -}); - -const driverError = ( - key: Pick, - message: string, - cause?: unknown, -): RuntimeDriverError => - new RuntimeDriverError({ - message, - stackId: key.stackId, - workloadId: key.workloadId, - ...(cause === undefined ? {} : { cause }), - }); - -const stateSecrets = (state: PersistedStackState): ReadonlyArray => - Object.values(state.secrets) - .map((entry) => entry.value) - .filter((value) => value.length > 0); - -const currentStateReader = (options: ProductionRuntimeOptions) => - options.stateStore.read(options.stackId).pipe( - Effect.provideContext(options.context), - Effect.flatMap((state) => - state === undefined - ? Effect.fail( - new StackStateInvalidError({ - message: "Persisted stack state is missing", - stackId: options.stackId, - }), - ) - : Effect.succeed(state), - ), - ); - -const artifactKey = (runtime: StackRuntime, workload: PlannedWorkload): string => - `${runtime.kind}:${runtime.kind === "container" ? runtime.engine : ""}:${workload.id}:${workload.selected.kind === "native" ? workload.selected.release : workload.selected.image}`; - -const runtimeMatches = (left: StackRuntime, right: StackRuntime): boolean => { - if (left.kind !== right.kind) return false; - if (left.kind === "native" || right.kind === "native") return true; - return left.engine === right.engine; -}; - -const urlHost = (host: string): string => { - const normalized = host === "0.0.0.0" ? "127.0.0.1" : host === "::" ? "::1" : host; - return normalized.includes(":") && !normalized.startsWith("[") ? `[${normalized}]` : normalized; -}; - -const DATABASE_WORKLOAD_ID = "database:database"; -// Native cold starts can spend more than 30 seconds loading shared libraries before serving. -const NATIVE_READINESS_DEADLINE = Duration.minutes(2); -const checkNativeDatabaseLockEvidence = ( - fileSystem: FileSystem.FileSystem, - lockPath: string, -): Effect.Effect => - Effect.gen(function* () { - if ( - !(yield* fileSystem - .exists(lockPath) - .pipe( - Effect.mapError((error) => - preparationError("Unable to inspect native database lock evidence", error), - ), - )) - ) - return; - const firstLine = (yield* fileSystem - .readFileString(lockPath) - .pipe( - Effect.mapError((error) => - preparationError("Unable to read native database lock evidence", error), - ), - )) - .split("\n", 1)[0] - ?.trim(); - if (firstLine === undefined || !/^[1-9]\d*$/u.test(firstLine)) - return yield* preparationError("Native database lock evidence contains an invalid PID"); - const pid = Number(firstLine); - if (!Number.isSafeInteger(pid)) - return yield* preparationError("Native database lock evidence contains an invalid PID"); - const probe = yield* Effect.sync( - (): { readonly alive: true } | { readonly alive: false; readonly cause: unknown } => { - try { - process.kill(pid, 0); - return { alive: true }; - } catch (cause) { - return { alive: false, cause }; - } - }, - ); - const alive = probe.alive - ? true - : typeof probe.cause === "object" && - probe.cause !== null && - "code" in probe.cause && - probe.cause.code === "ESRCH" - ? false - : yield* preparationError("Unable to verify native database lock owner", probe.cause); - if (alive) - return yield* preparationError( - `Native database lock is held by live process ${pid}; refusing a clean restart`, - ); - }); -const isDatabaseWorkload = (workload: PlannedWorkload): boolean => - workload.id === DATABASE_WORKLOAD_ID; -const configuredDatabaseReadinessDeadline = ( - definition: StackDefinition | undefined, -): Effect.Effect => { - if (definition === undefined) - return Effect.fail(preparationError("Persisted database definition is missing")); - const configured = definition.capabilities.database.settings.health_timeout; - if (configured === undefined || configured === null) - return Effect.fail(preparationError("Persisted database health_timeout is missing")); - return Effect.try({ - try: () => parseGoDuration(configured), - catch: (cause) => preparationError(`Invalid database health_timeout: ${configured}`, cause), - }).pipe( - Effect.flatMap((duration) => - Duration.isNegative(duration) || Duration.isZero(duration) - ? Effect.fail( - preparationError( - `Invalid database health_timeout: ${configured}; duration must be positive`, - ), - ) - : Effect.succeed(duration), - ), - ); -}; - -export const readinessDeadlineFor = ( - state: PersistedStackState, - workload: PlannedWorkload, -): Effect.Effect => - isDatabaseWorkload(workload) - ? configuredDatabaseReadinessDeadline(state.definition) - : Effect.succeed( - state.runtime.kind === "native" ? NATIVE_READINESS_DEADLINE : DEFAULT_READINESS_DEADLINE, - ); - -const validateDatabaseReadinessBudget = ( - definition: StackDefinition, - workloads: ReadonlyArray, -): Effect.Effect => - workloads.some(isDatabaseWorkload) - ? configuredDatabaseReadinessDeadline(definition).pipe(Effect.asVoid) - : Effect.void; - -const redactEntry = ( - entry: A, - secrets: ReadonlyArray, -): A => { - const message = redactKnownSecrets(entry.message, secrets); - return message === entry.message ? entry : { ...entry, message }; -}; - -/** Redacts against the monotonically growing set of secrets accepted by this owner. */ -const dynamicLogStore = (base: LogStore, knownSecrets: Ref.Ref>): LogStore => { - const secrets = Ref.get(knownSecrets).pipe(Effect.map((values) => [...values])); - return { - path: base.path, - append: (record: LogRecord) => - secrets.pipe( - Effect.flatMap((known) => - base.append({ ...record, message: redactKnownSecrets(record.message, known) }), - ), - ), - read: (options) => - secrets.pipe( - Effect.flatMap((known) => - base - .read(options) - .pipe(Effect.map((entries) => entries.map((entry) => redactEntry(entry, known)))), - ), - ), - }; -}; - -const rememberSecrets = ( - knownSecrets: Ref.Ref>, - secrets: PersistedSecretValues, -): Effect.Effect => - Ref.update(knownSecrets, (known) => { - const next = new Set(known); - for (const entry of Object.values(secrets)) { - if (entry.value.length > 0) next.add(entry.value); - } - return next; - }); - -const readinessFor = ( - state: PersistedStackState, - workload: PlannedWorkload, -): Effect.Effect => { - const spec = runtimeSpecFor(workload); - if (spec === undefined) - return Effect.fail(preparationError(`Unknown runtime specification for ${workload.id}`)); - const endpoint = spec.privateEndpoint(state, spec.readiness.binding, "native"); - if (endpoint === undefined) - return Effect.fail(preparationError(`Missing private readiness assignment for ${workload.id}`)); - return readinessDeadlineFor(state, workload).pipe( - Effect.flatMap((deadline) => - probeReadiness( - { - mode: spec.readiness.protocol, - host: "127.0.0.1", - port: endpoint.port, - ...(spec.readiness.path === undefined ? {} : { path: spec.readiness.path }), - ...(spec.readiness.headers === undefined ? {} : { headers: spec.readiness.headers }), - }, - { deadline }, - ), - ), - ); -}; - -declare const SUPABASE_FUNCTIONS_SERVE_MAIN_TEMPLATE: string | undefined; - -// Release builds inject the already-bundled Edge Runtime entrypoint. The -// source-only fallback keeps local development/tests convenient while keeping -// esbuild out of the shipped supervisor's runtime dependency graph. -const bootstrapContent = - typeof SUPABASE_FUNCTIONS_SERVE_MAIN_TEMPLATE === "string" - ? Effect.succeed(SUPABASE_FUNCTIONS_SERVE_MAIN_TEMPLATE) - : Effect.tryPromise({ - try: () => import("../functions/serve-main-bundler.ts"), - catch: (cause) => preparationError("Unable to bundle functions bootstrap", cause), - }).pipe( - Effect.flatMap(({ bundleServeMainTemplate }) => - bundleServeMainTemplate.pipe( - Effect.mapError((cause) => - preparationError("Unable to bundle functions bootstrap", cause), - ), - ), - ), - ); - -const mapDriverError = ( - key: Pick, - error: unknown, -): RuntimeDriverError => - driverError(key, error instanceof Error ? error.message : "Runtime operation failed", error); - -/** Ensures owner files are attempted even when the selected runtime cleanup fails. */ -export const withOwnedRuntimeFileCleanup = ( - driver: RuntimeDriver, - envFiles: RuntimeEnvFileOwner, - functionsBootstrap: FunctionsBootstrapOwner, - inputOwner?: RuntimeInputOwner, - preparationCleanup?: Effect.Effect, -): RuntimeDriver => { - const cleanupFiles = (stackId: StackId): Effect.Effect => - Effect.gen(function* () { - const key = { stackId, workloadId: "" }; - let cleanupCause: Cause.Cause = Cause.empty; - const attempts: ReadonlyArray> = [ - ...(preparationCleanup === undefined - ? [] - : [ - preparationCleanup.pipe( - Effect.mapError((error) => - driverError(key, "Unable to clean runtime preparation", error), - ), - ), - ]), - envFiles.cleanupAll.pipe( - Effect.mapError((error) => driverError(key, "Unable to clean runtime files", error)), - ), - functionsBootstrap.cleanupAll.pipe( - Effect.mapError((error) => driverError(key, "Unable to clean runtime files", error)), - ), - ...(inputOwner === undefined - ? [] - : [ - inputOwner.cleanupAll.pipe( - Effect.mapError((error) => - driverError(key, "Unable to clean runtime files", error), - ), - ), - ]), - ]; - for (const attempt of attempts) { - const result = yield* Effect.exit(attempt); - if (Exit.isFailure(result)) cleanupCause = Cause.combine(cleanupCause, result.cause); - } - if (cleanupCause.reasons.length > 0) return yield* Effect.failCause(cleanupCause); - }); - return { - ...driver, - cleanup: (request) => - Effect.gen(function* () { - const runtime = yield* Effect.exit(driver.cleanup(request)); - const files = yield* Effect.exit(cleanupFiles(request.stackId)); - if (Exit.isFailure(runtime) && Exit.isFailure(files)) - return yield* Effect.failCause(Cause.combine(runtime.cause, files.cause)); - if (Exit.isFailure(runtime)) return yield* Effect.failCause(runtime.cause); - if (Exit.isFailure(files)) return yield* Effect.failCause(files.cause); - }), - }; -}; - -/** Composes concrete runtime owners around one persisted stack identity. */ -export const makeProductionRuntime = ( - options: ProductionRuntimeOptions, -): Effect.Effect< - SupervisorRuntime, - StackError, - | Scope.Scope - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const state = yield* currentStateReader(options); - const fileSystem = yield* FileSystem.FileSystem; - const pathService = yield* Path.Path; - const paths = yield* resolveStackPaths({ - stateRoot: options.stateRoot, - stackId: options.stackId, - }).pipe( - Effect.mapError((error) => preparationError("Unable to resolve stack runtime paths", error)), - ); - const fetchJson: RuntimeJsonFetcher = - options.fetchJson ?? - ((url) => - Effect.gen(function* () { - const client = yield* HttpClient.HttpClient; - const response = yield* HttpClient.followRedirects(client, 20).get(url); - if (response.status < 200 || response.status >= 300) - return yield* preparationError(`HTTP ${response.status}`); - return yield* response.json; - }).pipe( - Effect.mapError((cause) => preparationError("Unable to fetch Auth OIDC metadata", cause)), - Effect.provide(NodeHttpClient.layerNodeHttp), - )); - const inputOwner = yield* makeRuntimeInputOwner({ - stateRoot: options.stateRoot, - stackId: options.stackId, - fetchJson, - }); - const hostRoute = yield* Ref.make(undefined); - const ingress = - options.ingress ?? - (yield* makeSupervisorIngress({ - stackId: options.stackId, - stateRoot: options.stateRoot, - store: options.stateStore, - context: options.context, - resolveAuthTemplates: inputOwner.resolveAuthTemplates, - resolveInternalApiBindAddress: () => - Ref.get(hostRoute).pipe(Effect.map((route) => route?.bindAddress)), - })); - const envFiles = - options.envFileOwner ?? - (yield* makeRuntimeEnvFileOwner({ stateRoot: options.stateRoot, stackId: options.stackId })); - const functionsBootstrap = - options.functionsBootstrapOwner ?? - (yield* makeFunctionsBootstrapOwner({ - stateRoot: options.stateRoot, - stackId: options.stackId, - })); - const knownSecrets = yield* Ref.make>(new Set(stateSecrets(state))); - const logStoreInitialization = yield* Effect.result( - options.logStore === undefined - ? makeLogStore({ path: paths.logs, knownSecrets: stateSecrets(state) }) - : Effect.succeed(options.logStore), - ); - const logStoreInitializationFailure = Result.isFailure(logStoreInitialization) - ? logStoreInitialization.failure - : undefined; - const baseLogs: LogStore = Result.isSuccess(logStoreInitialization) - ? logStoreInitialization.success - : unavailableLogStore(logStoreInitialization.failure, paths.logs); - const logs = dynamicLogStore(baseLogs, knownSecrets); - const selectedEngine = state.runtime.kind === "container" ? state.runtime.engine : undefined; - const containerEngine = - options.containerEngine ?? - (selectedEngine === undefined - ? undefined - : yield* resolveContainerEngine(selectedEngine).pipe( - Effect.mapError((error) => - containerEngineError( - selectedEngine, - `Unable to configure ${selectedEngine} artifact engine`, - error, - ), - ), - )); - const preparer = - options.artifactPreparer ?? - (yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: options.stateRoot, - artifactCacheRoot: options.artifactCacheRoot, - runtime: state.runtime, - ...(containerEngine === undefined ? {} : { containerEngine }), - })); - const serveTemplate = yield* Effect.cached(bootstrapContent); - const bootstrapDatabase = - options.bootstrapDatabase ?? ((state: PersistedStackState) => bootstrapDatabaseAt(state)); - - const artifacts = new Map(); - const preparationStatuses = new Map(); - const recordPreparationProgress = (progress: RuntimeArtifactPreparationProgress): void => { - preparationStatuses.set(progress.workloadId, { - workloadId: progress.workloadId, - capability: progress.capability, - state: progress.state, - ...(progress.error === undefined ? {} : { error: progress.error }), - }); - }; - const queuePreparation = (workload: PlannedWorkload): void => { - if (artifacts.has(artifactKey(state.runtime, workload))) return; - const current = preparationStatuses.get(workload.id); - if (current?.state === "preparing" || current?.state === "downloading") return; - recordPreparationProgress({ - workloadId: workload.id, - capability: workload.capability, - state: "queued", - }); - }; - const preparationGate = yield* Semaphore.make(1); - const parentScope = yield* Scope.Scope; - let preparationScope: Scope.Scope | undefined; - const preparationInFlight = new Map< - string, - Fiber.Fiber - >(); - yield* Scope.addFinalizer( - parentScope, - Effect.suspend(() => - preparationScope === undefined ? Effect.void : Scope.close(preparationScope, Exit.void), - ), - ); - // Runtime input materialization writes shared files and populates completed caches. Keep - // that short preparation boundary serialized while allowing the actual workloads to start - // concurrently after their inputs are ready. - const runtimeInputGate = yield* Semaphore.make(1); - const freshState = (key: Pick) => - currentStateReader(options).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - Effect.flatMap((fresh) => - runtimeMatches(fresh.runtime, state.runtime) - ? Effect.succeed(fresh) - : Effect.fail(driverError(key, "Persisted runtime changed while owner was active")), - ), - ); - const prepareOne = (runtime: StackRuntime, workload: PlannedWorkload) => { - return Effect.suspend(() => { - const key = artifactKey(runtime, workload); - const cached = artifacts.get(key); - return cached === undefined - ? Effect.sync(() => - recordPreparationProgress({ - workloadId: workload.id, - capability: workload.capability, - state: "preparing", - }), - ).pipe( - Effect.andThen(preparer.prepare(runtime, workload, recordPreparationProgress)), - Effect.tap((prepared) => - Effect.sync(() => { - artifacts.set(key, prepared); - recordPreparationProgress({ - workloadId: workload.id, - capability: workload.capability, - state: "ready", - }); - }), - ), - Effect.tapError((error) => - Effect.sync(() => - recordPreparationProgress({ - workloadId: workload.id, - capability: workload.capability, - state: "failed", - error: error.message, - }), - ), - ), - ) - : Effect.succeed(cached); - }); - }; - const prepare = (runtime: StackRuntime, workload: PlannedWorkload) => - Effect.gen(function* () { - const key = artifactKey(runtime, workload); - const joined = yield* Effect.uninterruptible( - preparationGate.withPermit( - Effect.gen(function* () { - const existing = preparationInFlight.get(key); - if (existing !== undefined) return existing; - const scope = preparationScope ?? (preparationScope = yield* Scope.make("parallel")); - let fiber: Fiber.Fiber | undefined; - const owner = prepareOne(runtime, workload).pipe( - Effect.ensuring( - Effect.sync(() => { - if (preparationInFlight.get(key) === fiber) preparationInFlight.delete(key); - }), - ), - ); - fiber = yield* Effect.forkIn(owner, scope, { startImmediately: false }); - preparationInFlight.set(key, fiber); - return fiber; - }), - ), - ); - return yield* Fiber.join(joined); - }); - const prepareArtifacts = (runtime: StackRuntime, workloads: ReadonlyArray) => - Effect.forEach(workloads, (workload) => prepare(runtime, workload), { - concurrency: "unbounded", - }); - const prepareFor = ( - input: LifecycleInput, - selected: ReadonlySet, - ): Effect.Effect => - Effect.gen(function* () { - const workloads = input.plan.workloads.filter((workload) => - selected.has(workload.capability), - ); - for (const workload of workloads) queuePreparation(workload); - yield* prepareArtifacts(input.state.runtime, workloads); - }); - const logPreparationFailure = (message: string): Effect.Effect => - logs.append({ source: "supervisor", stream: "internal", message }).pipe(Effect.ignore); - const prefetch = (persisted: PersistedStackState): Effect.Effect => - Effect.gen(function* () { - if (persisted.definition === undefined || persisted.definition.preparation === "on-demand") - return; - const plan = yield* rebuildExecutionPlan(persisted.runtime, persisted.definition).pipe( - Effect.mapError((error) => - preparationError("Unable to plan background preparation", error), - ), - ); - const workloads = plan.workloads.filter( - (workload) => - persisted.definition?.capabilities[workload.capability].activation === "lazy" && - !artifacts.has(artifactKey(persisted.runtime, workload)), - ); - for (const workload of workloads) queuePreparation(workload); - yield* Effect.forEach( - workloads, - (workload) => - prepare(persisted.runtime, workload).pipe( - Effect.catch((error) => - logPreparationFailure( - `Background preparation failed for ${workload.id}: ${error.message}`, - ), - ), - ), - { concurrency: "unbounded", discard: true }, - ); - }).pipe( - Effect.catch((error) => - logPreparationFailure( - `Background preparation failed: ${error instanceof Error ? error.message : String(error)}`, - ), - ), - ); - const cleanupPreparation: Effect.Effect = Effect.gen(function* () { - const scope = yield* preparationGate.withPermit( - Effect.sync(() => { - const current = preparationScope; - preparationScope = undefined; - preparationInFlight.clear(); - return current; - }), - ); - if (scope !== undefined) yield* Scope.close(scope, Exit.interrupt()); - yield* Effect.sync(() => { - preparationInFlight.clear(); - preparationStatuses.clear(); - artifacts.clear(); - }); - }).pipe(Effect.uninterruptible); - const functionsPath = (): Effect.Effect => - serveTemplate.pipe(Effect.flatMap((content) => functionsBootstrap.write({ content }))); - const runtimeInputs = ( - workload: PlannedWorkload, - fresh: PersistedStackState, - host: ContainerHostRoute | undefined, - ): Effect.Effect => - runtimeInputGate.withPermit( - Effect.gen(function* () { - const material = yield* inputOwner.resolve(fresh, workload.id); - const templates = material.auth?.templates; - const apiListener = fresh.definition?.listeners.api; - const apiAssignment = fresh.ports.find((assignment) => assignment.field === "api"); - const templateBaseUrl = - workload.id !== "auth:auth" || templates === undefined || templates.length === 0 - ? undefined - : apiListener?.enabled !== true || apiAssignment === undefined - ? yield* preparationError( - "Configured Auth email templates require a public API listener", - ) - : `http://${urlHost(host?.host ?? apiListener.address)}:${apiAssignment.port}`; - const auth = - material.auth === undefined - ? undefined - : { - ...material.auth, - ...(templateBaseUrl === undefined ? {} : { templateBaseUrl }), - }; - const functions = - workload.id === "functions:edge-runtime" - ? { - bootstrapPath: yield* functionsPath(), - ...(material.functions?.secrets === undefined - ? {} - : { secrets: material.functions.secrets }), - } - : undefined; - return { - ...(auth === undefined ? {} : { auth }), - ...(workload.id.startsWith("analytics:") && material.analytics !== undefined - ? { analytics: material.analytics } - : {}), - database: { dataPath: pathService.join(paths.data, "database") }, - storage: { dataPath: pathService.join(paths.data, "storage") }, - ...(functions === undefined ? {} : { functions }), - ...(host === undefined ? {} : { hostRoute: host }), - }; - }), - ); - - const preflight = (input: LifecycleInput): Effect.Effect => - Effect.gen(function* () { - if (logStoreInitializationFailure !== undefined) - return yield* preparationError( - `Unable to open stack logs: ${logStoreInitializationFailure.message}`, - logStoreInitializationFailure, - ); - if (!runtimeMatches(input.state.runtime, state.runtime)) - return yield* new StackRuntimeMismatchError({ - message: "Lifecycle runtime does not match persisted runtime", - }); - // Remember candidate secrets before any runtime preflight work can emit logs. - yield* rememberSecrets(knownSecrets, input.state.secrets); - yield* rememberSecrets(knownSecrets, input.secrets); - // Eagerly validate the candidate before any engine/artifact work. Start-time checks - // below revalidate the fresh persisted definition because it is runtime authority. - yield* validateDatabaseReadinessBudget(input.definition, input.plan.workloads); - const candidateState: PersistedStackState = { - ...input.state, - definition: input.definition, - secrets: input.secrets, - }; - yield* Effect.forEach(input.plan.workloads, (workload) => - validateMaterializedSecrets(candidateState, workload.capability), - ); - yield* Effect.forEach(input.plan.workloads, (workload) => - Effect.gen(function* () { - if (runtimeSpecFor(workload) === undefined) - return yield* preparationError(`Unknown runtime specification for ${workload.id}`); - if (workload.selected.kind !== input.state.runtime.kind) - return yield* preparationError(`Runtime artifact mismatch for ${workload.id}`); - }), - ); - if (input.state.runtime.kind === "container") { - const engineKind = input.state.runtime.engine; - if (containerEngine === undefined || containerEngine.kind !== engineKind) - return yield* containerEngineError( - engineKind, - "Selected container engine is unavailable", - ); - const route = yield* containerEngine.preflight.pipe( - Effect.mapError((error) => - containerEngineError(engineKind, "Container host route preflight failed", error), - ), - ); - yield* Ref.set(hostRoute, route); - } - if (input.state.runtime.kind === "native") { - const usableListeners = new Set(input.plan.routes.map(({ listener }) => listener)); - for (const assignment of input.state.ports) { - const listener = input.definition.listeners[assignment.field]; - if ( - !usableListeners.has(assignment.field) || - !listener.enabled || - (listener.port === "automatic" - ? assignment.intent !== "automatic" - : listener.port !== assignment.port) - ) - continue; - yield* checkHostPort(listener.address, assignment.port, assignment.field).pipe( - Effect.mapError( - (error) => - new PortUnavailableError({ - field: assignment.field, - port: assignment.port, - message: `Persisted ${assignment.field} port is unavailable`, - cause: error, - }), - ), - ); - } - const requestedPrivate = new Set( - privateBindingIntentsFor(input.plan, candidateState).map(privateBindingKey), - ); - for (const assignment of input.state.privatePorts) { - if (!requestedPrivate.has(privateBindingKey(assignment))) continue; - yield* checkHostPort( - "127.0.0.1", - assignment.port, - `${assignment.workloadId}:${assignment.binding}`, - ); - } - yield* checkNativeDatabaseLockEvidence( - fileSystem, - pathService.join(paths.data, "database", "postmaster.pid"), - ); - } - }); - - const activate = ( - capability: CapabilityName, - input: LifecycleInput, - ): Effect.Effect< - { readonly host: string; readonly port: number }, - GatewayActivationError | StackError - > => - Effect.gen(function* () { - const workload = input.plan.workloads.find( - (entry) => entry.capability === capability && entry.readiness.portField !== undefined, - ); - if (workload === undefined) - return yield* new GatewayActivationError({ - message: `Capability ${capability} has no workload`, - }); - const fresh = yield* freshState({ - stackId: options.stackId, - workloadId: workload.id, - }).pipe(Effect.mapError((error) => new GatewayActivationError({ message: error.message }))); - const spec = runtimeSpecFor(workload); - if (spec === undefined) - return yield* new GatewayActivationError({ - message: `Unknown runtime specification for ${workload.id}`, - }); - yield* validatePrivateAssignments(fresh, workload).pipe( - Effect.mapError((error) => new GatewayActivationError({ message: error.message })), - ); - const endpoint = spec.privateEndpoint(fresh, spec.readiness.binding, "native"); - if (endpoint === undefined) - return yield* new GatewayActivationError({ - message: `Missing private endpoint for ${workload.id}`, - }); - return endpoint; - }); - - const waitForReadiness = (key: RuntimeWorkloadKey, workload: PlannedWorkload) => - freshState(key).pipe( - Effect.flatMap((fresh) => readinessFor(fresh, workload)), - Effect.mapError((error) => - driverError(key, `Readiness check failed for ${workload.id}: ${error.message}`, error), - ), - ); - const bootstrapWorkloadDatabase = (key: RuntimeWorkloadKey, workload: PlannedWorkload) => - workload.bootstrap === "database" - ? freshState(key).pipe( - Effect.flatMap((fresh) => - readinessDeadlineFor(fresh, workload).pipe( - Effect.flatMap((deadline) => - Effect.timeout( - Effect.retry( - Effect.suspend(() => bootstrapDatabase(fresh)), - { - schedule: Schedule.spaced("100 millis"), - while: (error) => - error instanceof DatabaseBootstrapError && error.retryable === true, - }, - ), - deadline, - ), - ), - ), - ), - Effect.mapError((error) => mapDriverError(key, error)), - ) - : Effect.void; - - let driver: RuntimeDriver; - if (state.runtime.kind === "native") { - driver = yield* makeNativeRuntime({ - resolveProcess: (key, workload) => - freshState(key).pipe( - Effect.flatMap((fresh) => - Effect.gen(function* () { - const spec = runtimeSpecFor(workload); - if (spec === undefined) - return yield* driverError( - key, - `Unknown runtime specification for ${workload.id}`, - ); - // Revalidate the fresh persisted definition before spawning; preflight checks - // the candidate definition, while this state is the runtime authority. - const inputs = yield* runtimeInputs(workload, fresh, undefined).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - yield* validateWorkloadRuntimeInputs(fresh, workload, inputs).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - yield* validatePrivateAssignments(fresh, workload).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - const prepared = yield* prepare(fresh.runtime, workload).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - if (prepared.artifactRoot === undefined) - return yield* driverError( - key, - `Native artifact root is unavailable for ${workload.id}`, - ); - const endpoint = spec.privateEndpoint(fresh, spec.readiness.binding, "native"); - if (endpoint === undefined) - return yield* driverError( - key, - `Missing private port assignment for ${workload.id}`, - ); - const resolvedNativeProcess = spec.nativeProcess( - prepared.artifactRoot, - fresh, - workload, - endpoint.port, - inputs, - ); - const environment = spec.env(fresh, workload, endpoint.port, "native", inputs); - return { - startup: spec - .nativeStartupProcesses( - prepared.artifactRoot, - fresh, - workload, - endpoint.port, - inputs, - ) - .map((startup: NativeProcessSpec) => ({ - ...startup, - timeout: startup.timeout ?? Duration.minutes(5), - env: { ...environment, ...startup.env }, - })), - main: { ...resolvedNativeProcess, env: environment }, - }; - }), - ), - ), - waitForReadiness, - bootstrapDatabase: bootstrapWorkloadDatabase, - logStore: logs, - knownSecrets: Ref.get(knownSecrets).pipe(Effect.map((values) => [...values])), - wipeDatabaseData: Effect.gen(function* () { - const dataPath = pathService.join(paths.data, "database"); - const key = { stackId: options.stackId, workloadId: "database:database" }; - const exists = yield* fileSystem.exists(dataPath).pipe(Effect.orElseSucceed(() => false)); - if (exists) - yield* fileSystem - .remove(dataPath, { recursive: true }) - .pipe( - Effect.mapError((error) => - driverError(key, "Unable to wipe native database data", error), - ), - ); - yield* fileSystem - .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) - .pipe( - Effect.mapError((error) => - driverError(key, "Unable to recreate native database data directory", error), - ), - ); - }), - }).pipe( - Effect.mapError((error) => preparationError("Unable to initialize native runtime", error)), - ); - } else { - if (containerEngine === undefined || containerEngine.kind !== state.runtime.engine) - return yield* containerEngineError( - state.runtime.engine, - "Selected container engine is unavailable", - ); - driver = yield* makeContainerRuntime({ - engine: containerEngine, - ownerSessionId: options.ownerSessionId, - resolveWorkload: (key, workload) => - freshState(key).pipe( - Effect.flatMap((fresh) => - Effect.gen(function* () { - const spec = runtimeSpecFor(workload); - if (spec === undefined) - return yield* driverError( - key, - `Unknown runtime specification for ${workload.id}`, - ); - yield* prepare(fresh.runtime, workload).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - yield* readinessDeadlineFor(fresh, workload).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - let route = yield* Ref.get(hostRoute); - if (route === undefined) { - route = yield* containerEngine.preflight.pipe( - Effect.mapError((error) => - mapDriverError( - key, - containerEngineError( - containerEngine.kind, - "Container host route preflight failed", - error, - ), - ), - ), - ); - yield* Ref.set(hostRoute, route); - } - const inputs = yield* runtimeInputs(workload, fresh, route).pipe( - Effect.mapError((error) => mapDriverError(key, error)), - ); - const resolution = yield* resolveContainerResolutionFor( - fresh, - workload, - inputs, - ).pipe(Effect.mapError((error) => mapDriverError(key, error))); - if (resolution === undefined) - return yield* driverError( - key, - `Unknown container runtime specification for ${workload.id}`, - ); - const envFile = yield* envFiles - .write({ - workloadId: workload.id, - values: resolution.env, - }) - .pipe(Effect.mapError((error) => mapDriverError(key, error))); - const volume = - workload.id === "database:database" - ? { target: "/var/lib/postgresql/data", readOnly: false } - : workload.id === "storage:storage" - ? { - target: "/mnt", - readOnly: false, - ownerWorkloadId: "storage:storage", - } - : workload.id === "storage:imgproxy" - ? { - target: "/mnt", - readOnly: true, - ownerWorkloadId: "storage:storage", - } - : undefined; - const { env: _env, ...withoutEnv } = resolution; - return { - ...withoutEnv, - envFile, - ...(volume === undefined ? {} : { volume }), - } satisfies ContainerWorkloadResolution; - }), - ), - ), - waitForReadiness, - bootstrapDatabase: bootstrapWorkloadDatabase, - onNetworkReady: (network) => { - const resolveGateway = containerEngine.resolveNetworkGateway; - if (resolveGateway === undefined) return Effect.succeed(false); - return resolveGateway(network.id).pipe( - Effect.mapError( - (error) => - new RuntimeDriverError({ - message: "Unable to resolve container network gateway", - stackId: options.stackId, - workloadId: network.name, - cause: error, - }), - ), - Effect.flatMap((gateway) => - Effect.gen(function* () { - const current = yield* Ref.get(hostRoute); - if (current?.host === gateway && current.bindAddress === gateway) return false; - yield* Ref.set(hostRoute, { host: gateway, bindAddress: gateway }); - return true; - }), - ), - ); - }, - logStore: logs, - }); - } - const baseDriver = withOwnedRuntimeFileCleanup( - driver, - envFiles, - functionsBootstrap, - inputOwner, - cleanupPreparation, - ); - return { - driver: baseDriver, - preflight, - prepare: prepareFor, - prefetch, - artifacts: Effect.sync(() => [...preparationStatuses.values()]), - activate, - ingress, - logStore: logs, - } satisfies SupervisorRuntime; - }); - -const containerEngineError = ( - engine: ContainerEngineKind, - message: string, - cause?: unknown, -): ContainerEngineError => - new ContainerEngineError({ engine, message, ...(cause === undefined ? {} : { cause }) }); diff --git a/packages/stack/src/runtime/ReadinessProbe.ts b/packages/stack/src/runtime/ReadinessProbe.ts deleted file mode 100644 index 32bcc7ce0a..0000000000 --- a/packages/stack/src/runtime/ReadinessProbe.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { Duration, Effect, Option, Schedule, Stream } from "effect"; -import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient"; -import { HttpClient } from "effect/unstable/http"; -import { Socket } from "node:net"; -import { RuntimeDriverError } from "./RuntimeDriver.ts"; -import { StackPreparationError } from "../public/Errors.ts"; - -export interface ReadinessTarget { - readonly mode: "http" | "tcp"; - readonly host: string; - readonly port: number; - readonly path?: string; - readonly headers?: Readonly>; -} - -export interface ReadinessProbeOptions { - /** Optional maximum number of retries after the initial attempt. */ - readonly retries?: number; - /** Delay between attempts; defaults to a short spaced schedule. */ - readonly retryDelay?: Duration.Input; - /** Total time allowed for the initial attempt and all retries; zero ignores retries. */ - readonly deadline?: Duration.Input; -} - -/** Default total readiness budget for non-database workloads. */ -export const DEFAULT_READINESS_DEADLINE = Duration.seconds(30); - -const preparationError = (message: string): StackPreparationError => - new StackPreparationError({ message }); - -const runtimeError = ( - target: ReadinessTarget, - message: string, - cause?: unknown, -): RuntimeDriverError => - new RuntimeDriverError({ - message, - target, - cause, - }); - -const containsControl = (value: string): boolean => - [...value].some((character) => { - const code = character.charCodeAt(0); - return code <= 0x1f || code === 0x7f; - }); - -const validHeaderName = (value: string): boolean => - value.length > 0 && /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u.test(value); - -const validateTarget = (target: ReadinessTarget): Effect.Effect => { - if (target.host.length === 0 || containsControl(target.host)) - return Effect.fail(preparationError("Readiness host is invalid")); - if (!Number.isInteger(target.port) || target.port < 1 || target.port > 65_535) - return Effect.fail(preparationError("Readiness port is invalid")); - if (target.mode === "http" && target.path !== undefined) { - if (!target.path.startsWith("/")) - return Effect.fail(preparationError("Readiness HTTP path must begin with '/'")); - if (containsControl(target.path)) - return Effect.fail(preparationError("Readiness HTTP path contains control characters")); - } - if (target.headers !== undefined) { - for (const [name, value] of Object.entries(target.headers)) { - if (!validHeaderName(name)) - return Effect.fail(preparationError("Readiness HTTP header name is invalid")); - if (containsControl(value)) - return Effect.fail(preparationError("Readiness HTTP header value is invalid")); - } - } - return Effect.void; -}; - -const decodeDuration = ( - value: Duration.Input, - name: string, -): Effect.Effect => { - const decoded = Duration.fromInput(value); - if (Option.isNone(decoded)) return Effect.fail(preparationError(`${name} is invalid`)); - if (!Duration.isFinite(decoded.value) || Duration.isNegative(decoded.value)) - return Effect.fail(preparationError(`${name} must be finite and non-negative`)); - return Effect.succeed(decoded.value); -}; - -const httpAttempt = (target: ReadinessTarget) => { - const host = - target.host.includes(":") && !target.host.startsWith("[") ? `[${target.host}]` : target.host; - const url = `http://${host}:${target.port}${target.path ?? "/"}`; - return Effect.gen(function* () { - const client = yield* HttpClient.HttpClient; - const response = yield* client.get(url, { headers: target.headers }); - yield* response.stream.pipe(Stream.runDrain); - if (response.status < 200 || response.status >= 300) - return yield* runtimeError(target, "Readiness HTTP status was not successful"); - }).pipe( - Effect.mapError((cause) => - cause instanceof RuntimeDriverError - ? cause - : runtimeError(target, "Readiness HTTP request failed", cause), - ), - ); -}; - -const tcpAttempt = (target: ReadinessTarget): Effect.Effect => - Effect.callback((resume) => { - let settled = false; - let socket: Socket | undefined; - const cleanup = () => { - socket?.off("connect", onConnect); - socket?.off("error", onError); - socket?.off("close", onClose); - }; - const finish = (effect: Effect.Effect) => { - if (settled) return; - settled = true; - cleanup(); - socket?.destroy(); - resume(effect); - }; - const onConnect = () => finish(Effect.void); - const onError = (cause: Error) => - finish(Effect.fail(runtimeError(target, "Readiness TCP connection failed", cause))); - const onClose = () => { - finish(Effect.fail(runtimeError(target, "Readiness TCP connection closed"))); - }; - socket = new Socket(); - socket.once("connect", onConnect); - socket.once("error", onError); - socket.once("close", onClose); - socket.connect({ host: target.host, port: target.port }); - return Effect.sync(() => { - if (settled) return; - settled = true; - cleanup(); - socket?.destroy(); - }); - }); - -/** - * Probes one exact private endpoint. Retry is expressed through Effect's - * Schedule so cancellation interrupts the owned request/socket immediately. - */ -export const probeReadiness = ( - target: ReadinessTarget, - options: ReadinessProbeOptions = {}, -): Effect.Effect => - Effect.gen(function* () { - yield* validateTarget(target); - const retries = options.retries; - if (retries !== undefined && (!Number.isSafeInteger(retries) || retries < 0)) - return yield* preparationError("Readiness retry count is invalid"); - const retryDelay = yield* decodeDuration( - options.retryDelay ?? Duration.millis(100), - "Readiness retry delay", - ); - const deadline = yield* decodeDuration( - options.deadline ?? DEFAULT_READINESS_DEADLINE, - "Readiness deadline", - ); - const withReadinessBudget = (attempt: Effect.Effect) => { - // A zero deadline runs one probe; a positive deadline interrupts the active request/socket. - if (Duration.isZero(deadline)) return attempt; - const schedule = - retries === undefined - ? Schedule.spaced(retryDelay) - : Schedule.spaced(retryDelay).pipe(Schedule.upTo({ times: retries })); - return Effect.timeoutOrElse(Effect.retry(attempt, schedule), { - duration: deadline, - orElse: () => Effect.fail(runtimeError(target, "Readiness deadline exceeded")), - }); - }; - return yield* target.mode === "http" - ? withReadinessBudget(httpAttempt(target)).pipe(Effect.provide(NodeHttpClient.layerNodeHttp)) - : withReadinessBudget(tcpAttempt(target)); - }); diff --git a/packages/stack/src/runtime/RuntimeDriver.ts b/packages/stack/src/runtime/RuntimeDriver.ts deleted file mode 100644 index 30f4285303..0000000000 --- a/packages/stack/src/runtime/RuntimeDriver.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { Data, Effect } from "effect"; -import type { StackId } from "../public/StackId.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { ReadinessTarget } from "./ReadinessProbe.ts"; - -/** The exact identity used when touching a private runtime resource. */ -export interface RuntimeWorkloadKey { - readonly stackId: StackId; - readonly workloadId: string; -} - -/** Exact stack-wide runtime cleanup requested after lifecycle failure or owner recovery. */ -export interface RuntimeCleanupRequest { - readonly stackId: StackId; - /** Remove persistent runtime volumes in addition to ephemeral containers/networks. */ - readonly destroy: boolean; -} - -/** Runtime states projected to callers and status inspection. */ -type ObservedWorkloadState = "starting" | "ready" | "stopped" | "failed"; - -export interface ObservedWorkload extends RuntimeWorkloadKey { - readonly state: ObservedWorkloadState; - readonly error?: string; -} - -export interface RuntimeDriver { - /** Enumerates only private resources owned by this exact stack identity. */ - readonly observe: ( - stackId: StackId, - ) => Effect.Effect, RuntimeDriverError>; - /** - * Starts one exact resource and returns after its readiness probe succeeds. Callers must use - * stop or cleanup for ownership teardown; caller interruption is not part of this contract. - */ - readonly start: ( - key: RuntimeWorkloadKey, - workload: PlannedWorkload, - ) => Effect.Effect; - /** Stops one exact resource; no other stack may be touched. */ - readonly stop: (key: RuntimeWorkloadKey) => Effect.Effect; - /** Removes one exact resource; no other stack may be touched. */ - readonly remove: (key: RuntimeWorkloadKey) => Effect.Effect; - /** - * Cleans up exact resources belonging to one stack. Ordinary stop retains volumes; destructive - * cleanup removes them after containers and networks have been removed. - */ - readonly cleanup: (request: RuntimeCleanupRequest) => Effect.Effect; - /** - * Wipes persistent data for one stopped and removed workload. Native database data directories - * and container volumes owned by that workload are removed; other stack volumes stay. - */ - readonly wipePersistentData: (key: RuntimeWorkloadKey) => Effect.Effect; -} - -export class RuntimeDriverError extends Data.TaggedError("RuntimeDriverError")<{ - readonly message: string; - readonly stackId?: StackId; - readonly workloadId?: string; - /** Private endpoint used by readiness failures, when applicable. */ - readonly target?: ReadinessTarget; - readonly cause?: unknown; -}> {} diff --git a/packages/stack/src/runtime/RuntimeEnvFile.ts b/packages/stack/src/runtime/RuntimeEnvFile.ts deleted file mode 100644 index a89f25fffd..0000000000 --- a/packages/stack/src/runtime/RuntimeEnvFile.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { Crypto, Effect, FileSystem, Path, PlatformError } from "effect"; -import { StackPreparationError } from "../public/Errors.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import type { StackId } from "../public/StackId.ts"; - -export interface RuntimeEnvFileOwner { - /** Writes one workload file and returns its exact owned path. */ - readonly write: (input: { - readonly workloadId: string; - readonly values: Readonly>; - }) => Effect.Effect; - /** Removes only this owner's env-file directory; safe when already absent. */ - readonly cleanupAll: Effect.Effect; -} - -export interface RuntimeEnvFileOwnerOptions { - readonly stateRoot: string; - readonly stackId: StackId; -} - -const error = (message: string, fields: Readonly> = {}) => - new StackPreparationError({ message, ...fields }); - -const validWorkloadId = (workloadId: string): boolean => - /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/u.test(workloadId); - -const validName = (name: string): boolean => /^[A-Za-z_][A-Za-z0-9_]*$/u.test(name); - -const encodeWorkloadId = (workloadId: string): string => encodeURIComponent(workloadId); - -const mapFile = ( - path: string, - operation: string, - effect: Effect.Effect, -): Effect.Effect => - effect.pipe( - Effect.mapError(() => - error(`Unable to ${operation}`, { - path, - }), - ), - ); - -export const encodeRuntimeEnvFile = ( - values: Readonly>, -): Effect.Effect => { - const entries = Object.entries(values).sort(([left], [right]) => left.localeCompare(right)); - for (const [name, value] of entries) { - if (!validName(name) || name.includes("\u0000") || /[\r\n]/u.test(name)) - return Effect.fail(error("Invalid runtime environment variable name")); - if (value.includes("\u0000") || /[\r\n]/u.test(value)) - return Effect.fail(error("Invalid runtime environment variable value", { name })); - } - return Effect.succeed(entries.map(([name, value]) => `${name}=${value}\n`).join("")); -}; - -/** - * Owns container env files under `/runtime/env`. Native workloads use - * their fd4 environment and never pass through this owner. - */ -export const makeRuntimeEnvFileOwner = ( - options: RuntimeEnvFileOwnerOptions, -): Effect.Effect< - RuntimeEnvFileOwner, - StackPreparationError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const paths = yield* resolveStackPaths(options).pipe( - Effect.mapError((cause) => error("Unable to resolve runtime environment path", { cause })), - ); - const envRoot = path.join(paths.runtime, "env"); - - const write = (input: { - readonly workloadId: string; - readonly values: Readonly>; - }): Effect.Effect => { - if (!validWorkloadId(input.workloadId)) - return Effect.fail(error("Invalid runtime environment workload identity")); - return Effect.gen(function* () { - const text = yield* encodeRuntimeEnvFile(input.values); - const target = path.join(envRoot, `${encodeWorkloadId(input.workloadId)}.env`); - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError(() => error("Unable to allocate runtime environment file name")), - ); - const temporary = `${target}.${token}.tmp`; - return yield* Effect.gen(function* () { - yield* mapFile( - envRoot, - "create runtime environment directory", - fs.makeDirectory(envRoot, { recursive: true, mode: 0o700 }), - ); - yield* mapFile(envRoot, "secure runtime environment directory", fs.chmod(envRoot, 0o700)); - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* mapFile( - temporary, - "create runtime environment file", - fs.open(temporary, { flag: "w", mode: 0o600 }), - ); - yield* mapFile( - temporary, - "write runtime environment file", - file.writeAll(new TextEncoder().encode(text)), - ); - yield* mapFile(temporary, "sync runtime environment file", file.sync); - }), - ); - yield* mapFile(temporary, "secure runtime environment file", fs.chmod(temporary, 0o600)); - yield* mapFile(target, "publish runtime environment file", fs.rename(temporary, target)); - yield* mapFile( - target, - "secure published runtime environment file", - fs.chmod(target, 0o600), - ); - return target; - }).pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - }); - }; - - const cleanupAll = mapFile( - envRoot, - "clean runtime environment files", - fs.remove(envRoot, { recursive: true, force: true }), - ); - return { write, cleanupAll }; - }); diff --git a/packages/stack/src/runtime/RuntimeEnvFile.unit.test.ts b/packages/stack/src/runtime/RuntimeEnvFile.unit.test.ts deleted file mode 100644 index edc85189f4..0000000000 --- a/packages/stack/src/runtime/RuntimeEnvFile.unit.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Option } from "effect"; -import { StackPreparationError } from "../public/Errors.ts"; -import { encodeRuntimeEnvFile } from "./RuntimeEnvFile.ts"; - -describe("encodeRuntimeEnvFile", () => { - it.effect("encodes sorted NAME=value lines", () => - Effect.gen(function* () { - const text = yield* encodeRuntimeEnvFile({ ZETA: "2", ALPHA: "1" }); - expect(text).toBe("ALPHA=1\nZETA=2\n"); - }), - ); - - it.effect("rejects CR/LF in a value", () => - Effect.gen(function* () { - const exit = yield* encodeRuntimeEnvFile({ POSTGRES_PASSWORD: "x\ny" }).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(StackPreparationError); - if (!(error instanceof StackPreparationError)) return; - expect(error.message).toBe("Invalid runtime environment variable value"); - }), - ); - - it.effect("rejects CR/LF in a name", () => - Effect.gen(function* () { - const exit = yield* encodeRuntimeEnvFile({ "FOO\nBAR": "1" }).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(StackPreparationError); - if (!(error instanceof StackPreparationError)) return; - expect(error.message).toBe("Invalid runtime environment variable name"); - }), - ); -}); diff --git a/packages/stack/src/runtime/RuntimeInputOwner.ts b/packages/stack/src/runtime/RuntimeInputOwner.ts deleted file mode 100644 index 8d80ec9a28..0000000000 --- a/packages/stack/src/runtime/RuntimeInputOwner.ts +++ /dev/null @@ -1,560 +0,0 @@ -import { Crypto, Effect, FileSystem, Path, PlatformError, Schema } from "effect"; -import type { PersistedStackState } from "../state/StackState.ts"; -import type { StackId } from "../public/StackId.ts"; -import { StackPreparationError } from "../public/Errors.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { isRecord, settingValue, settingsFor } from "../state/MaterializedSettings.ts"; -import { - base64UrlEncode, - resolveSigningKeyMaterial, - type ResolvedSigningKeyMaterial, -} from "../state/SecretStore.ts"; -import { resolveThirdPartyIssuer } from "../model/capabilities/auth-third-party.ts"; -import { canonicalize } from "../model/Compiler.ts"; - -/** A parsed JSON document fetched by the owner for OIDC discovery. */ -export type RuntimeJsonFetcher = (url: string) => Effect.Effect; - -interface RuntimeAuthTemplate { - /** Stable URL id used by GoTrue's mailer template setting. */ - readonly id: string; - /** User-configured project-relative path. */ - readonly path: string; - /** Canonical host path, retained for live gateway serving. */ - readonly canonicalPath: string; - /** File extension including the dot, when one is present. */ - readonly extension: string; -} - -interface RuntimeInputMaterial { - readonly auth?: Readonly<{ - readonly jwtKeys?: string; - readonly jwks: string; - readonly templates?: ReadonlyArray; - }>; - readonly analytics?: Readonly<{ - readonly gcpJwtPath?: string; - /** Session-scoped Vector config owned by this stack runtime. */ - readonly vectorConfigPath?: string; - }>; - readonly functions?: Readonly<{ readonly secrets: Readonly> }>; -} - -export interface RuntimeInputOwner { - /** - * Resolves stack-owned inputs needed before a workload is created. - * - * The Supervisor serializes workload startup and runtime cleanup. This owner therefore keeps - * only completed material in its caches; the caller owns an in-progress resolution and its - * interruption. - */ - readonly resolve: ( - state: PersistedStackState, - workloadId: string, - ) => Effect.Effect; - /** Resolves one configured project-relative regular file without copying it. */ - readonly resolveProjectFile: ( - state: PersistedStackState, - configuredPath: string, - ) => Effect.Effect; - readonly resolveAuthTemplates: ( - state: PersistedStackState, - ) => Effect.Effect, StackPreparationError>; - readonly cleanupAll: Effect.Effect; -} - -export interface RuntimeInputOwnerOptions { - readonly stateRoot: string; - readonly stackId: StackId; - /** Injected OIDC JSON fetcher; no network service is hidden in this owner. */ - readonly fetchJson?: RuntimeJsonFetcher; -} - -const failure = (message: string, fields: Readonly> = {}) => - new StackPreparationError({ message, ...fields }); - -const mapFile = ( - target: string, - operation: string, - effect: Effect.Effect, -): Effect.Effect => - effect.pipe( - Effect.mapError((error) => failure(`Unable to ${operation}`, { path: target, cause: error })), - ); - -const relativeEscape = (path: Path.Path, root: string, candidate: string): boolean => { - const relative = path.relative(root, candidate); - return relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative); -}; - -const absoluteForPlatform = (path: Path.Path, value: string): boolean => - path.isAbsolute(value) || /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith("\\\\"); - -const extensionFor = (configuredPath: string): string => { - const file = configuredPath.slice( - Math.max(configuredPath.lastIndexOf("/"), configuredPath.lastIndexOf("\\")) + 1, - ); - const dot = file.lastIndexOf("."); - return dot <= 0 ? "" : file.slice(dot); -}; - -const publicRemoteJwks = Schema.Struct({ keys: Schema.Array(Schema.Unknown) }); -const oidcDiscovery = Schema.Struct({ jwks_uri: Schema.String }); - -const decodeJson = (value: unknown): Effect.Effect => - typeof value === "string" - ? Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(value) - : Effect.succeed(value); - -/** Keeps OIDC diagnostics useful without echoing userinfo, query, or fragment data. */ -const safeUrlLabel = (value: string): string => { - try { - const url = new URL(value); - return `${url.origin}${url.pathname}`; - } catch { - return ""; - } -}; - -const symmetricJwk = (secret: string): Readonly> => ({ - kty: "oct", - alg: "HS256", - use: "sig", - key_ops: ["verify"], - k: base64UrlEncode(new TextEncoder().encode(secret)), -}); - -/** Resolves materialized Edge Runtime secrets to their caller-visible names. */ -const resolveFunctionsEdgeRuntimeSecrets = ( - state: PersistedStackState, -): Effect.Effect>, StackPreparationError> => { - const settings = settingsFor(state, "functions"); - const edgeRuntime = - isRecord(settings) && isRecord(settings.edge_runtime) ? settings.edge_runtime : {}; - const configured = isRecord(edgeRuntime.secrets) ? edgeRuntime.secrets : {}; - const output: Record = {}; - for (const [name, raw] of Object.entries(configured)) { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)) - return Effect.fail(failure("Functions Edge Runtime secret name is invalid", { name })); - if (name.startsWith("SUPABASE_")) - return Effect.fail(failure("Functions Edge Runtime secret name is reserved", { name })); - const value = settingValue(state, raw); - if ( - [...value].some((character) => { - const code = character.codePointAt(0) ?? 0; - return code <= 0x1f || code === 0x7f; - }) - ) - return Effect.fail(failure("Functions Edge Runtime secret value is invalid", { name })); - output[name] = value; - } - return Effect.succeed(output); -}; - -const vectorConfig = `data_dir: /tmp -api: - enabled: true - address: "\${VECTOR_API_ADDRESS}" - -sources: - stack_heartbeat: - type: demo_logs - format: json - count: 1 - stack_keepalive: - type: internal_metrics - -transforms: - stack_marker: - type: remap - inputs: - - stack_heartbeat - source: | - .event_message = "supabase-stack-vector" - del(.message) - .project = "default" - -sinks: - analytics: - type: http - inputs: - - stack_marker - uri: "\${LOGFLARE_URL}/logs?source_name=postgres.logs" - method: post - request: - headers: - x-api-key: "\${LOGFLARE_PRIVATE_ACCESS_TOKEN}" - retry_attempts: 5 - retry_initial_backoff_secs: 1 - retry_max_duration_secs: 10 - encoding: - codec: json - keepalive: - type: blackhole - inputs: - - stack_keepalive -`; - -export const makeRuntimeInputOwner = ( - options: RuntimeInputOwnerOptions, -): Effect.Effect< - RuntimeInputOwner, - StackPreparationError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const stackPaths = yield* resolveStackPaths(options).pipe( - Effect.mapError((cause) => failure("Unable to resolve runtime input paths", { cause })), - ); - const vectorRoot = path.join(stackPaths.runtime, "inputs", "vector"); - - const resolveProjectFile = ( - state: PersistedStackState, - configuredPath: string, - ): Effect.Effect => { - if (configuredPath.length === 0 || absoluteForPlatform(path, configuredPath)) - return Effect.fail(failure("Configured project file path must be relative")); - const root = path.resolve(state.identity.projectRoot); - const candidate = path.resolve(root, configuredPath); - if (relativeEscape(path, root, candidate)) - return Effect.fail(failure("Configured project file escapes the project root")); - return Effect.gen(function* () { - const canonicalRoot = yield* mapFile(root, "resolve project root", fs.realPath(root)); - const canonicalCandidate = yield* mapFile( - candidate, - "resolve configured project file", - fs.realPath(candidate), - ); - if (relativeEscape(path, canonicalRoot, canonicalCandidate)) - return yield* failure("Configured project file escapes the project root"); - const info = yield* mapFile( - canonicalCandidate, - "inspect configured project file", - fs.stat(canonicalCandidate), - ); - if (info.type !== "File") - return yield* failure("Configured project path must resolve to a regular file"); - return canonicalCandidate; - }); - }; - - const ensureFunctionsRoot = ( - state: PersistedStackState, - ): Effect.Effect => { - const settings = settingsFor(state, "functions"); - const root = isRecord(settings) ? settingValue(state, settings.functions_root) : ""; - if (root.length === 0) return Effect.fail(failure("Persisted Functions root is missing")); - return mapFile( - root, - "create Functions root", - fs.makeDirectory(root, { recursive: true }), - ).pipe(Effect.asVoid); - }; - - const resolveAuthTemplates = ( - state: PersistedStackState, - ): Effect.Effect, StackPreparationError> => - Effect.gen(function* () { - const auth = settingsFor(state, "auth"); - const email = isRecord(auth) && isRecord(auth.email) ? auth.email : {}; - const result: RuntimeAuthTemplate[] = []; - const ids = new Set(); - const urls = new Set(); - const add = (id: string, raw: unknown): Effect.Effect => { - if (!isRecord(raw)) return Effect.void; - const configuredPath = settingValue(state, raw.content_path); - if (configuredPath.length === 0) return Effect.void; - if (ids.has(id)) return Effect.fail(failure("Duplicate Auth email template id", { id })); - const extension = extensionFor(configuredPath); - const url = `/email/${id}${extension}`; - if (urls.has(url)) return Effect.fail(failure("Duplicate Auth email URL", { url })); - return resolveProjectFile(state, configuredPath).pipe( - Effect.tap((canonicalPath) => - Effect.sync(() => { - ids.add(id); - urls.add(url); - result.push({ - id, - path: configuredPath, - canonicalPath, - extension, - }); - }), - ), - Effect.asVoid, - ); - }; - if (isRecord(email.template)) - for (const [name, value] of Object.entries(email.template)) yield* add(name, value); - if (isRecord(email.notification)) - for (const [name, value] of Object.entries(email.notification)) - if (isRecord(value) && value.enabled === true) - yield* add(`${name}_notification`, value); - return result; - }); - - const resolveRemoteKeys = ( - issuer: string, - ): Effect.Effect, StackPreparationError> => { - const fetchJson = options.fetchJson; - if (fetchJson === undefined) - return Effect.fail(failure("OIDC discovery requires an injected JSON fetcher")); - const discoveryUrl = `${issuer.replace(/\/+$/u, "")}/.well-known/openid-configuration`; - const fetchAt = ( - url: string, - operation: string, - ): Effect.Effect => - fetchJson(url).pipe( - Effect.mapError(() => failure(`${operation} request failed`, { url: safeUrlLabel(url) })), - ); - return fetchAt(discoveryUrl, "OIDC discovery").pipe( - Effect.flatMap((value) => - decodeJson(value).pipe( - Effect.mapError(() => - failure("OIDC discovery response is invalid", { url: safeUrlLabel(discoveryUrl) }), - ), - ), - ), - Effect.flatMap((value) => - Schema.decodeUnknownEffect(oidcDiscovery)(value).pipe( - Effect.mapError(() => - failure("OIDC discovery response is invalid", { url: safeUrlLabel(discoveryUrl) }), - ), - ), - ), - Effect.flatMap((discovery) => { - const jwksUrl = discovery.jwks_uri.trim(); - if (jwksUrl.length === 0) - return Effect.fail( - failure("OIDC discovery response does not expose jwks_uri", { - url: safeUrlLabel(discoveryUrl), - }), - ); - return fetchAt(jwksUrl, "OIDC JWKS").pipe( - Effect.flatMap((value) => - decodeJson(value).pipe( - Effect.mapError(() => - failure("OIDC JWKS response is invalid", { url: safeUrlLabel(jwksUrl) }), - ), - ), - ), - Effect.flatMap((value) => - Schema.decodeUnknownEffect(publicRemoteJwks)(value).pipe( - Effect.mapError(() => - failure("OIDC JWKS response is invalid", { url: safeUrlLabel(jwksUrl) }), - ), - ), - ), - Effect.flatMap((jwks) => - jwks.keys.length === 0 - ? Effect.fail( - failure("OIDC JWKS response contains no keys", { url: safeUrlLabel(jwksUrl) }), - ) - : Effect.succeed(jwks.keys), - ), - ); - }), - ); - }; - - const resolveAuth = ( - state: PersistedStackState, - ): Effect.Effect, StackPreparationError> => - Effect.gen(function* () { - const signing = state.definition?.security.jwt.signing; - let local: ResolvedSigningKeyMaterial | undefined; - if (signing?.kind === "jwks-file") { - local = yield* resolveSigningKeyMaterial({ - kind: "jwks-file", - projectRoot: state.identity.projectRoot, - path: signing.path, - }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.mapError(() => - failure("Unable to resolve Auth signing keys", { path: signing.path }), - ), - ); - } - const thirdParty = resolveThirdPartyIssuer(settingsFor(state, "auth")); - if (!thirdParty.ok) - return yield* failure("Unable to resolve Auth third-party issuer", { - provider: thirdParty.provider, - }); - const remote = - thirdParty.value === undefined ? [] : yield* resolveRemoteKeys(thirdParty.value.issuer); - const localPublic = local?.publicKeys ?? []; - const symmetric = - signing?.kind === "jwks-file" - ? [] - : (() => { - const secret = state.secrets["secret:auth.settings.jwt_secret"]?.value ?? ""; - return secret.length === 0 ? [] : [symmetricJwk(secret)]; - })(); - if (signing?.kind !== "jwks-file" && symmetric.length === 0) - return yield* failure("Persisted Auth JWT secret is missing"); - const publicKeys = [...remote, ...localPublic, ...symmetric]; - const templates = - state.definition?.capabilities.auth.enabled !== true - ? [] - : yield* resolveAuthTemplates(state); - return { - ...(local === undefined ? {} : { jwtKeys: local.privateKeysJson }), - jwks: yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ - keys: publicKeys, - }).pipe(Effect.mapError(() => failure("Unable to encode Auth JWKS"))), - ...(templates.length === 0 ? {} : { templates }), - }; - }); - - const writeVectorConfig = ( - state: PersistedStackState, - ): Effect.Effect => { - const assignment = state.privatePorts.find( - (entry) => entry.workloadId === "analytics:vector" && entry.binding === "primary", - ); - if (state.runtime.kind === "native" && assignment === undefined) - return Effect.fail(failure("Persisted native Vector assignment is missing")); - const target = path.join(vectorRoot, "vector.yaml"); - return Effect.gen(function* () { - yield* mapFile( - vectorRoot, - "create Vector config directory", - fs.makeDirectory(vectorRoot, { recursive: true, mode: 0o700 }), - ); - yield* mapFile(vectorRoot, "secure Vector config directory", fs.chmod(vectorRoot, 0o700)); - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError(() => failure("Unable to allocate Vector config file")), - ); - const temporary = path.join(vectorRoot, `.vector.yaml.${token}.tmp`); - yield* Effect.ensuring( - Effect.scoped( - Effect.gen(function* () { - const file = yield* mapFile( - temporary, - "create Vector config", - fs.open(temporary, { flag: "w", mode: 0o600 }), - ); - yield* mapFile( - temporary, - "write Vector config", - file.writeAll(new TextEncoder().encode(vectorConfig)), - ); - yield* mapFile(temporary, "sync Vector config", file.sync); - }), - ).pipe( - Effect.andThen(mapFile(temporary, "secure Vector config", fs.chmod(temporary, 0o600))), - Effect.andThen(mapFile(target, "publish Vector config", fs.rename(temporary, target))), - Effect.andThen( - mapFile(target, "secure published Vector config", fs.chmod(target, 0o600)), - ), - ), - fs.remove(temporary, { force: true }).pipe(Effect.catchCause(() => Effect.void)), - ); - return target; - }); - }; - - const commonCompleted = new Map(); - const authCompleted = new Map>(); - const keyFor = (state: PersistedStackState): string => - `${options.stackId}\u0000${canonicalize(state.runtime)}\u0000${canonicalize(state.definition ?? {})}`; - const needsAuthMaterial = (state: PersistedStackState, workloadId: string): boolean => - (["rest", "auth", "realtime", "storage", "functions"] as const).some( - (capability) => - state.definition?.capabilities[capability].enabled === true && - (workloadId === `${capability}:${capability}` || - (capability === "functions" && workloadId === "functions:edge-runtime")), - ); - - const materializeCommon = ( - state: PersistedStackState, - workloadId: string, - authMaterial: NonNullable | undefined, - ): Effect.Effect => - Effect.gen(function* () { - if (workloadId === "studio:studio" || workloadId === "functions:edge-runtime") - yield* ensureFunctionsRoot(state); - const auth = needsAuthMaterial(state, workloadId) ? authMaterial : undefined; - const resolvesAnalyticsMaterial = - state.definition?.capabilities.analytics.enabled === true && - workloadId.startsWith("analytics:"); - const analytics = !resolvesAnalyticsMaterial - ? undefined - : yield* Effect.gen(function* () { - const analyticsSettings = settingsFor(state, "analytics"); - const gcpPath = isRecord(analyticsSettings) - ? settingValue(state, analyticsSettings.gcp_jwt_path) - : ""; - const vectorConfigPath = - workloadId === "analytics:vector" ? yield* writeVectorConfig(state) : undefined; - return gcpPath.length === 0 && vectorConfigPath === undefined - ? undefined - : { - ...(gcpPath.length === 0 - ? {} - : { gcpJwtPath: yield* resolveProjectFile(state, gcpPath) }), - ...(vectorConfigPath === undefined ? {} : { vectorConfigPath }), - }; - }); - const resolvesFunctionsMaterial = - workloadId === "functions:edge-runtime" && - state.definition?.capabilities.functions.enabled === true; - const functions = resolvesFunctionsMaterial - ? { secrets: yield* resolveFunctionsEdgeRuntimeSecrets(state) } - : undefined; - return { - ...(auth === undefined ? {} : { auth }), - ...(analytics === undefined ? {} : { analytics }), - ...(functions === undefined ? {} : { functions }), - }; - }); - - const resolveCached = ( - key: string, - completed: Map, - materialize: Effect.Effect, - ): Effect.Effect => - Effect.suspend(() => { - const ready = completed.get(key); - return ready === undefined - ? materialize.pipe(Effect.tap((value) => Effect.sync(() => completed.set(key, value)))) - : Effect.succeed(ready); - }); - - const resolve = ( - state: PersistedStackState, - workloadId: string, - ): Effect.Effect => - Effect.gen(function* () { - const key = `${keyFor(state)}\u0000${workloadId}`; - const auth = needsAuthMaterial(state, workloadId) - ? yield* resolveCached(keyFor(state), authCompleted, resolveAuth(state)) - : undefined; - const common = yield* resolveCached( - key, - commonCompleted, - materializeCommon(state, workloadId, auth), - ); - return common; - }); - const cleanupAll = Effect.gen(function* () { - commonCompleted.clear(); - authCompleted.clear(); - yield* mapFile( - vectorRoot, - "clean Vector configs", - fs.remove(vectorRoot, { recursive: true, force: true }), - ); - }); - - return { - resolve, - resolveProjectFile, - resolveAuthTemplates, - cleanupAll, - }; - }); diff --git a/packages/stack/src/runtime/SchemaInit.ts b/packages/stack/src/runtime/SchemaInit.ts deleted file mode 100644 index dfb5b59704..0000000000 --- a/packages/stack/src/runtime/SchemaInit.ts +++ /dev/null @@ -1,705 +0,0 @@ -import { - Crypto, - Duration, - Effect, - FileSystem, - Option, - Path, - Redacted, - Schema, - Scope, - Stream, -} from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import { compileStack } from "../model/Compiler.ts"; -import { excludeStackCapabilities, type ExcludableCapabilityName } from "../model/Exclusions.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import type { - SchemaInitCapabilityName, - SchemaInitOptions, - SchemaInitTarget, -} from "../public/SchemaInit.ts"; -import type { SchemaInitError } from "../public/Errors.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; -import { - ContainerEngineError, - InvalidStackConfigError, - InvalidStackIdentityError, - RequiresActivatedProcessError, - StackPreparationError, - StackRuntimeError, -} from "../public/Errors.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import { - AUTH_JWT_SECRET_SLOT, - DATABASE_INTERNAL_PASSWORD_SLOT, - resolveSecrets, - type SecretDeclaration, -} from "../state/SecretStore.ts"; -import { - STACK_STATE_FORMAT, - type PersistedSecretValues, - type PersistedStackState, -} from "../state/StackState.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { resolveRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { makeProductionRuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; -import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; -import type { ContainerEngine, ContainerHostRoute } from "./ContainerEngine.ts"; -import { resolveContainerEngine } from "./ContainerEngineResolver.ts"; -import { runContainerStartupProcess, schemaInitContainerName } from "./ContainerRuntime.ts"; -import { - defaultNativeProcessLauncher, - spawnNativeProcess, - type NativeProcessSpec, -} from "./NativeProcess.ts"; -import { makeProcessOutputTail } from "./Diagnostics.ts"; -import { makeRuntimeInputOwner } from "./RuntimeInputOwner.ts"; -import { encodeRuntimeEnvFile } from "./RuntimeEnvFile.ts"; -import type { RuntimeWorkloadKey } from "./RuntimeDriver.ts"; -import { - runtimeSpecFor, - validateWorkloadRuntimeInputs, - type WorkloadRuntimeInputs, - type WorkloadRuntimeSpec, -} from "./WorkloadRuntimeSpec.ts"; - -const STARTUP_TIMEOUT = "5 minutes" satisfies Duration.Input; -const AUTH_TEMPLATE_BASE_URL = "http://127.0.0.1"; - -const PRIMARY_WORKLOAD: Record = { - auth: "auth:auth", - storage: "storage:storage", - realtime: "realtime:realtime", - analytics: "analytics:analytics", - pooler: "pooler:pooler", -}; - -const SCHEMA_INIT_BINDINGS = [ - "primary", - "admin", - "ui", - "smtp", - "pop3", - "inspector", - "rpc", -] as const; - -const schemaInitCompileConfig = ( - config: SchemaInitTarget["config"], - names: ReadonlyArray, -) => { - const requested = new Set(names); - return excludeStackCapabilities( - config, - CAPABILITY_NAMES.filter( - (name): name is ExcludableCapabilityName => name !== "database" && !requested.has(name), - ), - ); -}; - -// Live schema-init compiles a capability subset. Stamp declared managed values from -// persisted state; passing the full map into resolveSecrets fails on extra pass-through -// slots while the stack is running. -const overlayManagedSecrets = ( - declarations: ReadonlyArray, - persisted: PersistedSecretValues | undefined, -): ReadonlyArray => { - if (persisted === undefined) return declarations; - return declarations.map((entry) => { - if (entry.value !== undefined) return entry; - const existing = persisted[entry.slot]; - if (existing === undefined || existing.policy !== "managed") return entry; - return { ...entry, value: Redacted.make(existing.value) }; - }); -}; - -/** Catalog version and image for a schema-init one-shot; undefined when the pin is unknown. */ -export const schemaInitArtifactIdentity = ( - name: SchemaInitCapabilityName, - version?: string, -): string | undefined => { - const release = catalogReleaseFor(PRIMARY_WORKLOAD[name], version); - return release === undefined ? undefined : `${release.version}:${release.containerImage}`; -}; - -const schemaInitPrivatePorts = ( - databasePort: number, - workloadId: string, - bindings: WorkloadRuntimeSpec["bindings"], -): PersistedStackState["privatePorts"] => [ - { workloadId: "database:database", binding: "primary", port: databasePort }, - ...SCHEMA_INIT_BINDINGS.flatMap((binding) => { - const bound = bindings[binding]; - return bound === undefined ? [] : [{ workloadId, binding, port: bound.containerPort }]; - }), -]; - -/** Linux Engine extra hosts so `host.docker.internal` resolves (DNS even when URLs stay on the alias). */ -export const schemaInitHostGatewayExtraHosts = ( - platform: string, - host: string, -): ReadonlyArray => - platform === "linux" && host === "host.docker.internal" - ? ["host.docker.internal:host-gateway"] - : []; - -const DATABASE_HOST_KEYS = new Set([ - "DB_HOST", - "GOTRUE_DB_HOST", - "POSTGRES_HOST", - "PGHOST", - "PG_META_DB_HOST", -]); -const DATABASE_PORT_KEYS = new Set([ - "DB_PORT", - "GOTRUE_DB_PORT", - "POSTGRES_PORT", - "PGPORT", - "PG_META_DB_PORT", -]); -const DATABASE_PASSWORD_KEYS = new Set([ - "DB_PASSWORD", - "GOTRUE_DB_PASSWORD", - "POSTGRES_PASSWORD", - "PGPASSWORD", - "PG_META_DB_PASSWORD", -]); -const CONNECTION_PROTOCOLS = new Set(["postgres:", "postgresql:", "ecto:"]); - -export interface ParsedDatabaseUrl { - readonly host: string; - readonly port: number; - readonly password: string; - readonly database: string; -} - -export const parseSchemaInitDatabaseUrl = (url: string): ParsedDatabaseUrl | undefined => { - try { - const parsed = new URL(url); - if (!CONNECTION_PROTOCOLS.has(parsed.protocol)) return undefined; - const port = Number.parseInt(parsed.port === "" ? "5432" : parsed.port, 10); - if (!Number.isInteger(port) || parsed.hostname.length === 0) return undefined; - const database = decodeURIComponent(parsed.pathname.replace(/^\//, "")); - return { - host: parsed.hostname, - port, - password: decodeURIComponent(parsed.password), - database, - }; - } catch { - return undefined; - } -}; - -const rewriteConnectionUrl = ( - value: string, - target: { readonly host: string; readonly port: number; readonly password: string }, -): string | undefined => { - if (!value.includes("://")) return undefined; - try { - const parsed = new URL(value); - if (!CONNECTION_PROTOCOLS.has(parsed.protocol)) return undefined; - parsed.hostname = target.host; - parsed.port = String(target.port); - parsed.password = target.password; - return parsed.toString(); - } catch { - return undefined; - } -}; - -/** Rewrites only host/port/password on database connection settings; keeps user and database name. */ -export const rewriteDatabaseEnvironment = ( - env: Readonly>, - target: { readonly host: string; readonly port: number; readonly password: string }, -): Record => { - const rewritten: Record = {}; - for (const [key, value] of Object.entries(env)) { - if (DATABASE_HOST_KEYS.has(key)) { - rewritten[key] = target.host; - continue; - } - if (DATABASE_PORT_KEYS.has(key)) { - rewritten[key] = String(target.port); - continue; - } - if (DATABASE_PASSWORD_KEYS.has(key)) { - rewritten[key] = target.password; - continue; - } - rewritten[key] = rewriteConnectionUrl(value, target) ?? value; - } - return rewritten; -}; - -const loopbackHost = (host: string): boolean => host === "127.0.0.1" || host === "localhost"; - -const schemaInitIdentity = ( - crypto: Crypto.Crypto, -): Effect.Effect => - Effect.gen(function* () { - const first = yield* crypto.randomUUIDv4; - const second = yield* crypto.randomUUIDv4; - return yield* Schema.decodeEffect(StackIdSchema)(`${first}${second}`.replaceAll("-", "")); - }).pipe( - Effect.mapError( - (cause) => - new InvalidStackIdentityError({ - message: "Unable to allocate schema-init identity", - cause, - }), - ), - ); - -const runtimeError = ( - key: Pick, - message: string, - cause?: unknown, -): StackRuntimeError => - new StackRuntimeError({ - message, - stackId: key.stackId, - workloadId: key.workloadId, - ...(cause === undefined ? {} : { cause }), - }); - -const mapContainerEngineError = ( - engine: StackRuntime & { readonly kind: "container" }, - message: string, - cause: unknown, -): ContainerEngineError => - new ContainerEngineError({ - engine: engine.engine, - message, - cause, - }); - -const resolveEngine = ( - target: SchemaInitTarget, - options: SchemaInitOptions, -): Effect.Effect< - Option.Option, - ContainerEngineError, - ChildProcessSpawner.ChildProcessSpawner -> => { - if (target.runtime.kind !== "container") return Effect.succeed(Option.none()); - if (options.containerEngine !== undefined) - return Effect.succeed(Option.some(options.containerEngine)); - const runtime = target.runtime; - return resolveContainerEngine(runtime.engine).pipe( - Effect.map(Option.some), - Effect.mapError((cause) => - mapContainerEngineError( - runtime, - `Unable to configure ${runtime.engine} for schema init`, - cause, - ), - ), - ); -}; - -const resolveLiveNetwork = ( - engine: ContainerEngine, - stackId: StackId, - runtime: StackRuntime & { readonly kind: "container" }, -): Effect.Effect => - engine.listResources(stackId).pipe( - Effect.mapError((cause) => - mapContainerEngineError(runtime, "Unable to list stack resources for schema init", cause), - ), - Effect.flatMap((resources) => { - const network = resources.find((entry) => entry.kind === "network"); - return network === undefined - ? Effect.fail( - runtimeError( - { stackId, workloadId: "" }, - "Stack network is unavailable for schema init", - ), - ) - : Effect.succeed(network.id); - }), - ); - -const acquireEphemeralNetwork = ( - engine: ContainerEngine, - schemaInitId: StackId, - runtime: StackRuntime & { readonly kind: "container" }, -): Effect.Effect => - Effect.gen(function* () { - const name = `supabase-${schemaInitId.slice(0, 16)}-schema-init-net`; - yield* Effect.addFinalizer(() => engine.removeNetwork(name).pipe(Effect.ignore)); - const created = yield* engine - .createNetwork({ - name, - labels: { - stackId: schemaInitId, - ownerSessionId: schemaInitId.slice(0, 32), - role: "network", - }, - }) - .pipe( - Effect.mapError((cause) => - mapContainerEngineError(runtime, "Unable to create schema-init network", cause), - ), - ); - return created.id; - }); - -const runNativeStartup = ( - spec: NativeProcessSpec, - key: RuntimeWorkloadKey, - knownSecrets: ReadonlyArray = [], -): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const process = yield* spawnNativeProcess(spec, defaultNativeProcessLauncher(), key).pipe( - Effect.mapError((error) => runtimeError(key, error.message, error)), - ); - const tail = makeProcessOutputTail(); - const drain = Effect.all( - [ - Stream.runForEach(process.stdout, (bytes) => - Effect.sync(() => tail.pushBytes("stdout", bytes)), - ), - Stream.runForEach(process.stderr, (bytes) => - Effect.sync(() => tail.pushBytes("stderr", bytes)), - ), - ], - { concurrency: "unbounded", discard: true }, - ).pipe(Effect.mapError((error) => runtimeError(key, error.message, error))); - const exit = process.exitCode.pipe( - Effect.mapError((error) => runtimeError(key, error.message, error)), - ); - const [exitCode] = yield* Effect.all([exit, drain], { concurrency: "unbounded" }).pipe( - Effect.timeoutOrElse({ - duration: spec.timeout ?? STARTUP_TIMEOUT, - orElse: () => - Effect.fail(runtimeError(key, `Native schema init timed out for ${key.workloadId}`)), - }), - ); - if (exitCode !== 0) { - const diagnostic = tail.finish(knownSecrets); - const message = `Native schema init exited with code ${String(exitCode)} for ${key.workloadId}`; - return yield* runtimeError( - key, - diagnostic.length === 0 ? message : `${message}\n${diagnostic}`, - ); - } - }), - ); - -const capabilityInputs = ( - material: WorkloadRuntimeInputs, - hostRoute: ContainerHostRoute | undefined, -): WorkloadRuntimeInputs => { - const templateBaseUrl = material.auth?.templateBaseUrl ?? AUTH_TEMPLATE_BASE_URL; - const auth = { - ...material.auth, - templateBaseUrl, - }; - return { - ...material, - auth, - ...(hostRoute === undefined ? {} : { hostRoute }), - }; -}; - -export const schemaInitWorkloads = ( - names: ReadonlyArray, - target: SchemaInitTarget, - options: SchemaInitOptions = {}, -): Effect.Effect< - void, - SchemaInitError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawner.ChildProcessSpawner -> => - Effect.scoped( - Effect.gen(function* () { - const parsed = parseSchemaInitDatabaseUrl(target.databaseUrl); - if (parsed === undefined) - return yield* new InvalidStackConfigError({ - message: "Schema init requires a valid PostgreSQL URL", - }); - const password = Redacted.value(target.secrets.databasePassword); - const connection = { host: parsed.host, port: parsed.port, password }; - const compiled = yield* compileStack({ - projectRoot: target.projectRoot, - runtime: target.runtime, - config: schemaInitCompileConfig(target.config, names), - }); - const shared = yield* resolveRuntimeEnvironment; - let persistedSecrets: PersistedSecretValues | undefined; - if (target.kind === "live") { - const liveState = yield* (yield* makeStackStateStore({ stateRoot: shared.stateRoot })).read( - target.stackId, - ); - persistedSecrets = liveState?.secrets; - } - const declarations = overlayManagedSecrets( - compiled.secrets.map((entry) => { - if (entry.slot === DATABASE_INTERNAL_PASSWORD_SLOT) - return { ...entry, value: target.secrets.databasePassword }; - if (entry.slot === AUTH_JWT_SECRET_SLOT && target.secrets.jwtSecret !== undefined) - return { ...entry, value: target.secrets.jwtSecret }; - return entry; - }), - persistedSecrets, - ); - const resolved = yield* resolveSecrets({ declarations }, undefined, "unconfigured"); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const schemaInitId = yield* schemaInitIdentity(crypto); - const tempRoot = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-schema-init-" }).pipe( - Effect.mapError( - (cause) => - new StackPreparationError({ - message: "Unable to create schema-init workspace", - cause, - }), - ), - ); - const engine = Option.getOrUndefined(yield* resolveEngine(target, options)); - const preparer = - options.artifactPreparer ?? - (yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: shared.stateRoot, - ...(shared.artifactCacheRoot === undefined - ? {} - : { artifactCacheRoot: shared.artifactCacheRoot }), - runtime: target.runtime, - ...(engine === undefined ? {} : { containerEngine: engine }), - })); - const inputOwner = yield* makeRuntimeInputOwner({ - stateRoot: tempRoot, - stackId: schemaInitId, - }); - yield* Effect.addFinalizer(() => inputOwner.cleanupAll.pipe(Effect.ignore)); - const joinPostgresNetwork = target.kind === "ephemeral" && target.networkId !== undefined; - const envKind = - target.runtime.kind === "container" && (target.kind === "live" || joinPostgresNetwork) - ? "container" - : "native"; - const privatePorts: PersistedStackState["privatePorts"] = [ - { workloadId: "database:database", binding: "primary", port: connection.port }, - ]; - const state: PersistedStackState = { - format: STACK_STATE_FORMAT, - identity: { - projectRoot: target.projectRoot, - branchContext: "schema-init", - stackName: "schema-init", - }, - runtime: target.runtime, - desiredLifecycle: "stopped", - definition: compiled.definition, - ports: [], - privatePorts, - secrets: resolved.persisted, - }; - let hostRoute: ContainerHostRoute | undefined; - let networkId: string | undefined; - if (target.runtime.kind === "container") { - const runtime = target.runtime; - if (engine === undefined) - return yield* new ContainerEngineError({ - engine: runtime.engine, - message: "Container engine is unavailable for schema init", - }); - hostRoute = yield* engine.preflight.pipe( - Effect.mapError((cause) => - mapContainerEngineError(runtime, "Container host route preflight failed", cause), - ), - ); - networkId = - target.kind === "live" - ? yield* resolveLiveNetwork(engine, target.stackId, runtime) - : target.networkId !== undefined - ? target.networkId - : yield* acquireEphemeralNetwork(engine, schemaInitId, runtime); - } - const rewriteTarget = - target.runtime.kind === "container" && target.kind === "ephemeral" && !joinPostgresNetwork - ? { - host: - hostRoute !== undefined && loopbackHost(connection.host) - ? hostRoute.host - : connection.host, - port: connection.port, - password, - } - : undefined; - const extraHosts = - target.runtime.kind === "container" && target.kind === "ephemeral" - ? schemaInitHostGatewayExtraHosts( - options.platform ?? process.platform, - hostRoute?.host ?? "host.docker.internal", - ) - : []; - yield* Effect.forEach( - names, - (name) => - Effect.gen(function* () { - if (compiled.definition.capabilities[name].enabled !== true) { - yield* Effect.logWarning(`Skipping schema init for disabled capability ${name}`); - return; - } - const workloadId = PRIMARY_WORKLOAD[name]; - const workload = compiled.executionPlan.workloads.find( - (entry) => entry.id === workloadId, - ); - if (workload === undefined) - return yield* runtimeError( - { stackId: schemaInitId, workloadId }, - `Missing planned workload for ${name} schema init`, - ); - const spec = runtimeSpecFor(workload); - if (spec === undefined) - return yield* new StackPreparationError({ - message: `Unknown runtime specification for ${workload.id}`, - workload: workload.id, - }); - const key: RuntimeWorkloadKey = { stackId: schemaInitId, workloadId }; - const dummyPort = spec.containerPort; - const envState: PersistedStackState = { - ...state, - privatePorts: schemaInitPrivatePorts(connection.port, workloadId, spec.bindings), - }; - const material = yield* inputOwner.resolve(envState, workload.id); - const inputs = capabilityInputs(material, hostRoute); - yield* validateWorkloadRuntimeInputs(envState, workload, inputs); - const environment = yield* Effect.try({ - try: () => spec.env(envState, workload, dummyPort, envKind, inputs), - catch: (cause) => - runtimeError(key, cause instanceof Error ? cause.message : String(cause), cause), - }); - const env = - rewriteTarget === undefined - ? environment - : rewriteDatabaseEnvironment(environment, rewriteTarget); - if (target.runtime.kind === "native") { - const preview = spec.nativeStartupProcesses( - "", - envState, - workload, - dummyPort, - inputs, - ); - if (preview.length === 0) - return yield* new RequiresActivatedProcessError({ - capability: name, - message: `${name} schema init requires the activated ${name} process`, - }); - const prepared = yield* preparer.prepare(target.runtime, workload); - if (prepared.artifactRoot === undefined) - return yield* runtimeError( - key, - `Native artifact root is unavailable for ${workload.id}`, - ); - const startups = spec.nativeStartupProcesses( - prepared.artifactRoot, - envState, - workload, - dummyPort, - inputs, - ); - const knownSecrets = Object.values(resolved.persisted).flatMap((entry) => { - const value = entry.value; - if (value.length === 0) return []; - const encoded = encodeURIComponent(value); - return encoded === value ? [value] : [value, encoded]; - }); - yield* Effect.forEach( - startups, - (startup) => { - // Native prepare seeds when SEED_SELF_HOST is set; this one-shot must only migrate. - const nativeEnv = { ...env, ...startup.env }; - delete nativeEnv.SEED_SELF_HOST; - return runNativeStartup( - { - ...startup, - timeout: STARTUP_TIMEOUT, - env: nativeEnv, - }, - key, - knownSecrets, - ); - }, - { discard: true }, - ); - return; - } - const startups = spec.containerStartupProcesses(envState, workload, inputs); - if (startups.length === 0) - return yield* new RequiresActivatedProcessError({ - capability: name, - message: `${name} schema init requires the activated ${name} process`, - }); - if (engine === undefined || networkId === undefined) - return yield* new ContainerEngineError({ - engine: target.runtime.kind === "container" ? target.runtime.engine : "docker", - message: "Container engine is unavailable for schema init", - }); - const prepared = yield* preparer.prepare(target.runtime, workload); - if (prepared.image === undefined) - return yield* runtimeError(key, `Container image is unavailable for ${workload.id}`); - const encoded = yield* encodeRuntimeEnvFile(env); - const envFile = path.join(tempRoot, `${encodeURIComponent(workload.id)}.env`); - yield* fs.writeFileString(envFile, encoded, { mode: 0o600 }).pipe( - Effect.mapError( - (cause) => - new StackPreparationError({ - message: "Unable to write schema-init environment file", - path: envFile, - cause, - }), - ), - ); - yield* fs.chmod(envFile, 0o600).pipe( - Effect.mapError( - (cause) => - new StackPreparationError({ - message: "Unable to secure schema-init environment file", - path: envFile, - cause, - }), - ), - ); - const image = prepared.image; - const mounts = spec.containerMounts?.(envState, workload, inputs) ?? []; - yield* Effect.forEach( - startups, - (startup) => - runContainerStartupProcess({ - engine, - key, - timeout: STARTUP_TIMEOUT, - specification: { - name: schemaInitContainerName(key), - image, - labels: { - stackId: schemaInitId, - ownerSessionId: schemaInitId.slice(0, 32), - workloadId, - startup: true, - role: "workload", - }, - network: networkId, - mounts, - volumeMounts: [], - publications: [], - role: "workload", - entrypoint: startup.entrypoint, - command: startup.command, - envFile, - ...(extraHosts.length === 0 ? {} : { extraHosts }), - }, - }).pipe(Effect.mapError((error) => runtimeError(key, error.message, error))), - { discard: true }, - ); - }), - { discard: true }, - ); - }), - ); diff --git a/packages/stack/src/runtime/WorkloadRuntimeSpec.ts b/packages/stack/src/runtime/WorkloadRuntimeSpec.ts deleted file mode 100644 index 1bd5f03435..0000000000 --- a/packages/stack/src/runtime/WorkloadRuntimeSpec.ts +++ /dev/null @@ -1,1460 +0,0 @@ -import type { PersistedStackState } from "../state/StackState.ts"; -import type { ExecutionPlan, PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import { - flattenSettings, - isRecord, - secret, - settingValue, - settingsFor, - valueAt, -} from "../state/MaterializedSettings.ts"; -import type { - ContainerHostRoute, - ContainerMount, - ContainerStartupProcess, -} from "./ContainerEngine.ts"; -import { catalogEntryFor, containerAliasFor } from "../model/WorkloadCatalog.ts"; -import { parseFileSize } from "../model/capabilities/storage.ts"; -import { resolveThirdPartyIssuer } from "../model/capabilities/auth-third-party.ts"; -import { Effect, type Duration } from "effect"; -import { StackPreparationError, StackStateInvalidError } from "../public/Errors.ts"; -import { FUNCTIONS_CONTAINER_ROOT } from "../functions/serve-main-deps.ts"; -import { DATABASE_INTERNAL_PASSWORD_SLOT } from "../state/SecretStore.ts"; -export { FUNCTIONS_CONTAINER_ROOT } from "../functions/serve-main-deps.ts"; - -type WorkloadRuntimeKind = "native" | "container"; - -/** Closed set of private ports a workload may expose to the host gateway. */ -type WorkloadBindingName = "primary" | "admin" | "ui" | "smtp" | "pop3" | "inspector" | "rpc"; - -interface WorkloadBinding { - readonly containerPort: number; -} - -interface WorkloadBindings { - readonly primary?: WorkloadBinding; - readonly admin?: WorkloadBinding; - readonly ui?: WorkloadBinding; - readonly smtp?: WorkloadBinding; - readonly pop3?: WorkloadBinding; - readonly inspector?: WorkloadBinding; - readonly rpc?: WorkloadBinding; -} - -type BindingSelectionState = Pick; - -export interface WorkloadBindingIntent { - readonly workloadId: string; - readonly binding: WorkloadBindingName; -} - -/** Inputs resolved by the owner before a process/container is created. */ -export interface WorkloadRuntimeInputs { - /** Resolved GoTrue signing key JSON and public JWKS. */ - readonly auth?: Readonly<{ - readonly jwtKeys?: string; - readonly jwks?: string; - /** Base URL serving the configured Auth email templates through the gateway. */ - readonly templateBaseUrl?: string; - }>; - /** Resolved host path for the BigQuery service-account file. */ - readonly analytics?: Readonly<{ - readonly gcpJwtPath?: string; - /** Generation-scoped Vector config generated by the stack owner. */ - readonly vectorConfigPath?: string; - }>; - /** Stack-owned Edge Runtime bootstrap source. */ - readonly functions?: Readonly<{ - readonly bootstrapPath?: string; - readonly secrets?: Readonly>; - }>; - /** Stack-owned native persistent data paths. Containers use their named volumes instead. */ - readonly database?: Readonly<{ readonly dataPath?: string }>; - readonly storage?: Readonly<{ readonly dataPath?: string }>; - /** Host route used by containers to reach StackGateway. */ - readonly hostRoute?: ContainerHostRoute; -} - -interface NativeProcessResolution { - readonly executable: string; - readonly args: ReadonlyArray; - readonly cwd: string; - readonly env?: Readonly>; - readonly gracefulStopSignal?: "SIGTERM" | "SIGINT"; - readonly gracefulStopTimeout?: Duration.Input; -} - -/** Complete process-side contract for one catalog workload. */ -export interface WorkloadRuntimeSpec { - readonly bindings: WorkloadBindings; - readonly containerPort: number; - readonly cwd: (state: PersistedStackState, workload: PlannedWorkload) => string; - readonly privateEndpoint: ( - state: PersistedStackState, - binding?: WorkloadBindingName, - runtime?: WorkloadRuntimeKind, - ) => Readonly<{ readonly host: string; readonly port: number }> | undefined; - readonly args: ( - state: PersistedStackState, - workload: PlannedWorkload, - port: number, - runtime?: WorkloadRuntimeKind, - ) => ReadonlyArray; - readonly env: ( - state: PersistedStackState, - workload: PlannedWorkload, - port: number, - runtime?: WorkloadRuntimeKind, - inputs?: WorkloadRuntimeInputs, - ) => Readonly>; - readonly containerArgs: ( - state: PersistedStackState, - workload: PlannedWorkload, - port: number, - inputs?: WorkloadRuntimeInputs, - ) => ReadonlyArray; - /** Optional main-container entrypoint override; startup processes remain explicit. */ - readonly containerEntrypoint?: string; - /** Service-owned one-shot container processes run before the main container. */ - readonly containerStartupProcesses: ( - state: PersistedStackState, - workload: PlannedWorkload, - inputs?: WorkloadRuntimeInputs, - ) => ReadonlyArray; - readonly containerMounts?: ( - state: PersistedStackState, - workload: PlannedWorkload, - inputs?: WorkloadRuntimeInputs, - ) => ReadonlyArray; - readonly readiness: Readonly<{ - readonly protocol: "http" | "tcp"; - readonly path?: string; - readonly binding: WorkloadBindingName; - readonly headers?: Readonly>; - }>; - readonly nativeProcess: ( - artifactRoot: string, - state: PersistedStackState, - workload: PlannedWorkload, - port: number, - inputs?: WorkloadRuntimeInputs, - ) => NativeProcessResolution; - /** Service-owned, short-lived native processes to run before the main process. */ - readonly nativeStartupProcesses: ( - artifactRoot: string, - state: PersistedStackState, - workload: PlannedWorkload, - port: number, - inputs?: WorkloadRuntimeInputs, - ) => ReadonlyArray; -} - -type WorkloadRuntimeSpecDefinition = Omit< - WorkloadRuntimeSpec, - | "containerPort" - | "cwd" - | "privateEndpoint" - | "nativeProcess" - | "nativeStartupProcesses" - | "containerStartupProcesses" - | "readiness" -> & - Readonly<{ - readonly readiness: Omit & { - readonly binding?: WorkloadBindingName; - }; - }> & - Partial>; - -export interface ContainerWorkloadResolution { - readonly entrypoint?: string; - readonly command: ReadonlyArray; - readonly startup?: ReadonlyArray; - readonly env: Readonly>; - readonly mounts: ReadonlyArray; - readonly networkAliases: ReadonlyArray; - readonly publications: ReadonlyArray<{ - readonly address: "127.0.0.1"; - readonly hostPort: number; - readonly containerPort: number; - }>; - readonly bootstrap?: Readonly<{ readonly source: string; readonly destination: string }>; -} - -/** - * Validates owner-resolved material before a process/container is created. - * Network/file discovery stays outside this pure runtime specification; the - * owner supplies the resulting values through WorkloadRuntimeInputs. - */ -export const validateWorkloadRuntimeInputs = ( - state: PersistedStackState, - workload: PlannedWorkload, - inputs: WorkloadRuntimeInputs = {}, -): Effect.Effect => - Effect.gen(function* () { - const signing = state.definition?.security.jwt.signing; - const thirdParty = resolveThirdPartyIssuer(settingsFor(state, "auth")); - if (!thirdParty.ok) - return yield* new StackPreparationError({ - message: thirdParty.message, - workload: workload.id, - }); - const jwksConsumer = - workload.id === "rest:rest" || - workload.id === "auth:auth" || - workload.id === "realtime:realtime" || - workload.id === "storage:storage" || - workload.id === "functions:edge-runtime"; - if ( - jwksConsumer && - (signing?.kind === "jwks-file" || thirdParty.value !== undefined) && - (inputs.auth?.jwks === undefined || inputs.auth.jwks.length === 0) - ) - return yield* new StackPreparationError({ - message: "Resolved JWKS material is required for the configured auth mode", - workload: workload.id, - }); - if ( - jwksConsumer && - signing?.kind !== "jwks-file" && - thirdParty.value === undefined && - secret(state, "secret:auth.settings.jwt_secret").length === 0 - ) - return yield* new StackPreparationError({ - message: "Managed JWT signing secret is required for the configured auth mode", - workload: workload.id, - }); - if (workload.id === "analytics:analytics") { - const backend = valueAt(state, "analytics", "backend"); - if ( - backend === "bigquery" && - valueAt(state, "analytics", "gcp_jwt_path").length > 0 && - (inputs.analytics?.gcpJwtPath === undefined || inputs.analytics.gcpJwtPath.length === 0) - ) - return yield* new StackPreparationError({ - message: "Resolved Analytics service-account path is required", - workload: workload.id, - }); - } - if (workload.id === "auth:auth") { - if ( - signing?.kind === "jwks-file" && - (inputs.auth?.jwtKeys === undefined || inputs.auth.jwtKeys.length === 0) - ) - return yield* new StackPreparationError({ - message: "Resolved JWT signing keys are required for Auth", - workload: workload.id, - }); - const email = settingsFor(state, "auth"); - const emailSettings = isRecord(email) && isRecord(email.email) ? email.email : undefined; - const templates = emailSettings?.template; - const notifications = emailSettings?.notification; - const hasContent = - (isRecord(templates) && - Object.values(templates).some( - (value) => isRecord(value) && settingValue(state, value.content_path).length > 0, - )) || - (isRecord(notifications) && - Object.values(notifications).some( - (value) => isRecord(value) && settingValue(state, value.content_path).length > 0, - )); - if ( - hasContent && - (inputs.auth?.templateBaseUrl === undefined || inputs.auth.templateBaseUrl.length === 0) - ) - return yield* new StackPreparationError({ - message: "Resolved Auth template base URL is required", - workload: workload.id, - }); - } - }); - -export const FUNCTIONS_BOOTSTRAP_CONTAINER_PATH = "/root"; - -const compactEnvironment = ( - environment: Readonly>, -): Readonly> => - Object.fromEntries(Object.entries(environment).filter(([, value]) => value.length > 0)); - -const capabilityEnabled = (state: PersistedStackState, capability: CapabilityName): boolean => - state.definition?.capabilities[capability].enabled ?? true; - -const capabilityEnv = ( - state: PersistedStackState, - capability: CapabilityName, - prefix: string, - omit: (key: string) => boolean = () => false, -): Record => { - const out: Record = {}; - const settings = settingsFor(state, capability); - if (settings !== undefined) flattenSettings(state, settings, prefix, out); - for (const key of Object.keys(out)) if (omit(key)) delete out[key]; - return out; -}; - -const dbPort = (state: PersistedStackState): number => - state.privatePorts.find( - (assignment) => - assignment.workloadId === "database:database" && assignment.binding === "primary", - )?.port ?? 5432; - -const privatePortFor = ( - state: PersistedStackState, - workloadId: string, - binding: WorkloadBindingName, -): number | undefined => - state.privatePorts.find( - (assignment) => assignment.workloadId === workloadId && assignment.binding === binding, - )?.port; - -const bindingFor = ( - bindings: WorkloadBindings, - binding: WorkloadBindingName, -): WorkloadBinding | undefined => bindings[binding]; - -const workloadPort = ( - state: PersistedStackState, - workloadId: string, - binding: WorkloadBindingName, - runtime: WorkloadRuntimeKind, - containerPort: number, -): number => - runtime === "container" - ? containerPort - : (privatePortFor(state, workloadId, binding) ?? containerPort); - -const containerPortFor = ( - state: PersistedStackState, - workloadId: string, - binding: WorkloadBindingName, - fallback: number, -): number => { - if (workloadId === "pooler:pooler" && binding === "primary") - return valueAt(state, "pooler", "pool_mode") === "session" ? 5432 : 6543; - return fallback; -}; - -const dbHost = (runtime: WorkloadRuntimeKind): string => - runtime === "container" ? containerAliasFor("database:database") : "127.0.0.1"; - -const dbUrl = ( - state: PersistedStackState, - role: string, - runtime: WorkloadRuntimeKind, - database = "postgres", -): string => { - const port = runtime === "container" ? 5432 : dbPort(state); - return `postgresql://${role}:${secret(state, DATABASE_INTERNAL_PASSWORD_SLOT)}@${dbHost(runtime)}:${port}/${database}`; -}; - -const usesResolvedJwks = (state: PersistedStackState): boolean => { - const signing = state.definition?.security.jwt.signing; - const thirdParty = resolveThirdPartyIssuer(settingsFor(state, "auth")); - return signing?.kind === "jwks-file" || (thirdParty.ok && thirdParty.value !== undefined); -}; - -const edgeRuntimeJwtEnvironment = ( - state: PersistedStackState, - inputs: WorkloadRuntimeInputs, -): Record => { - const fixed = compactEnvironment({ - SUPABASE_INTERNAL_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - SUPABASE_INTERNAL_PUBLISHABLE_KEY: secret(state, "secret:auth.settings.publishable_key"), - SUPABASE_INTERNAL_SECRET_KEY: secret(state, "secret:auth.settings.secret_key"), - SUPABASE_INTERNAL_HOST_PORT: String( - state.ports.find((assignment) => assignment.field === "api")?.port ?? "", - ), - SUPABASE_JWKS: inputs.auth?.jwks ?? '{"keys":[]}', - }); - // Preserve explicitly empty owner-resolved values. Fixed runtime credentials - // must win over user function secrets with reserved names. - return { ...inputs.functions?.secrets, ...fixed }; -}; - -const functionsConfigEnvironment = (state: PersistedStackState): string => { - const settings = settingsFor(state, "functions"); - const edgeRuntime = - isRecord(settings) && isRecord(settings.edge_runtime) ? settings.edge_runtime : {}; - const configured = isRecord(settings) && isRecord(settings.functions) ? settings.functions : {}; - const result: Record = {}; - const defaults: Record = {}; - if (typeof edgeRuntime.verify_jwt_default === "boolean") - defaults.verify_jwt = edgeRuntime.verify_jwt_default; - if (typeof edgeRuntime.import_map_default === "string") - defaults.import_map_root = edgeRuntime.import_map_default; - if (Object.keys(defaults).length > 0) result.$default = defaults; - for (const [slug, value] of Object.entries(configured)) { - if (!isRecord(value)) continue; - const env = isRecord(value.env) - ? Object.fromEntries( - Object.entries(value.env).map(([key, entry]) => [key, settingValue(state, entry)]), - ) - : {}; - result[slug] = { - enabled: value.enabled ?? true, - verify_jwt: value.verify_jwt ?? true, - import_map: settingValue(state, value.import_map), - entrypoint: settingValue(state, value.entrypoint), - static_files: Array.isArray(value.static_files) - ? value.static_files.map((entry) => settingValue(state, entry)) - : [], - env, - }; - } - return JSON.stringify(result); -}; - -const functionsRoot = (state: PersistedStackState): string => - valueAt(state, "functions", "functions_root"); - -const privateEndpointFor = ( - state: PersistedStackState, - workloadId: string, - bindings: WorkloadBindings, - binding: WorkloadBindingName, - runtime: WorkloadRuntimeKind, - alias: string, -): Readonly<{ readonly host: string; readonly port: number }> | undefined => { - const declared = bindingFor(bindings, binding); - if (declared === undefined) return undefined; - const port = - runtime === "container" - ? containerPortFor(state, workloadId, binding, declared.containerPort) - : privatePortFor(state, workloadId, binding); - return port === undefined - ? undefined - : { - host: runtime === "container" ? alias : "127.0.0.1", - port, - }; -}; - -const artifactPath = (root: string, relative: string): string => - relative === "." ? root : `${root.replace(/\/+$/u, "")}/${relative}`; - -const nativeArgsFor = ( - workload: PlannedWorkload, - args: ReadonlyArray, - inputs: WorkloadRuntimeInputs, -): ReadonlyArray => { - if (workload.id === "analytics:vector" && inputs.analytics?.vectorConfigPath !== undefined) - return ["--config", inputs.analytics.vectorConfigPath]; - if (workload.id !== "functions:edge-runtime") return args; - return args.map((arg) => (arg.startsWith("--main-service=") ? "--main-service=." : arg)); -}; - -const nativeFunctionsDirectory = ( - state: PersistedStackState, - inputs: WorkloadRuntimeInputs, -): string => { - const bootstrapPath = inputs.functions?.bootstrapPath; - if (bootstrapPath === undefined) return functionsRoot(state); - return bootstrapPath.slice(0, bootstrapPath.lastIndexOf("/")) || bootstrapPath; -}; - -const functionsInspectorRequested = (state: Pick): boolean => { - const inspectorSettings = state.definition?.capabilities.functions.settings.inspector; - return ( - isRecord(inspectorSettings) || state.definition?.listeners.functionsInspector.enabled === true - ); -}; - -const functionsInspectorArgs = ( - state: PersistedStackState, - runtime: WorkloadRuntimeKind, -): ReadonlyArray => { - const configuredMode = valueAt(state, "functions", "inspector.mode"); - const inspectorRequested = functionsInspectorRequested(state); - const mode = - configuredMode === "run" || configuredMode === "brk" || configuredMode === "wait" - ? configuredMode - : inspectorRequested - ? "run" - : ""; - if (mode !== "run" && mode !== "brk" && mode !== "wait") return []; - const port = - runtime === "container" ? 9229 : privatePortFor(state, "functions:edge-runtime", "inspector"); - if (port === undefined) - throw new Error("Functions inspector private port assignment is required"); - const address = runtime === "container" ? "0.0.0.0" : "127.0.0.1"; - const flag = mode === "brk" ? "--inspect-brk" : mode === "wait" ? "--inspect-wait" : "--inspect"; - return [ - `${flag}=${address}:${port}`, - ...(valueAt(state, "functions", "inspector.main") === "true" ? ["--inspect-main"] : []), - ]; -}; - -const nativeProcessFor = ( - artifactRoot: string, - state: PersistedStackState, - workload: PlannedWorkload, - port: number, - spec: WorkloadRuntimeSpecDefinition, - inputs: WorkloadRuntimeInputs = {}, -): NativeProcessResolution => { - const catalog = catalogEntryFor(workload.id); - const executablePath = catalog?.executablePath; - const resolvedPort = privatePortFor(state, workload.id, "primary") ?? port; - const args = spec - .args(state, workload, resolvedPort, "native") - .map((arg) => - arg.startsWith("app/") || arg.startsWith("share/") ? artifactPath(artifactRoot, arg) : arg, - ); - - const nativeArgs = nativeArgsFor(workload, args, inputs); - return { - executable: - executablePath === undefined ? artifactRoot : artifactPath(artifactRoot, executablePath), - args: nativeArgs, - cwd: - workload.id === "functions:edge-runtime" - ? nativeFunctionsDirectory(state, inputs) - : (spec.cwd?.(state, workload) ?? state.identity.projectRoot), - ...(workload.id === "database:database" - ? { gracefulStopSignal: "SIGINT", gracefulStopTimeout: "15 seconds" } - : {}), - }; -}; - -const nativeStartupProcessesFor = ( - artifactRoot: string, - _state: PersistedStackState, - workload: PlannedWorkload, - _inputs: WorkloadRuntimeInputs = {}, -): ReadonlyArray => { - const artifact = (relative: string): string => artifactPath(artifactRoot, relative); - const cwd = artifactRoot; - switch (workload.id) { - case "auth:auth": - return [{ executable: artifact("bin/auth"), args: ["migrate"], cwd }]; - case "storage:storage": - return [{ executable: artifact("bin/prepare"), args: [], cwd }]; - case "realtime:realtime": - return [{ executable: artifact("bin/prepare"), args: [], cwd }]; - case "analytics:analytics": - return [{ executable: artifact("bin/prepare"), args: [], cwd }]; - case "pooler:pooler": - return [ - { executable: artifact("bin/prepare"), args: [], cwd }, - { executable: artifact("bin/provision-tenant"), args: [], cwd }, - ]; - default: - return []; - } -}; - -const withRestSettings = ( - state: PersistedStackState, - runtime: WorkloadRuntimeKind, - port: number, - inputs: WorkloadRuntimeInputs = {}, -): Record => - compactEnvironment({ - PGRST_DB_URI: dbUrl(state, "authenticator", runtime), - PGRST_DB_SCHEMAS: valueAt(state, "rest", "schemas"), - PGRST_DB_EXTRA_SEARCH_PATH: valueAt(state, "rest", "extra_search_path"), - PGRST_DB_ANON_ROLE: "anon", - PGRST_JWT_SECRET: usesResolvedJwks(state) - ? (inputs.auth?.jwks ?? "") - : secret(state, "secret:auth.settings.jwt_secret"), - PGRST_SERVER_PORT: String(port), - PGRST_DB_MAX_ROWS: valueAt(state, "rest", "max_rows"), - PGRST_ADMIN_SERVER_PORT: String(workloadPort(state, "rest:rest", "admin", runtime, 3001)), - PGRST_OPENAPI_SERVER_PROXY_URI: valueAt(state, "rest", "external_url"), - }); - -const authNestedEnvironment = ( - state: PersistedStackState, - jwtIssuer: string, - inputs: WorkloadRuntimeInputs, -): Record => { - const out: Record = {}; - const settings = settingsFor(state, "auth"); - if (!isRecord(settings)) return out; - const external = settings.external; - if (isRecord(external)) - for (const [provider, value] of Object.entries(external)) { - if (!isRecord(value)) continue; - const prefix = `GOTRUE_EXTERNAL_${provider.replace(/[^A-Za-z0-9]+/gu, "_").toUpperCase()}`; - const enabled = value.enabled === true; - out[`${prefix}_ENABLED`] = enabled ? "true" : "false"; - if (!enabled) continue; - for (const key of ["client_id", "secret", "url", "skip_nonce_check", "email_optional"]) { - if (!(key in value)) continue; - const rendered = settingValue(state, value[key]); - if (rendered.length > 0) out[`${prefix}_${key.toUpperCase()}`] = rendered; - } - const configuredRedirect = settingValue(state, value.redirect_uri); - out[`${prefix}_REDIRECT_URI`] = configuredRedirect || `${jwtIssuer}/callback`; - } - const hooks = settings.hook; - if (isRecord(hooks)) - for (const [hook, value] of Object.entries(hooks)) { - if (!isRecord(value)) continue; - if (value.enabled !== true) continue; - const prefix = `GOTRUE_HOOK_${hook.replace(/[^A-Za-z0-9]+/gu, "_").toUpperCase()}`; - for (const [key, entry] of Object.entries(value)) - out[`${prefix}_${key.replace(/[^A-Za-z0-9]+/gu, "_").toUpperCase()}`] = settingValue( - state, - entry, - ); - } - const email = settings.email; - if (isRecord(email)) { - const templateBaseUrl = inputs.auth?.templateBaseUrl?.replace(/\/+$/u, ""); - const fileExtension = (path: string): string => { - const file = path.slice(Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\")) + 1); - const dot = file.lastIndexOf("."); - return dot <= 0 ? "" : file.slice(dot); - }; - const templateUrl = (id: string, contentPath: string): string | undefined => - templateBaseUrl === undefined - ? undefined - : `${templateBaseUrl}/email/${id}${fileExtension(contentPath)}`; - const templates = email.template; - if (isRecord(templates)) - for (const [name, value] of Object.entries(templates)) { - if (!isRecord(value)) continue; - const normalized = name.replace(/[^A-Za-z0-9]+/gu, "_").toUpperCase(); - const subject = settingValue(state, value.subject); - const contentPath = settingValue(state, value.content_path); - if (value.subject !== null && value.subject !== undefined) - out[`GOTRUE_MAILER_SUBJECTS_${normalized}`] = subject; - const url = contentPath.length > 0 ? templateUrl(name, contentPath) : undefined; - if (url !== undefined) out[`GOTRUE_MAILER_TEMPLATES_${normalized}`] = url; - } - const notifications = email.notification; - if (isRecord(notifications)) - for (const [name, value] of Object.entries(notifications)) { - if (!isRecord(value)) continue; - if (value.enabled !== true) continue; - const normalized = name.replace(/[^A-Za-z0-9]+/gu, "_").toUpperCase(); - out[`GOTRUE_MAILER_NOTIFICATIONS_${normalized}_ENABLED`] = settingValue( - state, - value.enabled, - ); - const contentPath = settingValue(state, value.content_path); - const notificationUrl = - contentPath.length > 0 ? templateUrl(`${name}_notification`, contentPath) : undefined; - if (notificationUrl !== undefined) - out[`GOTRUE_MAILER_TEMPLATES_${normalized}_NOTIFICATION`] = notificationUrl; - if (value.subject !== null && value.subject !== undefined) - out[`GOTRUE_MAILER_SUBJECTS_${normalized}_NOTIFICATION`] = settingValue( - state, - value.subject, - ); - } - } - return out; -}; - -const authExternalUrl = (state: PersistedStackState): string => { - const apiPort = state.ports.find((assignment) => assignment.field === "api")?.port; - return `http://127.0.0.1${apiPort === undefined ? "" : `:${apiPort}`}/auth/v1`; -}; - -const apiListenerUrl = (state: PersistedStackState, inputs?: WorkloadRuntimeInputs): string => { - const apiPort = state.ports.find((assignment) => assignment.field === "api")?.port; - const host = inputs?.hostRoute?.host ?? "127.0.0.1"; - return `http://${host}${apiPort === undefined ? "" : `:${apiPort}`}`; -}; - -const apiGatewayUrl = (state: PersistedStackState, inputs?: WorkloadRuntimeInputs): string => { - if (inputs?.hostRoute !== undefined) return apiListenerUrl(state, inputs); - return valueAt(state, "studio", "api_url") || apiListenerUrl(state); -}; - -const authSmsProvider = (state: PersistedStackState): string => { - const sms = settingsFor(state, "auth"); - if (!isRecord(sms) || !isRecord(sms.sms)) return ""; - // Providers are checked in this fixed order; if multiple are enabled, the first one wins - // and only its credentials are passed to GoTrue. - const providers = ["twilio", "twilio_verify", "messagebird", "textlocal", "vonage"]; - for (const provider of providers) { - const value = sms.sms[provider]; - if (isRecord(value) && value.enabled === true) return provider; - } - return ""; -}; - -const authSmsTestOtp = (state: PersistedStackState): string => { - const value = valueAt(state, "auth", "sms.test_otp"); - if (value.length === 0) return ""; - const settings = settingsFor(state, "auth"); - if (!isRecord(settings) || !isRecord(settings.sms) || !isRecord(settings.sms.test_otp)) return ""; - return Object.entries(settings.sms.test_otp) - .map(([phone, otp]) => `${phone}:${settingValue(state, otp)}`) - .join(","); -}; - -const passwordRequiredCharacters = (state: PersistedStackState): string => { - const requirements = valueAt(state, "auth", "password_requirements"); - if (requirements === "letters_digits") - return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ:0123456789"; - if (requirements === "lower_upper_letters_digits") - return "abcdefghijklmnopqrstuvwxyz:ABCDEFGHIJKLMNOPQRSTUVWXYZ:0123456789"; - if (requirements === "lower_upper_letters_digits_symbols") - return "abcdefghijklmnopqrstuvwxyz:ABCDEFGHIJKLMNOPQRSTUVWXYZ:0123456789:!@#$%^&*()_+-=[]{};'\\\":|<>?,./`~"; - return ""; -}; - -const withAuthSettings = ( - state: PersistedStackState, - runtime: WorkloadRuntimeKind, - port: number, - inputs: WorkloadRuntimeInputs = {}, -): Record => - compactEnvironment({ - ...capabilityEnv( - state, - "auth", - "GOTRUE", - (key) => - key === "GOTRUE_SIGNING_KEYS_PATH" || - key.startsWith("GOTRUE_THIRD_PARTY_") || - key.startsWith("GOTRUE_EXTERNAL_") || - key.startsWith("GOTRUE_MFA_PHONE_"), - ), - ...authNestedEnvironment( - state, - valueAt(state, "auth", "jwt_issuer") || authExternalUrl(state), - inputs, - ), - GOTRUE_DB_DATABASE_URL: dbUrl(state, "supabase_auth_admin", runtime), - GOTRUE_DB_DRIVER: "postgres", - GOTRUE_SITE_URL: valueAt(state, "auth", "site_url"), - GOTRUE_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - GOTRUE_JWT_EXP: valueAt(state, "auth", "jwt_expiry"), - GOTRUE_JWT_AUD: "authenticated", - GOTRUE_JWT_ADMIN_ROLES: "service_role", - GOTRUE_JWT_DEFAULT_GROUP_NAME: "authenticated", - // GoTrue releases consume either spelling; keep both for cross-version compatibility. - GOTRUE_JWT_VALIDMETHODS: "HS256,RS256,ES256", - GOTRUE_JWT_VALID_METHODS: "HS256,RS256,ES256", - ...(inputs.auth?.jwtKeys === undefined ? {} : { GOTRUE_JWT_KEYS: inputs.auth.jwtKeys }), - GOTRUE_API_HOST: "0.0.0.0", - GOTRUE_API_PORT: String(port), - API_EXTERNAL_URL: authExternalUrl(state), - GOTRUE_MAILER_URLPATHS_INVITE: `${authExternalUrl(state)}/verify`, - GOTRUE_MAILER_URLPATHS_CONFIRMATION: `${authExternalUrl(state)}/verify`, - GOTRUE_MAILER_URLPATHS_RECOVERY: `${authExternalUrl(state)}/verify`, - GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE: `${authExternalUrl(state)}/verify`, - GOTRUE_URI_ALLOW_LIST: valueAt(state, "auth", "additional_redirect_urls"), - GOTRUE_REFRESH_TOKEN_ROTATION_ENABLED: valueAt(state, "auth", "enable_refresh_token_rotation"), - GOTRUE_REFRESH_TOKEN_REUSE_INTERVAL: valueAt(state, "auth", "refresh_token_reuse_interval"), - GOTRUE_DISABLE_SIGNUP: valueAt(state, "auth", "enable_signup") === "false" ? "true" : "false", - GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: valueAt(state, "auth", "enable_anonymous_sign_ins"), - GOTRUE_PASSWORD_MIN_LENGTH: valueAt(state, "auth", "minimum_password_length"), - GOTRUE_PASSWORD_REQUIREMENTS: valueAt(state, "auth", "password_requirements"), - GOTRUE_PASSWORD_REQUIRED_CHARACTERS: passwordRequiredCharacters(state), - GOTRUE_JWT_ISSUER: valueAt(state, "auth", "jwt_issuer") || authExternalUrl(state), - GOTRUE_SECURITY_MANUAL_LINKING_ENABLED: valueAt(state, "auth", "enable_manual_linking"), - GOTRUE_SECURITY_REFRESH_TOKEN_ROTATION_ENABLED: valueAt( - state, - "auth", - "enable_refresh_token_rotation", - ), - GOTRUE_SECURITY_REFRESH_TOKEN_REUSE_INTERVAL: valueAt( - state, - "auth", - "refresh_token_reuse_interval", - ), - GOTRUE_RATE_LIMIT_EMAIL_SENT: valueAt(state, "auth", "rate_limit.email_sent"), - GOTRUE_RATE_LIMIT_SMS_SENT: valueAt(state, "auth", "rate_limit.sms_sent"), - GOTRUE_RATE_LIMIT_ANONYMOUS_USERS: valueAt(state, "auth", "rate_limit.anonymous_users"), - GOTRUE_RATE_LIMIT_TOKEN_REFRESH: valueAt(state, "auth", "rate_limit.token_refresh"), - GOTRUE_RATE_LIMIT_VERIFY: valueAt(state, "auth", "rate_limit.token_verifications"), - GOTRUE_RATE_LIMIT_OTP: valueAt(state, "auth", "rate_limit.sign_in_sign_ups"), - GOTRUE_RATE_LIMIT_WEB3: valueAt(state, "auth", "rate_limit.web3"), - GOTRUE_SECURITY_CAPTCHA_ENABLED: valueAt(state, "auth", "captcha.enabled"), - GOTRUE_SECURITY_CAPTCHA_PROVIDER: valueAt(state, "auth", "captcha.provider"), - GOTRUE_SECURITY_CAPTCHA_SECRET: valueAt(state, "auth", "captcha.secret"), - GOTRUE_MFA_TOTP_ENROLL_ENABLED: valueAt(state, "auth", "mfa.totp.enroll_enabled"), - GOTRUE_MFA_TOTP_VERIFY_ENABLED: valueAt(state, "auth", "mfa.totp.verify_enabled"), - GOTRUE_MFA_PHONE_ENROLL_ENABLED: valueAt(state, "auth", "mfa.phone.enroll_enabled"), - GOTRUE_MFA_PHONE_VERIFY_ENABLED: valueAt(state, "auth", "mfa.phone.verify_enabled"), - ...(valueAt(state, "auth", "mfa.phone.enroll_enabled") === "true" || - valueAt(state, "auth", "mfa.phone.verify_enabled") === "true" - ? { - GOTRUE_MFA_PHONE_OTP_LENGTH: valueAt(state, "auth", "mfa.phone.otp_length"), - GOTRUE_MFA_PHONE_TEMPLATE: valueAt(state, "auth", "mfa.phone.template"), - GOTRUE_MFA_PHONE_MAX_FREQUENCY: valueAt(state, "auth", "mfa.phone.max_frequency"), - } - : {}), - GOTRUE_MFA_WEB_AUTHN_ENROLL_ENABLED: valueAt(state, "auth", "mfa.web_authn.enroll_enabled"), - GOTRUE_MFA_WEB_AUTHN_VERIFY_ENABLED: valueAt(state, "auth", "mfa.web_authn.verify_enabled"), - GOTRUE_MFA_MAX_ENROLLED_FACTORS: valueAt(state, "auth", "mfa.max_enrolled_factors"), - GOTRUE_SESSIONS_TIMEBOX: valueAt(state, "auth", "sessions.timebox"), - GOTRUE_SESSIONS_INACTIVITY_TIMEOUT: valueAt(state, "auth", "sessions.inactivity_timeout"), - GOTRUE_MAILER_AUTOCONFIRM: - valueAt(state, "auth", "email.enable_confirmations") === "false" ? "true" : "false", - GOTRUE_MAILER_TEMPLATE_RELOADING_ENABLED: "true", - GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: valueAt( - state, - "auth", - "email.double_confirm_changes", - ), - GOTRUE_SECURITY_UPDATE_PASSWORD_REQUIRE_REAUTHENTICATION: valueAt( - state, - "auth", - "email.secure_password_change", - ), - GOTRUE_MAILER_MAX_FREQUENCY: valueAt(state, "auth", "email.max_frequency"), - GOTRUE_SMTP_MAX_FREQUENCY: valueAt(state, "auth", "email.max_frequency"), - GOTRUE_MAILER_OTP_LENGTH: valueAt(state, "auth", "email.otp_length"), - GOTRUE_MAILER_OTP_EXP: valueAt(state, "auth", "email.otp_expiry"), - ...(valueAt(state, "auth", "email.smtp.enabled") === "true" - ? { - GOTRUE_SMTP_HOST: valueAt(state, "auth", "email.smtp.host"), - GOTRUE_SMTP_PORT: valueAt(state, "auth", "email.smtp.port"), - GOTRUE_SMTP_USER: valueAt(state, "auth", "email.smtp.user"), - GOTRUE_SMTP_PASS: valueAt(state, "auth", "email.smtp.pass"), - GOTRUE_SMTP_ADMIN_EMAIL: valueAt(state, "auth", "email.smtp.admin_email"), - GOTRUE_SMTP_SENDER_NAME: valueAt(state, "auth", "email.smtp.sender_name"), - } - : capabilityEnabled(state, "mail") - ? { - GOTRUE_SMTP_HOST: - runtime === "container" ? containerAliasFor("mail:mail") : "127.0.0.1", - GOTRUE_SMTP_PORT: String(workloadPort(state, "mail:mail", "smtp", runtime, 1025)), - GOTRUE_SMTP_ADMIN_EMAIL: valueAt(state, "mail", "admin_email"), - GOTRUE_SMTP_SENDER_NAME: valueAt(state, "mail", "sender_name"), - } - : {}), - GOTRUE_SMS_AUTOCONFIRM: - valueAt(state, "auth", "sms.enable_confirmations") === "false" ? "true" : "false", - GOTRUE_SMS_MAX_FREQUENCY: valueAt(state, "auth", "sms.max_frequency"), - GOTRUE_SMS_OTP_EXP: "6000", - GOTRUE_SMS_OTP_LENGTH: "6", - GOTRUE_SMS_TEMPLATE: valueAt(state, "auth", "sms.template"), - GOTRUE_SMS_PROVIDER: authSmsProvider(state), - GOTRUE_SMS_TEST_OTP: authSmsTestOtp(state), - GOTRUE_EXTERNAL_WEB3_SOLANA_ENABLED: valueAt(state, "auth", "web3.solana.enabled"), - GOTRUE_EXTERNAL_WEB3_ETHEREUM_ENABLED: valueAt(state, "auth", "web3.ethereum.enabled"), - GOTRUE_EXTERNAL_EMAIL_ENABLED: valueAt(state, "auth", "email.enable_signup"), - GOTRUE_EXTERNAL_PHONE_ENABLED: valueAt(state, "auth", "sms.enable_signup"), - GOTRUE_OAUTH_SERVER_ENABLED: valueAt(state, "auth", "oauth_server.enabled"), - GOTRUE_OAUTH_SERVER_AUTHORIZATION_PATH: valueAt( - state, - "auth", - "oauth_server.authorization_url_path", - ), - GOTRUE_OAUTH_SERVER_ALLOW_DYNAMIC_REGISTRATION: valueAt( - state, - "auth", - "oauth_server.allow_dynamic_registration", - ), - }); - -const withStorageSettings = ( - state: PersistedStackState, - runtime: WorkloadRuntimeKind, - port: number, - inputs: WorkloadRuntimeInputs = {}, -): Record => { - const fileSizeLimit = parseFileSize(valueAt(state, "storage", "file_size_limit")); - return compactEnvironment({ - ...capabilityEnv( - state, - "storage", - "STORAGE", - (key) => - key === "STORAGE_FILE_SIZE_LIMIT" || - key.startsWith("STORAGE_S3_PROTOCOL_") || - key.startsWith("STORAGE_VECTOR_") || - key.startsWith("STORAGE_IMAGE_TRANSFORMATION_"), - ), - PORT: String(port), - ANON_KEY: secret(state, "secret:auth.settings.anon_key"), - SERVICE_KEY: secret(state, "secret:auth.settings.service_role_key"), - AUTH_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - DATABASE_URL: dbUrl(state, "supabase_storage_admin", runtime), - ...(fileSizeLimit === undefined ? {} : { FILE_SIZE_LIMIT: fileSizeLimit }), - STORAGE_BACKEND: "file", - FILE_STORAGE_BACKEND_PATH: - runtime === "container" - ? "/mnt" - : (inputs.storage?.dataPath ?? `${state.identity.projectRoot}/.supabase/storage`), - STORAGE_FILE_BACKEND_PATH: - runtime === "container" - ? "/mnt" - : (inputs.storage?.dataPath ?? `${state.identity.projectRoot}/.supabase/storage`), - ENABLE_IMAGE_TRANSFORMATION: valueAt(state, "storage", "image_transformation.enabled"), - S3_PROTOCOL_ENABLED: valueAt(state, "storage", "s3_protocol.enabled"), - S3_PROTOCOL_ACCESS_KEY_ID: valueAt(state, "storage", "s3_protocol.access_key_id"), - S3_PROTOCOL_ACCESS_KEY_SECRET: valueAt(state, "storage", "s3_protocol.secret_access_key"), - STORAGE_S3_REGION: valueAt(state, "storage", "s3_protocol.region"), - GLOBAL_S3_BUCKET: "stub", - TENANT_ID: "stub", - S3_PROTOCOL_PREFIX: "/storage/v1", - UPLOAD_FILE_SIZE_LIMIT: "52428800000", - UPLOAD_FILE_SIZE_LIMIT_STANDARD: "5242880000", - SIGNED_UPLOAD_URL_EXPIRATION_TIME: "7200", - PGRST_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - ...(inputs.auth?.jwks === undefined ? {} : { JWT_JWKS: inputs.auth.jwks }), - TUS_URL_PATH: "/storage/v1/upload/resumable", - IMGPROXY_URL: - runtime === "container" - ? `http://${containerAliasFor("storage:imgproxy")}:5001` - : `http://127.0.0.1:${workloadPort(state, "storage:imgproxy", "primary", runtime, 5001)}`, - ...(valueAt(state, "storage", "vector.enabled") === "true" - ? { - VECTOR_ENABLED: "true", - VECTOR_BUCKET_PROVIDER: "pgvector", - VECTOR_STORE_MIGRATIONS_ENABLED: "true", - VECTOR_DATABASE_URL: dbUrl(state, "postgres", runtime), - } - : {}), - }); -}; - -const databaseArgs = ( - state: PersistedStackState, - port: number, - runtime: WorkloadRuntimeKind, -): ReadonlyArray => { - const settings = settingsFor(state, "database"); - const postgresSettings = isRecord(settings) ? settings.settings : undefined; - const tuning = isRecord(postgresSettings) ? postgresSettings : {}; - const tuned = Object.entries(tuning).flatMap(([key, value]) => { - const rendered = settingValue(state, value); - return rendered.length === 0 ? [] : ["-c", `${key}=${rendered}`]; - }); - return [ - "-p", - String(port), - "-c", - runtime === "container" ? "listen_addresses=*" : "listen_addresses=127.0.0.1", - ...tuned, - ]; -}; - -const analyticsEnv = ( - state: PersistedStackState, - runtime: WorkloadRuntimeKind, - port: number, - inputs: WorkloadRuntimeInputs = {}, -): Record => { - const backend = valueAt(state, "analytics", "backend"); - const gcpJwtPath = inputs.analytics?.gcpJwtPath ?? ""; - return compactEnvironment({ - ...capabilityEnv(state, "analytics", "ANALYTICS", (key) => key === "ANALYTICS_GCP_JWT_PATH"), - PORT: String(port), - PHX_HTTP_PORT: String(port), - DB_HOSTNAME: dbHost(runtime), - DB_PORT: String(runtime === "container" ? 5432 : dbPort(state)), - DB_DATABASE: "_supabase", - DB_SCHEMA: "_analytics", - DB_USERNAME: "supabase_admin", - DB_PASSWORD: secret(state, DATABASE_INTERNAL_PASSWORD_SLOT), - LOGFLARE_SUPABASE_MODE: "true", - LOGFLARE_SINGLE_TENANT: "true", - LOGFLARE_PRIVATE_ACCESS_TOKEN: valueAt(state, "analytics", "api_key"), - LOGFLARE_FEATURE_FLAG_OVERRIDE: "'multibackend=true'", - LOGFLARE_MIN_CLUSTER_SIZE: "1", - LOGFLARE_LOG_LEVEL: "warn", - LOGFLARE_NODE_HOST: runtime === "container" ? "0.0.0.0" : "127.0.0.1", - RELEASE_COOKIE: "cookie", - ...(backend === "postgres" - ? { - POSTGRES_BACKEND_URL: dbUrl(state, "postgres", runtime, "_supabase"), - POSTGRES_BACKEND_SCHEMA: "_analytics", - } - : { - GOOGLE_DATASET_ID_APPEND: "_prod", - GOOGLE_PROJECT_ID: valueAt(state, "analytics", "gcp_project_id"), - GOOGLE_PROJECT_NUMBER: valueAt(state, "analytics", "gcp_project_number"), - GOOGLE_APPLICATION_CREDENTIALS: - runtime === "container" && gcpJwtPath.length > 0 - ? "/opt/app/rel/logflare/bin/gcloud.json" - : gcpJwtPath, - }), - }); -}; - -const specs: Readonly> = { - "database:database": { - bindings: { primary: { containerPort: 5432 } }, - args: (state, _workload, port) => databaseArgs(state, port, "native"), - env: (state, _workload, _port, runtime = "native", inputs = {}) => - compactEnvironment({ - PGDATA: - runtime === "container" - ? "/var/lib/postgresql/data" - : (inputs.database?.dataPath ?? `${state.identity.projectRoot}/.supabase/db/data`), - POSTGRES_USER: "supabase_admin", - POSTGRES_DB: "postgres", - POSTGRES_PASSWORD: secret(state, DATABASE_INTERNAL_PASSWORD_SLOT), - TZDIR: "/var/db/timezone/zoneinfo", - }), - containerArgs: (state, _workload, port) => databaseArgs(state, port, "container"), - readiness: { protocol: "tcp" }, - }, - "rest:rest": { - bindings: { primary: { containerPort: 3000 }, admin: { containerPort: 3001 } }, - args: () => [], - env: (state, _workload, port, runtime = "native", inputs = {}) => - withRestSettings(state, runtime, port, inputs), - containerArgs: () => [], - readiness: { protocol: "http", path: "/" }, - }, - "auth:auth": { - bindings: { primary: { containerPort: 9999 } }, - args: () => [], - env: (state, _workload, port, runtime = "native", inputs = {}) => - withAuthSettings(state, runtime, port, inputs), - containerArgs: () => [], - containerStartupProcesses: () => [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - readiness: { protocol: "http", path: "/health" }, - }, - "realtime:realtime": { - bindings: { primary: { containerPort: 4000 }, rpc: { containerPort: 5369 } }, - args: () => [], - env: (state, _workload, port, runtime = "native", inputs = {}) => { - // Production Realtime defaults both gen_rpc TCP ports to 5369. Native stacks - // need a unique host binding; container netns already owns 5369 in-container. - const rpcPort = - state.runtime.kind === "native" - ? privatePortFor(state, "realtime:realtime", "rpc") - : undefined; - return compactEnvironment({ - ...capabilityEnv(state, "realtime", "REALTIME"), - PORT: String(port), - DB_HOST: dbHost(runtime), - DB_PORT: String(runtime === "container" ? 5432 : dbPort(state)), - DB_USER: "supabase_admin", - DB_PASSWORD: secret(state, DATABASE_INTERNAL_PASSWORD_SLOT), - DB_NAME: "postgres", - DB_AFTER_CONNECT_QUERY: "SET search_path TO _realtime", - API_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - ...(inputs.auth?.jwks === undefined ? {} : { API_JWT_JWKS: inputs.auth.jwks }), - METRICS_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - DB_ENC_KEY: secret(state, "secret:realtime.settings.db_enc_key"), - SECRET_KEY_BASE: secret(state, "secret:realtime.settings.secret_key_base"), - DNS_NODES: "''", - APP_NAME: "realtime", - SEED_SELF_HOST: "true", - MAX_HEADER_LENGTH: valueAt(state, "realtime", "max_header_length"), - ERL_AFLAGS: - valueAt(state, "realtime", "ip_version") === "IPv6" - ? "-proto_dist inet6_tcp" - : "-proto_dist inet_tcp", - RUN_JANITOR: "true", - ...(rpcPort === undefined - ? {} - : { - GEN_RPC_TCP_SERVER_PORT: String(rpcPort), - GEN_RPC_TCP_CLIENT_PORT: String(rpcPort), - GEN_RPC_SOCKET_IP: "127.0.0.1", - }), - }); - }, - containerEntrypoint: "/usr/bin/tini", - containerArgs: () => ["-s", "-g", "--", "/app/bin/server"], - containerStartupProcesses: () => [{ entrypoint: "/app/bin/prepare", command: [] }], - readiness: { protocol: "http", path: "/healthcheck" }, - }, - "storage:storage": { - bindings: { primary: { containerPort: 5000 } }, - args: () => [], - env: (state, _workload, port, runtime = "native", inputs = {}) => - withStorageSettings(state, runtime, port, inputs), - containerArgs: () => [], - containerStartupProcesses: () => [{ entrypoint: "/slim-runtime/bin/prepare", command: [] }], - readiness: { protocol: "http", path: "/status" }, - }, - "storage:imgproxy": { - bindings: { primary: { containerPort: 5001 } }, - args: () => [], - env: (_state, _workload, port, runtime = "native") => ({ - IMGPROXY_BIND: `${runtime === "container" ? "0.0.0.0" : "127.0.0.1"}:${port}`, - IMGPROXY_LOCAL_FILESYSTEM_ROOT: "/", - }), - containerArgs: () => [], - readiness: { protocol: "http", path: "/health" }, - }, - "functions:edge-runtime": { - bindings: { primary: { containerPort: 9000 }, inspector: { containerPort: 9229 } }, - cwd: functionsRoot, - args: (state, _workload, port, runtime = "native") => [ - "start", - `--main-service=${functionsRoot(state)}`, - `--port=${port}`, - `--policy=${valueAt(state, "functions", "edge_runtime.policy")}`, - ...functionsInspectorArgs(state, runtime), - ], - env: (state, _workload, port, runtime = "native", inputs = {}) => ({ - ...edgeRuntimeJwtEnvironment(state, inputs), - EDGE_RUNTIME_PORT: String(port), - FUNCTIONS_CONTAINER_ROOT, - SUPABASE_INTERNAL_FUNCTIONS_ROOT: - runtime === "container" ? FUNCTIONS_CONTAINER_ROOT : functionsRoot(state), - SUPABASE_INTERNAL_FUNCTIONS_CONFIG: functionsConfigEnvironment(state), - SUPABASE_URL: apiListenerUrl(state, runtime === "container" ? inputs : undefined), - EDGE_RUNTIME_POLICY: valueAt(state, "functions", "edge_runtime.policy"), - EDGE_RUNTIME_DENO_VERSION: valueAt(state, "functions", "edge_runtime.deno_version"), - INSPECTOR_MODE: valueAt(state, "functions", "inspector.mode"), - INSPECTOR_MAIN: valueAt(state, "functions", "inspector.main"), - }), - containerArgs: (state, _workload, port) => [ - "start", - `--main-service=${FUNCTIONS_BOOTSTRAP_CONTAINER_PATH}`, - `--port=${port}`, - `--policy=${valueAt(state, "functions", "edge_runtime.policy")}`, - ...functionsInspectorArgs(state, "container"), - ], - containerMounts: (state) => [ - { source: functionsRoot(state), target: FUNCTIONS_CONTAINER_ROOT, readOnly: true }, - ], - readiness: { protocol: "http", path: "/_internal/health" }, - }, - "studio:studio": { - bindings: { primary: { containerPort: 3000 } }, - args: () => [], - env: (state, _workload, port, runtime = "native", inputs = {}) => - compactEnvironment({ - ...capabilityEnv(state, "studio", "STUDIO"), - PORT: String(port), - HOSTNAME: "0.0.0.0", - STUDIO_PG_META_URL: - runtime === "container" - ? `http://${containerAliasFor("studio:pgmeta")}:8080` - : `http://127.0.0.1:${workloadPort(state, "studio:pgmeta", "primary", runtime, 8080)}`, - LOGFLARE_URL: - runtime === "container" - ? `http://${containerAliasFor("analytics:analytics")}:4000` - : `http://127.0.0.1:${workloadPort(state, "analytics:analytics", "primary", runtime, 4000)}`, - LOGFLARE_PRIVATE_ACCESS_TOKEN: valueAt(state, "analytics", "api_key"), - NEXT_PUBLIC_ENABLE_LOGS: capabilityEnabled(state, "analytics") ? "true" : "false", - NEXT_ANALYTICS_BACKEND_PROVIDER: valueAt(state, "analytics", "backend"), - SUPABASE_URL: apiGatewayUrl(state, runtime === "container" ? inputs : undefined), - SUPABASE_PUBLIC_URL: apiListenerUrl(state), - SUPABASE_ANON_KEY: secret(state, "secret:auth.settings.anon_key"), - SUPABASE_SERVICE_KEY: secret(state, "secret:auth.settings.service_role_key"), - SUPABASE_PUBLISHABLE_KEY: secret(state, "secret:auth.settings.publishable_key"), - SUPABASE_SECRET_KEY: secret(state, "secret:auth.settings.secret_key"), - EDGE_FUNCTIONS_MANAGEMENT_FOLDER: - runtime === "container" ? FUNCTIONS_CONTAINER_ROOT : functionsRoot(state), - OPENAI_API_KEY: secret(state, "secret:studio.settings.openai_api_key"), - CURRENT_CLI_VERSION: "local", - POSTGRES_PASSWORD: secret(state, DATABASE_INTERNAL_PASSWORD_SLOT), - POSTGRES_USER_READ_WRITE: "postgres", - PGRST_DB_SCHEMAS: "public,graphql_public", - PGRST_DB_EXTRA_SEARCH_PATH: "public,extensions", - PGRST_DB_MAX_ROWS: "1000", - }), - containerArgs: () => [], - containerMounts: (state) => [ - { source: functionsRoot(state), target: FUNCTIONS_CONTAINER_ROOT, readOnly: true }, - ], - readiness: { protocol: "http", path: "/api/platform/profile" }, - }, - "studio:pgmeta": { - bindings: { primary: { containerPort: 8080 } }, - args: () => [], - env: (state, _workload, port, runtime = "native") => ({ - ...capabilityEnv(state, "studio", "PG_META"), - PG_META_PORT: String(port), - PG_META_DB_HOST: dbHost(runtime), - PG_META_DB_PORT: String(runtime === "container" ? 5432 : dbPort(state)), - PG_META_DB_NAME: "postgres", - PG_META_DB_USER: "postgres", - PG_META_DB_PASSWORD: secret(state, DATABASE_INTERNAL_PASSWORD_SLOT), - }), - containerArgs: () => [], - readiness: { protocol: "http", path: "/health" }, - }, - "mail:mail": { - bindings: { - ui: { containerPort: 8025 }, - smtp: { containerPort: 1025 }, - pop3: { containerPort: 1110 }, - }, - args: () => [], - env: (state, _workload, _port, runtime = "native") => ({ - ...capabilityEnv(state, "mail", "MAIL"), - MP_UI_BIND_ADDR: `${runtime === "container" ? "0.0.0.0" : "127.0.0.1"}:${workloadPort(state, "mail:mail", "ui", runtime, 8025)}`, - MP_SMTP_BIND_ADDR: `${runtime === "container" ? "0.0.0.0" : "127.0.0.1"}:${workloadPort(state, "mail:mail", "smtp", runtime, 1025)}`, - MP_POP3_BIND_ADDR: `${runtime === "container" ? "0.0.0.0" : "127.0.0.1"}:${workloadPort(state, "mail:mail", "pop3", runtime, 1110)}`, - MP_SMTP_DISABLE_RDNS: "true", - }), - containerArgs: () => [], - readiness: { protocol: "http", path: "/readyz", binding: "ui" }, - }, - "analytics:analytics": { - bindings: { primary: { containerPort: 4000 } }, - args: () => ["start"], - env: (state, _workload, port, runtime = "native", inputs = {}) => - analyticsEnv(state, runtime, port, inputs), - // The slim container entrypoint performs Logflare migrations before start. - containerArgs: () => [], - containerMounts: (state, _workload, inputs = {}) => { - const backend = valueAt(state, "analytics", "backend"); - const source = inputs.analytics?.gcpJwtPath ?? ""; - return backend === "bigquery" && source.length > 0 - ? [ - { - source, - target: "/opt/app/rel/logflare/bin/gcloud.json", - readOnly: true, - }, - ] - : []; - }, - readiness: { protocol: "http", path: "/health" }, - }, - "analytics:vector": { - bindings: { primary: { containerPort: 9001 } }, - args: (_state, _workload, _port) => ["--config", "share/doc/vector/config/vector.yaml"], - env: (state, _workload, port, runtime = "native") => ({ - ...capabilityEnv(state, "analytics", "VECTOR"), - VECTOR_API_ADDRESS: `${runtime === "container" ? "0.0.0.0" : "127.0.0.1"}:${port}`, - VECTOR_API_PORT: String(port), - LOGFLARE_URL: - runtime === "container" - ? `http://${containerAliasFor("analytics:analytics")}:4000` - : `http://127.0.0.1:${workloadPort(state, "analytics:analytics", "primary", runtime, 4000)}`, - LOGFLARE_PRIVATE_ACCESS_TOKEN: valueAt(state, "analytics", "api_key"), - }), - containerArgs: (_state, _workload, _port, inputs = {}) => - inputs.analytics?.vectorConfigPath === undefined - ? [] - : ["--config", "/etc/vector/vector.yaml"], - containerMounts: (_state, _workload, inputs = {}) => - inputs.analytics?.vectorConfigPath === undefined - ? [] - : [ - { - source: inputs.analytics.vectorConfigPath, - target: "/etc/vector/vector.yaml", - readOnly: true, - }, - ], - readiness: { protocol: "http", path: "/health" }, - }, - "pooler:pooler": { - bindings: { primary: { containerPort: 6543 }, admin: { containerPort: 4000 } }, - args: () => ["start"], - env: (state, _workload, _port, runtime = "native") => { - const adminPort = - runtime === "container" ? 4000 : privatePortFor(state, "pooler:pooler", "admin"); - const primaryPort = privatePortFor(state, "pooler:pooler", "primary"); - if (adminPort === undefined || primaryPort === undefined) - throw new Error("Pooler private port assignments must be validated before env resolution"); - return { - ...capabilityEnv(state, "pooler", "POOLER"), - // `port` is the readiness binding (admin); proxy listeners use the primary SQL binding. - // validatePrivateAssignments guarantees both native bindings are assigned. - PORT: String(adminPort), - PROXY_PORT_SESSION: - runtime === "native" && valueAt(state, "pooler", "pool_mode") === "session" - ? String(primaryPort) - : "5432", - PROXY_PORT_TRANSACTION: - runtime === "native" && valueAt(state, "pooler", "pool_mode") !== "session" - ? String(primaryPort) - : "6543", - DATABASE_URL: `ecto://postgres:${secret(state, DATABASE_INTERNAL_PASSWORD_SLOT)}@${dbHost(runtime)}:${runtime === "container" ? 5432 : dbPort(state)}/_supabase`, - POSTGRES_HOST: dbHost(runtime), - POSTGRES_PORT: String(runtime === "container" ? 5432 : dbPort(state)), - POSTGRES_PASSWORD: secret(state, DATABASE_INTERNAL_PASSWORD_SLOT), - API_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - REGION: "local", - TENANT_ID: valueAt(state, "pooler", "tenant_id"), - CLUSTER_POSTGRES: "true", - SECRET_KEY_BASE: valueAt(state, "pooler", "secret_key_base"), - VAULT_ENC_KEY: valueAt(state, "pooler", "encryption_key"), - METRICS_JWT_SECRET: secret(state, "secret:auth.settings.jwt_secret"), - DEFAULT_POOL_SIZE: valueAt(state, "pooler", "default_pool_size"), - MAX_CLIENT_CONN: valueAt(state, "pooler", "max_client_conn"), - POOL_MODE: valueAt(state, "pooler", "pool_mode"), - }; - }, - // Mirror native convergence explicitly: prepare and provision before the main server. - containerEntrypoint: "/usr/bin/tini", - containerStartupProcesses: () => [ - { entrypoint: "/app/bin/prepare", command: [] }, - { entrypoint: "/app/bin/provision-tenant", command: [] }, - ], - containerArgs: () => ["-s", "-g", "--", "/app/bin/server"], - readiness: { protocol: "http", path: "/api/health", binding: "admin" }, - }, -}; - -const WORKLOAD_BINDING_NAMES: ReadonlyArray = [ - "primary", - "admin", - "ui", - "smtp", - "pop3", - "inspector", - "rpc", -]; - -const declaredBindings = ( - bindings: WorkloadBindings, -): ReadonlyArray => - WORKLOAD_BINDING_NAMES.flatMap((name) => { - const binding = bindings[name]; - return binding === undefined ? [] : [[name, binding] as const]; - }); - -const selectedBindings = ( - state: BindingSelectionState, - bindings: WorkloadBindings, -): ReadonlyArray => { - return declaredBindings(bindings).filter(([binding]) => { - if (binding === "inspector") return functionsInspectorRequested(state); - // Docker already owns 5369 in the container netns; publishing it on the host - // collides when two stacks share a host. - if (binding === "rpc") return state.runtime.kind === "native"; - return true; - }); -}; - -/** Derive the exact private endpoint reservations required by a compiled plan. */ -export const privateBindingIntentsFor = ( - plan: ExecutionPlan, - state: BindingSelectionState, -): ReadonlyArray => - plan.workloads.flatMap((workload) => { - const spec = specs[workload.id]; - if (spec === undefined) return []; - return selectedBindings(state, spec.bindings).map(([binding]) => ({ - workloadId: workload.id, - binding, - })); - }); - -export const validatePrivateAssignments = ( - state: PersistedStackState, - workload: PlannedWorkload, -): Effect.Effect => { - const spec = specs[workload.id]; - if (spec === undefined) return Effect.void; - // Runtime env resolution assumes every declared binding was assigned here. - for (const [binding] of selectedBindings(state, spec.bindings)) { - if ( - !state.privatePorts.some( - (assignment) => assignment.workloadId === workload.id && assignment.binding === binding, - ) - ) - return Effect.fail( - new StackPreparationError({ - message: `Missing private port assignment for ${workload.id} (${binding})`, - workload: workload.id, - }), - ); - } - return Effect.void; -}; - -export const runtimeSpecFor = (workload: PlannedWorkload): WorkloadRuntimeSpec | undefined => { - const spec = specs[workload.id]; - const catalog = catalogEntryFor(workload.id); - if (spec === undefined || catalog === undefined) return undefined; - const primary = - spec.bindings.primary ?? - spec.bindings.admin ?? - spec.bindings.ui ?? - spec.bindings.smtp ?? - spec.bindings.pop3 ?? - spec.bindings.inspector; - if (primary === undefined) return undefined; - return { - ...spec, - containerPort: primary.containerPort, - cwd: spec.cwd ?? ((state) => state.identity.projectRoot), - args: spec.args, - env: spec.env, - containerStartupProcesses: (state, currentWorkload, inputs = {}) => - spec.containerStartupProcesses?.(state, currentWorkload, inputs) ?? [], - readiness: { ...spec.readiness, binding: spec.readiness.binding ?? "primary" }, - privateEndpoint: (state, binding = "primary", runtime = "native") => - privateEndpointFor( - state, - workload.id, - spec.bindings, - binding, - runtime, - catalog.containerAlias, - ), - nativeProcess: (artifactRoot, state, currentWorkload, port, inputs = {}) => - nativeProcessFor(artifactRoot, state, currentWorkload, port, spec, inputs), - nativeStartupProcesses: (artifactRoot, state, currentWorkload, _port, inputs = {}) => - nativeStartupProcessesFor(artifactRoot, state, currentWorkload, inputs), - }; -}; - -/** Adapter consumed by ContainerRuntime; env is written to an env-file by its owner. */ -export const containerResolutionFor = ( - state: PersistedStackState, - workload: PlannedWorkload, - inputs: WorkloadRuntimeInputs = {}, -): ContainerWorkloadResolution | undefined => { - const spec = runtimeSpecFor(workload); - if (spec === undefined) return undefined; - const catalog = catalogEntryFor(workload.id); - if (catalog === undefined) return undefined; - return { - ...(spec.containerEntrypoint === undefined ? {} : { entrypoint: spec.containerEntrypoint }), - command: spec.containerArgs( - state, - workload, - containerPortFor(state, workload.id, "primary", spec.containerPort), - inputs, - ), - startup: spec.containerStartupProcesses(state, workload, inputs), - env: spec.env( - state, - workload, - containerPortFor(state, workload.id, "primary", spec.containerPort), - "container", - inputs, - ), - mounts: spec.containerMounts?.(state, workload, inputs) ?? [], - networkAliases: [catalog.containerAlias], - publications: selectedBindings(state, spec.bindings).flatMap(([binding, definition]) => { - const assignment = state.privatePorts.find( - (entry) => entry.workloadId === workload.id && entry.binding === binding, - ); - return assignment === undefined - ? [] - : [ - { - address: "127.0.0.1" as const, - hostPort: assignment.port, - containerPort: containerPortFor( - state, - workload.id, - binding, - definition.containerPort, - ), - }, - ]; - }), - ...(workload.id === "functions:edge-runtime" && inputs.functions?.bootstrapPath !== undefined - ? { - bootstrap: { - source: inputs.functions.bootstrapPath, - destination: FUNCTIONS_BOOTSTRAP_CONTAINER_PATH, - }, - } - : {}), - }; -}; - -/** Effect boundary used by owners that need typed validation before creating a container. */ -export const resolveContainerResolutionFor = ( - state: PersistedStackState, - workload: PlannedWorkload, - inputs: WorkloadRuntimeInputs = {}, -): Effect.Effect< - ContainerWorkloadResolution | undefined, - StackPreparationError | StackStateInvalidError -> => - Effect.all([ - validateWorkloadRuntimeInputs(state, workload, inputs), - validatePrivateAssignments(state, workload), - ]).pipe(Effect.map(() => containerResolutionFor(state, workload, inputs))); diff --git a/packages/stack/src/runtime/container-runtime.integration.test.ts b/packages/stack/src/runtime/container-runtime.integration.test.ts deleted file mode 100644 index a83d2f90d6..0000000000 --- a/packages/stack/src/runtime/container-runtime.integration.test.ts +++ /dev/null @@ -1,2886 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Crypto, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Path, - Stream, - Sink, -} from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import { NodeServices } from "@effect/platform-node"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { ContainerArtifact } from "../model/CapabilityModule.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import { - ContainerCommandError, - ContainerEngineProtocolError, - makeProcessCommandRunner, - type ContainerProcessRequest, - type ContainerCommandResult, - type ContainerContainerSpec, - type ContainerLogLine, - type ContainerEngine, - type ContainerCommandRunner, - type ContainerNetworkSpec, - type ContainerResource, - type ContainerVolumeSpec, -} from "./ContainerEngine.ts"; -import { makeDockerEngine, serializeDockerCommand } from "./DockerEngine.ts"; -import { defaultContainerEngineResolver } from "./ContainerEngineResolver.ts"; -import { makePodmanEngine, serializePodmanCommand } from "./PodmanEngine.ts"; -import { makeContainerRuntime } from "./ContainerRuntime.ts"; -import { RuntimeDriverError, type RuntimeWorkloadKey } from "./RuntimeDriver.ts"; -import { LogStoreError, type LogRecord, type LogStore } from "../supervisor/LogStore.ts"; -import { ContainerEngineError } from "../public/Errors.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { makeSupervisor, type SupervisorRuntime } from "../supervisor/Supervisor.ts"; -import type { SupervisorIngress } from "../supervisor/Ingress.ts"; -import { deriveStackId } from "../identity/Identity.ts"; - -const makeControlledCommandRunner = ( - options: Pick & Partial>, -): ContainerCommandRunner => ({ - run: options.run, - stream: options.stream ?? (() => Stream.empty), -}); - -const stackId = StackIdSchema.make("a".repeat(64)); -const key: RuntimeWorkloadKey = { - stackId, - workloadId: "database:database", -}; - -const containerArtifact: ContainerArtifact = { - kind: "container", - image: "example/database:1", -}; - -const workload = (selected: PlannedWorkload["selected"] = containerArtifact): PlannedWorkload => ({ - id: key.workloadId, - capability: key.workloadId.startsWith("functions:") ? "functions" : "database", - dependencies: [], - readiness: {}, - artifacts: { - native: { kind: "native", release: "1" }, - container: containerArtifact, - }, - selected, -}); - -const commandResult = (value: unknown): ContainerCommandResult => ({ - stdout: typeof value === "string" ? value : JSON.stringify(value), - stderr: "", - exitCode: 0, -}); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -interface FakeContainerState { - resources: Array; - imagePresent: boolean; - calls: Array; - createdSpecs: Array; - nextId: number; - inspectImageFailure?: ContainerCommandError; - copyFailure?: ContainerCommandError; - waitExitCode?: number; -} - -const fakeContainerEngine = (state: FakeContainerState): ContainerEngine => { - const id = (prefix: string): string => `${prefix}-${state.nextId++}`; - const find = (resourceId: string): ContainerResource | undefined => - state.resources.find((resource) => resource.id === resourceId); - return { - kind: "docker", - preflight: Effect.succeed({ host: "host.docker.internal" }), - probe: Effect.void, - inspectImage: () => - state.inspectImageFailure !== undefined - ? Effect.fail(state.inspectImageFailure) - : Effect.sync(() => { - state.calls.push("inspect-image"); - return { present: state.imagePresent }; - }), - pullImage: () => - Effect.sync(() => { - state.calls.push("pull-image"); - state.imagePresent = true; - }), - listResources: () => - Effect.sync(() => { - state.calls.push("list-resources"); - return [...state.resources]; - }), - createNetwork: (spec: ContainerNetworkSpec) => - Effect.sync(() => { - state.calls.push("create-network"); - const resource: ContainerResource = { - id: id("network"), - name: spec.name, - kind: "network", - labels: spec.labels, - }; - state.resources.push(resource); - return resource; - }), - removeNetwork: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`remove-network:${resourceId}`); - state.resources = state.resources.filter((resource) => resource.id !== resourceId); - }), - createVolume: (spec: ContainerVolumeSpec) => - Effect.sync(() => { - state.calls.push(`create-volume:${spec.name}`); - const resource: ContainerResource = { - id: id("volume"), - name: spec.name, - kind: "volume", - labels: spec.labels, - }; - state.resources.push(resource); - return resource; - }), - removeVolume: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`remove-volume:${resourceId}`); - state.resources = state.resources.filter((resource) => resource.id !== resourceId); - }), - createContainer: (spec: ContainerContainerSpec) => - Effect.sync(() => { - state.calls.push("create-container"); - state.createdSpecs.push(spec); - const resource: ContainerResource = { - id: id("container"), - name: spec.name, - kind: spec.role, - labels: spec.labels, - state: "created", - }; - state.resources.push(resource); - return resource; - }), - copyToContainer: (resourceId: string, source: string, destination: string) => - state.copyFailure !== undefined - ? Effect.fail(state.copyFailure) - : Effect.sync(() => { - state.calls.push(`copy:${resourceId}:${source}:${destination}`); - }), - startContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`start:${resourceId}`); - const resource = find(resourceId); - if (resource !== undefined) - state.resources = state.resources.map((entry) => - entry.id === resourceId ? { ...entry, state: "running" } : entry, - ); - }), - waitContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`wait:${resourceId}`); - return state.waitExitCode ?? 0; - }), - stopContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`stop:${resourceId}`); - const resource = find(resourceId); - if (resource !== undefined) - state.resources = state.resources.map((entry) => - entry.id === resourceId ? { ...entry, state: "stopped" } : entry, - ); - }), - removeContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`remove:${resourceId}`); - state.resources = state.resources.filter((resource) => resource.id !== resourceId); - }), - streamLogs: () => Stream.empty, - }; -}; - -const memoryLogStore = (records: LogRecord[]): LogStore => ({ - path: "memory://container-logs", - append: (record) => - Effect.sync(() => ({ - cursor: { opaque: `v1_${records.length + 1}` }, - timestamp: "2026-01-01T00:00:00.000Z", - ...record, - })).pipe(Effect.tap((entry) => Effect.sync(() => records.push(entry)))), - read: () => Effect.succeed([]), -}); - -const pendingStartRace = (operation: "stop" | "remove") => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const startEntered = yield* Deferred.make(); - const releaseStart = yield* Deferred.make(); - let removeCalls = 0; - let stopCalls = 0; - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - startContainer: (id) => - Effect.gen(function* () { - yield* Deferred.succeed(startEntered, undefined); - yield* Deferred.await(releaseStart); - yield* base.startContainer(id); - }), - stopContainer: (id) => - Effect.gen(function* () { - stopCalls += 1; - yield* base.stopContainer(id); - }), - removeContainer: (id) => - Effect.gen(function* () { - removeCalls += 1; - yield* base.removeContainer(id); - }), - }, - ownerSessionId: "owner-session", - }); - const starting = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(startEntered); - yield* runtime[operation](key); - yield* Deferred.succeed(releaseStart, undefined); - const started = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(started)).toBe(true); - expect(state.resources.filter((resource) => resource.kind === "workload")).toEqual([]); - expect(removeCalls).toBe(1); - expect(stopCalls).toBe(operation === "stop" ? 1 : 0); - yield* runtime.remove(key); - expect(removeCalls).toBe(1); - }); - -describe("container runtime", () => { - it.live("checks only the Docker client version when selecting an installed runtime", () => { - const commands: Array> = []; - const spawner = ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); - commands.push([command.command, ...command.args]); - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(1), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ); - }); - return Effect.gen(function* () { - expect(yield* defaultContainerEngineResolver.isInstalled("docker")).toBe(true); - expect(commands).toEqual([["docker", "--version"]]); - }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)); - }); - - it.live("executes the exact command argv through a bounded process boundary", () => - Effect.gen(function* () { - const runner = yield* makeProcessCommandRunner({ - executable: process.execPath, - baseArgs: [ - "-e", - "if (process.argv[1] !== 'version' || process.argv[2] !== '--format' || process.argv[3] !== '{{json .}}') process.exit(9); process.stdout.write(JSON.stringify({ok:true}))", - ], - }); - const result = yield* runner.run({ args: ["version", "--format", "{{json .}}"] }); - expect(result.exitCode).toBe(0); - expect(result.stdout).toBe('{"ok":true}'); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("interrupts an owned Docker log follower subprocess", () => - Effect.gen(function* () { - const firstChunk = yield* Deferred.make(); - const runner = yield* makeProcessCommandRunner({ - executable: process.execPath, - baseArgs: ["-e", "process.stdout.write('follower-line\\n'); setInterval(() => {}, 1000)"], - }); - const follower = yield* runner - .stream({ - args: ["logs", "--follow", "--tail", "0", "container-id"], - }) - .pipe( - Stream.runForEach(() => Deferred.succeed(firstChunk, undefined)), - Effect.forkChild({ startImmediately: true }), - ); - yield* Deferred.await(firstChunk); - yield* Fiber.interrupt(follower); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("serializes an entrypoint override before the image", () => - Effect.sync(() => { - const spec: ContainerContainerSpec = { - name: "auth-init", - image: "registry.example.test/auth:fixture", - labels: { - stackId, - ownerSessionId: "owner", - workloadId: "auth:auth", - role: "workload", - }, - network: "private", - mounts: [], - volumeMounts: [], - publications: [], - role: "workload", - entrypoint: "/usr/local/bin/auth", - command: ["migrate"], - }; - const docker = serializeDockerCommand({ operation: "create-container", spec }); - expect(docker.args.slice(docker.args.indexOf("--entrypoint"), -2)).toEqual([ - "--entrypoint", - "/usr/local/bin/auth", - ]); - expect(docker.args.indexOf("--entrypoint")).toBeLessThan(docker.args.indexOf(spec.image)); - expect(docker.args.slice(-2)).toEqual([spec.image, "migrate"]); - const podman = serializePodmanCommand({ operation: "create-container", spec }); - expect(podman.args.indexOf("--entrypoint")).toBeLessThan(podman.args.indexOf(spec.image)); - expect(podman.args.slice(-2)).toEqual([spec.image, "migrate"]); - }), - ); - - it.live("waits for one container and rejects malformed exit output", () => - Effect.gen(function* () { - const runner = makeControlledCommandRunner({ - run: (request) => - Effect.succeed(commandResult(request.args[0] === "wait" ? "17\n" : "27.0.0\n")), - }); - const docker = makeDockerEngine({ - runner, - platform: { os: "linux", desktop: false }, - }); - expect(yield* docker.waitContainer("container-id")).toBe(17); - const podman = makePodmanEngine({ - runner: makeControlledCommandRunner({ - run: (request) => - Effect.succeed(commandResult(request.args[0] === "wait" ? "17\n" : "27.0.0\n")), - }), - platform: { os: "linux", rootless: true }, - }); - expect(yield* podman.waitContainer("container-id")).toBe(17); - const malformed = makeDockerEngine({ - runner: makeControlledCommandRunner({ - run: () => Effect.succeed(commandResult("17\n18\n")), - }), - platform: { os: "linux", desktop: false }, - }); - const result = yield* malformed.waitContainer("container-id").pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(ContainerEngineProtocolError); - } - const malformedPodman = makePodmanEngine({ - runner: makeControlledCommandRunner({ - run: () => Effect.succeed(commandResult("17\n18\n")), - }), - platform: { os: "linux", rootless: true }, - }); - const podmanResult = yield* malformedPodman.waitContainer("container-id").pipe(Effect.exit); - expect(Exit.isFailure(podmanResult)).toBe(true); - if (Exit.isFailure(podmanResult)) { - const error = Cause.findErrorOption(podmanResult.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(ContainerEngineProtocolError); - } - }), - ); - - it.live("derives host routes before mutation and rejects unsupported hosts", () => - Effect.gen(function* () { - const calls: ContainerProcessRequest[] = []; - const runner = makeControlledCommandRunner({ - run: (command) => - Effect.sync(() => { - calls.push(command); - return commandResult({ ok: true }); - }), - }); - const desktop = makeDockerEngine({ - runner, - platform: { os: "darwin", desktop: true }, - }); - expect((yield* desktop.preflight).host).toBe("host.docker.internal"); - const linux = makeDockerEngine({ - runner, - platform: { os: "linux", desktop: false }, - }); - expect((yield* linux.preflight).host).toBe("host.docker.internal"); - const podman = makePodmanEngine({ - runner, - platform: { os: "linux", rootless: true }, - }); - expect((yield* podman.preflight).host).toBe("host.containers.internal"); - const unsupported = makeDockerEngine({ - runner, - platform: { os: "linux", desktop: false, remote: true }, - }); - const result = yield* unsupported.preflight.pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - } - expect(calls).toHaveLength(0); - }), - ); - - it.live("resolves a rootful Docker network gateway without enabling host binds elsewhere", () => - Effect.gen(function* () { - const requests: ContainerProcessRequest[] = []; - const docker = makeDockerEngine({ - runner: makeControlledCommandRunner({ - run: (request) => - Effect.sync(() => { - requests.push(request); - return commandResult([{ Gateway: "172.18.0.1" }]); - }), - }), - platform: { os: "linux", desktop: false, rootless: false }, - }); - expect(docker.resolveNetworkGateway).toBeDefined(); - expect(yield* docker.resolveNetworkGateway!("network-id")).toBe("172.18.0.1"); - expect(requests[0]?.args).toEqual([ - "network", - "inspect", - "network-id", - "--format", - "{{json .IPAM.Config}}", - ]); - const desktop = makeDockerEngine({ - runner: makeControlledCommandRunner({ run: () => Effect.succeed(commandResult([])) }), - platform: { os: "darwin", desktop: true }, - }); - expect(desktop.resolveNetworkGateway).toBeUndefined(); - const rootless = makeDockerEngine({ - runner: makeControlledCommandRunner({ run: () => Effect.succeed(commandResult([])) }), - platform: { os: "linux", rootless: true }, - }); - expect(rootless.resolveNetworkGateway).toBeUndefined(); - }), - ); - - it.live("runs only container artifacts with exact labels, private backend and RO mounts", () => - Effect.gen(function* () { - const commands: ContainerProcessRequest[] = []; - const runner = makeControlledCommandRunner({ - run: (command) => - Effect.sync(() => { - commands.push(command); - if ( - command.args[0] === "create" || - (command.args[0] === "network" && command.args[1] === "create") - ) - return commandResult(command.args[0] === "create" ? "resource-id" : "network-id"); - if (command.args[0] === "image") return commandResult(""); - if ( - command.args[0] === "ps" || - command.args[0] === "network" || - command.args[0] === "volume" - ) - return commandResult(""); - return commandResult(""); - }), - }); - const engine = makeDockerEngine({ - runner, - platform: { os: "darwin", desktop: true }, - }); - const functionsWorkload = { - ...workload(), - id: "functions:edge-runtime" as const, - capability: "functions" as const, - }; - const runtime = yield* makeContainerRuntime({ - engine, - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - mounts: [{ source: "/functions", target: "/__supabase_functions", readOnly: true }], - }), - }); - const functionsKey = { ...key, workloadId: "functions:edge-runtime" }; - const ready = yield* runtime.start(functionsKey, functionsWorkload); - expect(ready.state).toBe("ready"); - const create = commands.find((entry) => entry.args[0] === "create"); - expect(create).toBeDefined(); - if (create?.args !== undefined) { - expect(create.args).not.toContain("--publish"); - expect(create.args.join(" ")).toContain("dst=/__supabase_functions,ro"); - expect(create.args.join(" ")).not.toContain("secret"); - } - yield* runtime.stop(functionsKey); - yield* runtime.remove(functionsKey); - }), - ); - - it.live("resolves workload inputs again after the network gateway is ready", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - let routeReady = false; - let resolutions = 0; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.sync(() => { - resolutions += 1; - return { command: [routeReady ? "updated" : "initial"] }; - }), - onNetworkReady: () => - Effect.sync(() => { - routeReady = true; - return true; - }), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - expect(resolutions).toBe(2); - expect(state.createdSpecs.at(-1)?.command).toEqual(["updated"]); - yield* runtime.cleanup({ stackId, destroy: true }); - }), - ); - - it.live("creates one shared network when workloads start concurrently", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const networkEntered = yield* Deferred.make(); - const releaseNetwork = yield* Deferred.make(); - let networkCreates = 0; - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - createNetwork: (spec) => - Effect.gen(function* () { - networkCreates += 1; - yield* Deferred.succeed(networkEntered, undefined); - yield* Deferred.await(releaseNetwork); - return yield* base.createNetwork(spec); - }), - }, - ownerSessionId: "owner-session", - }); - const secondKey = { ...key, workloadId: "rest:rest" }; - const secondWorkload = { - ...workload(), - id: secondKey.workloadId, - capability: "rest" as const, - }; - const first = yield* Effect.forkChild(runtime.start(key, workload()), { - startImmediately: true, - }); - yield* Deferred.await(networkEntered); - const second = yield* Effect.forkChild(runtime.start(secondKey, secondWorkload), { - startImmediately: true, - }); - yield* Deferred.succeed(releaseNetwork, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); - expect(networkCreates).toBe(1); - yield* runtime.cleanup({ stackId, destroy: true }); - }), - ); - - it.live("runs independent startup migrations concurrently after sharing the network", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const firstEntered = yield* Deferred.make(); - const secondEntered = yield* Deferred.make(); - const releaseMigrations = yield* Deferred.make(); - const migrationIds = new Map(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - createContainer: (spec) => - Effect.gen(function* () { - const resource = yield* base.createContainer(spec); - const command = spec.command?.[0]; - if (command === "first" || command === "second") - migrationIds.set(resource.id, command); - return resource; - }), - waitContainer: (resourceId) => { - const migration = migrationIds.get(resourceId); - if (migration === undefined) return base.waitContainer(resourceId); - return Effect.gen(function* () { - yield* Deferred.succeed( - migration === "first" ? firstEntered : secondEntered, - undefined, - ); - yield* Deferred.await(releaseMigrations); - return 0; - }); - }, - }, - ownerSessionId: "owner-session", - resolveWorkload: (workloadKey) => - Effect.succeed({ - startup: [ - { - entrypoint: "/usr/local/bin/migrate", - command: [workloadKey.workloadId === "database:database" ? "first" : "second"], - }, - ], - }), - }); - const secondKey = { ...key, workloadId: "rest:rest" }; - const secondWorkload = { - ...workload(), - id: secondKey.workloadId, - capability: "rest" as const, - }; - const first = yield* Effect.forkChild(runtime.start(key, workload()), { - startImmediately: true, - }); - const second = yield* Effect.forkChild(runtime.start(secondKey, secondWorkload), { - startImmediately: true, - }); - yield* Deferred.await(firstEntered); - yield* Deferred.await(secondEntered); - yield* Deferred.succeed(releaseMigrations, undefined); - expect(yield* Fiber.join(first)).toEqual({ ...key, state: "ready" }); - expect(yield* Fiber.join(second)).toEqual({ ...secondKey, state: "ready" }); - expect(state.calls.filter((call) => call.startsWith("create-network"))).toHaveLength(1); - yield* runtime.cleanup({ stackId, destroy: true }); - }), - ); - - it.live("interrupts a setup blocked on shared network creation during cleanup", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const networkEntered = yield* Deferred.make(); - const releaseNetwork = yield* Deferred.make(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - createNetwork: (spec) => - Effect.gen(function* () { - yield* Deferred.succeed(networkEntered, undefined); - yield* Deferred.await(releaseNetwork); - return yield* base.createNetwork(spec); - }), - }, - ownerSessionId: "owner-session", - }); - const starting = yield* Effect.forkChild(runtime.start(key, workload()), { - startImmediately: true, - }); - yield* Deferred.await(networkEntered); - const secondKey = { ...key, workloadId: "rest:rest" }; - const secondWorkload = { - ...workload(), - id: secondKey.workloadId, - capability: "rest" as const, - }; - const queued = yield* Effect.forkChild(runtime.start(secondKey, secondWorkload), { - startImmediately: true, - }); - yield* runtime.cleanup({ stackId, destroy: true }); - yield* Deferred.succeed(releaseNetwork, undefined); - expect(Exit.isFailure(yield* Fiber.join(starting).pipe(Effect.exit))).toBe(true); - expect(Exit.isFailure(yield* Fiber.join(queued).pipe(Effect.exit))).toBe(true); - expect(state.resources).toEqual([]); - }), - ); - - it.live("runs container startup migrations before the main workload", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - envFile: "/tmp/auth.env", - networkAliases: ["auth"], - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - expect(state.createdSpecs).toHaveLength(2); - const [startup, main] = state.createdSpecs; - expect(startup?.entrypoint).toBe("/usr/local/bin/auth"); - expect(startup?.command).toEqual(["migrate"]); - expect(startup?.envFile).toBe("/tmp/auth.env"); - expect(startup?.networkAliases).toBeUndefined(); - expect(startup?.publications).toEqual([]); - expect(main?.entrypoint).toBeUndefined(); - expect(main?.command).toBeUndefined(); - expect(main?.envFile).toBe("/tmp/auth.env"); - expect(main?.networkAliases).toEqual(["auth"]); - const startupStart = state.calls.findIndex((call) => call.startsWith("start:")); - const startupWait = state.calls.findIndex((call) => call.startsWith("wait:")); - const startupRemove = state.calls.findIndex((call) => call.startsWith("remove:")); - expect(startupStart).toBeGreaterThanOrEqual(0); - expect(startupWait).toBeGreaterThan(startupStart); - expect(startupRemove).toBeGreaterThan(startupWait); - yield* runtime.stop(key); - }), - ); - - it.live("preserves the container engine cause when startup wait fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const protocol = new ContainerEngineProtocolError({ - operation: "wait-container", - message: "wait endpoint failed", - }); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - waitContainer: () => Effect.fail(protocol), - }, - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - const error = errorOf(result); - expect(error).toBeInstanceOf(RuntimeDriverError); - expect(error?.cause).toBeInstanceOf(ContainerEngineError); - }), - ); - - it.live("publishes an unexpected container workload exit after readiness", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const exit = yield* Deferred.make(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - waitContainer: (resourceId) => - state.calls.includes(`wait:${resourceId}`) - ? Deferred.await(exit) - : base.waitContainer(resourceId), - }, - ownerSessionId: "owner-session", - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - yield* Deferred.succeed(exit, 17); - yield* Effect.yieldNow; - const observed = yield* runtime.observe(stackId); - expect(observed).toEqual([ - expect.objectContaining({ workloadId: key.workloadId, state: "failed" }), - ]); - yield* runtime.remove(key); - }), - ); - - it.live("ignores an interrupt-only one-shot log follower exit", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - let streamCalls = 0; - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - streamLogs: () => { - streamCalls += 1; - return streamCalls === 1 ? Stream.failCause(Cause.interrupt()) : Stream.empty; - }, - }, - ownerSessionId: "owner-session", - logStore: memoryLogStore([]), - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - expect(streamCalls).toBe(2); - yield* runtime.stop(key); - }), - ); - - it.live("hides in-flight startup containers from observation", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const waitEntered = yield* Deferred.make(); - const gate = yield* Deferred.make(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - waitContainer: () => - Effect.gen(function* () { - yield* Deferred.succeed(waitEntered, undefined); - return yield* Deferred.await(gate); - }), - }, - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - envFile: "/tmp/auth.env", - networkAliases: ["auth"], - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const running = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(waitEntered); - expect(yield* runtime.observe(key.stackId)).toEqual([]); - yield* Deferred.succeed(gate, 0); - expect(yield* Fiber.join(running)).toEqual({ ...key, state: "ready" }); - expect(yield* runtime.observe(key.stackId)).toEqual([{ ...key, state: "ready" }]); - const [startup, main] = state.createdSpecs; - expect(startup?.envFile).toBe("/tmp/auth.env"); - expect(startup?.networkAliases).toBeUndefined(); - expect(startup?.publications).toEqual([]); - expect(main?.networkAliases).toEqual(["auth"]); - yield* runtime.stop(key); - }), - ); - - it.live("runs multiple startup processes sequentially before the main workload", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - startup: [ - { entrypoint: "/usr/local/bin/auth", command: ["first"] }, - { entrypoint: "/usr/local/bin/auth", command: ["second"] }, - ], - }), - }); - expect(yield* runtime.start(key, workload())).toEqual({ ...key, state: "ready" }); - expect(state.createdSpecs.map((spec) => spec.command)).toEqual([ - ["first"], - ["second"], - undefined, - ]); - const waits = state.calls.filter((call) => call.startsWith("wait:")); - const removes = state.calls.filter((call) => call.startsWith("remove:")); - expect(waits).toHaveLength(2); - expect(removes).toHaveLength(2); - const firstRemove = removes[0]; - const secondWait = waits[1]; - expect(firstRemove).toBeDefined(); - expect(secondWait).toBeDefined(); - if (firstRemove !== undefined && secondWait !== undefined) - expect(state.calls.indexOf(secondWait)).toBeGreaterThan(state.calls.indexOf(firstRemove)); - yield* runtime.stop(key); - }), - ); - - it.live("blocks the main workload and removes the init container on migration failure", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - waitExitCode: 17, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(state.createdSpecs).toHaveLength(1); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - expect(state.calls.filter((call) => call.startsWith("start:"))).toHaveLength(1); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - }), - ); - - it.effect("times out a startup migration and removes the exact init container", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const waitEntered = yield* Deferred.make(); - const waitForever = yield* Deferred.make(); - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - waitContainer: () => - Effect.gen(function* () { - yield* Deferred.succeed(waitEntered, undefined); - return yield* Deferred.await(waitForever); - }), - }, - ownerSessionId: "owner-session", - startupProcessTimeout: "5 seconds", - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const running = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(waitEntered); - yield* TestClock.adjust("5 seconds"); - const result = yield* Fiber.join(running).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(RuntimeDriverError); - expect(error.value.message).toContain("startup process timed out"); - } - } - expect(state.createdSpecs).toHaveLength(1); - expect(state.calls.filter((call) => call.startsWith("start:")).length).toBe(1); - expect(state.calls.filter((call) => call.startsWith("remove:")).length).toBe(1); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - }), - ); - - it.live("keeps a nonzero migration exit authoritative when log persistence also fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - waitExitCode: 17, - }; - const logError = new LogStoreError({ - path: "memory://container-logs-failure", - message: "disk full", - }); - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - streamLogs: () => - Stream.fromIterable([ - { stream: "stdout", message: "migration-output" } satisfies ContainerLogLine, - ]), - }, - ownerSessionId: "owner-session", - logStore: { - path: logError.path, - append: () => Effect.fail(logError), - read: () => Effect.succeed([]), - }, - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const message = Cause.pretty(result.cause); - expect(message).toContain("code 17"); - expect(message).toContain("disk full"); - } - expect(state.createdSpecs).toHaveLength(1); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - }), - ); - - it.live("blocks the main workload when startup log persistence fails after exit zero", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const logError = new LogStoreError({ - path: "memory://container-logs-failure", - message: "disk full", - }); - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - streamLogs: () => - Stream.fromIterable([ - { stream: "stdout", message: "migration-output" } satisfies ContainerLogLine, - ]), - }, - ownerSessionId: "owner-session", - logStore: { - path: logError.path, - append: () => Effect.fail(logError), - read: () => Effect.succeed([]), - }, - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(state.createdSpecs).toHaveLength(1); - expect(state.calls.filter((call) => call.startsWith("start:"))).toHaveLength(1); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - if (Exit.isFailure(result)) expect(Cause.pretty(result.cause)).toContain("disk full"); - }), - ); - - it.live("persists startup stdout and stderr before removing the init container", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const records: LogRecord[] = []; - const streamStates: Array = []; - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - streamLogs: (id) => { - const resource = state.resources.find((entry) => entry.id === id); - streamStates.push(resource?.state); - return resource?.state === "running" && - resource.labels.role === "workload" && - resource.labels.startup === true - ? Stream.fromIterable([ - { stream: "stdout", message: "migrated" } satisfies ContainerLogLine, - { stream: "stderr", message: "notice" } satisfies ContainerLogLine, - ]) - : Stream.empty; - }, - }, - ownerSessionId: "owner-session", - logStore: memoryLogStore(records), - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - yield* runtime.start(key, workload()); - expect(streamStates).toEqual(["running", "running"]); - expect(records.map((record) => [record.stream, record.message])).toEqual([ - ["stdout", "migrated"], - ["stderr", "notice"], - ]); - }), - ); - - it.live("cleans an interrupted init container", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const waitEntered = yield* Deferred.make(); - const gate = yield* Deferred.make(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - waitContainer: () => - Effect.gen(function* () { - yield* Deferred.succeed(waitEntered, undefined); - return yield* Deferred.await(gate).pipe(Effect.as(0)); - }), - }, - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const running = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(waitEntered); - yield* Fiber.interrupt(running); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - expect(state.createdSpecs).toHaveLength(1); - }), - ); - - it.live("reports a running container as starting until readiness completes", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const readinessEntered = yield* Deferred.make(); - const readinessRelease = yield* Deferred.make(); - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - waitForReadiness: () => - Effect.gen(function* () { - yield* Deferred.succeed(readinessEntered, undefined); - yield* Deferred.await(readinessRelease); - }), - }); - const starting = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(readinessEntered); - const observed = yield* runtime.observe(stackId); - expect(observed).toEqual([{ ...key, state: "starting" }]); - yield* Deferred.succeed(readinessRelease, undefined); - expect((yield* Fiber.join(starting)).state).toBe("ready"); - }), - ); - - it.live("cleans a resource interrupted after registration", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const readinessEntered = yield* Deferred.make(); - const readinessRelease = yield* Deferred.make(); - const cleanupEntered = yield* Deferred.make(); - let removeCalls = 0; - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - removeContainer: (id) => - Effect.gen(function* () { - removeCalls += 1; - yield* base.removeContainer(id); - yield* Deferred.succeed(cleanupEntered, undefined); - }), - }, - ownerSessionId: "owner-session", - waitForReadiness: () => - Effect.gen(function* () { - yield* Deferred.succeed(readinessEntered, undefined); - yield* Deferred.await(readinessRelease); - }), - }); - const starting = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(readinessEntered); - yield* Fiber.interrupt(starting); - yield* Deferred.await(cleanupEntered); - expect(state.resources.filter((resource) => resource.kind === "workload")).toEqual([]); - expect(removeCalls).toBe(1); - yield* runtime.remove(key); - expect(removeCalls).toBe(1); - yield* Deferred.succeed(readinessRelease, undefined); - }), - ); - - it.live("allows stop to interrupt a blocked container readiness", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - waitForReadiness: () => - Effect.gen(function* () { - yield* Deferred.succeed(entered, undefined); - yield* Deferred.await(release); - }), - }), - }); - const starting = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(entered); - yield* runtime.stop(key); - const failed = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - yield* Deferred.succeed(release, undefined); - }), - ); - - it.live("does not resurrect a resource removed by a failed start", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const readinessEntered = yield* Deferred.make(); - const failReadiness = yield* Deferred.make(); - const removeEntered = yield* Deferred.make(); - const stopEntered = yield* Deferred.make(); - const releaseStop = yield* Deferred.make(); - const removed = yield* Deferred.make(); - let stopCalls = 0; - let removeCalls = 0; - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - stopContainer: (id) => - Effect.gen(function* () { - stopCalls += 1; - if (stopCalls === 1) { - yield* Deferred.succeed(stopEntered, undefined); - yield* Deferred.await(releaseStop); - } - yield* base.stopContainer(id); - }), - removeContainer: (id) => - Effect.gen(function* () { - removeCalls += 1; - if (removeCalls === 1) { - yield* Deferred.succeed(removeEntered, undefined); - } - yield* base.removeContainer(id); - yield* Deferred.succeed(removed, undefined); - }), - }, - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - waitForReadiness: () => - Effect.gen(function* () { - yield* Deferred.succeed(readinessEntered, undefined); - yield* Deferred.await(failReadiness); - return yield* new RuntimeDriverError({ - message: "readiness failed", - stackId: key.stackId, - workloadId: key.workloadId, - }); - }), - }), - }); - const starting = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(readinessEntered); - const stopping = yield* runtime.stop(key).pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(stopEntered); - yield* Deferred.succeed(failReadiness, undefined); - yield* Deferred.succeed(releaseStop, undefined); - yield* Deferred.await(removed); - expect(yield* runtime.observe(stackId)).toEqual([]); - yield* Fiber.join(stopping); - const failed = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - yield* runtime.remove(key); - expect(removeCalls).toBe(1); - const finalObserved = yield* runtime.observe(stackId); - expect(finalObserved).toEqual([]); - }), - ); - - it.live("does not register a container removed while start is pending", () => - pendingStartRace("remove"), - ); - - it.live("does not register a container stopped while start is pending", () => - pendingStartRace("stop"), - ); - - it.live("does not let cleanup race an in-flight container creation", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const createEntered = yield* Deferred.make(); - const releaseCreate = yield* Deferred.make(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - createContainer: (spec) => - Effect.gen(function* () { - yield* Deferred.succeed(createEntered, undefined); - yield* Deferred.await(releaseCreate); - return yield* base.createContainer(spec); - }), - }, - ownerSessionId: "owner-session", - }); - const starting = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(createEntered); - - // Cleanup must interrupt the start before it can create an unowned container. - yield* runtime.cleanup({ stackId, destroy: true }); - yield* Deferred.succeed(releaseCreate, undefined); - const startExit = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(startExit)).toBe(true); - - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - }), - ); - - it.live("interrupts multiple pending starts during cleanup without deadlocking", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const authKey: RuntimeWorkloadKey = { ...key, workloadId: "auth:auth" }; - const databaseReadinessEntered = yield* Deferred.make(); - const authReadinessEntered = yield* Deferred.make(); - const waitForever = yield* Deferred.make(); - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - waitContainer: () => Deferred.await(waitForever).pipe(Effect.as(0)), - }, - ownerSessionId: "owner-session", - resolveWorkload: (requestKey) => - Effect.succeed({ - waitForReadiness: () => - Effect.gen(function* () { - yield* requestKey.workloadId === key.workloadId - ? Deferred.succeed(databaseReadinessEntered, undefined) - : Deferred.succeed(authReadinessEntered, undefined); - yield* Deferred.await(waitForever); - }), - }), - }); - const databaseStart = yield* runtime - .start(key, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - const authStart = yield* runtime - .start(authKey, workload()) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(databaseReadinessEntered); - yield* Deferred.await(authReadinessEntered); - - const completed = yield* runtime - .cleanup({ stackId, destroy: true }) - .pipe(Effect.as("completed"), Effect.timeout("1 second")); - expect(completed).toBe("completed"); - expect(Exit.isFailure(yield* Fiber.join(databaseStart).pipe(Effect.exit))).toBe(true); - expect(Exit.isFailure(yield* Fiber.join(authStart).pipe(Effect.exit))).toBe(true); - expect(state.resources.filter((resource) => resource.kind === "workload")).toEqual([]); - }), - ); - - it.live("does not adopt an exact main container when starting a new session", () => - Effect.gen(function* () { - const existing: ContainerResource = { - id: "existing-main", - name: "supabase-aaaaaaaaaaaaaaaa-database_database-workload", - kind: "workload", - state: "running", - labels: { - stackId, - ownerSessionId: "owner-session", - workloadId: key.workloadId, - role: "workload", - }, - }; - const state: FakeContainerState = { - resources: [existing], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - expect(state.createdSpecs).toHaveLength(2); - expect(state.calls.some((call) => call.startsWith("wait:"))).toBe(true); - }), - ); - - it.live("removes an interrupted startup container before retrying the workload", () => - Effect.gen(function* () { - const stale: ContainerResource = { - id: "stale-startup", - name: `supabase-${stackId.slice(0, 16)}-${key.workloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-workload`, - kind: "workload", - state: "stopped", - labels: { - stackId, - ownerSessionId: "owner-session", - workloadId: key.workloadId, - startup: true, - role: "workload", - }, - }; - const state: FakeContainerState = { - resources: [stale], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - startup: [{ entrypoint: "/usr/local/bin/auth", command: ["migrate"] }], - }), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - expect(state.calls).toContain("remove:stale-startup"); - expect(state.createdSpecs).toHaveLength(2); - }), - ); - - it.live("rejects native artifacts before invoking the engine", () => - Effect.gen(function* () { - let called = false; - const runner = makeControlledCommandRunner({ - run: () => - Effect.sync(() => { - called = true; - return commandResult({}); - }), - }); - const runtime = yield* makeContainerRuntime({ - engine: makeDockerEngine({ runner, platform: { os: "darwin", desktop: true } }), - ownerSessionId: "owner-session", - }); - const result = yield* runtime - .start(key, workload({ kind: "native", release: "1" })) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(called).toBe(false); - }), - ); - - it.live( - "captures container logs before readiness and does not duplicate same-resource followers", - () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const records: LogRecord[] = []; - const appended = yield* Deferred.make(); - let followerCalls = 0; - let followerAttached = false; - const base = fakeContainerEngine(state); - const baseLogStore = memoryLogStore(records); - const logStore: LogStore = { - ...baseLogStore, - append: (record) => - baseLogStore - .append(record) - .pipe(Effect.tap(() => Deferred.succeed(appended, undefined))), - }; - const engine: ContainerEngine = { - ...base, - streamLogs: () => { - followerCalls += 1; - followerAttached = true; - return Stream.fromIterable([ - { stream: "stdout", message: "container-started" } satisfies ContainerLogLine, - ]); - }, - }; - const runtime = yield* makeContainerRuntime({ - engine, - ownerSessionId: "owner-session", - logStore, - resolveWorkload: () => - Effect.succeed({ - waitForReadiness: () => - followerAttached - ? Effect.void - : Effect.fail( - new RuntimeDriverError({ - message: "logs not attached", - stackId: key.stackId, - workloadId: key.workloadId, - }), - ), - }), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - expect(followerCalls).toBe(1); - yield* Deferred.await(appended); - expect(records.map((record) => record.message)).toEqual(["container-started"]); - yield* runtime.start(key, workload()); - expect(followerCalls).toBe(1); - yield* runtime.stop(key); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - }), - ); - - it.live("keeps following logs after a non-destructive cleanup and restart", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const records: LogRecord[] = []; - const firstAppended = yield* Deferred.make(); - const secondAppended = yield* Deferred.make(); - let followerCalls = 0; - const base = fakeContainerEngine(state); - const baseLogStore = memoryLogStore(records); - const logStore: LogStore = { - ...baseLogStore, - append: (record) => - baseLogStore - .append(record) - .pipe( - Effect.tap(() => - records.length === 1 - ? Deferred.succeed(firstAppended, undefined) - : Deferred.succeed(secondAppended, undefined), - ), - ), - }; - const engine: ContainerEngine = { - ...base, - streamLogs: () => { - const sequence = followerCalls + 1; - followerCalls += 1; - return Stream.fromIterable([ - { - stream: "stdout", - message: `container-started-${sequence}`, - } satisfies ContainerLogLine, - ]); - }, - }; - const runtime = yield* makeContainerRuntime({ - engine, - ownerSessionId: "owner-session", - logStore, - }); - - yield* runtime.start(key, workload()); - yield* Deferred.await(firstAppended); - yield* runtime.cleanup({ stackId: key.stackId, destroy: false }); - - const restartedKey: RuntimeWorkloadKey = key; - const restartedWorkload: PlannedWorkload = workload(); - yield* runtime.start(restartedKey, restartedWorkload); - yield* Deferred.await(secondAppended); - - expect(followerCalls).toBe(2); - expect(records.map((record) => record.message)).toEqual([ - "container-started-1", - "container-started-2", - ]); - }), - ); - - it.live("propagates restart-session log failures after a non-destructive cleanup", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const readiness = yield* Deferred.make(); - const followerAttached = yield* Deferred.make(); - const followerFailure = yield* Deferred.make(); - const restartedKey: RuntimeWorkloadKey = key; - const restartedWorkload: PlannedWorkload = workload(); - let starts = 0; - let followerCalls = 0; - const engine: ContainerEngine = { - ...fakeContainerEngine(state), - waitContainer: () => Effect.never, - streamLogs: () => { - followerCalls += 1; - return followerCalls === 1 - ? Stream.empty - : Stream.fromEffect( - Effect.gen(function* () { - yield* Deferred.succeed(followerAttached, undefined); - return yield* Deferred.await(followerFailure); - }), - ); - }, - }; - const runtime = yield* makeContainerRuntime({ - engine, - ownerSessionId: "owner-session", - logStore: memoryLogStore([]), - resolveWorkload: (requestKey) => - Effect.succeed({ - waitForReadiness: - requestKey.workloadId === restartedKey.workloadId - ? () => { - starts += 1; - return starts === 1 ? Effect.void : Deferred.await(readiness); - } - : undefined, - }), - }); - - yield* runtime.start(key, workload()); - yield* runtime.cleanup({ stackId: key.stackId, destroy: false }); - - const starting = yield* runtime - .start(restartedKey, restartedWorkload) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(followerAttached); - yield* Deferred.fail( - followerFailure, - new ContainerEngineProtocolError({ - operation: "logs", - message: "restart-session follower disconnected", - }), - ); - const result = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(RuntimeDriverError); - expect(error.value.message).toBe("restart-session follower disconnected"); - expect(error.value.cause).toBeInstanceOf(ContainerEngineError); - } - } - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - }), - ); - - it.live("fails startup and removes a new container when log persistence fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const logError = new LogStoreError({ - path: "memory://container-logs-failure", - message: "disk full", - }); - const readiness = yield* Deferred.make(); - const logStore: LogStore = { - path: logError.path, - append: () => Effect.fail(logError), - read: () => Effect.succeed([]), - }; - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - streamLogs: () => - Stream.fromIterable([ - { stream: "stdout", message: "will-not-persist" } satisfies ContainerLogLine, - ]), - }, - ownerSessionId: "owner-session", - logStore, - resolveWorkload: () => - Effect.succeed({ waitForReadiness: () => Deferred.await(readiness) }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - }), - ); - - it.live("marks a running container failed when its log follower fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const followerFailure = yield* Deferred.make(); - const runtime = yield* makeContainerRuntime({ - engine: { - ...fakeContainerEngine(state), - streamLogs: () => Stream.fromEffect(Deferred.await(followerFailure)), - }, - ownerSessionId: "owner-session", - logStore: memoryLogStore([]), - }); - const ready = yield* runtime.start(key, workload()); - expect(ready.state).toBe("ready"); - yield* Deferred.fail( - followerFailure, - new ContainerEngineProtocolError({ - operation: "logs", - message: "follower disconnected", - }), - ); - yield* Effect.yieldNow; - const observed = yield* runtime.observe(key.stackId); - expect(observed[0]?.state).toBe("failed"); - yield* runtime.stop(key); - }), - ); - - it.live("rejects malformed inspect output and workload port publication", () => - Effect.gen(function* () { - let calls = 0; - const malformed = makeDockerEngine({ - runner: makeControlledCommandRunner({ - run: () => - Effect.sync(() => { - calls += 1; - return commandResult( - '{"ID":"bad","Names":"bad","Labels":{},"State":"running","extra":true}', - ); - }), - }), - platform: { os: "linux", desktop: false }, - }); - const malformedExit = yield* malformed.listResources(stackId).pipe(Effect.exit); - expect(Exit.isFailure(malformedExit)).toBe(true); - if (Exit.isFailure(malformedExit)) { - const error = Cause.findErrorOption(malformedExit.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(ContainerEngineProtocolError); - } - const engine = makeDockerEngine({ - runner: makeControlledCommandRunner({ - run: () => - Effect.sync(() => { - calls += 1; - return commandResult(""); - }), - }), - platform: { os: "linux", desktop: false }, - }); - const result = yield* engine - .createContainer({ - name: "backend", - image: "example/backend:1", - labels: { - stackId, - ownerSessionId: "owner-session", - workloadId: "database:database", - role: "workload", - }, - network: "private", - mounts: [], - publications: [{ address: "127.0.0.1", hostPort: 5432, containerPort: 5432 }], - volumeMounts: [], - role: "workload", - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(calls).toBe(2); - }), - ); - - it.live("decodes Docker closed rows and image outcomes", () => - Effect.gen(function* () { - const dockerJsonRow = (values: ReadonlyArray): string => - values.map((value) => JSON.stringify(value)).join("\t"); - const runner = makeControlledCommandRunner({ - run: (request) => - Effect.succeed({ - stdout: - request.args[0] === "image" - ? request.args[2] === "present" - ? '"sha256:present"\n' - : request.args[2] === "malformed" - ? "not-json\n" - : "" - : request.args[0] === "ps" - ? `${dockerJsonRow(["container-id", "backend", stackId, "owner", key.workloadId, "false", "workload", "running"])}\n` - : request.args[0] === "network" && request.args[1] === "create" - ? "created-id\nsecond\n" - : request.args[0] === "network" - ? `${dockerJsonRow(["network-id", "private", stackId, "owner", "network"])}\n` - : request.args[0] === "volume" - ? `${dockerJsonRow(["volume-name", stackId, key.workloadId, "volume"])}\n` - : request.args[0] === "version" - ? '"27.0.0"\n' - : "created-id\n", - stderr: "", - exitCode: request.args[0] === "nonzero" ? 17 : 0, - }), - }); - const engine = makeDockerEngine({ - runner, - platform: { os: "linux", desktop: false }, - }); - yield* engine.probe; - expect((yield* engine.inspectImage("absent")).present).toBe(false); - expect((yield* engine.inspectImage("present")).present).toBe(true); - const malformed = yield* engine.inspectImage("malformed").pipe(Effect.exit); - expect(Exit.isFailure(malformed)).toBe(true); - const resources = yield* engine.listResources(stackId); - expect(resources.map((resource) => resource.kind)).toEqual(["workload", "network", "volume"]); - expect(resources[0]?.labels.role).toBe("workload"); - expect(resources[1]?.labels.role).toBe("network"); - expect(resources[2]?.labels.role).toBe("volume"); - const multiline = yield* engine - .createNetwork({ - name: "private", - labels: { stackId, ownerSessionId: "owner", role: "network" }, - }) - .pipe(Effect.exit); - expect(Exit.isFailure(multiline)).toBe(true); - }), - ); - - const commandSerializationWorkload = { - name: "backend", - image: "example/backend:1", - labels: { - stackId, - ownerSessionId: "owner", - workloadId: "backend", - role: "workload" as const, - }, - network: "private", - mounts: [{ source: "/tmp/backend", target: "/app/backend", readOnly: true }], - volumeMounts: [{ volume: "backend-data", target: "/var/lib/backend", readOnly: false }], - publications: [{ address: "127.0.0.1" as const, hostPort: 54321, containerPort: 8000 }], - envFile: "/tmp/supabase-owned.env", - networkAliases: ["supabase-database"], - command: ["serve", "--port", "8000"], - role: "workload" as const, - }; - - it.live( - "serializes Docker workload creation with environment, mounts, aliases, and command", - () => - Effect.sync(() => { - const docker = serializeDockerCommand({ - operation: "create-container", - spec: commandSerializationWorkload, - }); - expect(docker.args).toEqual( - expect.arrayContaining([ - "--publish", - "127.0.0.1:54321:8000", - "--network-alias", - "supabase-database", - "--env-file", - "/tmp/supabase-owned.env", - "type=bind,src=/tmp/backend,dst=/app/backend,ro", - "type=volume,src=backend-data,dst=/var/lib/backend", - ]), - ); - expect(docker.args.slice(-3)).toEqual(["serve", "--port", "8000"]); - }), - ); - - it.live("serializes Docker container copy arguments", () => - Effect.sync(() => { - expect( - serializeDockerCommand({ - operation: "copy-container", - id: "container-id", - source: "/tmp/functions-main.ts", - destination: "/root", - }).args, - ).toEqual(["cp", "/tmp/functions-main.ts", "container-id:/root"]); - }), - ); - - it.live.each([ - { - name: "quotes commas in bind mount paths", - mount: { source: '/tmp/a,b"c', target: "/app/a,b", readOnly: false }, - expected: 'type=bind,"src=/tmp/a,b""c","dst=/app/a,b"', - }, - { - name: "preserves equals signs in bind mount paths", - mount: { source: "/tmp/a=b", target: "/app/a=b", readOnly: false }, - expected: "type=bind,src=/tmp/a=b,dst=/app/a=b", - }, - ])("serializes Docker bind mounts when it $name", ({ mount, expected }) => - Effect.sync(() => { - expect( - serializeDockerCommand({ - operation: "create-container", - spec: { ...commandSerializationWorkload, mounts: [mount] }, - }).args, - ).toContain(expected); - }), - ); - - it.live( - "serializes Podman workload creation with environment, mounts, aliases, and command", - () => - Effect.sync(() => { - const podman = serializePodmanCommand({ - operation: "create-container", - spec: commandSerializationWorkload, - }); - expect(podman.args).toEqual( - expect.arrayContaining([ - "--env-file", - "/tmp/supabase-owned.env", - "127.0.0.1:54321:8000", - "type=bind,src=/tmp/backend,dst=/app/backend,ro", - "type=volume,src=backend-data,dst=/var/lib/backend", - "--network-alias", - "supabase-database", - ]), - ); - expect(podman.args.slice(-3)).toEqual(["serve", "--port", "8000"]); - }), - ); - - it.live("serializes Podman container copy arguments", () => - Effect.sync(() => { - expect( - serializePodmanCommand({ - operation: "copy-container", - id: "container-id", - source: "/tmp/functions-main.ts", - destination: "/root", - }).args, - ).toEqual(["cp", "/tmp/functions-main.ts", "container-id:/root"]); - }), - ); - - it.live("serializes Podman network inspection templates", () => - Effect.sync(() => { - expect( - serializePodmanCommand({ operation: "inspect-networks", stackId }).args.join(" "), - ).toContain("{{index .Labels"); - }), - ); - - it.live("rejects Podman on unsupported hosts before resource parsing", () => - Effect.gen(function* () { - const unsupported = yield* makePodmanEngine({ - runner: makeControlledCommandRunner({ run: () => Effect.succeed(commandResult("ok")) }), - platform: { os: "darwin" }, - }).preflight.pipe(Effect.exit); - expect(Exit.isFailure(unsupported)).toBe(true); - }), - ); - - it.live("parses Podman resource rows after a successful preflight", () => - Effect.gen(function* () { - const rowRunner = makeControlledCommandRunner({ - run: (request) => - Effect.succeed({ - stdout: - request.args[0] === "version" - ? "5.0.0\n" - : request.args[0] === "image" - ? request.args[2] === "present" - ? "sha256:present\n" - : request.args[2] === "malformed" - ? "bad\trow\n" - : "" - : request.args[0] === "ps" - ? `container-id\tbackend\t${stackId}\towner\t${key.workloadId}\tfalse\tworkload\trunning\n` - : request.args[0] === "network" - ? `network-id\tprivate\t${stackId}\towner\tnetwork\n` - : request.args[0] === "volume" - ? `volume-name\t${stackId}\t${key.workloadId}\tvolume\n` - : "created-id\n", - stderr: "", - exitCode: 0, - }), - }); - const podmanEngine = makePodmanEngine({ - runner: rowRunner, - platform: { os: "linux", rootless: true }, - }); - yield* podmanEngine.probe; - expect((yield* podmanEngine.inspectImage("absent")).present).toBe(false); - expect((yield* podmanEngine.inspectImage("present")).present).toBe(true); - const malformedImage = yield* podmanEngine.inspectImage("malformed").pipe(Effect.exit); - expect(Exit.isFailure(malformedImage)).toBe(true); - const podmanResources = yield* podmanEngine.listResources(stackId); - expect(podmanResources.map((resource) => resource.kind)).toEqual([ - "workload", - "network", - "volume", - ]); - }), - ); - - it.live("follows one Docker or Podman log process and joins split stdout/stderr lines", () => - Effect.gen(function* () { - const requests: ContainerProcessRequest[] = []; - const chunks: ReadonlyArray<{ - readonly stream: "stdout" | "stderr"; - readonly bytes: Uint8Array; - }> = [ - { stream: "stdout", bytes: new TextEncoder().encode("first\nsec") }, - { stream: "stderr", bytes: new TextEncoder().encode("error\nlast") }, - { stream: "stdout", bytes: new TextEncoder().encode("ond\n") }, - ]; - const runner = makeControlledCommandRunner({ - run: () => Effect.succeed(commandResult("ok")), - stream: (request) => { - requests.push(request); - return Stream.fromIterable(chunks); - }, - }); - const docker = makeDockerEngine({ - runner, - platform: { os: "linux", desktop: false }, - }); - const podman = makePodmanEngine({ - runner, - platform: { os: "linux", rootless: true }, - }); - const dockerLogs = yield* Stream.runCollect(docker.streamLogs("container-id", { tail: 0 })); - const podmanLogs = yield* Stream.runCollect(podman.streamLogs("podman-id")); - expect(dockerLogs).toEqual([ - { stream: "stdout", message: "first" }, - { stream: "stderr", message: "error" }, - { stream: "stdout", message: "second" }, - { stream: "stderr", message: "last" }, - ] satisfies ReadonlyArray); - expect(podmanLogs).toEqual(dockerLogs); - expect(requests.map((request) => request.args)).toEqual([ - ["logs", "--follow", "--tail", "0", "container-id"], - ["logs", "--follow", "--tail", "all", "podman-id"], - ]); - }), - ); - - it.live("rejects non-loopback publications before preflight or daemon mutation", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - let preflightCalls = 0; - const base = fakeContainerEngine(state); - const runtime = yield* makeContainerRuntime({ - engine: { - ...base, - preflight: Effect.sync(() => { - preflightCalls += 1; - return { host: "host.docker.internal" }; - }), - }, - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ - publications: [{ address: "0.0.0.0", hostPort: 30_000, containerPort: 8080 }], - }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(preflightCalls).toBe(0); - expect(state.calls).toEqual([]); - expect(state.createdSpecs).toEqual([]); - }), - ); - - it.live("fails process output overflow and nonzero commands with typed failures", () => - Effect.gen(function* () { - const runner = yield* makeProcessCommandRunner({ - executable: process.execPath, - baseArgs: ["-e", "process.stdout.write('123456'); process.stderr.write('abcdef')"], - maxOutputBytes: 4, - }); - const overflow = yield* runner.run({ args: ["probe"] }).pipe(Effect.exit); - expect(Exit.isFailure(overflow)).toBe(true); - const nonzero = makeControlledCommandRunner({ - run: () => Effect.succeed({ stdout: "", stderr: "daemon unavailable", exitCode: 17 }), - }); - const engine = makeDockerEngine({ runner: nonzero, platform: { os: "linux" } }); - const failure = yield* engine.probe.pipe(Effect.exit); - expect(Exit.isFailure(failure)).toBe(true); - if (Exit.isFailure(failure)) { - const error = Cause.findErrorOption(failure.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBeInstanceOf(ContainerCommandError); - } - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live( - "fails fast on a noisy process and force-reaps one that ignores SIGTERM", - () => - Effect.gen(function* () { - const runner = yield* makeProcessCommandRunner({ - executable: process.execPath, - baseArgs: [ - "-e", - "process.on('SIGTERM', () => {}); process.stdout.write('x'.repeat(1024 * 1024)); setInterval(() => {}, 1000)", - ], - maxOutputBytes: 16, - }); - const result = yield* runner - .run({ args: ["probe"] }) - .pipe(Effect.timeout("8 seconds"), Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) - expect(error.value).toBeInstanceOf(ContainerEngineProtocolError); - } - }).pipe(Effect.provide(NodeServices.layer)), - 10_000, - ); - - it.live("replaces an exact running container for each new start session", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - }); - yield* runtime.start(key, workload()); - const nextRuntime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "next-owner-session", - }); - state.calls.length = 0; - yield* nextRuntime.start(key, workload()); - expect(state.calls).toContain("inspect-image"); - expect(state.calls).toContain("create-container"); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - }), - ); - - it.live("recreates a same-stack network owned by a previous start session", () => - Effect.gen(function* () { - const networkName = `supabase-${key.stackId.slice(0, 16)}-network`; - const state: FakeContainerState = { - resources: [ - { - id: "stale-network", - name: networkName, - kind: "network", - labels: { - stackId: key.stackId, - ownerSessionId: "previous-owner", - role: "network", - }, - }, - ], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "current-owner", - }); - - yield* runtime.start(key, workload()); - - const removeIndex = state.calls.findIndex((call) => call === "remove-network:stale-network"); - const createIndex = state.calls.findIndex((call) => call === "create-network"); - expect(removeIndex).toBeGreaterThanOrEqual(0); - expect(createIndex).toBeGreaterThan(removeIndex); - expect( - state.resources.some( - (resource) => - resource.kind === "network" && - resource.labels.role === "network" && - resource.labels.ownerSessionId === "current-owner", - ), - ).toBe(true); - }), - ); - - it.live("leaves foreign and sanitized-name collisions untouched", () => - Effect.gen(function* () { - const foreignStackId = StackIdSchema.make("c".repeat(64)); - const foreignKey = { ...key, stackId: foreignStackId, workloadId: "api:api" }; - const foreignName = `supabase-${stackId.slice(0, 16)}-api-api-workload`; - const state: FakeContainerState = { - resources: [ - { - id: "foreign-container", - name: foreignName, - kind: "workload", - state: "running", - labels: { - stackId: foreignStackId, - ownerSessionId: "other-session", - workloadId: foreignKey.workloadId, - role: "workload", - }, - }, - ], - imagePresent: false, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - }); - const foreign = yield* runtime - .start({ ...foreignKey, stackId }, workload()) - .pipe(Effect.exit); - expect(Exit.isFailure(foreign)).toBe(true); - expect(state.calls).not.toContain("pull-image"); - expect(state.calls).not.toContain("create-container"); - expect(state.resources[0]?.id).toBe("foreign-container"); - - const firstKey = { ...key, workloadId: "foo/bar" }; - const secondKey = { ...key, workloadId: "foo?bar" }; - state.resources = []; - state.calls.length = 0; - yield* runtime.start(firstKey, workload()); - state.calls.length = 0; - const sanitized = yield* runtime.start(secondKey, workload()).pipe(Effect.exit); - expect(Exit.isFailure(sanitized)).toBe(true); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(false); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(false); - expect(state.resources.filter((resource) => resource.kind === "workload")).toHaveLength(1); - }), - ); - - it.live( - "derives volume mounts, preserves volumes on remove, and validates mappings before mutation", - () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const volume = { target: "/var/lib/postgresql/data", readOnly: false }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => Effect.succeed({ volume }), - }); - yield* runtime.start(key, workload()); - const physicalVolumeName = `supabase-${key.stackId}-${key.workloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-volume`; - expect(state.createdSpecs[0]?.volumeMounts).toEqual([ - { volume: physicalVolumeName, target: volume.target, readOnly: false }, - ]); - expect(state.resources.some((resource) => resource.kind === "volume")).toBe(true); - yield* runtime.remove(key); - expect(state.resources.some((resource) => resource.kind === "volume")).toBe(true); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - - const otherStackKey = { - ...key, - stackId: StackIdSchema.make("b".repeat(64)), - }; - yield* runtime.start(otherStackKey, workload()); - expect(state.resources.filter((resource) => resource.kind === "volume")).toHaveLength(2); - state.calls.length = 0; - const nextRuntime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "next-owner-session", - resolveWorkload: () => - Effect.succeed({ bootstrap: { source: "/tmp/main.ts", destination: "/root" } }), - }); - yield* nextRuntime.start(key, workload()); - expect(state.calls).not.toContain(`create-volume:${physicalVolumeName}`); - - const invalidState: FakeContainerState = { - resources: [], - imagePresent: false, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const invalidRuntime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(invalidState), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ volume: { target: "", readOnly: volume.readOnly } }), - }); - const invalid = yield* invalidRuntime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(invalid)).toBe(true); - expect(invalidState.calls).not.toContain("inspect-image"); - expect(invalidState.calls).not.toContain("create-network"); - }), - ); - - it.live("shares one owner volume across read-only and read-write workloads", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const ownerKey: RuntimeWorkloadKey = { - ...key, - workloadId: "storage:storage", - }; - const secondaryKey: RuntimeWorkloadKey = { - ...key, - workloadId: "storage:imgproxy", - }; - const ownerWorkload = { - ...workload(), - id: ownerKey.workloadId, - capability: "storage" as const, - }; - const secondaryWorkload = { - ...workload(), - id: secondaryKey.workloadId, - capability: "storage" as const, - }; - const ownerWorkloadId = ownerKey.workloadId; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: (requestKey) => - Effect.succeed({ - volume: - requestKey.workloadId === ownerWorkloadId - ? { target: "/var/lib/storage", readOnly: false, ownerWorkloadId } - : { target: "/mnt", readOnly: true, ownerWorkloadId }, - }), - }); - - // A read-only dependent can materialize and own the shared volume first. - yield* runtime.start(secondaryKey, secondaryWorkload); - yield* runtime.start(ownerKey, ownerWorkload); - - const expectedVolume = `supabase-${stackId}-${ownerWorkloadId.replace(/[^A-Za-z0-9_.-]/g, "-")}-volume`; - expect(state.resources.filter((resource) => resource.kind === "volume")).toHaveLength(1); - expect(state.resources.find((resource) => resource.kind === "volume")?.name).toBe( - expectedVolume, - ); - expect(state.calls.filter((call) => call.startsWith("create-volume:"))).toEqual([ - `create-volume:${expectedVolume}`, - ]); - expect(state.createdSpecs.map((spec) => spec.volumeMounts[0])).toEqual([ - { volume: expectedVolume, target: "/mnt", readOnly: true }, - { volume: expectedVolume, target: "/var/lib/storage", readOnly: false }, - ]); - - yield* runtime.stop(secondaryKey); - yield* runtime.stop(ownerKey); - expect(state.resources.filter((resource) => resource.kind === "volume")).toHaveLength(1); - const sharedVolume = state.resources.find((resource) => resource.kind === "volume"); - expect(sharedVolume).toBeDefined(); - if (sharedVolume === undefined) return; - yield* runtime.cleanup({ stackId, destroy: true }); - expect(state.calls.filter((call) => call.startsWith("remove-volume:"))).toEqual([ - `remove-volume:${sharedVolume.id}`, - ]); - }), - ); - - it.live( - "cleans up a newly created container when readiness fails and keeps the original cause", - () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const readinessError = new RuntimeDriverError({ - message: "database did not become ready", - stackId: key.stackId, - workloadId: key.workloadId, - }); - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ waitForReadiness: () => Effect.fail(readinessError) }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBe(readinessError); - } - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - expect(state.resources.some((resource) => resource.kind === "network")).toBe(true); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - }), - ); - - it.live("runs database bootstrap after readiness and before reporting ready", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const order: string[] = []; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ waitForReadiness: () => Effect.sync(() => order.push("readiness")) }), - bootstrapDatabase: () => Effect.sync(() => order.push("bootstrap")), - }); - const databaseWorkload = { ...workload(), bootstrap: "database" as const }; - const databaseReady = yield* runtime.start(key, databaseWorkload); - expect(databaseReady.state).toBe("ready"); - expect(order).toEqual(["readiness", "bootstrap"]); - }), - ); - - it.live("cleans up a newly created container when database bootstrap fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const bootstrapError = new RuntimeDriverError({ - message: "database bootstrap failed", - stackId: key.stackId, - workloadId: key.workloadId, - }); - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - bootstrapDatabase: () => Effect.fail(bootstrapError), - }); - const databaseWorkload = { ...workload(), bootstrap: "database" as const }; - const result = yield* runtime.start(key, databaseWorkload).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Cause.findErrorOption(result.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) expect(error.value).toBe(bootstrapError); - } - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - }), - ); - - it.live("removes a replaced running database container on bootstrap failure", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const owner = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "previous-owner", - }); - yield* owner.start(key, workload()); - state.calls.length = 0; - const bootstrapError = new RuntimeDriverError({ - message: "replaced database bootstrap failed", - stackId: key.stackId, - workloadId: key.workloadId, - }); - let attempts = 0; - const replacementRuntime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "new-owner", - bootstrapDatabase: () => - Effect.gen(function* () { - attempts += 1; - if (attempts === 1) return yield* bootstrapError; - }), - }); - const result = yield* replacementRuntime - .start(key, { ...workload(), bootstrap: "database" as const }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(state.calls.some((call) => call.startsWith("stop:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - const retry = yield* replacementRuntime.start(key, { - ...workload(), - bootstrap: "database" as const, - }); - expect(retry.state).toBe("ready"); - expect(attempts).toBe(2); - }), - ); - - it.live("copies a new functions bootstrap on each fresh start", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ bootstrap: { source: "/tmp/main.ts", destination: "/root" } }), - }); - yield* runtime.start(key, workload()); - const copyIndex = state.calls.findIndex((call) => call.startsWith("copy:")); - const startIndex = state.calls.findIndex((call) => call.startsWith("start:")); - expect(copyIndex).toBeGreaterThanOrEqual(0); - expect(startIndex).toBeGreaterThan(copyIndex); - state.calls.length = 0; - const nextRuntime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "next-owner-session", - resolveWorkload: () => - Effect.succeed({ bootstrap: { source: "/tmp/main.ts", destination: "/root" } }), - }); - yield* nextRuntime.start(key, workload()); - expect(state.calls.some((call) => call.startsWith("copy:"))).toBe(true); - }), - ); - - it.live("removes a newly-created container when bootstrap copy fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - copyFailure: new ContainerCommandError({ - operation: "copy-container", - message: "bootstrap copy failed", - }), - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - resolveWorkload: () => - Effect.succeed({ bootstrap: { source: "/tmp/main.ts", destination: "/root" } }), - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(state.calls.some((call) => call.startsWith("copy:"))).toBe(false); - expect(state.calls.some((call) => call.startsWith("remove:"))).toBe(true); - expect(state.calls.some((call) => call.startsWith("start:"))).toBe(false); - expect(state.resources.some((resource) => resource.kind === "workload")).toBe(false); - }), - ); - - it.live("serializes concurrent starts and does not mutate when image inspection fails", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: false, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - }); - yield* Effect.all([runtime.start(key, workload()), runtime.start(key, workload())], { - concurrency: 2, - }); - expect(state.calls.filter((call) => call === "create-network")).toHaveLength(1); - expect(state.calls.filter((call) => call === "create-container")).toHaveLength(1); - - const failedState: FakeContainerState = { - resources: [], - imagePresent: false, - calls: [], - createdSpecs: [], - nextId: 1, - inspectImageFailure: new ContainerCommandError({ - operation: "inspect-image", - message: "registry unavailable", - }), - }; - const failedRuntime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(failedState), - ownerSessionId: "owner-session", - }); - const failed = yield* failedRuntime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(failedState.calls).not.toContain("pull-image"); - expect(failedState.calls).not.toContain("create-network"); - expect(failedState.calls).not.toContain("create-container"); - }), - ); - - it.live("preserves container engine identity for runtime operations after preflight", () => - Effect.gen(function* () { - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - inspectImageFailure: new ContainerCommandError({ - operation: "inspect-image", - message: "daemon rejected image inspection", - }), - }; - const runtime = yield* makeContainerRuntime({ - engine: fakeContainerEngine(state), - ownerSessionId: "owner-session", - }); - const result = yield* runtime.start(key, workload()).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - const failure = errorOf(result); - expect(failure).toBeInstanceOf(RuntimeDriverError); - expect(failure?.cause).toBeInstanceOf(ContainerEngineError); - expect(failure?.cause).toMatchObject({ message: "daemon rejected image inspection" }); - }), - ); - - it.live("reports container engine identity when a log follower fails before readiness", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-container-follower-" }); - const testStackId = yield* deriveStackId({ - projectRoot: root, - branchContext: "ordinary-workspace", - stackName: "container-follower", - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(testStackId, { - format: "supabase-stack-state-v1", - identity: { - projectRoot: root, - branchContext: "ordinary-workspace", - stackName: "container-follower", - }, - runtime: { kind: "container", engine: "docker" }, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, - }); - const state: FakeContainerState = { - resources: [], - imagePresent: true, - calls: [], - createdSpecs: [], - nextId: 1, - }; - const engine: ContainerEngine = { - ...fakeContainerEngine(state), - waitContainer: () => Effect.never, - streamLogs: () => - Stream.fail( - new ContainerEngineProtocolError({ - operation: "logs", - message: "follower disconnected before readiness", - }), - ), - }; - const logStore = memoryLogStore([]); - const driver = yield* makeContainerRuntime({ - engine, - ownerSessionId: "owner-session", - logStore, - resolveWorkload: () => Effect.succeed({ waitForReadiness: () => Effect.never }), - }); - const ingress: SupervisorIngress = { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: () => Effect.void, - close: Effect.void, - }; - const runtime: SupervisorRuntime = { - driver, - preflight: () => Effect.void, - prepare: () => Effect.void, - prefetch: () => Effect.void, - artifacts: Effect.succeed([]), - activate: () => Effect.succeed({ host: "127.0.0.1", port: 9999 }), - ingress, - logStore, - }; - const context = yield* Effect.context(); - const supervisor = yield* makeSupervisor({ - stackId: testStackId, - ownerSessionId: "owner-session", - stateStore: store, - context, - runtime, - }); - const result = yield* supervisor.start().pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(ContainerEngineError); - expect(failure.value.message).toContain("follower disconnected before readiness"); - } - } - expect((yield* supervisor.status).lifecycle).toBe("unconfigured"); - yield* supervisor.shutdownIfIdle; - yield* supervisor.shutdown; - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/runtime/database-bootstrap-catalog.integration.test.ts b/packages/stack/src/runtime/database-bootstrap-catalog.integration.test.ts deleted file mode 100644 index d22e950ff4..0000000000 --- a/packages/stack/src/runtime/database-bootstrap-catalog.integration.test.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Option, Redacted } from "effect"; -import { compileStack } from "../model/Compiler.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { StackPreparationError } from "../public/Errors.ts"; -import { databaseBootstrapPlan } from "./DatabaseBootstrapCatalog.ts"; - -const stateFrom = (definition: PersistedStackState["definition"]): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity: { - projectRoot: "/tmp/project", - branchContext: "ordinary-workspace", - stackName: "default", - }, - runtime: { kind: "native" }, - desiredLifecycle: "stopped", - definition, - ports: [], - privatePorts: [{ workloadId: "database:database", binding: "primary", port: 54_321 }], - secrets: { - "secret:database.internal.password": { policy: "managed", value: "database-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - }, -}); - -const compileDefinition = compileStack({ - projectRoot: "/tmp/project", - runtime: { kind: "native" }, -}).pipe( - Effect.provide(NodeServices.layer), - Effect.map((result) => stateFrom(result.definition)), -); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("database bootstrap catalog", () => { - it.live("returns the managed database material required for reconciliation", () => - Effect.gen(function* () { - const state = yield* compileDefinition; - const plan = yield* databaseBootstrapPlan(state); - expect(Redacted.value(plan.databasePassword)).toBe("database-secret"); - expect(Redacted.value(plan.jwtSecret)).toBe("jwt-secret"); - expect(plan.jwtExpiry).toBe(3600); - }), - ); - - it.live("rejects database bootstrap when the managed database password is absent", () => - Effect.gen(function* () { - const state = yield* compileDefinition; - const missing = yield* databaseBootstrapPlan({ - ...state, - secrets: Object.fromEntries( - Object.entries(state.secrets).filter( - ([slot]) => slot !== "secret:database.internal.password", - ), - ), - }).pipe(Effect.exit); - const missingError = errorOf(missing); - expect(missingError).toMatchObject({ - message: "Managed database password is unavailable for bootstrap", - }); - expect(missingError).toBeInstanceOf(StackPreparationError); - }), - ); - - it.live("rejects database bootstrap when the managed JWT secret is absent", () => - Effect.gen(function* () { - const state = yield* compileDefinition; - const missing = yield* databaseBootstrapPlan({ - ...state, - secrets: Object.fromEntries( - Object.entries(state.secrets).filter( - ([slot]) => slot !== "secret:auth.settings.jwt_secret", - ), - ), - }).pipe(Effect.exit); - const missingError = errorOf(missing); - expect(missingError).toMatchObject({ - message: "Managed JWT secret is unavailable for database bootstrap", - }); - expect(missingError).toBeInstanceOf(StackPreparationError); - }), - ); - - it.live("rejects database bootstrap when the Auth JWT expiry is invalid", () => - Effect.gen(function* () { - const state = yield* compileDefinition; - if (state.definition === undefined) throw new Error("compiled state has no definition"); - const invalidDefinition = { - ...state.definition, - capabilities: { - ...state.definition.capabilities, - auth: { - ...state.definition.capabilities.auth, - settings: { ...state.definition.capabilities.auth.settings, jwt_expiry: 0 }, - }, - }, - }; - const invalid = yield* databaseBootstrapPlan({ - ...state, - definition: invalidDefinition, - }).pipe(Effect.exit); - const invalidError = errorOf(invalid); - expect(invalidError).toMatchObject({ - message: "Auth JWT expiry must be a finite positive integer", - }); - expect(invalidError).toBeInstanceOf(StackPreparationError); - }), - ); -}); diff --git a/packages/stack/src/runtime/native-launcher.ts b/packages/stack/src/runtime/native-launcher.ts index da171c7764..a2c6845067 100644 --- a/packages/stack/src/runtime/native-launcher.ts +++ b/packages/stack/src/runtime/native-launcher.ts @@ -153,7 +153,7 @@ export const runNativeLauncher = (): void => { } ownerLossGraceful = true; // The fiber is process-scoped: the launcher exits on group termination, so - // no detached work can outlive this supervisor process. + // no detached work can outlive its owner process. Effect.runFork( Effect.sleep(Duration.millis(specGracefulStopTimeoutMs)).pipe( Effect.andThen(Effect.sync(() => terminateGroup("SIGKILL"))), @@ -171,7 +171,7 @@ export const runNativeLauncher = (): void => { fd: 3, autoClose: false, }); - // Owner-loss is an abrupt supervisor crash, not an explicit graceful stop. + // Owner-loss is an abrupt owner crash, not an explicit graceful stop. // Apply a workload's bounded graceful policy when present, then guarantee // tree termination so descendants cannot be orphaned after owner loss. ownerPipe.on("end", handleOwnerLoss); diff --git a/packages/stack/src/runtime/native-process.integration.test.ts b/packages/stack/src/runtime/native-process.integration.test.ts index 528adb80bb..7776e25efc 100644 --- a/packages/stack/src/runtime/native-process.integration.test.ts +++ b/packages/stack/src/runtime/native-process.integration.test.ts @@ -5,6 +5,7 @@ import { NodeServices } from "@effect/platform-node"; import { systemError } from "effect/PlatformError"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerType } from "effect/unstable/process/ChildProcessSpawner"; +import type { ExitCode } from "effect/unstable/process/ChildProcessSpawner"; import { spawnNativeProcess, type NativeProcess, type NativeProcessSpec } from "./NativeProcess.ts"; const targetPid = 87_035; @@ -19,6 +20,8 @@ interface FakeProcessOptions { readonly groupForeignSpawnDefect?: boolean; readonly groupStallReady?: Deferred.Deferred; readonly groupStallClosed?: Deferred.Deferred; + readonly exitStarted?: Deferred.Deferred; + readonly exitCode?: Deferred.Deferred; } const makeSpawner = (options: FakeProcessOptions) => { @@ -84,7 +87,16 @@ const makeSpawner = (options: FakeProcessOptions) => { return Effect.succeed( ChildProcessSpawner.makeHandle({ pid: ChildProcessSpawner.ProcessId(options.targetPid ?? targetPid), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + exitCode: (() => { + const exit = options.exitCode; + return exit === undefined + ? Effect.succeed(ChildProcessSpawner.ExitCode(0)) + : Effect.gen(function* () { + if (options.exitStarted !== undefined) + yield* Deferred.succeed(options.exitStarted, undefined); + return yield* Deferred.await(exit); + }); + })(), isRunning: Effect.succeed(false), kill: () => Effect.void, stdin: Sink.drain, @@ -150,6 +162,15 @@ describe("native process group cleanup", () => { }), ); + it.live.skipIf(process.platform !== "darwin")( + "accepts EPERM when a valid process-group listing contains only exiting members", + () => + Effect.gen(function* () { + const { result } = yield* runKill({ groupOutput: `${targetPid} ?E\n` }); + expect(Exit.isSuccess(result)).toBe(true); + }), + ); + it.effect.skipIf(process.platform !== "darwin")( "preserves EPERM when process-group inspection stalls", () => @@ -211,6 +232,19 @@ describe("native process group cleanup", () => { }), ); + it.live.skipIf(process.platform !== "darwin")( + "preserves EPERM when an exiting group also has a live member", + () => + Effect.gen(function* () { + const { result } = yield* runKill({ groupOutput: `${targetPid} ?E\n${targetPid} S\n` }); + expect(Exit.isFailure(result)).toBe(true); + if (Exit.isFailure(result)) { + const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); + expect(error).toMatchObject({ cause: { code: "EPERM" } }); + } + }), + ); + it.live.skipIf(process.platform !== "darwin")( "preserves EPERM when process-group inspection is malformed or empty", () => @@ -319,4 +353,33 @@ describe("native process group cleanup", () => { Effect.tap(({ result }) => Effect.sync(() => expect(Exit.isSuccess(result)).toBe(true))), ), ); + + it.live("keeps the shared exit observation alive after a canceled waiter", () => + withMockedTargetKill( + Effect.gen(function* () { + const exitStarted = yield* Deferred.make(); + const exitCode = yield* Deferred.make(); + yield* Effect.scoped( + Effect.gen(function* () { + const native = yield* spawnNativeProcess(spec, { + command: "test-launcher", + args: [], + }); + const first = yield* Effect.forkChild(native.exitCode); + yield* Deferred.await(exitStarted); + yield* Fiber.interrupt(first); + yield* Deferred.succeed(exitCode, ChildProcessSpawner.ExitCode(17)); + expect(yield* native.exitCode).toBe(ChildProcessSpawner.ExitCode(17)); + }).pipe( + Effect.provideService( + ChildProcessSpawner.ChildProcessSpawner, + makeSpawner({ exitStarted, exitCode }), + ), + ), + ); + }), + targetPid, + "ESRCH", + ), + ); }); diff --git a/packages/stack/src/runtime/native-runtime.integration.test.ts b/packages/stack/src/runtime/native-runtime.integration.test.ts deleted file mode 100644 index 0f2a24f928..0000000000 --- a/packages/stack/src/runtime/native-runtime.integration.test.ts +++ /dev/null @@ -1,1239 +0,0 @@ -import { NodeServices, NodeSocket } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Data, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Path, - Ref, - Schedule, - Schema, - Scope, - Stream, -} from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { fileURLToPath } from "node:url"; -import { LogStoreError, makeLogStore, type LogStore } from "../supervisor/LogStore.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import type { RuntimeWorkloadKey } from "./RuntimeDriver.ts"; -import { RuntimeDriverError } from "./RuntimeDriver.ts"; -import { makeNativeRuntime } from "./NativeRuntime.ts"; -import { - defaultNativeProcessLauncher, - nativeLauncherEntrypointFor, - NATIVE_PROCESS_DISPATCH_SENTINEL, - spawnNativeProcess, - type NativeProcess, -} from "./NativeProcess.ts"; - -const stackId = StackIdSchema.make("d".repeat(64)); -const encodeJson = (value: unknown): string => - Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))(value); - -class ProcessTreeTestError extends Data.TaggedError("ProcessTreeTestError")<{ - readonly message: string; -}> {} - -const workload = (id: string, bootstrap?: "database"): PlannedWorkload => ({ - id, - capability: "database", - ...(bootstrap === undefined ? {} : { bootstrap }), - dependencies: [], - readiness: {}, - artifacts: { - native: { kind: "native", release: "test" }, - container: { kind: "container", image: `test/${id}` }, - }, - selected: { kind: "native", release: "test" }, -}); - -const keyFor = (id: string): RuntimeWorkloadKey => ({ - stackId, - workloadId: `database:${id}`, -}); - -const fixtureProcess = (message: string) => ({ - executable: process.execPath, - args: [ - "-e", - `process.stdout.write(${JSON.stringify(`${message}\n`)}); process.stderr.write(${JSON.stringify("stderr\n")}); setInterval(() => {}, 1000)`, - ], -}); - -const oneShotProcess = (stdout: string, exitCode = 0) => ({ - executable: process.execPath, - args: [ - "-e", - `process.stdout.write(${JSON.stringify(`${stdout}\n`)}); process.stderr.write(${JSON.stringify("startup-stderr\n")}); process.exit(${exitCode})`, - ], -}); - -const processPlan = (main: A) => ({ startup: [], main }); - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const signalOnLog = ( - base: LogStore, - message: string, - signal: Deferred.Deferred, -): LogStore => ({ - ...base, - append: (record) => - base - .append(record) - .pipe( - Effect.tap((entry) => - entry.message === message ? Deferred.succeed(signal, undefined) : Effect.void, - ), - ), -}); - -// Native launcher tests create real process trees. Keep their timeout local to -// this suite so file-level parallelism does not turn the default 5s guard into -// a false failure while leaving the rest of the integration suite unchanged. -describe("native runtime", { timeout: 15_000 }, () => { - it("uses a private dispatch marker for compiled Bun launcher paths", () => { - expect( - nativeLauncherEntrypointFor("file:///$bunfs/packages/stack/src/runtime/NativeProcess.ts"), - ).toBe(NATIVE_PROCESS_DISPATCH_SENTINEL); - expect( - nativeLauncherEntrypointFor("C:\\$bunfs\\packages\\stack\\src\\runtime\\NativeProcess.ts"), - ).toBe(NATIVE_PROCESS_DISPATCH_SENTINEL); - }); - - it.live("rejects container artifacts before invoking native process resolution", () => - withPlatform( - Effect.gen(function* () { - let resolved = false; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.sync(() => { - resolved = true; - return processPlan(fixtureProcess("unexpected")); - }), - waitForReadiness: () => Effect.void, - }); - const containerSelected = workload("container"); - const result = yield* runtime - .start(keyFor("container"), { - ...containerSelected, - selected: containerSelected.artifacts.container, - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(resolved).toBe(false); - }), - ), - ); - - it.live("starts, captures redacted output, and terminates exact workloads", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-runtime-" }); - const logStore = yield* makeLogStore({ - path: path.join(root, "logs.json"), - knownSecrets: ["secret-value"], - }); - const outputLogged = yield* Deferred.make(); - const signaledLogStore = signalOnLog(logStore, "[REDACTED]", outputLogged); - let startedProcess: NativeProcess | undefined; - const runtime = yield* makeNativeRuntime({ - resolveProcess: (_key, _entry) => - Effect.succeed(processPlan(fixtureProcess("secret-value"))), - logStore: signaledLogStore, - waitForReadiness: (_key, _workload, process) => - Effect.sync(() => { - startedProcess = process; - }), - }); - const key = keyFor("one"); - const ready = yield* runtime.start(key, workload("one")); - expect(ready.state).toBe("ready"); - yield* Deferred.await(outputLogged); - const values = yield* runtime.observe(stackId); - expect(values).toHaveLength(1); - yield* runtime.stop(key); - expect((yield* runtime.observe(stackId))[0]?.state).toBe("stopped"); - expect(startedProcess).toBeDefined(); - if (startedProcess !== undefined) expect(yield* startedProcess.isRunning).toBe(false); - expect((yield* logStore.read()).map((entry) => entry.message)).toContain("[REDACTED]"); - yield* runtime.remove(key); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("publishes an unexpected native workload exit after readiness", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-exit-" }); - const triggerPath = path.join(root, "trigger"); - const readySignal = yield* Deferred.make(); - const logStore = yield* makeLogStore({ path: path.join(root, "logs.json") }); - const signaledLogStore = signalOnLog(logStore, "watch-ready", readySignal); - let startedProcess: NativeProcess | undefined; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [], - main: { - executable: process.execPath, - args: [ - "-e", - `const fs=require("node:fs"); fs.watch(${JSON.stringify(root)}, () => { if (fs.existsSync(${JSON.stringify(triggerPath)})) process.exit(1) }); process.stdout.write("watch-ready\\n")`, - ], - }, - }), - logStore: signaledLogStore, - waitForReadiness: (_key, _workload, process) => - Effect.sync(() => { - startedProcess = process; - }).pipe(Effect.andThen(Deferred.await(readySignal))), - }); - const ready = yield* runtime.start(keyFor("watch"), workload("watch")); - expect(ready.state).toBe("ready"); - expect(startedProcess).toBeDefined(); - yield* fs.writeFileString(triggerPath, "exit"); - if (startedProcess !== undefined) yield* startedProcess.exitCode; - const observed = yield* runtime.observe(stackId).pipe( - Effect.tap(() => Effect.yieldNow), - Effect.repeat({ - until: (values) => values[0]?.state === "failed", - schedule: Schedule.forever, - }), - Effect.timeout("5 seconds"), - ); - expect(observed).toEqual([ - expect.objectContaining({ - workloadId: keyFor("watch").workloadId, - state: "failed", - error: "Native workload exited before an explicit stop (code 1)", - }), - ]); - yield* runtime.remove(keyFor("watch")); - }), - ), - ); - - it.live("reports the signal that terminated a native workload", () => - withPlatform( - Effect.gen(function* () { - const native = yield* spawnNativeProcess({ - executable: process.execPath, - args: ["-e", 'process.kill(process.pid, "SIGTERM")'], - }); - const stderr = yield* native.stderr.pipe(Stream.decodeText, Stream.runCollect); - expect(yield* native.exitCode).toBe(1); - expect(Array.from(stderr).join("")).toContain( - "Native workload exited due to signal SIGTERM", - ); - }), - ), - ); - - it.live.each([ - { name: "SIGTERM stops", signal: "SIGTERM", closeScope: false }, - { name: "SIGINT stops", signal: "SIGINT", closeScope: false }, - { name: "scope closures", signal: "SIGTERM", closeScope: true }, - ] as const)( - "does not report unexpected exits during concurrent $name", - ({ signal, closeScope }) => - withPlatform( - Effect.forEach( - Array.from({ length: 25 }), - () => - Effect.scoped( - Effect.gen(function* () { - const ready = yield* Deferred.make(); - const processScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope) => - Scope.close(scope, Exit.void), - ); - const native = yield* spawnNativeProcess({ - gracefulStopSignal: signal, - executable: process.execPath, - args: ["-e", 'process.stdout.write("ready\\n"); setInterval(() => {}, 1000)'], - }).pipe(Effect.provideService(Scope.Scope, processScope)); - const stdout = yield* native.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - line === "ready" ? Deferred.succeed(ready, undefined) : Effect.void, - ), - Effect.forkChild({ startImmediately: true }), - ); - const stderr = yield* Effect.forkChild( - native.stderr.pipe(Stream.decodeText, Stream.runCollect), - { startImmediately: true }, - ); - yield* Deferred.await(ready).pipe(Effect.timeout("10 seconds")); - yield* closeScope ? Scope.close(processScope, Exit.void) : native.kill; - const stderrOutput = yield* Fiber.join(stderr); - yield* Fiber.join(stdout); - expect(Array.from(stderrOutput).join("")).not.toContain( - "Native workload exited due to signal", - ); - expect(yield* native.isRunning).toBe(false); - }), - ), - { concurrency: 4, discard: true }, - ), - ), - { timeout: 60_000 }, - ); - - it.live("keeps a ready workload after the losing exit observer is interrupted", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-native-exit-observer-", - }); - const logStore = yield* makeLogStore({ path: path.join(root, "logs.json") }); - const readinessEntered = yield* Deferred.make(); - const readinessRelease = yield* Deferred.make(); - let startedProcess: NativeProcess | undefined; - const outputReady = yield* Deferred.make(); - const signaledLogStore = signalOnLog(logStore, "main-started", outputReady); - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => Effect.succeed(processPlan(fixtureProcess("main-started"))), - logStore: signaledLogStore, - waitForReadiness: (_key, _workload, process) => - Effect.sync(() => { - startedProcess = process; - }).pipe( - Effect.andThen(Deferred.succeed(readinessEntered, undefined)), - Effect.andThen(Deferred.await(readinessRelease)), - ), - }); - const key = keyFor("exit-observer"); - const caller = yield* runtime - .start(key, workload("exit-observer")) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(readinessEntered); - yield* Deferred.await(outputReady); - yield* Deferred.succeed(readinessRelease, undefined); - const ready = yield* Fiber.join(caller); - expect(ready.state).toBe("ready"); - expect(startedProcess).toBeDefined(); - if (startedProcess !== undefined) expect(yield* startedProcess.isRunning).toBe(true); - expect((yield* runtime.observe(stackId))[0]?.state).toBe("ready"); - yield* runtime.stop(key); - }), - ), - ); - - it.live("cleans up when readiness fails", () => - withPlatform( - Effect.gen(function* () { - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => Effect.succeed(processPlan(fixtureProcess("fails"))), - waitForReadiness: (key) => - Effect.fail( - new RuntimeDriverError({ - message: "fixture readiness failed", - stackId: key.stackId, - workloadId: key.workloadId, - }), - ), - }); - const result = yield* runtime.start(keyFor("fails"), workload("fails")).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("fails readiness when a native process exits nonzero", () => - withPlatform( - Effect.gen(function* () { - const readiness = yield* Deferred.make(); - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed( - processPlan({ - executable: process.execPath, - args: ["-e", "process.stderr.write('native failed\\n'); process.exit(3)"], - }), - ), - waitForReadiness: () => Deferred.await(readiness), - }); - const result = yield* runtime - .start(keyFor("nonzero"), workload("nonzero")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - expect(Cause.pretty(result.cause)).toContain("stderr: native failed"); - } - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("attaches one-shot stderr and leftover wipe guidance to a startup failure", () => - withPlatform( - Effect.gen(function* () { - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [ - { - executable: process.execPath, - args: [ - "-e", - "process.stderr.write('initdb: directory exists but is not empty\\n'); process.exit(1)", - ], - }, - ], - main: fixtureProcess("unused-main"), - }), - waitForReadiness: () => Effect.void, - }); - const result = yield* runtime - .start(keyFor("startup-leftover"), workload("startup-leftover")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const pretty = Cause.pretty(result.cause); - expect(pretty).toContain("stderr: initdb: directory exists but is not empty"); - expect(pretty).toContain("stack destroy"); - expect(pretty).not.toContain("db reset --local"); - } - }), - ), - ); - - it.live("restarts a stopped exact key with a fresh process", () => - withPlatform( - Effect.gen(function* () { - let launches = 0; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.sync(() => { - launches += 1; - return processPlan(fixtureProcess(`restart-${launches}`)); - }), - waitForReadiness: () => Effect.void, - }); - const key = keyFor("restart"); - yield* runtime.start(key, workload("restart")); - yield* runtime.stop(key); - const restarted = yield* runtime.start(key, workload("restart")); - expect(restarted.state).toBe("ready"); - expect(launches).toBe(2); - yield* runtime.remove(key); - }), - ), - ); - - it.live("returns the existing observation when an exact key is already ready", () => - withPlatform( - Effect.gen(function* () { - let launches = 0; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.sync(() => { - launches += 1; - return processPlan(fixtureProcess(`ready-${launches}`)); - }), - waitForReadiness: () => Effect.void, - }); - const key = keyFor("already-ready"); - const first = yield* runtime.start(key, workload("already-ready")); - const second = yield* runtime.start(key, workload("already-ready")); - expect(second).toEqual(first); - expect(launches).toBe(1); - yield* runtime.stop(key); - yield* runtime.remove(key); - }), - ), - ); - - it.live("keeps a shared start alive when one caller is interrupted", () => - withPlatform( - Effect.gen(function* () { - const spawned = yield* Deferred.make(); - const readiness = yield* Deferred.make(); - let launches = 0; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.sync(() => { - launches += 1; - return processPlan(fixtureProcess("shared-start")); - }).pipe(Effect.tap(() => Deferred.succeed(spawned, undefined))), - waitForReadiness: () => Deferred.await(readiness), - }); - const key = keyFor("shared-start"); - const firstCaller = yield* runtime - .start(key, workload("shared-start")) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(spawned); - yield* Fiber.interrupt(firstCaller); - yield* Deferred.succeed(readiness, undefined); - const ready = yield* runtime.start(key, workload("shared-start")); - expect(ready.state).toBe("ready"); - expect(launches).toBe(1); - yield* runtime.remove(key); - }), - ), - ); - - it.live("fails startup when log persistence fails before readiness", () => - withPlatform( - Effect.gen(function* () { - const readiness = yield* Deferred.make(); - const logStore: LogStore = { - path: "memory://failing-log-store", - append: () => - Effect.fail( - new LogStoreError({ path: "memory://failing-log-store", message: "disk full" }), - ), - read: () => Effect.succeed([]), - }; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => Effect.succeed(processPlan(fixtureProcess("log-failure"))), - waitForReadiness: () => Deferred.await(readiness), - logStore, - }); - const result = yield* runtime - .start(keyFor("log-failure"), workload("log-failure")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("runs native startup processes before main readiness and captures their logs", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-startup-" }); - const logStore = yield* makeLogStore({ path: path.join(root, "logs.json") }); - const mainLogged = yield* Deferred.make(); - const signaledLogStore = signalOnLog(logStore, "main", mainLogged); - let observedReady = false; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [oneShotProcess("startup")], - main: fixtureProcess("main"), - }), - logStore: signaledLogStore, - waitForReadiness: () => - Effect.sync(() => { - observedReady = true; - }), - }); - const key = keyFor("startup-order"); - const ready = yield* runtime.start(key, workload("startup-order")); - expect(ready.state).toBe("ready"); - expect(observedReady).toBe(true); - yield* Deferred.await(mainLogged); - const messages = (yield* logStore.read()).map((entry) => entry.message); - expect(messages.indexOf("startup")).toBeGreaterThanOrEqual(0); - expect(messages).toContain("startup-stderr"); - expect(messages.indexOf("main")).toBeGreaterThan(messages.indexOf("startup")); - yield* runtime.remove(key); - }), - ), - ); - - it.live("bounds a native one-shot before it can report readiness", () => - withPlatform( - Effect.gen(function* () { - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [ - { - executable: process.execPath, - args: ["-e", "setInterval(() => {}, 1000)"], - timeout: "20 millis", - }, - ], - main: fixtureProcess("never-main"), - }), - waitForReadiness: () => Effect.void, - }); - const result = yield* runtime - .start(keyFor("startup-timeout"), workload("startup-timeout")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Cause.pretty(result.cause)).toContain("Native startup process timed out"); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("does not leak host credentials into native workload children", () => - withPlatform( - Effect.gen(function* () { - const path = yield* Path.Path; - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "supabase-native-env-", - }); - const logStore = yield* makeLogStore({ path: path.join(root, "logs.json") }); - const launcherPath = fileURLToPath(new URL("./native-launcher.ts", import.meta.url)); - const launcher = { - command: process.execPath, - args: [ - "-e", - `process.env.PGPASSWORD="host-secret"; const { runNativeLauncher } = await import(${encodeJson(launcherPath)}); runNativeLauncher()`, - ], - }; - const runtime = yield* makeNativeRuntime({ - resolveLauncher: () => Effect.succeed(launcher), - resolveProcess: () => - Effect.succeed({ - startup: [ - { - executable: process.execPath, - args: [ - "-e", - 'process.stdout.write(`${JSON.stringify({ password: process.env.PGPASSWORD ?? "missing", path: (process.env.PATH ?? "").length > 0 })}\\n`)', - ], - }, - ], - main: fixtureProcess("env-main"), - }), - waitForReadiness: () => Effect.void, - logStore, - }); - const key = keyFor("env-allowlist"); - yield* runtime.start(key, workload("env-allowlist")); - const messages = (yield* logStore.read()).map((entry) => entry.message); - expect(messages).toContain('{"password":"missing","path":true}'); - yield* runtime.stop(key); - }), - ), - ); - - it.live("does not spawn the native main process when startup exits nonzero", () => - withPlatform( - Effect.gen(function* () { - let readinessCalled = false; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [oneShotProcess("startup-failed", 7)], - // Invalid on purpose: the runtime must fail the startup process before spawning it. - main: { executable: "/missing/native-main" }, - }), - waitForReadiness: () => - Effect.sync(() => { - readinessCalled = true; - }), - }); - const result = yield* runtime - .start(keyFor("startup-failed"), workload("startup-failed")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(readinessCalled).toBe(false); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("drains startup output after the child exits", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-drain-" }); - const logStore = yield* makeLogStore({ path: path.join(root, "logs.json") }); - const startupOutputSignal = yield* Deferred.make(); - const signaledLogStore = signalOnLog(logStore, "startup-late", startupOutputSignal); - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [ - { - executable: process.execPath, - args: ["-e", 'process.stdout.write("startup-late\\n"); process.exit(0)'], - }, - ], - main: fixtureProcess("drained-main"), - }), - logStore: signaledLogStore, - waitForReadiness: () => Effect.void, - }); - const caller = yield* runtime - .start(keyFor("startup-drain"), workload("startup-drain")) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(startupOutputSignal); - const ready = yield* Fiber.join(caller); - expect(ready.state).toBe("ready"); - yield* runtime.remove(keyFor("startup-drain")); - }), - ), - ); - it.live("interrupts a startup process through its exact child scope", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-interrupt-" }); - const pidPath = path.join(root, "pid"); - const stoppedPath = path.join(root, "stopped"); - const logStore = yield* makeLogStore({ path: path.join(root, "logs.json") }); - const startedSignal = yield* Deferred.make(); - const signaledLogStore = signalOnLog(logStore, "started", startedSignal); - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [ - { - executable: process.execPath, - args: [ - "-e", - `const fs=require("node:fs"); fs.writeFileSync(${JSON.stringify(pidPath)},String(process.pid)); const stop=()=>{fs.writeFileSync(${JSON.stringify(stoppedPath)},"stopped"); process.exit(0)}; process.on("SIGTERM",stop); process.on("SIGINT",stop); process.stdout.write("started\\n"); setInterval(()=>{},1000)`, - ], - }, - ], - main: fixtureProcess("never-main"), - }), - logStore: signaledLogStore, - waitForReadiness: () => Effect.void, - }); - const key = keyFor("startup-interrupted"); - const caller = yield* runtime - .start(key, workload("startup-interrupted")) - .pipe(Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(startedSignal); - yield* runtime.stop(key); - yield* Fiber.interrupt(caller); - expect(Number.isSafeInteger(Number.parseInt(yield* fs.readFileString(pidPath), 10))).toBe( - true, - ); - expect(yield* fs.readFileString(stoppedPath)).toBe("stopped"); - }), - ), - ); - - it.live("marks a ready workload failed when log persistence fails afterward", () => - withPlatform( - Effect.gen(function* () { - const appendStarted = yield* Deferred.make(); - const appendFailure = yield* Deferred.make(); - const logStore: LogStore = { - path: "memory://failing-log-store-after-ready", - append: () => - Effect.gen(function* () { - yield* Deferred.succeed(appendStarted, undefined); - return yield* Deferred.await(appendFailure); - }), - read: () => Effect.succeed([]), - }; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed(processPlan(fixtureProcess("log-failure-after-ready"))), - waitForReadiness: () => Effect.void, - logStore, - }); - const key = keyFor("log-failure-after-ready"); - const ready = yield* runtime.start(key, workload("log-failure-after-ready")); - expect(ready.state).toBe("ready"); - yield* Deferred.await(appendStarted); - yield* Deferred.fail( - appendFailure, - new LogStoreError({ - path: logStore.path, - message: "disk full", - }), - ); - yield* Effect.yieldNow; - const observed = yield* runtime.observe(stackId); - expect(observed[0]?.state).toBe("failed"); - yield* runtime.remove(key); - }), - ), - ); - - it.live("runs database bootstrap before reporting readiness", () => - withPlatform( - Effect.gen(function* () { - let readinessCalled = false; - let bootstrapCalledAfterReadiness = false; - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => - Effect.succeed({ - startup: [], - main: fixtureProcess("database"), - }), - waitForReadiness: () => - Effect.sync(() => { - readinessCalled = true; - }), - bootstrapDatabase: () => - Effect.sync(() => { - bootstrapCalledAfterReadiness = readinessCalled; - }), - }); - const key = keyFor("bootstrap"); - const ready = yield* runtime.start(key, workload("bootstrap", "database")); - expect(bootstrapCalledAfterReadiness).toBe(true); - expect(ready.state).toBe("ready"); - yield* runtime.stop(key); - yield* runtime.remove(key); - }), - ), - ); - - it.live("requires a database session and plan for marked workloads", () => - withPlatform( - Effect.gen(function* () { - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => Effect.succeed(processPlan(fixtureProcess("database"))), - waitForReadiness: () => Effect.void, - }); - const result = yield* runtime - .start(keyFor("missing-bootstrap"), workload("missing-bootstrap", "database")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("cleans up the native process when database bootstrap fails", () => - withPlatform( - Effect.gen(function* () { - const bootstrapError = new RuntimeDriverError({ - message: "database bootstrap failed", - stackId, - workloadId: "database:failed-bootstrap", - }); - const runtime = yield* makeNativeRuntime({ - resolveProcess: () => Effect.succeed(processPlan(fixtureProcess("database"))), - waitForReadiness: () => Effect.void, - bootstrapDatabase: () => Effect.fail(bootstrapError), - }); - const result = yield* runtime - .start(keyFor("failed-bootstrap"), workload("failed-bootstrap", "database")) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(yield* runtime.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("isolates exact workload identities while stopping one", () => - withPlatform( - Effect.gen(function* () { - const runtime = yield* makeNativeRuntime({ - resolveProcess: (_key, entry) => Effect.succeed(processPlan(fixtureProcess(entry.id))), - waitForReadiness: () => Effect.void, - }); - const first = keyFor("isolated-one"); - const second = keyFor("isolated-two"); - yield* Effect.all( - [ - runtime.start(first, workload("isolated-one")), - runtime.start(second, workload("isolated-two")), - ], - { concurrency: "unbounded" }, - ); - yield* runtime.stop(first); - expect( - (yield* runtime.observe(stackId)).map((entry) => [entry.workloadId, entry.state]), - ).toEqual([ - ["database:isolated-one", "stopped"], - ["database:isolated-two", "ready"], - ]); - yield* runtime.remove(first); - yield* runtime.remove(second); - }), - ), - ); - - // The helper below is a raw Node launcher fixture; its JSON is the same fd4 - // payload covered by NativeProcess integration, not product serialization. - const runProcessTreeScenario = (runtimeCommand: string) => - withPlatform( - Effect.gen(function* () { - const launcherArgs = defaultNativeProcessLauncher().args; - const targetLauncher = { command: runtimeCommand, args: launcherArgs }; - const descendantCode = ` - const net = require("node:net"); - const server = net.createServer(); - server.listen({ host: "127.0.0.1", port: 0 }, () => { - const address = server.address(); - if (typeof address === "object" && address !== null) { - process.stdout.write(\`DESC_READY \${address.port}\\n\`); - } - }); - `; - const targetCode = ` - const net = require("node:net"); - const { spawn } = require("node:child_process"); - process.stdout.on("error", () => {}); - const server = net.createServer(); - server.listen({ host: "127.0.0.1", port: 0 }, () => { - const address = server.address(); - if (typeof address === "object" && address !== null) { - process.stdout.write(\`TARGET_READY \${address.port}\\n\`); - } - const child = spawn(process.execPath, ["-e", ${encodeJson(descendantCode)}], { - stdio: ["ignore", "pipe", "inherit"] - }); - child.stdout.on("data", (chunk) => { - if (!process.stdout.destroyed) process.stdout.write(chunk); - }); - }); - `; - const ownerCode = ` - const { spawn } = require("node:child_process"); - const launcherProcess = spawn(${encodeJson(targetLauncher.command)}, ${encodeJson(targetLauncher.args)}, { - detached: true, - stdio: ["ignore", "inherit", "inherit", "pipe", "pipe"] - }); - launcherProcess.stdio[4].end(JSON.stringify({ - executable: ${encodeJson(runtimeCommand)}, - args: ["-e", ${encodeJson(targetCode)}] - })); - setInterval(() => {}, 1000); - `; - const owner = yield* ChildProcess.make(process.execPath, ["-e", ownerCode], { - stdout: "pipe", - stderr: "pipe", - detached: true, - }); - const stderr = yield* Ref.make(""); - yield* owner.stderr.pipe( - Stream.decodeText, - Stream.runForEach((chunk) => Ref.update(stderr, (current) => current + chunk)), - Effect.forkChild({ startImmediately: true }), - ); - const targetReady = yield* Deferred.make(); - const descendantReady = yield* Deferred.make(); - const outputFiber = yield* owner.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - Effect.gen(function* () { - if (line.startsWith("TARGET_READY ")) yield* Deferred.succeed(targetReady, line); - else if (line.startsWith("DESC_READY ")) - yield* Deferred.succeed(descendantReady, line); - }), - ), - Effect.forkChild({ startImmediately: true }), - ); - const output = yield* Deferred.await(descendantReady).pipe( - Effect.timeoutOrElse({ - duration: "3 seconds", - orElse: () => - Effect.gen(function* () { - const diagnostics = yield* Ref.get(stderr); - return yield* new ProcessTreeTestError({ - message: `native launcher readiness timed out: ${diagnostics}`, - }); - }), - }), - ); - const targetLine = yield* Deferred.await(targetReady); - const targetPort = Number.parseInt(targetLine.slice("TARGET_READY ".length), 10); - const descendantPort = Number.parseInt(output.slice("DESC_READY ".length), 10); - expect(Number.isSafeInteger(targetPort)).toBe(true); - expect(Number.isSafeInteger(descendantPort)).toBe(true); - const targetSocket = yield* NodeSocket.makeNet({ host: "127.0.0.1", port: targetPort }); - const descendantSocket = yield* NodeSocket.makeNet({ - host: "127.0.0.1", - port: descendantPort, - }); - const targetClosedSignal = yield* Deferred.make(); - const targetClosed = yield* targetSocket - .runRaw(() => Effect.void, { - onOpen: Deferred.succeed(targetClosedSignal, undefined), - }) - .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(targetClosedSignal); - const descendantClosedSignal = yield* Deferred.make(); - const descendantClosed = yield* descendantSocket - .runRaw(() => Effect.void, { - onOpen: Deferred.succeed(descendantClosedSignal, undefined), - }) - .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(descendantClosedSignal); - yield* owner.kill({ killSignal: "SIGKILL" }); - yield* Fiber.join(targetClosed).pipe( - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => - Effect.fail( - new ProcessTreeTestError({ message: "target process tree did not close" }), - ), - }), - ); - yield* Fiber.join(descendantClosed).pipe( - Effect.timeoutOrElse({ - duration: "10 seconds", - orElse: () => - Effect.fail( - new ProcessTreeTestError({ message: "descendant process tree did not close" }), - ), - }), - ); - yield* Fiber.interrupt(outputFiber); - }), - ); - - it.live("kills a Bun native process tree when its owner pipe closes", () => - runProcessTreeScenario(process.execPath), - ); - - it.live("kills a Node native process tree when its owner pipe closes", () => - runProcessTreeScenario("node"), - ); - - const ownerLossTargetCode = (signalAware: boolean): string => { - const descendantCode = ` - const net = require("node:net"); - const server = net.createServer(); - server.listen({ host: "127.0.0.1", port: 0 }, () => { - const address = server.address(); - if (typeof address === "object" && address !== null) { - process.stdout.write("DESC_READY " + address.port + "\\n"); - } - }); - `; - return ` - const net = require("node:net"); - const { spawn } = require("node:child_process"); - process.stdout.on("error", () => {}); - const server = net.createServer(); - server.listen({ host: "127.0.0.1", port: 0 }, () => { - const address = server.address(); - if (typeof address === "object" && address !== null) { - process.stdout.write("TARGET_READY " + address.port + "\\n"); - } - const child = spawn(process.execPath, ["-e", ${JSON.stringify(descendantCode)}], { - stdio: ["ignore", "pipe", "inherit"] - }); - child.stdout.on("data", (chunk) => { - if (!process.stdout.destroyed) process.stdout.write(chunk); - }); - }); - process.once("SIGINT", () => { - ${signalAware ? 'process.stdout.write("TARGET_SIGINT\\n", () => { server.close(); process.exit(0); });' : 'process.stdout.write("TARGET_SIGINT\\n");'} - }); - `; - }; - - const runOwnerLossScenario = (options: { - readonly targetCode: string; - readonly gracefulStopTimeoutMs: number; - }) => - Effect.gen(function* () { - const launcherArgs = defaultNativeProcessLauncher().args; - const ownerCode = ` - const { spawn } = require("node:child_process"); - const launcherProcess = spawn(${encodeJson(process.execPath)}, ${encodeJson(launcherArgs)}, { - detached: true, - stdio: ["ignore", "inherit", "inherit", "pipe", "pipe"] - }); - launcherProcess.stdio[4].end(JSON.stringify({ - executable: ${encodeJson(process.execPath)}, - args: ["-e", ${encodeJson(options.targetCode)}], - gracefulStopSignal: "SIGINT", - gracefulStopTimeoutMs: ${String(options.gracefulStopTimeoutMs)} - })); - setInterval(() => {}, 1000); - `; - const owner = yield* ChildProcess.make(process.execPath, ["-e", ownerCode], { - stdout: "pipe", - stderr: "pipe", - detached: true, - }); - const stderr = yield* Ref.make(""); - yield* owner.stderr.pipe( - Stream.decodeText, - Stream.runForEach((chunk) => Ref.update(stderr, (current) => current + chunk)), - Effect.forkChild({ startImmediately: true }), - ); - const targetReady = yield* Deferred.make(); - const descendantReady = yield* Deferred.make(); - const signalDelivered = yield* Deferred.make(); - const outputFiber = yield* owner.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - Effect.gen(function* () { - if (line.startsWith("TARGET_READY ")) yield* Deferred.succeed(targetReady, line); - else if (line.startsWith("DESC_READY ")) yield* Deferred.succeed(descendantReady, line); - else if (line === "TARGET_SIGINT") yield* Deferred.succeed(signalDelivered, undefined); - }), - ), - Effect.forkChild({ startImmediately: true }), - ); - const diagnostics = yield* Ref.get(stderr); - const targetLine = yield* Deferred.await(targetReady).pipe( - Effect.timeoutOrElse({ - duration: "3 seconds", - orElse: () => Effect.fail(new ProcessTreeTestError({ message: diagnostics })), - }), - ); - const descendantLine = yield* Deferred.await(descendantReady).pipe( - Effect.timeoutOrElse({ - duration: "3 seconds", - orElse: () => Effect.fail(new ProcessTreeTestError({ message: diagnostics })), - }), - ); - const targetPort = Number.parseInt(targetLine.slice("TARGET_READY ".length), 10); - const descendantPort = Number.parseInt(descendantLine.slice("DESC_READY ".length), 10); - expect(Number.isSafeInteger(targetPort)).toBe(true); - expect(Number.isSafeInteger(descendantPort)).toBe(true); - const targetSocket = yield* NodeSocket.makeNet({ host: "127.0.0.1", port: targetPort }); - const descendantSocket = yield* NodeSocket.makeNet({ - host: "127.0.0.1", - port: descendantPort, - }); - const targetOpened = yield* Deferred.make(); - const targetClosed = yield* targetSocket - .runRaw(() => Effect.void, { onOpen: Deferred.succeed(targetOpened, undefined) }) - .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(targetOpened); - const descendantOpened = yield* Deferred.make(); - const descendantClosed = yield* descendantSocket - .runRaw(() => Effect.void, { onOpen: Deferred.succeed(descendantOpened, undefined) }) - .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(descendantOpened); - yield* owner.kill({ killSignal: "SIGKILL" }); - yield* Deferred.await(signalDelivered).pipe(Effect.timeout("3 seconds")); - yield* Fiber.join(targetClosed).pipe(Effect.timeout("5 seconds")); - yield* Fiber.join(descendantClosed).pipe(Effect.timeout("5 seconds")); - yield* Fiber.interrupt(outputFiber); - }); - - it.live("delivers graceful owner-loss SIGINT and reaps descendants", () => - withPlatform( - runOwnerLossScenario({ targetCode: ownerLossTargetCode(true), gracefulStopTimeoutMs: 5_000 }), - ), - ); - - it.live("force-kills an owner-loss workload after its graceful timeout", () => - withPlatform( - runOwnerLossScenario({ targetCode: ownerLossTargetCode(false), gracefulStopTimeoutMs: 100 }), - ), - ); - - it.live.each([ - { name: "normal workload exit", mode: "exit" }, - { name: "a graceful stop", mode: "graceful" }, - { name: "a forced stop", mode: "forced" }, - { name: "a scope closing an unresponsive workload", mode: "scope" }, - ] as const)("terminates descendants after $name", ({ mode }) => - withPlatform( - Effect.gen(function* () { - let launcherPid: number | undefined; - yield* Effect.gen(function* () { - const descendantReady = yield* Deferred.make(); - const workloadExited = yield* Deferred.make(); - const descendantCode = ` - const net = require("node:net"); - const server = net.createServer(); - server.listen({ host: "127.0.0.1", port: 0 }, () => { - const address = server.address(); - if (typeof address === "object" && address !== null) { - process.stdout.write("DESC_READY " + address.port + "\\n"); - } - }); - `; - const targetCode = ` - const { spawn } = require("node:child_process"); - ${mode === "forced" || mode === "scope" ? 'process.on("SIGTERM", () => {});' : ""} - const child = spawn(process.execPath, ["-e", ${encodeJson(descendantCode)}], { - stdio: ["ignore", "pipe", "inherit"] - }); - child.stdout.on("data", (chunk) => { - process.stdout.write(chunk); - if (${String(mode === "exit")} && chunk.toString().includes("DESC_READY ")) { - process.stdout.write("DIRECT_EXITING\\n"); - process.exit(0); - } - }); - child.on("error", () => process.exit(1)); - `; - const processScope = yield* Effect.acquireRelease(Scope.make("parallel"), (scope) => - Scope.close(scope, Exit.void), - ); - const native = yield* spawnNativeProcess({ - executable: process.execPath, - args: ["-e", targetCode], - gracefulStopTimeout: "100 millis", - }).pipe(Effect.provideService(Scope.Scope, processScope)); - launcherPid = Number(native.pid); - const output = yield* native.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => { - if (line.startsWith("DESC_READY ")) { - return Deferred.succeed( - descendantReady, - Number.parseInt(line.slice("DESC_READY ".length), 10), - ); - } - if (line === "DIRECT_EXITING") return Deferred.succeed(workloadExited, undefined); - return Effect.void; - }), - Effect.forkChild({ startImmediately: true }), - ); - const port = yield* Deferred.await(descendantReady).pipe(Effect.timeout("5 seconds")); - const socket = yield* NodeSocket.makeNet({ host: "127.0.0.1", port }); - const opened = yield* Deferred.make(); - const closed = yield* socket - .runRaw(() => Effect.void, { onOpen: Deferred.succeed(opened, undefined) }) - .pipe(Effect.exit, Effect.forkChild({ startImmediately: true })); - yield* Deferred.await(opened).pipe(Effect.timeout("5 seconds")); - if (mode === "exit") { - yield* Deferred.await(workloadExited).pipe(Effect.timeout("5 seconds")); - expect(yield* native.exitCode).toBe(0); - } else { - yield* mode === "scope" ? Scope.close(processScope, Exit.void) : native.kill; - expect(yield* native.isRunning).toBe(false); - } - yield* Fiber.join(closed).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => - Effect.fail( - new ProcessTreeTestError({ - message: `native launcher left descendant listener alive after ${mode}`, - }), - ), - }), - ); - yield* Fiber.interrupt(output); - }).pipe( - Effect.ensuring( - Effect.gen(function* () { - if (launcherPid !== undefined && process.platform !== "win32") { - const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; - yield* spawner - .exitCode( - ChildProcess.make("kill", ["-KILL", `-${launcherPid}`], { - stdout: "ignore", - stderr: "ignore", - }), - ) - .pipe(Effect.ignore); - } - }), - ), - ); - }), - ), - ); -}); diff --git a/packages/stack/src/runtime/postgres-client-args.ts b/packages/stack/src/runtime/postgres-client-args.ts deleted file mode 100644 index c131d764d6..0000000000 --- a/packages/stack/src/runtime/postgres-client-args.ts +++ /dev/null @@ -1,61 +0,0 @@ -export const POSTGRES_DUMP_BINS = ["pg_dump", "pg_dumpall", "psql"] as const; -export const POSTGRES_PROVE_BINS = ["pg_prove"] as const; -export const POSTGRES_CLIENT_BINS = [...POSTGRES_DUMP_BINS, ...POSTGRES_PROVE_BINS] as const; - -/** Dump scripts need pg_dump/psql; `pg_prove` is required only when that is the argv. */ -export const requiredPostgresClientBins = ( - argv: ReadonlyArray, -): ReadonlyArray<(typeof POSTGRES_CLIENT_BINS)[number]> => - argv[0] === "pg_prove" ? POSTGRES_PROVE_BINS : POSTGRES_DUMP_BINS; - -export interface PostgresClientMount { - readonly source: string; - readonly target: string; - readonly readOnly?: boolean; -} - -export const nativePostgresClientEnv = ( - binDir: string, - env: Readonly>, - pathValue: string | undefined, - delimiter: string, -): Record => ({ - ...env, - PATH: - pathValue === undefined || pathValue.length === 0 - ? binDir - : `${binDir}${delimiter}${pathValue}`, -}); - -export const postgresClientContainerArgs = (opts: { - readonly image: string; - readonly argv: ReadonlyArray; - readonly env: Readonly>; - readonly mounts?: ReadonlyArray; - readonly network?: "host" | { readonly name: string }; - readonly extraHosts?: ReadonlyArray; - readonly cwd?: string; - readonly securityOpt?: ReadonlyArray; -}): ReadonlyArray => { - const networkArgs = - opts.network === undefined - ? [] - : opts.network === "host" - ? ["--network", "host"] - : ["--network", opts.network.name]; - return [ - "run", - "--rm", - ...networkArgs, - ...(opts.extraHosts ?? []).flatMap((host) => ["--add-host", host]), - ...(opts.mounts ?? []).flatMap((mount) => [ - "-v", - `${mount.source}:${mount.target}${mount.readOnly === true ? ":ro" : ""}`, - ]), - ...Object.keys(opts.env).flatMap((key) => ["-e", key]), - ...(opts.securityOpt ?? []).flatMap((opt) => ["--security-opt", opt]), - ...(opts.cwd === undefined ? [] : ["-w", opts.cwd]), - opts.image, - ...opts.argv, - ]; -}; diff --git a/packages/stack/src/runtime/postgres-client-args.unit.test.ts b/packages/stack/src/runtime/postgres-client-args.unit.test.ts deleted file mode 100644 index 6d9d45ff16..0000000000 --- a/packages/stack/src/runtime/postgres-client-args.unit.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; - -import { - nativePostgresClientEnv, - postgresClientContainerArgs, - POSTGRES_DUMP_BINS, - POSTGRES_PROVE_BINS, - requiredPostgresClientBins, -} from "./postgres-client-args.ts"; - -describe("postgres client argv", () => { - it("prepends the artifact bin directory to PATH", () => { - expect( - nativePostgresClientEnv("/opt/postgres/bin", { PGHOST: "127.0.0.1" }, "/usr/bin", ":"), - ).toEqual({ - PGHOST: "127.0.0.1", - PATH: "/opt/postgres/bin:/usr/bin", - }); - }); - - it("emits docker run --rm with key-only -e flags", () => { - expect( - postgresClientContainerArgs({ - image: "ghcr.io/supabase/cli/postgres:17.6.1.168", - argv: ["bash", "-c", "pg_dump --version", "--"], - env: { PGPASSWORD: "secret", PGHOST: "host.docker.internal" }, - network: "host", - extraHosts: ["host.docker.internal:host-gateway"], - mounts: [{ source: "/tests", target: "/tests", readOnly: true }], - cwd: "/tests", - }), - ).toEqual([ - "run", - "--rm", - "--network", - "host", - "--add-host", - "host.docker.internal:host-gateway", - "-v", - "/tests:/tests:ro", - "-e", - "PGPASSWORD", - "-e", - "PGHOST", - "-w", - "/tests", - "ghcr.io/supabase/cli/postgres:17.6.1.168", - "bash", - "-c", - "pg_dump --version", - "--", - ]); - }); - - it("requires dump clients for bash pipelines and pg_prove only for prove argv", () => { - expect([...requiredPostgresClientBins(["bash", "-c", "pg_dump --version"])]).toEqual([ - ...POSTGRES_DUMP_BINS, - ]); - expect([...requiredPostgresClientBins(["pg_prove", "--ext", ".sql"])]).toEqual([ - ...POSTGRES_PROVE_BINS, - ]); - }); -}); diff --git a/packages/stack/src/runtime/postgres-client.integration.test.ts b/packages/stack/src/runtime/postgres-client.integration.test.ts deleted file mode 100644 index cf5af9078d..0000000000 --- a/packages/stack/src/runtime/postgres-client.integration.test.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, FileSystem, Layer, Option, Path, Sink, Stream } from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { PostgresClientError } from "../public/Errors.ts"; -import { runPostgresClient } from "../public/PostgresClient.ts"; -import { POSTGRES_CLIENT_BINS, POSTGRES_DUMP_BINS } from "./postgres-client-args.ts"; -import { PostgresClientPreparer } from "./postgres-client.ts"; - -const capturingSpawner = ( - calls: Array<{ - readonly command: string; - readonly args: ReadonlyArray; - readonly env?: Record; - }>, -) => - ChildProcessSpawner.make((command) => { - if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); - calls.push({ command: command.command, args: command.args, env: command.options.env }); - return Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(1), - stdout: Stream.fromIterable([new TextEncoder().encode("ok\n")]), - stderr: Stream.empty, - all: Stream.empty, - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - stdin: Sink.drain, - kill: () => Effect.void, - unref: Effect.succeed(Effect.void), - getInputFd: () => Sink.drain, - getOutputFd: () => Stream.empty, - }), - ); - }); - -const writeFakeBins = (root: string, bins: ReadonlyArray = POSTGRES_CLIENT_BINS) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const binDir = path.join(root, "bin"); - yield* fs.makeDirectory(binDir, { recursive: true }); - for (const bin of bins) { - yield* fs.writeFileString(path.join(binDir, bin), ""); - } - return binDir; - }); - -const nativePreparer = (bins: ReadonlyArray = POSTGRES_CLIENT_BINS) => - Layer.effect( - PostgresClientPreparer, - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectory({ prefix: "pg-client-" }); - yield* writeFakeBins(root, bins); - return PostgresClientPreparer.of({ - prepare: (runtime) => - Effect.succeed({ - runtime, - version: "17.6.1.168", - artifactRoot: root, - }), - }); - }), - ); - -const containerPreparer = Layer.succeed( - PostgresClientPreparer, - PostgresClientPreparer.of({ - prepare: (runtime) => - Effect.succeed({ - runtime, - version: "17.6.1.168", - image: "ghcr.io/supabase/cli/postgres:17.6.1.168", - }), - }), -); - -const nativeClientLayer = ( - calls: Array<{ - readonly command: string; - readonly args: ReadonlyArray; - readonly env?: Record; - }>, - bins?: ReadonlyArray, -) => - Layer.mergeAll( - NodeServices.layer, - nativePreparer(bins).pipe(Layer.provide(NodeServices.layer)), - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, capturingSpawner(calls)), - ); - -describe("runPostgresClient", () => { - it.live("native prepends artifact bin to PATH and never starts postgres", () => { - const calls: Array<{ - readonly command: string; - readonly args: ReadonlyArray; - readonly env?: Record; - }> = []; - return Effect.gen(function* () { - const chunks: Array = []; - const result = yield* runPostgresClient({ - version: "17.6.1.168", - runtime: { kind: "native" }, - argv: ["bash", "-c", "pg_dump --version", "--"], - env: { PGHOST: "127.0.0.1" }, - onStdout: (chunk) => Effect.sync(() => chunks.push(new TextDecoder().decode(chunk))), - }); - expect(result.exitCode).toBe(0); - expect(chunks.join("")).toBe("ok\n"); - expect(calls).toHaveLength(1); - expect(calls[0]?.command).toBe("bash"); - expect(calls[0]?.args).toEqual(["-c", "pg_dump --version", "--"]); - const firstPath = (calls[0]?.env?.["PATH"] ?? "").split(/[:;]/)[0] ?? ""; - expect(firstPath.endsWith("bin")).toBe(true); - }).pipe(Effect.provide(nativeClientLayer(calls))); - }); - - it.live("native fails closed when a required client binary is missing", () => - Effect.gen(function* () { - const exit = yield* runPostgresClient({ - version: "17.6.1.168", - runtime: { kind: "native" }, - argv: ["pg_dump", "--version"], - onStdout: () => Effect.void, - }).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(PostgresClientError); - if (!(error instanceof PostgresClientError)) return; - expect(error.reason).toBe("missing-bin"); - expect(error.bin).toBe("pg_dumpall"); - }).pipe(Effect.provide(nativeClientLayer([], ["pg_dump"]))), - ); - - it.live("native dump does not require pg_prove", () => { - const calls: Array<{ - readonly command: string; - readonly args: ReadonlyArray; - readonly env?: Record; - }> = []; - return Effect.gen(function* () { - const result = yield* runPostgresClient({ - version: "17.6.1.168", - runtime: { kind: "native" }, - argv: ["bash", "-c", "pg_dump --version", "--"], - onStdout: () => Effect.void, - }); - expect(result.exitCode).toBe(0); - expect(calls).toHaveLength(1); - }).pipe(Effect.provide(nativeClientLayer(calls, [...POSTGRES_DUMP_BINS]))); - }); - - it.live("container spawns docker run --rm with the caller argv", () => { - const calls: Array<{ - readonly command: string; - readonly args: ReadonlyArray; - readonly env?: Record; - }> = []; - return Effect.gen(function* () { - const result = yield* runPostgresClient({ - version: "17.6.1.168", - runtime: { kind: "container", engine: "docker" }, - argv: ["pg_prove", "--ext", ".sql"], - env: { PGPASSWORD: "secret" }, - network: "host", - onStdout: () => Effect.void, - }); - expect(result.exitCode).toBe(0); - expect(calls[0]?.command).toBe("docker"); - expect(calls[0]?.args.slice(0, 2)).toEqual(["run", "--rm"]); - expect(calls[0]?.args).toContain("-e"); - expect(calls[0]?.args).toContain("PGPASSWORD"); - expect(calls[0]?.args).not.toContain("secret"); - expect(calls[0]?.args.slice(-3)).toEqual(["pg_prove", "--ext", ".sql"]); - }).pipe( - Effect.provide( - Layer.mergeAll( - NodeServices.layer, - containerPreparer, - Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, capturingSpawner(calls)), - ), - ), - ); - }); -}); diff --git a/packages/stack/src/runtime/postgres-client.ts b/packages/stack/src/runtime/postgres-client.ts deleted file mode 100644 index 8443ef8c8e..0000000000 --- a/packages/stack/src/runtime/postgres-client.ts +++ /dev/null @@ -1,330 +0,0 @@ -import { Config, Context, Crypto, Effect, FileSystem, Option, Path, Stream } from "effect"; -import { ChildProcess } from "effect/unstable/process"; -import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { - ContainerEngineError, - PostgresClientError, - type PostgresClientRunError, -} from "../public/Errors.ts"; -import { - resolveEphemeralPostgresRelease, - type EphemeralPostgresRelease, -} from "../public/EphemeralPostgres.ts"; -import type { StackRuntime, StackRuntimePreference } from "../public/Runtime.ts"; -import { - makeProductionRuntimeArtifactPreparer, - type RuntimeArtifactPreparationError, -} from "../preparation/RuntimeArtifacts.ts"; -import { defaultRuntimeEnvironment, StackRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import { selectDefaultRuntime, ContainerEngineResolver } from "./ContainerEngineResolver.ts"; -import { - nativePostgresClientEnv, - postgresClientContainerArgs, - requiredPostgresClientBins, - type PostgresClientMount, -} from "./postgres-client-args.ts"; - -const DATABASE_WORKLOAD_ID = "database:database"; -const NATIVE_PATH_DELIMITER = process.platform === "win32" ? ";" : ":"; - -export interface RunPostgresClientOptions { - readonly version?: string; - readonly runtime?: StackRuntimePreference; - readonly argv: ReadonlyArray; - readonly env?: Readonly>; - readonly cwd?: string; - readonly mounts?: ReadonlyArray; - readonly network?: "host" | { readonly name: string }; - readonly extraHosts?: ReadonlyArray; - readonly securityOpt?: ReadonlyArray; - readonly onStdout: (chunk: Uint8Array) => Effect.Effect; - readonly teeStderr?: boolean; - readonly captureStderr?: boolean; -} - -export interface PostgresClientResult { - readonly exitCode: number; - readonly stderr: string; -} - -export interface PreparedPostgresClient { - readonly runtime: StackRuntime; - readonly version: string; - readonly artifactRoot?: string; - readonly image?: string; -} - -export class PostgresClientPreparer extends Context.Service< - PostgresClientPreparer, - { - readonly prepare: ( - runtime: StackRuntime, - release: EphemeralPostgresRelease, - ) => Effect.Effect; - } ->()("@supabase/stack/PostgresClientPreparer") {} - -const plannedWorkload = ( - version: string, - image: string, - runtime: StackRuntime, -): PlannedWorkload => ({ - id: DATABASE_WORKLOAD_ID, - capability: "database", - bootstrap: "database", - dependencies: [], - readiness: { portField: "database" }, - artifacts: { - native: { kind: "native", release: version }, - container: { kind: "container", image }, - }, - selected: - runtime.kind === "native" ? { kind: "native", release: version } : { kind: "container", image }, -}); - -const concatUtf8 = (chunks: ReadonlyArray): string => { - const total = chunks.reduce((size, chunk) => size + chunk.length, 0); - const bytes = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.length; - } - return new TextDecoder().decode(bytes); -}; - -const resolvedRuntime = ( - preference: StackRuntimePreference | undefined, -): Effect.Effect => - Effect.gen(function* () { - if (preference !== undefined) { - return preference.kind === "native" - ? { kind: "native" } - : { kind: "container", engine: preference.engine ?? "docker" }; - } - const resolver = yield* Effect.serviceOption(ContainerEngineResolver); - return yield* selectDefaultRuntime(Option.getOrUndefined(resolver)); - }); - -const defaultPrepare = ( - runtime: StackRuntime, - release: EphemeralPostgresRelease, -): Effect.Effect< - PreparedPostgresClient, - RuntimeArtifactPreparationError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => - Effect.gen(function* () { - const envOption = yield* Effect.serviceOption(StackRuntimeEnvironment); - const env = Option.isSome(envOption) ? envOption.value : yield* defaultRuntimeEnvironment; - const preparer = yield* makeProductionRuntimeArtifactPreparer({ - stateRoot: env.stateRoot, - ...(env.artifactCacheRoot === undefined ? {} : { artifactCacheRoot: env.artifactCacheRoot }), - runtime, - }); - const prepared = yield* preparer.prepare( - runtime, - plannedWorkload(release.version, release.image, runtime), - ); - return { - runtime, - version: release.version, - ...(prepared.artifactRoot === undefined ? {} : { artifactRoot: prepared.artifactRoot }), - ...(prepared.image === undefined ? {} : { image: prepared.image }), - }; - }); - -const prepareClient = ( - runtime: StackRuntime, - release: EphemeralPostgresRelease, -): Effect.Effect< - PreparedPostgresClient, - RuntimeArtifactPreparationError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => - Effect.gen(function* () { - const override = yield* Effect.serviceOption(PostgresClientPreparer); - return Option.isSome(override) - ? yield* override.value.prepare(runtime, release) - : yield* defaultPrepare(runtime, release); - }); - -const requireNativeClientBins = ( - artifactRoot: string, - argv: ReadonlyArray, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - for (const bin of requiredPostgresClientBins(argv)) { - const location = path.join(artifactRoot, "bin", bin); - const exists = yield* fs.exists(location).pipe(Effect.orElseSucceed(() => false)); - if (!exists) { - return yield* new PostgresClientError({ - message: `${bin} is missing from the prepared Postgres artifact.`, - reason: "missing-bin", - bin, - }); - } - } - }); - -const streamCommand = (params: { - readonly command: string; - readonly args: ReadonlyArray; - readonly env: Readonly>; - readonly cwd?: string; - readonly onStdout: (chunk: Uint8Array) => Effect.Effect; - readonly teeStderr: boolean; - readonly captureStderr: boolean; - readonly spawnError: (cause: unknown) => SE; -}): Effect.Effect => - Effect.scoped( - Effect.gen(function* () { - const handle = yield* ChildProcess.make(params.command, [...params.args], { - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", - cwd: params.cwd, - env: params.env, - extendEnv: true, - }).pipe(Effect.mapError(params.spawnError)); - const stderrChunks: Array = []; - yield* Effect.all( - [ - Stream.runForEach( - handle.stdout.pipe(Stream.mapError(params.spawnError)), - params.onStdout, - ), - Stream.runForEach(handle.stderr, (chunk) => - Effect.sync(() => { - if (params.captureStderr) stderrChunks.push(chunk); - if (params.teeStderr) globalThis.process.stderr.write(chunk); - }), - ).pipe(Effect.mapError(params.spawnError)), - ], - { concurrency: "unbounded" }, - ); - const exitCode = yield* handle.exitCode.pipe( - Effect.map(Number), - Effect.mapError(params.spawnError), - ); - return { exitCode, stderr: concatUtf8(stderrChunks) }; - }), - ); - -const runNative = ( - prepared: PreparedPostgresClient, - options: RunPostgresClientOptions, -): Effect.Effect< - PostgresClientResult, - E | PostgresClientError, - FileSystem.FileSystem | Path.Path | ChildProcessSpawnerService -> => - Effect.gen(function* () { - const path = yield* Path.Path; - const artifactRoot = prepared.artifactRoot; - if (artifactRoot === undefined) - return yield* new PostgresClientError({ - message: "Native Postgres client preparation did not produce an artifact root.", - reason: "missing-bin", - }); - yield* requireNativeClientBins(artifactRoot, options.argv); - const env = options.env ?? {}; - const inheritedPath = - env["PATH"] ?? - Option.getOrUndefined( - yield* Config.option(Config.string("PATH")).pipe(Effect.orElseSucceed(() => Option.none())), - ); - const nativeEnv = nativePostgresClientEnv( - path.join(artifactRoot, "bin"), - env, - inheritedPath, - NATIVE_PATH_DELIMITER, - ); - const argv = options.argv; - const command = argv[0]; - if (command === undefined) - return yield* new PostgresClientError({ message: "Postgres client argv must not be empty." }); - return yield* streamCommand({ - command, - args: argv.slice(1), - env: nativeEnv, - cwd: options.cwd, - onStdout: options.onStdout, - teeStderr: options.teeStderr ?? false, - captureStderr: options.captureStderr ?? true, - spawnError: () => - new PostgresClientError({ - message: `Unable to spawn ${command} from the prepared Postgres artifact.`, - reason: "spawn", - bin: command, - }), - }); - }); - -const runContainer = ( - prepared: PreparedPostgresClient, - options: RunPostgresClientOptions, -): Effect.Effect< - PostgresClientResult, - E | PostgresClientError | ContainerEngineError, - ChildProcessSpawnerService -> => - Effect.gen(function* () { - if (prepared.runtime.kind !== "container") - return yield* new PostgresClientError({ - message: "Container Postgres client preparation did not select a container engine.", - reason: "spawn", - }); - const image = prepared.image; - if (image === undefined) - return yield* new PostgresClientError({ - message: "Container Postgres client preparation did not produce an image.", - reason: "spawn", - }); - const engine = prepared.runtime.engine; - const env = options.env ?? {}; - return yield* streamCommand({ - command: engine, - args: postgresClientContainerArgs({ - image, - argv: options.argv, - env, - mounts: options.mounts, - network: options.network, - extraHosts: options.extraHosts, - cwd: options.cwd, - securityOpt: options.securityOpt, - }), - env, - onStdout: options.onStdout, - teeStderr: options.teeStderr ?? false, - captureStderr: options.captureStderr ?? true, - spawnError: () => - new ContainerEngineError({ - message: `Unable to spawn ${engine} for a Postgres client one-shot.`, - engine, - }), - }); - }); - -/** Runs argv against a prepared catalog Postgres artifact or image. Never starts the server. */ -export const runPostgresClient = ( - options: RunPostgresClientOptions, -): Effect.Effect< - PostgresClientResult, - E | PostgresClientRunError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawnerService -> => - Effect.gen(function* () { - if (options.argv.length === 0) - return yield* new PostgresClientError({ message: "Postgres client argv must not be empty." }); - const release = yield* resolveEphemeralPostgresRelease(options.version); - const runtime = yield* resolvedRuntime(options.runtime); - const prepared = yield* prepareClient(runtime, release); - return runtime.kind === "native" - ? yield* runNative(prepared, options) - : yield* runContainer(prepared, options); - }); diff --git a/packages/stack/src/runtime/postgres-database-session.integration.test.ts b/packages/stack/src/runtime/postgres-database-session.integration.test.ts index 2127b06131..2b41980f38 100644 --- a/packages/stack/src/runtime/postgres-database-session.integration.test.ts +++ b/packages/stack/src/runtime/postgres-database-session.integration.test.ts @@ -12,7 +12,7 @@ import { makeDatabaseSessionFromSqlClient, type DatabaseSqlClient, } from "./PostgresDatabaseSession.ts"; -import { DatabaseBootstrapError } from "../model/DatabaseBootstrap.ts"; +import { DatabaseBootstrapError } from "./DatabaseBootstrap.ts"; describe("Postgres database session", () => { it.live("classifies SQL connection failures without retaining raw causes", () => diff --git a/packages/stack/src/runtime/production-runtime.integration.test.ts b/packages/stack/src/runtime/production-runtime.integration.test.ts deleted file mode 100644 index 122188fb93..0000000000 --- a/packages/stack/src/runtime/production-runtime.integration.test.ts +++ /dev/null @@ -1,2968 +0,0 @@ -import { NodeHttpServer, NodeHttpServerRequest, NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Context, - Deferred, - Duration, - Effect, - Exit, - FileSystem, - Fiber, - Path, - Crypto, - Redacted, - Schema, - Option, - Predicate, - Stream, - Layer, -} from "effect"; -import * as TestClock from "effect/testing/TestClock"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- NodeHttpServer.layer requires a native factory to bind loopback; layerTest does not expose a host option. -import { createServer as createHttpServer } from "node:http"; -import { createServer as createNetServer } from "node:net"; -import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; -import type { StackLogEntry } from "../public/Logs.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import { NetworkPortSchema } from "../public/Status.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import type { StackStateStore } from "../state/StackStateStore.ts"; -import { resolveSecrets } from "../state/SecretStore.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import type { SupervisorIngress } from "../supervisor/Ingress.ts"; -import type { LogStore } from "../supervisor/LogStore.ts"; -import { LogStoreError } from "../supervisor/LogStore.ts"; -import type { LifecycleInput } from "../supervisor/Lifecycle.ts"; -import { - makeProductionRuntime, - readinessDeadlineFor, - withOwnedRuntimeFileCleanup, -} from "./ProductionRuntime.ts"; -import { makeSupervisor } from "../supervisor/Supervisor.ts"; -import { RuntimeDriverError, type RuntimeDriver } from "./RuntimeDriver.ts"; -import { - InvalidStackConfigError, - PortUnavailableError, - StackPreparationError, - StackStateInvalidError, -} from "../public/Errors.ts"; -import { DatabaseBootstrapError } from "../model/DatabaseBootstrap.ts"; -import type { RuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; -import { makeRuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; -import { makeArtifactStore, type ArtifactSource } from "../preparation/ArtifactStore.ts"; -import { compileStack } from "../model/Compiler.ts"; -import type { - ContainerContainerSpec, - ContainerEngine, - ContainerResource, - ContainerNetworkSpec, - ContainerVolumeSpec, -} from "./ContainerEngine.ts"; -import { ContainerEngineProtocolError } from "./ContainerEngine.ts"; -import { - makeFunctionsBootstrapOwner, - type FunctionsBootstrapOwner, -} from "../functions/FunctionsBootstrap.ts"; -import type { RuntimeEnvFileOwner } from "./RuntimeEnvFile.ts"; -import { makeRuntimeEnvFileOwner } from "./RuntimeEnvFile.ts"; -import { probeReadiness } from "./ReadinessProbe.ts"; - -const encodeJson = (value: unknown): string => - Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))(value); - -const stackId = StackIdSchema.make("a".repeat(64)); - -const stateFor = ( - secrets: PersistedStackState["secrets"], - runtime: StackRuntime = { kind: "native" }, -): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity: { - projectRoot: "/tmp/production-runtime", - branchContext: "ordinary-workspace", - stackName: "production-runtime", - }, - runtime, - desiredLifecycle: "stopped", - ports: [], - privatePorts: [], - secrets, -}); - -const stateStoreFor = ( - current: { value: PersistedStackState }, - onRead?: () => void, -): StackStateStore => ({ - read: () => - Effect.sync(() => { - onRead?.(); - return current.value; - }), - initialize: () => Effect.die("unused"), - replace: () => Effect.die("unused"), - replaceUnlocked: () => Effect.die("unused"), - cleanup: () => Effect.die("unused"), - recoverRuntimeRemnant: () => Effect.die("unused"), -}); - -const mutableStateStoreFor = (current: { - value: PersistedStackState | undefined; -}): StackStateStore => ({ - read: () => Effect.succeed(current.value), - initialize: () => Effect.die("unused"), - replace: (_stackId, state) => - Effect.sync(() => { - current.value = state; - }), - replaceUnlocked: (_stackId, state) => - Effect.sync(() => { - current.value = state; - }), - cleanup: () => - Effect.sync(() => { - current.value = undefined; - }), - recoverRuntimeRemnant: () => Effect.void, -}); - -const memoryLogStore = (entries: StackLogEntry[]): LogStore => ({ - path: "memory://production-runtime", - append: (record) => - Effect.sync(() => { - const entry: StackLogEntry = { - cursor: { opaque: `v1_${entries.length + 1}` }, - timestamp: record.timestamp ?? "2026-01-01T00:00:00.000Z", - source: record.source, - stream: record.stream, - message: record.message, - }; - entries.push(entry); - return entry; - }), - read: () => Effect.succeed(entries), -}); - -const ingress: SupervisorIngress = { - acquire: () => Effect.die("unused"), - open: () => Effect.die("unused"), - close: Effect.void, -}; - -const artifacts: RuntimeArtifactPreparer = { - prepare: () => Effect.die("unused"), -}; - -const writeNativeDatabaseFixture = ( - fs: FileSystem.FileSystem, - path: Path.Path, - root: string, - eventsPath: string, -) => - Effect.gen(function* () { - const binDirectory = path.join(root, "bin"); - yield* fs.makeDirectory(binDirectory, { recursive: true }); - const main = path.join(binDirectory, "supabase-postgres-start"); - const helperScript = [ - 'const fs = require("node:fs");', - 'const net = require("node:net");', - "const args = process.argv.slice(1);", - 'const port = Number(args[args.indexOf("-p") + 1]);', - `fs.appendFileSync(${encodeJson(eventsPath)}, "postgres-start|data=" + process.env.PGDATA + "|user=" + process.env.POSTGRES_USER + "|db=" + process.env.POSTGRES_DB + "|password=" + process.env.POSTGRES_PASSWORD + "|args=" + args.join(" ") + "\\n");`, - "const server = net.createServer((socket) => socket.end());", - 'server.listen(port, "127.0.0.1");', - "const stop = () => server.close(() => process.exit(0));", - 'process.on("SIGTERM", stop);', - 'process.on("SIGINT", stop);', - ].join(""); - yield* fs.writeFileString( - main, - `#!/bin/sh -set -eu -exec ${encodeJson(process.execPath)} -e ${encodeJson(helperScript)} -- "$@" -`, - ); - yield* fs.chmod(main, 0o755); - return { main }; - }); - -const writeNativeRealtimeFixture = ( - fs: FileSystem.FileSystem, - path: Path.Path, - root: string, - eventsPath: string, -) => - Effect.gen(function* () { - const bin = path.join(root, "bin"); - yield* fs.makeDirectory(bin, { recursive: true }); - const profile = path.join(bin, ".runtime-env.sh"); - yield* fs.writeFileString( - profile, - `if [ -z "\${RELEASE_DISTRIBUTION:-}" ]; then - export RELEASE_DISTRIBUTION=name -fi -`, - ); - const migrate = path.join(bin, "migrate"); - yield* fs.writeFileString( - migrate, - `#!/bin/sh -. "$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)/.runtime-env.sh" -printf 'realtime-migrate|RELEASE_DISTRIBUTION=%s\\n' "\${RELEASE_DISTRIBUTION:-missing}" >> ${encodeJson(eventsPath)} -`, - ); - yield* fs.chmod(migrate, 0o755); - const prepare = path.join(bin, "prepare"); - yield* fs.writeFileString( - prepare, - `#!/bin/sh -set -eu -SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" -"$SCRIPT_DIR/migrate" -if [ "\${SEED_SELF_HOST:-}" = true ]; then - "$SCRIPT_DIR/realtime" eval 'Realtime.Release.seeds(Realtime.Repo)' -fi -`, - ); - yield* fs.chmod(prepare, 0o755); - const realtime = path.join(bin, "realtime"); - yield* fs.writeFileString(realtime, "#!/bin/sh\nexit 0\n"); - yield* fs.chmod(realtime, 0o755); - const server = path.join(bin, "server"); - yield* fs.writeFileString( - server, - "#!/bin/sh\ntrap 'exit 0' TERM INT\nwhile :; do sleep 1; done\n", - ); - yield* fs.chmod(server, 0o755); - return { migrate, realtime, server }; - }); - -const listenForNativeReadiness = (server: ReturnType) => - Effect.acquireRelease( - Effect.callback((resume) => { - server.once("error", (error: Error) => resume(Effect.fail(error))); - server.listen(0, "127.0.0.1", () => resume(Effect.void)); - }), - () => - Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }), - ); - -const listenForHttpFixture = ( - handler: ( - request: HttpServerRequest.HttpServerRequest, - ) => Effect.Effect, -) => - Effect.gen(function* () { - const scope = yield* Effect.scope; - const context = yield* Layer.buildWithScope( - Layer.fresh(NodeHttpServer.layer(createHttpServer, { port: 0, host: "127.0.0.1" })), - scope, - ); - const server = Context.get(context, HttpServer.HttpServer); - yield* server.serve(Effect.flatMap(HttpServerRequest.HttpServerRequest, handler)); - return server; - }); - -const httpFixturePort = (server: HttpServer.HttpServer["Service"]): Effect.Effect => - Predicate.isTagged(server.address, "TcpAddress") - ? Effect.succeed(server.address.port) - : Effect.die("HTTP fixture did not expose TCP address"); - -const envFiles: RuntimeEnvFileOwner = { - write: () => Effect.die("unused"), - cleanupAll: Effect.void, -}; - -const bootstrap: FunctionsBootstrapOwner = { - write: () => Effect.die("unused"), - cleanupAll: Effect.void, -}; - -const ownerInputContainerEngine = ( - createdSpecs: ContainerContainerSpec[], - copiedFiles: Array> = [], -): ContainerEngine => { - const resources: ContainerResource[] = []; - const oneShot = new Set(); - let nextId = 1; - const resource = ( - kind: ContainerResource["kind"], - name: string, - labels: ContainerResource["labels"], - ): ContainerResource => ({ - id: `${kind}-${nextId++}`, - name, - kind, - labels, - ...(kind === "workload" ? { state: "created" as const } : {}), - }); - const updateState = (id: string, state: "running" | "stopped") => { - const index = resources.findIndex((entry) => entry.id === id); - const entry = resources[index]; - if (entry !== undefined) resources[index] = { ...entry, state }; - }; - return { - kind: "docker", - preflight: Effect.sync(() => { - return { host: "host.docker.internal" }; - }), - probe: Effect.void, - inspectImage: () => Effect.succeed({ present: true }), - pullImage: () => Effect.void, - listResources: () => Effect.sync(() => [...resources]), - createNetwork: (spec: ContainerNetworkSpec) => - Effect.sync(() => { - const created = resource("network", spec.name, spec.labels); - resources.push(created); - return created; - }), - removeNetwork: (id) => - Effect.sync(() => { - const index = resources.findIndex((entry) => entry.id === id); - if (index >= 0) resources.splice(index, 1); - }), - createVolume: (spec: ContainerVolumeSpec) => - Effect.sync(() => { - const created = resource("volume", spec.name, spec.labels); - resources.push(created); - return created; - }), - removeVolume: (id) => - Effect.sync(() => { - const index = resources.findIndex((entry) => entry.id === id); - if (index >= 0) resources.splice(index, 1); - }), - createContainer: (spec) => - Effect.sync(() => { - createdSpecs.push(spec); - const created = resource("workload", spec.name, spec.labels); - if ( - spec.entrypoint?.startsWith("/app/bin/") || - spec.command?.includes("migrate") || - spec.command?.join(" ").includes("eval") || - spec.entrypoint === "/node/bin/node" - ) - oneShot.add(created.id); - resources.push(created); - return created; - }), - copyToContainer: (_id, source, destination) => - Effect.sync(() => { - copiedFiles.push({ source, destination }); - }), - startContainer: (id) => - Effect.sync(() => { - updateState(id, "running"); - }), - waitContainer: (id) => (oneShot.has(id) ? Effect.succeed(0) : Effect.never), - stopContainer: (id) => Effect.sync(() => updateState(id, "stopped")), - removeContainer: (id) => - Effect.sync(() => { - const index = resources.findIndex((entry) => entry.id === id); - if (index >= 0) resources.splice(index, 1); - }), - streamLogs: () => Stream.empty, - }; -}; - -describe("production runtime", () => { - it.live("validates materialized secrets from the candidate definition and values", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-candidate-", - }); - const previous = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const previousSecrets = yield* resolveSecrets( - { declarations: previous.secrets }, - undefined, - "stopped", - ); - const candidate = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - storage: { - enabled: true, - settings: { - s3_protocol: { - secret_access_key: Redacted.make("candidate-secret"), - }, - }, - }, - }, - }, - }); - const candidateSecrets = yield* resolveSecrets( - { declarations: candidate.secrets }, - undefined, - "stopped", - ); - const missing = Object.fromEntries( - Object.entries(candidateSecrets.persisted).filter( - ([slot]) => slot !== "secret:storage.settings.s3_protocol.secret_access_key", - ), - ); - const current = { - value: { - ...stateFor({}, { kind: "native" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - definition: previous.definition, - secrets: previousSecrets.persisted, - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "candidate-secrets", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - }), - }, - logStore: memoryLogStore([]), - }); - const result = yield* runtime - .preflight({ - stackId, - state: current.value, - definition: candidate.definition, - secrets: missing, - plan: candidate.executionPlan, - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(StackStateInvalidError); - } - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("keeps cleanup available when retained logs are corrupted", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-production-bad-logs-" }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - }); - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(path.dirname(paths.logs), { recursive: true }); - yield* fs.writeFileString(paths.logs, "not-json\n"); - const current = { - value: { - ...stateFor({}, { kind: "container", engine: "docker" }), - desiredLifecycle: "running" as const, - definition: compiled.definition, - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - }, - } satisfies { value: PersistedStackState | undefined }; - const store = mutableStateStoreFor(current); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "bad-logs-owner", - stateStore: store, - context, - ingress, - artifactPreparer: artifacts, - containerEngine: ownerInputContainerEngine([]), - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - }); - const supervisor = yield* makeSupervisor({ - stackId, - ownerSessionId: "bad-logs-owner", - stateStore: store, - context, - runtime, - }); - const failed = yield* supervisor.start().pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - if (Exit.isFailure(failed)) { - const error = Cause.findErrorOption(failed.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(StackPreparationError); - expect(error.value.cause).toBeInstanceOf(LogStoreError); - } - } - const logs = yield* supervisor.logs().pipe(Effect.exit); - expect(Exit.isFailure(logs)).toBe(true); - if (Exit.isFailure(logs)) { - const error = Cause.findErrorOption(logs.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(StackStateInvalidError); - expect(error.value.cause).toBeInstanceOf(LogStoreError); - } - } - expect(yield* fs.readFileString(paths.logs)).toBe("not-json\n"); - expect((yield* supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect(yield* fs.readFileString(paths.logs)).toBe("not-json\n"); - yield* supervisor.destroy; - expect(current.value).toBeUndefined(); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("retries a transient database bootstrap connection failure", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-bootstrap-", - }); - const readinessServer = createNetServer((socket) => socket.end()); - yield* listenForNativeReadiness(readinessServer); - const address = readinessServer.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("Database readiness server did not expose an address"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - }); - const current = { - value: { - ...stateFor( - { - "secret:database.internal.password": { policy: "managed", value: "db-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - }, - { kind: "container", engine: "docker" }, - ), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: address.port }, - ], - }, - } satisfies { value: PersistedStackState }; - const database = compiled.executionPlan.workloads.find( - (workload) => workload.id === "database:database", - ); - if (database === undefined) return yield* Effect.die("Expected database workload"); - const createdSpecs: ContainerContainerSpec[] = []; - let attempts = 0; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: ownerInputContainerEngine(createdSpecs), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - image: workload.selected.kind === "container" ? workload.selected.image : undefined, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: () => { - attempts += 1; - return attempts === 1 - ? Effect.fail( - new DatabaseBootstrapError({ - message: "Database is still accepting connections", - retryable: true, - }), - ) - : Effect.void; - }, - }); - const ready = yield* runtime.driver.start( - { - stackId, - workloadId: database.id, - }, - database, - ); - expect(ready.state).toBe("ready"); - expect(attempts).toBe(2); - yield* runtime.driver.stop({ - stackId, - workloadId: database.id, - }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("defers unreachable Auth OIDC resolution until Auth activation", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-oidc-lazy-", - }); - const readinessServer = createNetServer((socket) => socket.end()); - yield* listenForNativeReadiness(readinessServer); - const address = readinessServer.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("Database readiness server did not expose an address"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - auth: { - enabled: true, - settings: { - third_party: { - workos: { enabled: true, issuer_url: "https://issuer.example" }, - }, - }, - }, - }, - }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const current = { - value: { - ...stateFor(resolved.persisted, { kind: "container", engine: "docker" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: address.port }, - ], - }, - } satisfies { value: PersistedStackState }; - const database = compiled.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const auth = compiled.executionPlan.workloads.find(({ id }) => id === "auth:auth"); - if (database === undefined || auth === undefined) - return yield* Effect.die("Expected database and Auth workloads"); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "oidc-lazy", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: ownerInputContainerEngine([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - image: workload.selected.kind === "container" ? workload.selected.image : undefined, - }), - }, - logStore: memoryLogStore([]), - fetchJson: () => Effect.fail(new StackPreparationError({ message: "OIDC unavailable" })), - bootstrapDatabase: () => Effect.void, - }); - const databaseReady = yield* runtime.driver.start( - { stackId, workloadId: database.id }, - database, - ); - expect(databaseReady.state).toBe("ready"); - const authResult = yield* runtime.driver - .start({ stackId, workloadId: auth.id }, auth) - .pipe(Effect.exit); - expect(Exit.isFailure(authResult)).toBe(true); - if (Exit.isFailure(authResult)) { - const error = Option.getOrUndefined(Cause.findErrorOption(authResult.cause)); - expect(error).toBeInstanceOf(RuntimeDriverError); - expect(error?.message).toContain("OIDC discovery request failed"); - expect(error?.cause).toBeInstanceOf(StackPreparationError); - } - yield* runtime.driver.stop({ stackId, workloadId: database.id }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("follows redirects while resolving Auth OIDC metadata", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-oidc-lazy-", - }); - let issuer = ""; - const oidc = yield* listenForHttpFixture((request) => { - if (request.url === "/.well-known/openid-configuration") - return Effect.succeed(HttpServerResponse.redirect("/discovery")); - if (request.url === "/discovery") - return Effect.succeed( - HttpServerResponse.text(encodeJson({ jwks_uri: `${issuer}/keys` }), { - headers: { "content-type": "application/json" }, - }), - ); - if (request.url === "/keys") - return Effect.succeed(HttpServerResponse.redirect("/keys-final")); - return Effect.succeed( - HttpServerResponse.text(encodeJson({ keys: [{ kty: "RSA", n: "n", e: "AQAB" }] }), { - headers: { "content-type": "application/json" }, - }), - ); - }); - const oidcPort = yield* httpFixturePort(oidc); - issuer = `http://127.0.0.1:${oidcPort}`; - const readinessServer = createNetServer((socket) => socket.end()); - yield* listenForNativeReadiness(readinessServer); - const address = readinessServer.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("Database readiness server did not expose an address"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - auth: { - enabled: true, - settings: { - third_party: { - workos: { enabled: true, issuer_url: issuer }, - }, - }, - }, - }, - }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const current = { - value: { - ...stateFor(resolved.persisted, { kind: "container", engine: "docker" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: address.port }, - { workloadId: "auth:auth", binding: "primary", port: oidcPort }, - ], - }, - } satisfies { value: PersistedStackState }; - const database = compiled.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const auth = compiled.executionPlan.workloads.find(({ id }) => id === "auth:auth"); - if (database === undefined || auth === undefined) - return yield* Effect.die("Expected database and Auth workloads"); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "oidc-lazy", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: ownerInputContainerEngine([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - image: workload.selected.kind === "container" ? workload.selected.image : undefined, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: () => Effect.void, - }); - const databaseReady = yield* runtime.driver.start( - { stackId, workloadId: database.id }, - database, - ); - expect(databaseReady.state).toBe("ready"); - const authResult = yield* runtime.driver.start({ stackId, workloadId: auth.id }, auth); - expect(authResult.state).toBe("ready"); - yield* runtime.driver.stop({ stackId, workloadId: database.id }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("shares background preparation with a concurrent studio activation closure", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-shared-preparation-", - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - preparation: "background", - capabilities: { - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - mail: { enabled: false }, - pooler: { enabled: false }, - rest: { activation: "eager" }, - analytics: { activation: "eager" }, - }, - }, - }); - const studioWorkloads = compiled.executionPlan.workloads.filter( - (workload) => workload.capability === "studio", - ); - const studioImages = new Set( - studioWorkloads.map((workload) => - workload.selected.kind === "container" ? workload.selected.image : "", - ), - ); - if (studioImages.has("") || studioImages.size === 0) - return yield* Effect.die("Expected container Studio workloads"); - const closure: ReadonlySet = new Set([ - "database", - "rest", - "analytics", - "studio", - ]); - const closureWorkloads = compiled.executionPlan.workloads.filter( - (workload) => - closure.has(workload.capability) && - !studioImages.has( - workload.selected.kind === "container" ? workload.selected.image : "", - ), - ); - if (closureWorkloads.length < 2) - return yield* Effect.die("Expected at least two non-Studio closure workloads"); - const blocked = yield* Deferred.make(); - const release = yield* Deferred.make(); - const releaseClosure = yield* Deferred.make(); - const newClosureWorkStarted = yield* Deferred.make(); - let blockedPullsStarted = 0; - let closurePullsStarted = 0; - const pulls = new Map(); - const createdSpecs: ContainerContainerSpec[] = []; - const baseEngine = ownerInputContainerEngine(createdSpecs); - const engine: ContainerEngine = { - ...baseEngine, - inspectImage: () => Effect.succeed({ present: false }), - pullImage: (image) => - Effect.gen(function* () { - pulls.set(image, (pulls.get(image) ?? 0) + 1); - if (studioImages.has(image)) { - blockedPullsStarted += 1; - if (blockedPullsStarted === studioImages.size) - yield* Deferred.succeed(blocked, undefined); - yield* Deferred.await(release); - } else { - closurePullsStarted += 1; - if (closurePullsStarted === closureWorkloads.length) - yield* Deferred.succeed(newClosureWorkStarted, undefined); - yield* Deferred.await(releaseClosure); - } - }), - }; - const current = { - value: { - ...stateFor({}, { kind: "container", engine: "docker" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "shared-preparation", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: engine, - artifactPreparer: makeRuntimeArtifactPreparer({ containerEngine: engine }), - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore([]), - }); - const input: LifecycleInput = { - stackId, - state: current.value, - definition: compiled.definition, - secrets: current.value.secrets, - plan: compiled.executionPlan, - }; - const background = yield* Effect.forkChild(runtime.prefetch(current.value), { - startImmediately: true, - }); - yield* Deferred.await(blocked); - expect(yield* runtime.artifacts).toEqual( - expect.arrayContaining( - studioWorkloads.map((workload) => - expect.objectContaining({ - workloadId: workload.id, - state: "downloading", - }), - ), - ), - ); - const foreground = yield* Effect.forkChild(runtime.prepare(input, closure), { - startImmediately: true, - }); - const interruptedWaiter = yield* Effect.forkChild(runtime.prepare(input, closure), { - startImmediately: true, - }); - yield* Deferred.await(newClosureWorkStarted); - const interrupted = yield* Fiber.interrupt(interruptedWaiter).pipe(Effect.exit); - expect(Exit.isSuccess(interrupted)).toBe(true); - const waiterExit = yield* Fiber.join(interruptedWaiter).pipe(Effect.exit); - expect(Exit.isFailure(waiterExit)).toBe(true); - expect(yield* Deferred.isDone(blocked)).toBe(true); - expect(closurePullsStarted).toBe(closureWorkloads.length); - yield* Deferred.succeed(releaseClosure, undefined); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(background); - yield* Fiber.join(foreground); - for (const workload of compiled.executionPlan.workloads.filter((entry) => - closure.has(entry.capability), - )) { - if (workload.selected.kind === "container") - expect(pulls.get(workload.selected.image)).toBe(1); - } - expect(createdSpecs).toHaveLength(0); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* runtime.artifacts).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live( - "keeps background preparation best effort and retries a failed artifact on activation", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-preparation-retry-", - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - preparation: "background", - capabilities: { - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const rest = compiled.executionPlan.workloads.find( - (workload) => workload.capability === "rest", - ); - const analytics = compiled.executionPlan.workloads.find( - (workload) => workload.capability === "analytics", - ); - if ( - rest === undefined || - analytics === undefined || - rest.selected.kind !== "container" || - analytics.selected.kind !== "container" - ) - return yield* Effect.die("Expected REST and analytics container workloads"); - const restImage = rest.selected.image; - const pullAttempts = new Map(); - const logs: StackLogEntry[] = []; - const createdSpecs: ContainerContainerSpec[] = []; - const baseEngine = ownerInputContainerEngine(createdSpecs); - const engine: ContainerEngine = { - ...baseEngine, - inspectImage: () => Effect.succeed({ present: false }), - pullImage: (image) => - Effect.gen(function* () { - const attempt = pullAttempts.get(image) ?? 0; - pullAttempts.set(image, attempt + 1); - if (image === restImage && attempt === 0) - return yield* new ContainerEngineProtocolError({ - operation: "pull-image", - message: "temporary image registry failure", - }); - }), - }; - const current = { - value: { - ...stateFor({}, { kind: "container", engine: "docker" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context< - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "preparation-retry", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: engine, - artifactPreparer: makeRuntimeArtifactPreparer({ containerEngine: engine }), - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore(logs), - }); - const background = yield* runtime.prefetch(current.value); - expect(background).toBeUndefined(); - expect(pullAttempts.get(restImage)).toBe(1); - expect(pullAttempts.get(analytics.selected.image)).toBe(1); - const statusesAfterBackground = yield* runtime.artifacts; - expect(statusesAfterBackground).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - workloadId: rest.id, - state: "failed", - error: expect.stringContaining("Unable to pull container image"), - }), - expect.objectContaining({ workloadId: analytics.id, state: "ready" }), - ]), - ); - expect( - logs.some((entry) => - entry.message.includes(`Background preparation failed for ${rest.id}`), - ), - ).toBe(true); - const input: LifecycleInput = { - stackId, - state: current.value, - definition: compiled.definition, - secrets: current.value.secrets, - plan: compiled.executionPlan, - }; - yield* runtime.prepare(input, new Set(["rest"])); - expect(pullAttempts.get(restImage)).toBe(2); - expect( - (yield* runtime.artifacts).find(({ workloadId }) => workloadId === rest.id), - ).toEqual(expect.objectContaining({ workloadId: rest.id, state: "ready" })); - expect(createdSpecs).toHaveLength(0); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* runtime.artifacts).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("reports artifact progress while launch preparation is waiting for a pull", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-eager-preparation-", - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - preparation: "on-demand", - capabilities: { - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const database = compiled.executionPlan.workloads.find( - (workload) => workload.capability === "database", - ); - if (database === undefined || database.selected.kind !== "container") - return yield* Effect.die("Expected container database workload"); - const databaseImage = database.selected.image; - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const current = { - value: { - ...stateFor(resolved.persisted, { kind: "container", engine: "docker" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - definition: compiled.definition, - secrets: resolved.persisted, - }, - } satisfies { value: PersistedStackState }; - const pullStarted = yield* Deferred.make(); - const release = yield* Deferred.make(); - const createdSpecs: ContainerContainerSpec[] = []; - let pullCount = 0; - const baseEngine = ownerInputContainerEngine(createdSpecs); - const engine: ContainerEngine = { - ...baseEngine, - inspectImage: () => Effect.succeed({ present: false }), - pullImage: (image) => - image === databaseImage - ? Effect.gen(function* () { - pullCount += 1; - yield* Deferred.succeed(pullStarted, undefined); - yield* Deferred.await(release); - }) - : Effect.void, - }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "eager-preparation", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: engine, - artifactPreparer: makeRuntimeArtifactPreparer({ containerEngine: engine }), - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore([]), - }); - const input: LifecycleInput = { - stackId, - state: current.value, - definition: compiled.definition, - secrets: resolved.persisted, - plan: compiled.executionPlan, - }; - yield* runtime.preflight(input); - const launching = yield* Effect.forkChild( - runtime.driver.start({ stackId, workloadId: database.id }, database), - { startImmediately: true }, - ); - yield* Deferred.await(pullStarted); - expect(yield* runtime.artifacts).toEqual([ - expect.objectContaining({ workloadId: database.id, state: "downloading" }), - ]); - const preparing = yield* Effect.forkChild(runtime.prepare(input, new Set(["database"])), { - startImmediately: true, - }); - expect(yield* runtime.artifacts).toEqual([ - expect.objectContaining({ workloadId: database.id, state: "downloading" }), - ]); - yield* Fiber.interrupt(launching); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(preparing); - expect(yield* runtime.artifacts).toEqual([ - expect.objectContaining({ workloadId: database.id, state: "ready" }), - ]); - expect(createdSpecs).toHaveLength(0); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* runtime.artifacts).toEqual([]); - yield* runtime.prefetch(current.value); - expect(pullCount).toBe(1); - expect(yield* runtime.artifacts).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("cancels an in-flight native artifact transfer during runtime cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-native-artifact-stop-", - }); - const transferStarted = yield* Deferred.make(); - const transferInterrupted = yield* Deferred.make(); - const archiveSha256 = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; - const checksumFailure = new StackPreparationError({ - message: "published checksum lookup is temporarily unavailable", - }); - let checksumCalls = 0; - const source: ArtifactSource = { - checksum: () => - Effect.sync(() => { - checksumCalls += 1; - return checksumCalls; - }).pipe( - Effect.flatMap((calls) => - calls === 1 ? Effect.fail(checksumFailure) : Effect.succeed(archiveSha256), - ), - ), - materialize: (_request, destination, _expectedSha256, onProgress) => - Effect.gen(function* () { - const sourceFile = path.join(destination, "partial-source"); - onProgress?.("downloading"); - yield* fs.writeFileString(sourceFile, "partial"); - yield* Deferred.succeed(transferStarted, undefined); - return yield* Effect.never; - }).pipe( - Effect.ensuring(Deferred.succeed(transferInterrupted, undefined)), - Effect.mapError( - (cause) => new StackPreparationError({ message: "native transfer failed", cause }), - ), - ), - }; - const store = yield* makeArtifactStore({ cacheRoot: path.join(root, "artifacts"), source }); - const preparer = makeRuntimeArtifactPreparer({ - native: { - store, - platform: { os: "darwin", arch: "arm64" }, - }, - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const current = { - value: { - ...stateFor({}, { kind: "native" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - }, - } satisfies { value: PersistedStackState }; - const workload = compiled.executionPlan.workloads.find( - (entry) => entry.capability === "database", - ); - if (workload === undefined) return yield* Effect.die("Expected database workload"); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "native-artifact-stop", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - artifactPreparer: preparer, - logStore: memoryLogStore([]), - }); - const input: LifecycleInput = { - stackId, - state: current.value, - definition: compiled.definition, - secrets: current.value.secrets, - plan: compiled.executionPlan, - }; - const firstAttempt = yield* runtime - .prepare(input, new Set([workload.capability])) - .pipe(Effect.exit); - expect(Exit.isFailure(firstAttempt)).toBe(true); - if (Exit.isFailure(firstAttempt)) { - const error = Cause.findErrorOption(firstAttempt.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(StackPreparationError); - expect(error.value.message).toBe(checksumFailure.message); - } - } - expect(yield* runtime.artifacts).toEqual([ - expect.objectContaining({ - workloadId: workload.id, - state: "failed", - error: checksumFailure.message, - }), - ]); - const preparing = yield* Effect.forkChild( - runtime.prepare(input, new Set([workload.capability])), - { startImmediately: true }, - ); - yield* Deferred.await(transferStarted); - expect(yield* runtime.artifacts).toEqual([ - expect.objectContaining({ workloadId: workload.id, state: "downloading" }), - ]); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* Deferred.isDone(transferInterrupted)).toBe(true); - const result = yield* Fiber.join(preparing).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - const entries = yield* fs.readDirectory(root, { recursive: true }); - expect(entries.some((entry) => entry.endsWith(".tmp"))).toBe(false); - expect(yield* runtime.artifacts).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("cancels an in-flight container image pull during runtime cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-container-pull-stop-", - }); - const pullStarted = yield* Deferred.make(); - const pullInterrupted = yield* Deferred.make(); - const createdSpecs: ContainerContainerSpec[] = []; - const baseEngine = ownerInputContainerEngine(createdSpecs); - const engine: ContainerEngine = { - ...baseEngine, - inspectImage: () => Effect.succeed({ present: false }), - pullImage: () => - Effect.gen(function* () { - yield* Deferred.succeed(pullStarted, undefined); - return yield* Effect.never; - }).pipe(Effect.ensuring(Deferred.succeed(pullInterrupted, undefined))), - }; - const preparer = makeRuntimeArtifactPreparer({ containerEngine: engine }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - }); - const current = { - value: { - ...stateFor({}, { kind: "container", engine: "docker" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - }, - } satisfies { value: PersistedStackState }; - const workload = compiled.executionPlan.workloads.find( - (entry) => entry.capability === "database", - ); - if (workload === undefined) return yield* Effect.die("Expected database workload"); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "container-pull-stop", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: engine, - artifactPreparer: preparer, - logStore: memoryLogStore([]), - }); - const input: LifecycleInput = { - stackId, - state: current.value, - definition: compiled.definition, - secrets: current.value.secrets, - plan: compiled.executionPlan, - }; - const preparing = yield* Effect.forkChild( - runtime.prepare(input, new Set([workload.capability])), - { startImmediately: true }, - ); - yield* Deferred.await(pullStarted); - expect(yield* runtime.artifacts).toEqual([ - expect.objectContaining({ workloadId: workload.id, state: "downloading" }), - ]); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* Deferred.isDone(pullInterrupted)).toBe(true); - const result = yield* Fiber.join(preparing).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(createdSpecs).toHaveLength(0); - expect(yield* runtime.artifacts).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("interrupts native input materialization before cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-oidc-stop-", - }); - const started = yield* Deferred.make(); - const release = yield* Deferred.make(); - const interrupted = yield* Deferred.make(); - yield* Effect.addFinalizer(() => Deferred.succeed(release, undefined)); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { - enabled: true, - settings: { - third_party: { - workos: { enabled: true, issuer_url: "https://issuer.example" }, - }, - }, - }, - }, - }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const current = { - value: { - ...stateFor(resolved.persisted, { kind: "native" }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - }, - } satisfies { value: PersistedStackState }; - const auth = compiled.executionPlan.workloads.find(({ id }) => id === "auth:auth"); - if (auth === undefined) return yield* Effect.die("Expected Auth workload"); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "oidc-stop", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - artifactPreparer: artifacts, - logStore: memoryLogStore([]), - fetchJson: (url) => - Effect.gen(function* () { - yield* Deferred.succeed(started, undefined); - yield* Deferred.await(release); - return url.endsWith("openid-configuration") - ? { jwks_uri: "https://issuer.example/keys" } - : { keys: [{ kty: "RSA", n: "n", e: "AQAB" }] }; - }).pipe(Effect.ensuring(Deferred.succeed(interrupted, undefined))), - }); - const key = { stackId, workloadId: auth.id }; - const starting = yield* Effect.forkChild(runtime.driver.start(key, auth), { - startImmediately: true, - }); - yield* Deferred.await(started); - yield* runtime.driver.stop(key); - expect(yield* Deferred.isDone(interrupted)).toBe(true); - // The release is only a guard for the interrupted implementation under test. A correct - // owner propagates cancellation from NativeRuntime.startFiber without this handoff. - yield* Deferred.succeed(release, undefined); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - const startExit = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(startExit)).toBe(true); - expect(yield* runtime.driver.observe(stackId)).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("does not retry non-retryable database bootstrap failures", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-bootstrap-fail-", - }); - const readinessServer = createNetServer((socket) => socket.end()); - yield* listenForNativeReadiness(readinessServer); - const address = readinessServer.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("Database readiness server did not expose an address"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - }); - const current = { - value: { - ...stateFor( - { - "secret:database.internal.password": { policy: "managed", value: "db-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - }, - { kind: "container", engine: "docker" }, - ), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: address.port }, - ], - }, - } satisfies { value: PersistedStackState }; - const database = compiled.executionPlan.workloads.find( - (workload) => workload.id === "database:database", - ); - if (database === undefined) return yield* Effect.die("Expected database workload"); - let attempts = 0; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: ownerInputContainerEngine([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - image: workload.selected.kind === "container" ? workload.selected.image : undefined, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: () => { - attempts += 1; - return Effect.fail(new DatabaseBootstrapError({ message: "invalid password" })); - }, - }); - const result = yield* runtime.driver - .start( - { - stackId, - workloadId: database.id, - }, - database, - ) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(attempts).toBe(1); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("activates a capability through its public listener workload", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-activation-endpoint-", - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - database: {}, - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { - enabled: true, - settings: { image_transformation: { enabled: true } }, - }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const current = { - value: { - ...stateFor({}), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "storage:imgproxy", binding: "primary", port: 41_001 }, - { workloadId: "storage:storage", binding: "primary", port: 41_002 }, - ], - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - artifactPreparer: artifacts, - logStore: memoryLogStore([]), - bootstrapDatabase: () => Effect.void, - }); - const input = { - stackId, - state: current.value, - definition: compiled.definition, - secrets: current.value.secrets, - plan: compiled.executionPlan, - } satisfies LifecycleInput; - if (runtime.activate === undefined) return yield* Effect.die("activation seam missing"); - const endpoint = yield* runtime.activate("storage", input); - expect(endpoint).toEqual({ host: "127.0.0.1", port: 41_002 }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("loads BEAM defaults from the prepared realtime artifact", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-production-beam-" }); - const eventsPath = path.join(root, "events"); - const artifactRoot = path.join(root, "artifact"); - yield* writeNativeRealtimeFixture(fs, path, artifactRoot, eventsPath); - const readinessServer = yield* listenForHttpFixture(() => - Effect.succeed(HttpServerResponse.text("ok", { headers: { Connection: "close" } })), - ); - const port = yield* httpFixturePort(readinessServer); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - database: {}, - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: true }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const current = { - value: { - ...stateFor({ - "secret:database.internal.password": { policy: "managed", value: "db-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "realtime:realtime", binding: "primary", port }, - { workloadId: "realtime:realtime", binding: "rpc", port: 41_069 }, - ], - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - artifactPreparer: { - prepare: () => - Effect.succeed({ - workloadId: "realtime:realtime", - capability: "realtime" as const, - version: compiled.definition.capabilities.realtime.version, - outcome: "cached" as const, - artifactRoot, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: () => Effect.void, - }); - const realtime = compiled.executionPlan.workloads.find( - (workload) => workload.id === "realtime:realtime", - ); - if (realtime === undefined) return yield* Effect.die("Expected Realtime workload"); - const key = { - stackId, - workloadId: realtime.id, - }; - const ready = yield* runtime.driver.start(key, realtime); - expect(ready.state).toBe("ready"); - expect(yield* fs.readFileString(eventsPath)).toContain( - "realtime-migrate|RELEASE_DISTRIBUTION=name", - ); - yield* runtime.driver.stop(key); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("starts the canonical PostgreSQL helper before bootstrap", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-production-pg-first-" }); - const eventsPath = path.join(root, "events"); - const artifactRoot = path.join(root, "artifact"); - yield* fs.writeFileString(eventsPath, ""); - yield* writeNativeDatabaseFixture(fs, path, artifactRoot, eventsPath); - const readinessServer = createNetServer(); - yield* listenForNativeReadiness(readinessServer); - const address = readinessServer.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("Native readiness server did not expose an address"); - yield* Effect.callback((resume) => { - readinessServer.close((error) => - error === undefined ? resume(Effect.void) : resume(Effect.fail(error)), - ); - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const current = { - value: { - ...stateFor({ - "secret:database.internal.password": { policy: "managed", value: "db-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - }), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: address.port }, - ], - }, - } satisfies { value: PersistedStackState }; - const runtimePaths = yield* resolveStackPaths({ stateRoot: root, stackId }); - const databaseDataPath = path.join(runtimePaths.data, "database"); - const bootstrap = () => - Effect.gen(function* () { - const previous = yield* fs.readFileString(eventsPath); - yield* fs.writeFileString(eventsPath, `${previous}bootstrap\n`); - }).pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: "bootstrap fixture failed", cause }), - ), - ); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - artifactPreparer: { - prepare: () => - Effect.succeed({ - workloadId: "database:database", - capability: "database" as const, - version: compiled.definition.capabilities.database.version, - outcome: "cached" as const, - artifactRoot, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: bootstrap, - }); - const database = compiled.executionPlan.workloads.find( - (workload) => workload.id === "database:database", - ); - if (database === undefined) return yield* Effect.die("Expected database workload"); - const ready = yield* runtime.driver.start( - { - stackId, - workloadId: database.id, - }, - database, - ); - expect(ready.state).toBe("ready"); - const events = yield* fs.readFileString(eventsPath); - expect(events).toContain(`postgres-start|data=${databaseDataPath}`); - expect(events).toContain("user=supabase_admin"); - expect(events).toContain("db=postgres"); - expect(events).toContain("password=db-secret"); - expect(events).toContain(`args=-p ${address.port}`); - expect(events.indexOf("postgres-start")).toBeLessThan(events.indexOf("bootstrap")); - yield* runtime.driver.stop({ - stackId, - workloadId: database.id, - }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("wires persisted database and generic readiness deadlines", () => - Effect.gen(function* () { - const configured = yield* compileStack({ - projectRoot: "/tmp/production-runtime-readiness-budget", - runtime: { kind: "native" }, - config: { capabilities: { database: { settings: { health_timeout: "90s" } } } }, - }); - const defaulted = yield* compileStack({ - projectRoot: "/tmp/production-runtime-readiness-budget-default", - runtime: { kind: "native" }, - }); - const configuredDatabase = configured.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const defaultDatabase = defaulted.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - const generic = configured.executionPlan.workloads.find(({ id }) => id === "rest:rest"); - if ( - configuredDatabase === undefined || - defaultDatabase === undefined || - generic === undefined - ) - return; - const configuredState = { - ...stateFor({}), - definition: configured.definition, - }; - const defaultState = { - ...stateFor({}), - definition: defaulted.definition, - }; - expect( - Duration.toMillis(yield* readinessDeadlineFor(configuredState, configuredDatabase)), - ).toBe(90_000); - expect(Duration.toMillis(yield* readinessDeadlineFor(defaultState, defaultDatabase))).toBe( - 120_000, - ); - expect(Duration.toMillis(yield* readinessDeadlineFor(configuredState, generic))).toBe( - 120_000, - ); - const configuredContainerState = { - ...stateFor({}, { kind: "container", engine: "docker" }), - definition: configured.definition, - }; - expect( - Duration.toMillis(yield* readinessDeadlineFor(configuredContainerState, generic)), - ).toBe(30_000); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.effect("allows cold native services to become ready after library loading", () => - Effect.scoped( - Effect.gen(function* () { - const root = "/tmp/production-runtime-native-readiness-delay"; - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const rest = compiled.executionPlan.workloads.find(({ id }) => id === "rest:rest"); - if (rest === undefined) return yield* Effect.die("Expected REST workload"); - const deadline = yield* readinessDeadlineFor( - { ...stateFor({}), definition: compiled.definition }, - rest, - ); - const received = yield* Deferred.make(); - const context = yield* Effect.context(); - let response: ReturnType | undefined; - const server = yield* listenForHttpFixture((request) => { - const nativeResponse = NodeHttpServerRequest.toServerResponse(request); - response = nativeResponse; - Effect.runSyncWith(context)(Deferred.succeed(received, undefined)); - return Effect.callback((resume) => { - let settled = false; - const onDone = () => { - if (settled) return; - settled = true; - nativeResponse.off("finish", onDone); - nativeResponse.off("close", onDone); - resume(Effect.succeed(HttpServerResponse.empty())); - }; - nativeResponse.once("finish", onDone); - nativeResponse.once("close", onDone); - return Effect.sync(() => { - settled = true; - nativeResponse.off("finish", onDone); - nativeResponse.off("close", onDone); - }); - }).pipe(Effect.interruptible); - }); - const port = yield* httpFixturePort(server); - const fiber = yield* Effect.forkChild( - probeReadiness({ mode: "http", host: "127.0.0.1", port }, { deadline }), - { startImmediately: true }, - ); - yield* Deferred.await(received); - yield* TestClock.adjust(Duration.seconds(52)); - const pending = yield* Effect.sync(() => fiber.pollUnsafe() === undefined); - response?.setHeader("Connection", "close"); - response?.end("ok"); - const result = yield* Fiber.join(fiber).pipe(Effect.exit); - expect(pending).toBe(true); - expect(Exit.isSuccess(result)).toBe(true); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.effect("bounds native HTTP readiness at two minutes and cancels the request", () => - Effect.scoped( - Effect.gen(function* () { - const root = "/tmp/production-runtime-native-readiness-bound"; - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const rest = compiled.executionPlan.workloads.find(({ id }) => id === "rest:rest"); - if (rest === undefined) return yield* Effect.die("Expected REST workload"); - const deadline = yield* readinessDeadlineFor( - { ...stateFor({}), definition: compiled.definition }, - rest, - ); - expect(Duration.toMillis(deadline)).toBe(120_000); - const received = yield* Deferred.make(); - const closed = yield* Deferred.make(); - const context = yield* Effect.context(); - const server = yield* listenForHttpFixture((request) => { - const response = NodeHttpServerRequest.toServerResponse(request); - response.once("close", () => { - Effect.runSyncWith(context)(Deferred.succeed(closed, undefined)); - }); - Effect.runSyncWith(context)(Deferred.succeed(received, undefined)); - return Effect.callback((resume) => { - const onClose = () => resume(Effect.succeed(HttpServerResponse.empty())); - response.once("close", onClose); - return Effect.sync(() => response.off("close", onClose)); - }).pipe(Effect.interruptible); - }); - const port = yield* httpFixturePort(server); - const fiber = yield* Effect.forkChild( - probeReadiness({ mode: "http", host: "127.0.0.1", port }, { deadline }), - { startImmediately: true }, - ); - yield* Deferred.await(received); - yield* TestClock.adjust(Duration.seconds(121)); - const result = yield* Fiber.join(fiber).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toMatchObject({ message: "Readiness deadline exceeded" }); - } - yield* Deferred.await(closed); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("rejects an invalid database readiness budget before creating a workload", () => - Effect.scoped( - Effect.gen(function* () { - const result = yield* compileStack({ - projectRoot: "/tmp/production-runtime-invalid-health-timeout", - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { database: { settings: { health_timeout: "not-a-duration" } } }, - }, - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const cause = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(cause).toBeInstanceOf(InvalidStackConfigError); - } - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("rejects a zero persisted database readiness budget", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: "/tmp/production-runtime-zero-health-timeout", - runtime: { kind: "native" }, - }); - const database = compiled.executionPlan.workloads.find( - ({ id }) => id === "database:database", - ); - if (database === undefined) return; - const state = { - ...stateFor({}), - definition: { - ...compiled.definition, - capabilities: { - ...compiled.definition.capabilities, - database: { - ...compiled.definition.capabilities.database, - settings: { - ...compiled.definition.capabilities.database.settings, - health_timeout: "0", - }, - }, - }, - }, - }; - const result = yield* readinessDeadlineFor(state, database).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( - StackPreparationError, - ); - } - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("does not create the Functions root during preflight", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-functions-root-", - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const secrets = Object.fromEntries( - compiled.secrets.map((entry) => [ - entry.slot, - { policy: entry.policy, value: "test-secret" }, - ]), - ); - const current = { - value: { - ...stateFor(secrets), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - definition: compiled.definition, - secrets, - }, - } satisfies { value: PersistedStackState }; - const functionsRoot = compiled.definition.capabilities.functions.settings.functions_root; - if (functionsRoot === null) return yield* Effect.die("Functions root was not materialized"); - expect(yield* fs.exists(functionsRoot)).toBe(false); - const prepared: string[] = []; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.sync(() => { - prepared.push(workload.id); - return { - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - }; - }), - }, - bootstrapDatabase: () => Effect.void, - }); - yield* runtime.preflight({ - stackId, - state: current.value, - definition: compiled.definition, - secrets, - plan: compiled.executionPlan, - }); - expect(yield* fs.exists(functionsRoot)).toBe(false); - expect(prepared).toEqual([]); - yield* runtime.prepare( - { - stackId, - state: current.value, - definition: compiled.definition, - secrets, - plan: compiled.executionPlan, - }, - new Set(["database"]), - ); - expect(prepared).toEqual(["database:database"]); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("rejects an occupied persisted native port during a cold preflight", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-preflight-" }); - const occupied = createNetServer(); - yield* listenForNativeReadiness(occupied); - const address = occupied.address(); - if (address === null || typeof address === "string") - return yield* Effect.die("occupied listener has no TCP address"); - const port = yield* Schema.decodeEffect(NetworkPortSchema)(address.port).pipe(Effect.orDie); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const secrets = Object.fromEntries( - compiled.secrets.map((entry) => [ - entry.slot, - { policy: entry.policy, value: "test-secret" }, - ]), - ); - const current = { - value: { - ...stateFor(secrets), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - definition: compiled.definition, - ports: [{ field: "database", port, intent: "automatic" }], - secrets, - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached", - }), - }, - bootstrapDatabase: () => Effect.void, - }); - const result = yield* runtime - .preflight({ - stackId, - state: current.value, - definition: compiled.definition, - secrets, - plan: compiled.executionPlan, - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(PortUnavailableError); - } - const changedPort = port === 65_535 ? 65_534 : port + 1; - const candidates = [ - { - ...compiled.definition, - listeners: { - ...compiled.definition.listeners, - database: { ...compiled.definition.listeners.database, enabled: false }, - }, - }, - { - ...compiled.definition, - listeners: { - ...compiled.definition.listeners, - database: { ...compiled.definition.listeners.database, port: changedPort }, - }, - }, - ]; - for (const definition of candidates) { - const accepted = yield* runtime - .preflight({ - stackId, - state: current.value, - definition, - secrets, - plan: compiled.executionPlan, - }) - .pipe(Effect.exit); - expect(Exit.isSuccess(accepted)).toBe(true); - } - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("ignores obsolete inspector bindings while retaining requested bindings", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-native-private-preflight-", - }); - const occupied = createNetServer(); - yield* listenForNativeReadiness(occupied); - const address = occupied.address(); - if (address === null || typeof address === "string") - return yield* Effect.die("occupied listener has no TCP address"); - const port = yield* Schema.decodeEffect(NetworkPortSchema)(address.port).pipe(Effect.orDie); - const previous = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { capabilities: { functions: { settings: { inspector: { mode: "run" } } } } }, - }); - const disabled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const requestedLazy = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { capabilities: { functions: { settings: { inspector: { mode: "run" } } } } }, - }); - const secrets = Object.fromEntries( - disabled.secrets.map((entry) => [ - entry.slot, - { policy: entry.policy, value: "test-secret" }, - ]), - ); - const current = { - value: { - ...stateFor(secrets), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - definition: previous.definition, - privatePorts: [{ workloadId: "functions:edge-runtime", binding: "inspector", port }], - secrets, - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "private-port-preflight", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - }), - }, - bootstrapDatabase: () => Effect.void, - }); - const input = (candidate: typeof disabled) => ({ - stackId, - state: current.value, - definition: candidate.definition, - secrets, - plan: candidate.executionPlan, - }); - const obsolete = yield* runtime.preflight(input(disabled)).pipe(Effect.exit); - expect(Exit.isSuccess(obsolete)).toBe(true); - const requested = yield* runtime.preflight(input(requestedLazy)).pipe(Effect.exit); - expect(Exit.isFailure(requested)).toBe(true); - if (Exit.isFailure(requested)) { - const error = Option.getOrUndefined(Cause.findErrorOption(requested.cause)); - expect(error).toBeInstanceOf(PortUnavailableError); - } - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("rejects native database lock evidence owned by a live process", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-native-lock-" }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const secrets = Object.fromEntries( - compiled.secrets.map((entry) => [ - entry.slot, - { policy: entry.policy, value: "test-secret" }, - ]), - ); - const current = { - value: { - ...stateFor(secrets), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - definition: compiled.definition, - secrets, - }, - } satisfies { value: PersistedStackState }; - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(path.join(paths.data, "database"), { recursive: true }); - yield* fs.writeFileString( - path.join(paths.data, "database", "postmaster.pid"), - `${process.pid}\n`, - ); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore([]), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached", - }), - }, - bootstrapDatabase: () => Effect.void, - }); - const result = yield* runtime - .preflight({ - stackId, - state: current.value, - definition: compiled.definition, - secrets, - plan: compiled.executionPlan, - }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(StackPreparationError); - expect(error?.message).toContain("live process"); - } - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("redacts logs using secret slots materialized after factory creation", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-production-logs-" }); - const current = { value: stateFor({}) }; - const reads = { value: 0 }; - const entries: StackLogEntry[] = []; - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - }); - const completeSecrets = Object.fromEntries( - compiled.secrets.map((entry) => [ - entry.slot, - { policy: entry.policy, value: "placeholder" }, - ]), - ); - const candidate = { - ...stateFor({ - ...completeSecrets, - "secret:auth.settings.jwt_secret": { - policy: "managed" as const, - value: "rotated-secret", - }, - }), - desiredLifecycle: "running" as const, - definition: compiled.definition, - } satisfies PersistedStackState; - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current, () => reads.value++), - context, - ingress, - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - }), - }, - envFileOwner: envFiles, - functionsBootstrapOwner: bootstrap, - logStore: memoryLogStore(entries), - bootstrapDatabase: () => Effect.void, - }); - yield* runtime.preflight({ - stackId, - state: candidate, - definition: compiled.definition, - secrets: candidate.secrets, - plan: compiled.executionPlan, - }); - const logStore = runtime.logStore; - expect(logStore).toBeDefined(); - if (logStore === undefined) return; - yield* logStore.append({ - source: "auth", - stream: "stdout", - message: "token=rotated-secret", - }); - yield* logStore.append({ - source: "auth", - stream: "stdout", - message: "token=rotated-secret again", - }); - expect(entries[0]?.message).toBe("token=[REDACTED]"); - expect(entries[1]?.message).toBe("token=[REDACTED] again"); - expect(reads.value).toBe(1); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - const makeOwnerMaterialFixture = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-production-inputs-" }); - const template = path.join(root, "templates", "confirmation.html"); - yield* fs.makeDirectory(path.dirname(template), { recursive: true }); - yield* fs.writeFileString(template, "confirmation"); - const gcpCredentials = path.join(root, "gcp.json"); - yield* fs.writeFileString(gcpCredentials, "{}"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - auth: { - settings: { - email: { - template: { confirmation: { content_path: "templates/confirmation.html" } }, - }, - }, - }, - pooler: { enabled: true }, - rest: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { - enabled: true, - settings: { - edge_runtime: { secrets: { FACTORY_SECRET: Redacted.make("factory-secret") } }, - }, - }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { - enabled: true, - settings: { backend: "bigquery", gcp_jwt_path: "gcp.json" }, - }, - }, - listeners: { - database: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const authServer = yield* listenForHttpFixture(() => - Effect.succeed(HttpServerResponse.text("ok", { headers: { Connection: "close" } })), - ); - const authPort = yield* httpFixturePort(authServer); - const functionsServer = yield* listenForHttpFixture(() => - Effect.succeed(HttpServerResponse.text("ok", { headers: { Connection: "close" } })), - ); - const functionsPort = yield* httpFixturePort(functionsServer); - const analyticsServer = yield* listenForHttpFixture(() => - Effect.succeed(HttpServerResponse.text("ok", { headers: { Connection: "close" } })), - ); - const analyticsPort = yield* httpFixturePort(analyticsServer); - const poolerServer = yield* listenForHttpFixture(() => - Effect.succeed(HttpServerResponse.empty({ status: 204 })), - ); - const poolerPort = yield* httpFixturePort(poolerServer); - const current = { - value: { - ...stateFor( - { - "secret:database.internal.password": { policy: "managed", value: "db-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - "secret:auth.settings.publishable_key": { - policy: "managed", - value: "sb_publishable_test", - }, - "secret:auth.settings.secret_key": { - policy: "managed", - value: "sb_secret_test", - }, - "secret:functions.settings.edge_runtime.secrets.FACTORY_SECRET": { - policy: "managed", - value: "factory-secret", - }, - }, - { kind: "container", engine: "docker" }, - ), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - ports: [{ field: "api" as const, port: 40_000, intent: "exact" as const }], - privatePorts: [ - { - workloadId: "auth:auth", - binding: "primary", - port: authPort, - }, - { - workloadId: "pooler:pooler", - binding: "primary", - port: poolerPort + 1, - }, - { - workloadId: "pooler:pooler", - binding: "admin", - port: poolerPort, - }, - { - workloadId: "functions:edge-runtime", - binding: "primary", - port: functionsPort, - }, - { - workloadId: "functions:edge-runtime", - binding: "inspector", - port: functionsPort + 1, - }, - { - workloadId: "analytics:analytics", - binding: "primary", - port: analyticsPort, - }, - ], - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const createdSpecs: ContainerContainerSpec[] = []; - const copiedFiles: Array> = []; - const engine = ownerInputContainerEngine(createdSpecs, copiedFiles); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: engine, - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - image: workload.selected.kind === "container" ? workload.selected.image : undefined, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: () => Effect.void, - }); - return { fs, runtime, createdSpecs, copiedFiles, compiled }; - }).pipe(Effect.provide(NodeServices.layer)); - - it.live("writes Auth owner material and confirmation template settings", () => - Effect.scoped( - Effect.gen(function* () { - const { fs, runtime, createdSpecs, compiled } = yield* makeOwnerMaterialFixture(); - const auth = compiled.executionPlan.workloads.find( - (workload) => workload.id === "auth:auth", - ); - if (auth === undefined) return yield* Effect.die("Expected Auth workload"); - - yield* runtime.driver.start({ stackId, workloadId: auth.id }, auth); - const authSpec = createdSpecs.find((spec) => spec.labels.workloadId === auth.id); - if (authSpec?.envFile === undefined) - return yield* Effect.die("Auth container was not captured"); - const authEnvironment = yield* fs.readFileString(authSpec.envFile); - - expect(authEnvironment).toContain("GOTRUE_JWT_SECRET=jwt-secret"); - expect(authEnvironment).toContain( - "GOTRUE_MAILER_TEMPLATES_CONFIRMATION=http://host.docker.internal:40000/email/confirmation.html", - ); - }), - ), - ); - - it.live("writes Functions owner material and bootstrap source", () => - Effect.scoped( - Effect.gen(function* () { - const { fs, runtime, createdSpecs, copiedFiles, compiled } = - yield* makeOwnerMaterialFixture(); - const functions = compiled.executionPlan.workloads.find( - (workload) => workload.id === "functions:edge-runtime", - ); - if (functions === undefined) return yield* Effect.die("Expected Functions workload"); - - yield* runtime.driver.start({ stackId, workloadId: functions.id }, functions); - const functionsSpec = createdSpecs.find((spec) => spec.labels.workloadId === functions.id); - if (functionsSpec?.envFile === undefined) - return yield* Effect.die("Functions container was not captured"); - const firstBootstrap = copiedFiles.at(-1); - if (firstBootstrap === undefined) - return yield* Effect.die("Functions bootstrap was not captured"); - const functionsEnvironment = yield* fs.readFileString(functionsSpec.envFile); - - expect(functionsEnvironment).toContain("FACTORY_SECRET=factory-secret"); - expect(yield* fs.exists(firstBootstrap.source)).toBe(true); - }), - ), - ); - - it.live("mounts the Analytics credentials file for the owner", () => - Effect.scoped( - Effect.gen(function* () { - const { runtime, createdSpecs, compiled } = yield* makeOwnerMaterialFixture(); - const analytics = compiled.executionPlan.workloads.find( - (workload) => workload.id === "analytics:analytics", - ); - if (analytics === undefined) return yield* Effect.die("Expected Analytics workload"); - - yield* runtime.driver.start({ stackId, workloadId: analytics.id }, analytics); - const analyticsSpec = createdSpecs.find((spec) => spec.labels.workloadId === analytics.id); - - expect(analyticsSpec?.mounts).toContainEqual({ - source: expect.stringContaining("gcp.json"), - target: "/opt/app/rel/logflare/bin/gcloud.json", - readOnly: true, - }); - }), - ), - ); - - it.live("starts Pooler with its owner startup entrypoints", () => - Effect.scoped( - Effect.gen(function* () { - const { runtime, createdSpecs, compiled } = yield* makeOwnerMaterialFixture(); - const pooler = compiled.executionPlan.workloads.find( - (workload) => workload.id === "pooler:pooler", - ); - if (pooler === undefined) return yield* Effect.die("Expected Pooler workload"); - - yield* runtime.driver.start({ stackId, workloadId: pooler.id }, pooler); - const poolerSpec = createdSpecs.find( - (spec) => spec.labels.workloadId === pooler.id && spec.labels.startup !== true, - ); - const poolerStartupSpecs = createdSpecs.filter( - (spec) => spec.labels.workloadId === pooler.id && spec.labels.startup === true, - ); - - expect(poolerSpec?.mounts).toEqual([]); - expect(poolerStartupSpecs.map((spec) => spec.entrypoint)).toEqual([ - "/app/bin/prepare", - "/app/bin/provision-tenant", - ]); - }), - ), - ); - - it.live("removes and recreates the Functions bootstrap across container cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const { fs, runtime, copiedFiles, compiled } = yield* makeOwnerMaterialFixture(); - const functions = compiled.executionPlan.workloads.find( - (workload) => workload.id === "functions:edge-runtime", - ); - if (functions === undefined) return yield* Effect.die("Expected Functions workload"); - - yield* runtime.driver.start({ stackId, workloadId: functions.id }, functions); - const firstBootstrap = copiedFiles.at(-1); - if (firstBootstrap === undefined) - return yield* Effect.die("Functions bootstrap was not captured"); - expect(yield* fs.exists(firstBootstrap.source)).toBe(true); - - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* fs.exists(firstBootstrap.source)).toBe(false); - - yield* runtime.driver.start({ stackId, workloadId: functions.id }, functions); - const restartedBootstrap = copiedFiles.at(-1); - if (restartedBootstrap === undefined || restartedBootstrap === firstBootstrap) - return yield* Effect.die("Restarted Functions bootstrap was not captured"); - expect(yield* fs.exists(restartedBootstrap.source)).toBe(true); - }), - ), - ); - - it.live("scopes Auth template URL requirements to the Auth workload", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-production-auth-scope-", - }); - const template = path.join(root, "confirmation.html"); - yield* fs.writeFileString(template, "confirmation"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - auth: { - settings: { - email: { template: { confirmation: { content_path: "confirmation.html" } } }, - }, - }, - rest: { enabled: true }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - listeners: { - api: { enabled: false }, - database: { enabled: false }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const restServer = yield* listenForHttpFixture(() => - Effect.succeed(HttpServerResponse.text("ok", { headers: { Connection: "close" } })), - ); - const port = yield* httpFixturePort(restServer); - const current = { - value: { - ...stateFor( - { - "secret:database.internal.password": { policy: "managed", value: "db-secret" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "jwt-secret" }, - }, - { kind: "container", engine: "docker" }, - ), - identity: { - ...stateFor({}).identity, - projectRoot: root, - }, - desiredLifecycle: "running" as const, - definition: compiled.definition, - privatePorts: [ - { workloadId: "rest:rest", binding: "primary", port }, - { workloadId: "rest:rest", binding: "admin", port: port + 1 }, - { workloadId: "auth:auth", binding: "primary", port: port + 2 }, - ], - }, - } satisfies { value: PersistedStackState }; - const context = yield* Effect.context(); - const createdSpecs: ContainerContainerSpec[] = []; - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - containerEngine: ownerInputContainerEngine(createdSpecs), - artifactPreparer: { - prepare: (_runtime, workload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "test", - outcome: "cached" as const, - image: workload.selected.kind === "container" ? workload.selected.image : undefined, - }), - }, - logStore: memoryLogStore([]), - bootstrapDatabase: () => Effect.void, - }); - const rest = compiled.executionPlan.workloads.find( - (workload) => workload.id === "rest:rest", - ); - const auth = compiled.executionPlan.workloads.find( - (workload) => workload.id === "auth:auth", - ); - if (rest === undefined || auth === undefined) - return yield* Effect.die("Expected REST and Auth workloads"); - yield* runtime.driver.start({ stackId, workloadId: rest.id }, rest); - expect(createdSpecs.some((spec) => spec.labels.workloadId === rest.id)).toBe(true); - const authStart = yield* runtime.driver - .start({ stackId, workloadId: auth.id }, auth) - .pipe(Effect.exit); - expect(Exit.isFailure(authStart)).toBe(true); - expect(createdSpecs.some((spec) => spec.labels.workloadId === auth.id)).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it.live("cleans owner env and Functions files only during stack cleanup", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-production-cleanup-" }); - const current = { value: stateFor({}) }; - const envOwner = yield* makeRuntimeEnvFileOwner({ stateRoot: root, stackId }); - const functionsOwner = yield* makeFunctionsBootstrapOwner({ - stateRoot: root, - stackId, - }); - const envPath = yield* envOwner.write({ - workloadId: "database:database", - values: { TOKEN: "stale" }, - }); - const bootstrapPath = yield* functionsOwner.write({ - content: "export default 1", - }); - const context = yield* Effect.context(); - const runtime = yield* makeProductionRuntime({ - stateRoot: root, - stackId, - ownerSessionId: "owner", - stateStore: stateStoreFor(current), - context, - ingress, - artifactPreparer: artifacts, - logStore: memoryLogStore([]), - bootstrapDatabase: () => Effect.void, - }); - yield* runtime.driver.stop({ - stackId, - workloadId: "database:database", - }); - expect(yield* fs.exists(envPath)).toBe(true); - expect(yield* fs.exists(bootstrapPath)).toBe(true); - yield* runtime.driver.cleanup({ stackId, destroy: false }); - expect(yield* fs.exists(envPath)).toBe(false); - expect(yield* fs.exists(bootstrapPath)).toBe(false); - }).pipe(Effect.provide(NodeServices.layer)), - ), - ); - - it("attempts both owner cleanups when runtime cleanup fails", () => { - const calls: string[] = []; - const runtimeFailure = new RuntimeDriverError({ - message: "runtime cleanup failed", - stackId, - }); - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: () => Effect.die("unused"), - stop: () => Effect.void, - remove: () => Effect.void, - cleanup: () => Effect.fail(runtimeFailure), - wipePersistentData: () => Effect.void, - }; - const envOwner: RuntimeEnvFileOwner = { - write: () => Effect.die("unused"), - cleanupAll: Effect.sync(() => { - calls.push("env"); - }), - }; - const functionsOwner: FunctionsBootstrapOwner = { - write: () => Effect.die("unused"), - cleanupAll: Effect.sync(() => { - calls.push("functions"); - }), - }; - const wrapped = withOwnedRuntimeFileCleanup(driver, envOwner, functionsOwner); - const result = Effect.runSyncExit(wrapped.cleanup({ stackId, destroy: false })); - expect(Exit.isFailure(result)).toBe(true); - expect(calls.sort()).toEqual(["env", "functions"]); - }); - - it("continues owner cleanup when one file owner fails", () => { - const calls: string[] = []; - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: () => Effect.die("unused"), - stop: () => Effect.void, - remove: () => Effect.void, - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const envOwner: RuntimeEnvFileOwner = { - write: () => Effect.die("unused"), - cleanupAll: Effect.gen(function* () { - calls.push("env"); - return yield* new StackPreparationError({ message: "env cleanup failed" }); - }), - }; - const functionsOwner: FunctionsBootstrapOwner = { - write: () => Effect.die("unused"), - cleanupAll: Effect.sync(() => { - calls.push("functions"); - }), - }; - const wrapped = withOwnedRuntimeFileCleanup(driver, envOwner, functionsOwner); - const result = Effect.runSyncExit(wrapped.cleanup({ stackId, destroy: false })); - expect(Exit.isFailure(result)).toBe(true); - expect(calls).toEqual(["env", "functions"]); - }); -}); diff --git a/packages/stack/src/runtime/readiness-probe.integration.test.ts b/packages/stack/src/runtime/readiness-probe.integration.test.ts deleted file mode 100644 index fe66e99832..0000000000 --- a/packages/stack/src/runtime/readiness-probe.integration.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Data, Deferred, Duration, Effect, Exit, Fiber, Option, Predicate } from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import { NodeHttpServer, NodeHttpServerRequest } from "@effect/platform-node"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- NodeHttpServer.layer requires a native factory to bind loopback; layerTest does not expose a host option. -import { createServer as createHttpServer } from "node:http"; -import { createServer as createTcpServer, type Server as TcpServer } from "node:net"; -import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; -import { StackPreparationError } from "../public/Errors.ts"; -import { parseGoDuration } from "../model/capabilities/database.ts"; -import { probeReadiness } from "./ReadinessProbe.ts"; - -class ListenerError extends Data.TaggedError("ListenerError")<{ readonly message: string }> {} - -const serveHttp = ( - handler: Effect.Effect< - HttpServerResponse.HttpServerResponse, - never, - HttpServerRequest.HttpServerRequest - >, -) => - Effect.gen(function* () { - const server = yield* HttpServer.HttpServer; - yield* server.serve(handler); - if (!Predicate.isTagged(server.address, "TcpAddress")) - return yield* new ListenerError({ message: "missing HTTP address" }); - return server.address.port; - }); - -const loopbackHttpLayer = NodeHttpServer.layer(createHttpServer, { - port: 0, - host: "127.0.0.1", -}); - -const listenTcp = (server: TcpServer): Effect.Effect => - Effect.callback((resume) => { - const onError = (cause: Error) => - resume(Effect.fail(new ListenerError({ message: cause.message }))); - server.once("error", onError); - server.listen(0, "127.0.0.1", () => { - server.off("error", onError); - const address = server.address(); - if (typeof address !== "object" || address === null) - return resume(Effect.fail(new ListenerError({ message: "missing TCP address" }))); - resume(Effect.succeed(address.port)); - }); - return Effect.sync(() => { - server.off("error", onError); - server.close(); - }); - }); - -describe("private endpoint readiness probe", () => { - it("parses supported Go duration strings without accepting negative budgets", () => { - expect(Duration.toMillis(parseGoDuration("1h30m2.5s"))).toBe(5_402_500); - expect(Duration.toMillis(parseGoDuration("250ms"))).toBe(250); - expect(Duration.toNanosUnsafe(parseGoDuration("1µs"))).toEqual(1_000n); - expect(Duration.toNanosUnsafe(parseGoDuration("1μs"))).toEqual(1_000n); - expect(Duration.toMillis(parseGoDuration("0s"))).toBe(0); - expect(() => parseGoDuration("-1s")).not.toThrow(); - expect(Duration.isNegative(parseGoDuration("-1s"))).toBe(true); - for (const invalid of ["", "1", "9223372036854775808ns", "Infinity"]) - expect(() => parseGoDuration(invalid)).toThrow(); - }); - - it.live("rejects negative and non-finite readiness deadlines as typed preparation failures", () => - Effect.gen(function* () { - for (const deadline of [Duration.seconds(-1), Duration.infinity]) { - const result = yield* probeReadiness( - { mode: "tcp", host: "127.0.0.1", port: 1 }, - { deadline }, - ).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const cause = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(cause).toBeInstanceOf(StackPreparationError); - } - } - }), - ); - - it.live("passes validated request headers to HTTP readiness endpoints", () => - Effect.scoped( - Effect.gen(function* () { - const received = yield* Deferred.make(); - const port = yield* serveHttp( - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest; - const host = request.headers?.host ?? ""; - yield* Deferred.succeed(received, host); - return HttpServerResponse.text("ok", { status: host === "realtime-dev" ? 200 : 400 }); - }), - ); - yield* probeReadiness({ - mode: "http", - host: "127.0.0.1", - port, - path: "/api/ping", - headers: { Host: "realtime-dev" }, - }); - expect(yield* Deferred.await(received)).toBe("realtime-dev"); - }), - ).pipe(Effect.provide(loopbackHttpLayer)), - ); - - it.live("rejects invalid readiness header names and values", () => - Effect.gen(function* () { - const cases: ReadonlyArray>> = [ - { "bad name": "ok" }, - { Host: "bad\nvalue" }, - ]; - for (const headers of cases) { - const result = yield* probeReadiness({ - mode: "http", - host: "127.0.0.1", - port: 1, - headers, - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const cause = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(cause).toBeInstanceOf(StackPreparationError); - } - } - }), - ); - - it.live("rejects control characters in HTTP paths as typed preparation failures", () => - Effect.gen(function* () { - for (const path of ["/ready\nX", "/ready\rX", "/ready\u0000X"]) { - const result = yield* probeReadiness({ - mode: "http", - host: "127.0.0.1", - port: 1, - path, - }).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const cause = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(cause).toBeInstanceOf(StackPreparationError); - } - } - }), - ); - - it.live("probes HTTP and TCP endpoints with a Schedule retry policy", () => - Effect.scoped( - Effect.gen(function* () { - const httpPort = yield* serveHttp( - Effect.succeed(HttpServerResponse.text("ok", { status: 200 })), - ); - yield* probeReadiness( - { mode: "http", host: "127.0.0.1", port: httpPort, path: "/health" }, - { retries: 1, retryDelay: 0 }, - ); - - const tcp = createTcpServer((socket) => socket.end()); - const tcpPort = yield* listenTcp(tcp); - yield* probeReadiness( - { mode: "tcp", host: "127.0.0.1", port: tcpPort }, - { retries: 1, retryDelay: 0 }, - ); - }), - ).pipe(Effect.provide(loopbackHttpLayer)), - ); - - it.live("reports a failed endpoint after bounded retries", () => - Effect.scoped( - Effect.gen(function* () { - let attempts = 0; - const port = yield* serveHttp( - Effect.sync(() => { - attempts += 1; - return HttpServerResponse.text("not ready", { status: 503 }); - }), - ); - const result = yield* probeReadiness( - { mode: "http", host: "127.0.0.1", port }, - { retries: 1, retryDelay: 0 }, - ).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(attempts).toBe(2); - }), - ).pipe(Effect.provide(loopbackHttpLayer)), - ); - - it.live("performs exactly one immediate probe for a zero readiness budget", () => - Effect.scoped( - Effect.gen(function* () { - let count = 0; - const port = yield* serveHttp( - Effect.sync(() => { - count += 1; - return HttpServerResponse.text("not ready", { status: 503 }); - }), - ); - const result = yield* probeReadiness( - { mode: "http", host: "127.0.0.1", port }, - { deadline: Duration.zero }, - ).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const cause = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(cause).toMatchObject({ message: "Readiness HTTP status was not successful" }); - expect(cause).toMatchObject({ - target: { mode: "http", host: "127.0.0.1", port }, - }); - } - expect(count).toBe(1); - }), - ).pipe(Effect.provide(loopbackHttpLayer)), - ); - - it.live("interrupts an in-flight HTTP request and closes its owned socket", () => - Effect.scoped( - Effect.gen(function* () { - const received = yield* Deferred.make(); - const port = yield* serveHttp( - Effect.gen(function* () { - const serverRequest = yield* HttpServerRequest.HttpServerRequest; - const request = NodeHttpServerRequest.toIncomingMessage(serverRequest); - yield* Deferred.succeed(received, undefined); - request.once("error", () => undefined); - return yield* Effect.never.pipe(Effect.interruptible); - }), - ); - const fiber = yield* Effect.forkChild( - probeReadiness({ mode: "http", host: "127.0.0.1", port, path: "/hang" }, { retries: 0 }), - ); - yield* Deferred.await(received); - yield* Fiber.interrupt(fiber); - }), - ).pipe(Effect.provide(loopbackHttpLayer)), - ); - - it.effect("interrupts an in-flight HTTP request at its total deadline", () => - Effect.scoped( - Effect.gen(function* () { - const received = yield* Deferred.make(); - const closed = yield* Deferred.make(); - const port = yield* serveHttp( - Effect.gen(function* () { - const serverRequest = yield* HttpServerRequest.HttpServerRequest; - const request = NodeHttpServerRequest.toIncomingMessage(serverRequest); - yield* Deferred.succeed(received, undefined); - request.once("close", () => { - Deferred.doneUnsafe(closed, Effect.void); - }); - return yield* Effect.never.pipe(Effect.interruptible); - }), - ); - const fiber = yield* Effect.forkChild( - probeReadiness( - { mode: "http", host: "127.0.0.1", port, path: "/deadline" }, - { deadline: Duration.millis(10) }, - ), - { startImmediately: true }, - ); - yield* Deferred.await(received); - yield* TestClock.adjust(Duration.millis(10)); - const result = yield* Fiber.join(fiber).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const cause = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(cause).toMatchObject({ message: "Readiness deadline exceeded" }); - } - yield* Deferred.await(closed); - }), - ).pipe(Effect.provide(loopbackHttpLayer)), - ); -}); diff --git a/packages/stack/src/runtime/runtime-env-file.integration.test.ts b/packages/stack/src/runtime/runtime-env-file.integration.test.ts deleted file mode 100644 index ef200b1903..0000000000 --- a/packages/stack/src/runtime/runtime-env-file.integration.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Path } from "effect"; -import { StackIdSchema } from "../public/StackId.ts"; -import { makeRuntimeEnvFileOwner } from "./RuntimeEnvFile.ts"; - -const stackId = StackIdSchema.make("e".repeat(64)); - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const setupEnvOwner = (prefix: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix }); - const owner = yield* makeRuntimeEnvFileOwner({ stateRoot: root, stackId }); - return { fs, root, owner }; - }); - -describe("runtime environment file owner", () => { - it.live("writes deterministic owner-only session files", () => - withPlatform( - Effect.gen(function* () { - const path = yield* Path.Path; - const { fs, owner } = yield* setupEnvOwner("stack-env-"); - const first = yield* owner.write({ - workloadId: "database:database", - values: { Z_LAST: "two", A_FIRST: "one" }, - }); - expect(yield* fs.readFileString(first)).toBe("A_FIRST=one\nZ_LAST=two\n"); - expect((yield* fs.stat(first)).mode & 0o777).toBe(0o600); - expect((yield* fs.stat(path.dirname(first))).mode & 0o777).toBe(0o700); - }), - ), - ); - - it.live("replaces one workload file without changing its path", () => - withPlatform( - Effect.gen(function* () { - const { fs, owner } = yield* setupEnvOwner("stack-env-replace-"); - const first = yield* owner.write({ - workloadId: "database:database", - values: { A_FIRST: "one" }, - }); - const second = yield* owner.write({ - workloadId: "database:database", - values: { A_FIRST: "updated" }, - }); - expect(second).toBe(first); - expect(yield* fs.readFileString(second)).toBe("A_FIRST=updated\n"); - }), - ), - ); - - it.live("rejects unsafe values without revealing the value", () => - withPlatform( - Effect.gen(function* () { - const { fs, root, owner } = yield* setupEnvOwner("stack-env-invalid-"); - const secret = "very-secret-value"; - const invalid = yield* owner - .write({ - workloadId: "database:database", - values: { VALID: `${secret}\nINJECTED` }, - }) - .pipe(Effect.exit); - expect(Exit.isFailure(invalid)).toBe(true); - if (Exit.isFailure(invalid)) expect(String(invalid.cause)).not.toContain(secret); - expect(yield* fs.exists(`${root}/${stackId}/runtime/env`)).toBe(false); - }), - ), - ); - - it.live("rejects invalid environment variable names", () => - withPlatform( - Effect.gen(function* () { - const { owner } = yield* setupEnvOwner("stack-env-invalid-name-"); - const invalid = yield* owner - .write({ workloadId: "database:database", values: { "BAD-NAME": "ok" } }) - .pipe(Effect.exit); - expect(Exit.isFailure(invalid)).toBe(true); - }), - ), - ); - - it.live("rejects workload identities that could escape the env root", () => - withPlatform( - Effect.gen(function* () { - const { owner } = yield* setupEnvOwner("stack-env-invalid-workload-"); - const invalid = yield* owner - .write({ workloadId: "../escape", values: { SAFE: "ok" } }) - .pipe(Effect.exit); - expect(Exit.isFailure(invalid)).toBe(true); - }), - ), - ); - - it.live("cleans exact session files idempotently", () => - withPlatform( - Effect.gen(function* () { - const path = yield* Path.Path; - const { fs, root, owner } = yield* setupEnvOwner("stack-env-cleanup-"); - const file = yield* owner.write({ - workloadId: "rest:rest", - values: { X: "y" }, - }); - - expect(yield* fs.exists(file)).toBe(true); - - yield* owner.cleanupAll; - - expect(yield* fs.exists(file)).toBe(false); - expect(yield* fs.exists(path.join(root, stackId, "runtime", "env"))).toBe(false); - yield* owner.cleanupAll; - }), - ), - ); - - it.live("recreates a workload file after cleanup", () => - withPlatform( - Effect.gen(function* () { - const { fs, owner } = yield* setupEnvOwner("stack-env-recreate-"); - const original = yield* owner.write({ workloadId: "rest:rest", values: { X: "y" } }); - expect(yield* fs.exists(original)).toBe(true); - yield* owner.cleanupAll; - expect(yield* fs.exists(original)).toBe(false); - const recreated = yield* owner.write({ workloadId: "rest:rest", values: { X: "z" } }); - expect(yield* fs.readFileString(recreated)).toBe("X=z\n"); - }), - ), - ); - - it.live("keeps concurrent workload files collision-free", () => - withPlatform( - Effect.gen(function* () { - const { fs, owner } = yield* setupEnvOwner("stack-env-concurrent-"); - const files = yield* Effect.all( - [ - owner.write({ workloadId: "rest:rest", values: { A: "one" } }), - owner.write({ workloadId: "auth:auth", values: { B: "two" } }), - ], - { concurrency: "unbounded" }, - ); - expect(new Set(files).size).toBe(2); - expect(yield* fs.readFileString(files[0])).toBe("A=one\n"); - expect(yield* fs.readFileString(files[1])).toBe("B=two\n"); - }), - ), - ); -}); diff --git a/packages/stack/src/runtime/runtime-input-owner.integration.test.ts b/packages/stack/src/runtime/runtime-input-owner.integration.test.ts deleted file mode 100644 index d4988e945b..0000000000 --- a/packages/stack/src/runtime/runtime-input-owner.integration.test.ts +++ /dev/null @@ -1,819 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, FileSystem, Option, Path, Redacted, Schema } from "effect"; -import { generateKeyPairSync } from "node:crypto"; -import { compileStack, type CompiledStack } from "../model/Compiler.ts"; -import { StackPreparationError } from "../public/Errors.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { resolveSecrets } from "../state/SecretStore.ts"; -import { makeRuntimeInputOwner } from "./RuntimeInputOwner.ts"; -import { resolveContainerResolutionFor } from "./WorkloadRuntimeSpec.ts"; - -const stackId = StackIdSchema.make("f".repeat(64)); -const jsonSchema = Schema.fromJsonString(Schema.Unknown); - -const encodeJson = (value: unknown): string => Schema.encodeSync(jsonSchema)(value); -const decodeJson = (value: string): unknown => Schema.decodeSync(jsonSchema)(value); -const decodeJwks = (value: string): { readonly keys: ReadonlyArray } => - Schema.decodeUnknownSync(Schema.Struct({ keys: Schema.Array(Schema.Unknown) }))( - decodeJson(value), - ); - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const identityFor = (projectRoot: string): PersistedStackState["identity"] => ({ - projectRoot, - branchContext: "ordinary-workspace", - stackName: "runtime-input-owner", -}); - -const stateFor = ( - root: string, - compiled: CompiledStack, - secrets: PersistedStackState["secrets"], - runtime: StackRuntime = { kind: "native" }, -): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity: identityFor(root), - runtime, - desiredLifecycle: "stopped", - definition: compiled.definition, - ports: [], - privatePorts: [], - secrets, -}); - -const compiledState = (root: string, config: Parameters[0]["config"] = {}) => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - return stateFor(root, compiled, resolved.persisted); - }); - -const vectorFixture = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-vector-" }); - const base = yield* compiledState(root, { - capabilities: { analytics: { settings: {} } }, - }); - const state: PersistedStackState = { - ...base, - privatePorts: [{ workloadId: "analytics:vector", binding: "primary", port: 30_008 }], - }; - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const resolveConfigPath = () => - Effect.gen(function* () { - const material = yield* owner.resolve(state, "analytics:vector"); - const configPath = material.analytics?.vectorConfigPath; - if (configPath === undefined) return yield* Effect.die("Vector config path is missing"); - return configPath; - }); - return { fs, owner, resolveConfigPath }; - }); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("runtime input owner", () => { - it.live("resolves contained regular files and rejects escapes, symlinks, and directories", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-files-" }); - const outside = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-outside-" }); - yield* fs.makeDirectory(path.join(root, "nested"), { recursive: true }); - yield* fs.writeFileString(path.join(root, "nested", "config.json"), "{}"); - yield* fs.writeFileString(path.join(outside, "secret.json"), "secret"); - yield* fs.symlink(path.join(outside, "secret.json"), path.join(root, "linked.json")); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const state = yield* compiledState(root); - const canonicalRoot = yield* fs.realPath(root); - expect(yield* owner.resolveProjectFile(state, "nested/config.json")).toBe( - path.join(canonicalRoot, "nested", "config.json"), - ); - for (const configured of ["/etc/passwd", "../outside.json", "linked.json"] as const) { - const failed = yield* owner.resolveProjectFile(state, configured).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - } - const directory = yield* owner.resolveProjectFile(state, "nested").pipe(Effect.exit); - expect(Exit.isFailure(directory)).toBe(true); - }), - ), - ); - - it.live("returns all private local keys and a public-only JWKS", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-jwks-" }); - const first = generateKeyPairSync("ec", { namedCurve: "prime256v1" }).privateKey.export({ - format: "jwk", - }); - const second = generateKeyPairSync("rsa", { modulusLength: 2048 }).privateKey.export({ - format: "jwk", - }); - yield* fs.writeFileString( - path.join(root, "keys.json"), - encodeJson([ - { ...first, alg: "ES256", kid: "ec-key" }, - { ...second, alg: "RS256", kid: "rsa-key" }, - ]), - ); - const base = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { - workos: { enabled: true, issuer_url: "https://issuer.example" }, - }, - }, - }, - }, - }); - if (base.definition === undefined) return yield* Effect.die("compiled definition missing"); - const state: PersistedStackState = { - ...base, - definition: { - ...base.definition, - security: { - ...base.definition.security, - jwt: { - ...base.definition.security.jwt, - signing: { kind: "jwks-file", path: "keys.json" }, - }, - }, - }, - }; - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: (url) => - Effect.succeed( - url.endsWith("openid-configuration") - ? { jwks_uri: "https://issuer.example/keys" } - : { keys: [{ kty: "RSA", n: "n", e: "AQAB" }] }, - ), - }); - const material = yield* owner.resolve(state, "auth:auth"); - expect(decodeJson(material.auth?.jwtKeys ?? "[]")).toHaveLength(2); - const jwks = decodeJwks(material.auth?.jwks ?? '{"keys":[]}'); - expect(jwks.keys).toHaveLength(3); - expect( - jwks.keys.every( - (key) => typeof key === "object" && key !== null && !Object.hasOwn(key, "d"), - ), - ).toBe(true); - }), - ), - ); - - it.live("rejects a JWKS file when any configured key is invalid", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-jwks-invalid-" }); - const valid = generateKeyPairSync("ec", { namedCurve: "prime256v1" }).privateKey.export({ - format: "jwk", - }); - yield* fs.writeFileString( - path.join(root, "keys.json"), - encodeJson([ - { ...valid, alg: "ES256" }, - { kty: "EC", alg: "ES256", d: "bad" }, - ]), - ); - const base = yield* compiledState(root); - if (base.definition === undefined) return yield* Effect.die("compiled definition missing"); - const state: PersistedStackState = { - ...base, - definition: { - ...base.definition, - security: { - ...base.definition.security, - jwt: { - ...base.definition.security.jwt, - signing: { kind: "jwks-file", path: "keys.json" }, - }, - }, - }, - }; - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const failed = yield* owner.resolve(state, "auth:auth").pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - }), - ), - ); - - it.live("merges injected third-party keys with the canonical symmetric key", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-oidc-" }); - const state = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { firebase: { enabled: true, project_id: "demo" } }, - }, - }, - }, - }); - const requested: string[] = []; - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: (url) => - Effect.sync(() => { - requested.push(url); - return url.endsWith("openid-configuration") - ? { jwks_uri: "https://issuer.example/keys" } - : { keys: [{ kty: "RSA", n: "n", e: "AQAB" }] }; - }), - }); - const material = yield* owner.resolve(state, "auth:auth"); - expect(requested).toEqual([ - "https://securetoken.google.com/demo/.well-known/openid-configuration", - "https://issuer.example/keys", - ]); - expect(decodeJwks(material.auth?.jwks ?? '{"keys":[]}').keys).toHaveLength(2); - }), - ), - ); - - it.live("publishes the persisted symmetric JWT secret as an oct JWK", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-symmetric-", - }); - const state = yield* compiledState(root, { - capabilities: { - auth: { settings: { jwt_secret: Redacted.make("symmetric-secret") } }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const jwks = decodeJwks( - (yield* owner.resolve(state, "auth:auth")).auth?.jwks ?? '{"keys":[]}', - ); - expect(jwks.keys).toEqual([ - { - kty: "oct", - alg: "HS256", - use: "sig", - key_ops: ["verify"], - k: "c3ltbWV0cmljLXNlY3JldA", - }, - ]); - }), - ), - ); - - it.live("skips JWT file and OIDC resolution when every JWT consumer is disabled", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-no-jwt-" }); - const base = yield* compiledState(root); - if (base.definition === undefined) return yield* Effect.die("compiled definition missing"); - const definition = base.definition; - const state: PersistedStackState = { - ...base, - definition: { - ...definition, - capabilities: { - ...definition.capabilities, - rest: { ...definition.capabilities.rest, enabled: false }, - auth: { ...definition.capabilities.auth, enabled: false }, - realtime: { ...definition.capabilities.realtime, enabled: false }, - storage: { ...definition.capabilities.storage, enabled: false }, - functions: { ...definition.capabilities.functions, enabled: false }, - }, - security: { - ...definition.security, - jwt: { - ...definition.security.jwt, - signing: { kind: "jwks-file", path: "missing.json" }, - }, - }, - }, - }; - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: () => Effect.die("OIDC fetch should not run"), - }); - const material = yield* owner.resolve(state, "auth:auth"); - expect(material.auth).toBeUndefined(); - }), - ), - ); - - it.live("resolves only material needed by the requested workload", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-workload-scope-", - }); - const state = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { - workos: { enabled: true, issuer_url: "https://issuer.example" }, - }, - }, - }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: () => Effect.fail(new StackPreparationError({ message: "OIDC unavailable" })), - }); - const database = yield* owner.resolve(state, "database:database"); - expect(database.auth).toBeUndefined(); - const auth = yield* owner.resolve(state, "auth:auth").pipe(Effect.exit); - expect(Exit.isFailure(auth)).toBe(true); - }), - ), - ); - - it.live("creates the configured Functions root only for mounted workloads", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-functions-root-" }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - functions: { enabled: true }, - studio: { enabled: true }, - }, - }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const state = stateFor(root, compiled, resolved.persisted, { - kind: "container", - engine: "docker", - }); - const withPrivatePorts: PersistedStackState = { - ...state, - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: 30_001 }, - { workloadId: "functions:edge-runtime", binding: "primary", port: 30_002 }, - { workloadId: "functions:edge-runtime", binding: "inspector", port: 30_004 }, - { workloadId: "studio:studio", binding: "primary", port: 30_003 }, - ], - }; - const functionsRoot = compiled.definition?.capabilities.functions.settings.functions_root; - if (functionsRoot === undefined || functionsRoot === null) - return yield* Effect.die("Compiled Functions root is missing"); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - yield* owner.resolve(withPrivatePorts, "database:database"); - expect(yield* fs.exists(functionsRoot)).toBe(false); - - const functionsMaterial = yield* owner.resolve(withPrivatePorts, "functions:edge-runtime"); - const functions = compiled.executionPlan.workloads.find( - ({ id }) => id === "functions:edge-runtime", - ); - if (functions === undefined) return yield* Effect.die("Functions workload is missing"); - const functionsResolution = yield* resolveContainerResolutionFor( - withPrivatePorts, - functions, - functionsMaterial, - ); - expect(yield* fs.exists(functionsRoot)).toBe(true); - expect(functionsResolution?.mounts[0]?.source).toBe(functionsRoot); - - yield* fs.remove(functionsRoot, { recursive: true, force: true }); - expect(yield* fs.exists(functionsRoot)).toBe(false); - const studioMaterial = yield* owner.resolve(withPrivatePorts, "studio:studio"); - const studio = compiled.executionPlan.workloads.find(({ id }) => id === "studio:studio"); - if (studio === undefined) return yield* Effect.die("Studio workload is missing"); - const studioResolution = yield* resolveContainerResolutionFor( - withPrivatePorts, - studio, - studioMaterial, - ); - expect(yield* fs.exists(functionsRoot)).toBe(true); - expect(studioResolution?.mounts[0]?.source).toBe(functionsRoot); - }), - ), - ); - - it.live("resolves JWT material when Auth is disabled but Rest remains enabled", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-rest-jwt-", - }); - const base = yield* compiledState(root); - if (base.definition === undefined) return yield* Effect.die("compiled definition missing"); - const state: PersistedStackState = { - ...base, - definition: { - ...base.definition, - capabilities: { - ...base.definition.capabilities, - auth: { ...base.definition.capabilities.auth, enabled: false }, - }, - }, - }; - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const material = yield* owner.resolve(state, "rest:rest"); - expect(material.auth?.jwks).toContain('"kty":"oct"'); - }), - ), - ); - - it.live("fails when REST needs signing keys from a missing JWKS file", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-rest-jwt-missing-", - }); - const base = yield* compiledState(root, { - capabilities: { - auth: { enabled: false }, - rest: { enabled: true }, - }, - security: { jwt: { signing: { kind: "jwks-file", path: "missing.json" } } }, - }); - if (base.definition === undefined) return yield* Effect.die("compiled definition missing"); - expect(base.definition.capabilities.auth.enabled).toBe(false); - expect(base.definition.capabilities.rest.enabled).toBe(true); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const failed = yield* owner.resolve(base, "rest:rest").pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(errorOf(failed)).toBeInstanceOf(StackPreparationError); - expect(errorOf(failed)?.message).toContain("Unable to resolve Auth signing keys"); - }), - ), - ); - - it.live("sanitizes OIDC URL labels in transport failures", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-oidc-secret-", - }); - const state = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { - workos: { - enabled: true, - issuer_url: "https://issuer.example/tenant?token=secret-token#fragment", - }, - }, - }, - }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: () => Effect.fail(new StackPreparationError({ message: "transport failure" })), - }); - const failed = yield* owner.resolve(state, "auth:auth").pipe(Effect.exit); - const error = errorOf(failed); - expect(error?.message).toContain("OIDC discovery request failed"); - expect(encodeJson(error)).not.toContain("secret-token"); - expect(encodeJson(error)).not.toContain("fragment"); - }), - ), - ); - - it.live("fails closed on malformed or empty third-party OIDC responses", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-oidc-invalid-" }); - const state = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { workos: { enabled: true, issuer_url: "https://issuer.example" } }, - }, - }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: (url) => - Effect.succeed( - url.endsWith("openid-configuration") - ? { jwks_uri: "https://issuer.example/keys" } - : { keys: [] }, - ), - }); - const failed = yield* owner.resolve(state, "auth:auth").pipe(Effect.exit); - expect(errorOf(failed)?.message).toContain("contains no keys"); - }), - ), - ); - - it.live("provides readable Vector config retaining runtime environment placeholders", () => - withPlatform( - Effect.gen(function* () { - const { fs, resolveConfigPath } = yield* vectorFixture(); - - const configPath = yield* resolveConfigPath(); - - const configText = yield* fs.readFileString(configPath); - const config = yield* Effect.try(() => Bun.YAML.parse(configText)); - expect(config).toMatchObject({ - api: { address: "${VECTOR_API_ADDRESS}" }, - sinks: { - analytics: { - uri: "${LOGFLARE_URL}/logs?source_name=postgres.logs", - request: { headers: { "x-api-key": "${LOGFLARE_PRIVATE_ACCESS_TOKEN}" } }, - }, - }, - }); - }), - ), - ); - - it.live("removes the returned Vector config file on cleanup", () => - withPlatform( - Effect.gen(function* () { - const { fs, owner, resolveConfigPath } = yield* vectorFixture(); - const configPath = yield* resolveConfigPath(); - - expect(yield* fs.exists(configPath)).toBe(true); - - yield* owner.cleanupAll; - - expect(yield* fs.exists(configPath)).toBe(false); - }), - ), - ); - - it.live("recreates readable Vector config after cleanup", () => - withPlatform( - Effect.gen(function* () { - const { fs, owner, resolveConfigPath } = yield* vectorFixture(); - const firstConfigPath = yield* resolveConfigPath(); - - yield* owner.cleanupAll; - expect(yield* fs.exists(firstConfigPath)).toBe(false); - - const configPath = yield* resolveConfigPath(); - const config = yield* fs.readFileString(configPath); - - const parsed = yield* Effect.try(() => Bun.YAML.parse(config)); - expect(parsed).toMatchObject({ - api: { address: "${VECTOR_API_ADDRESS}" }, - sinks: { - analytics: { - uri: "${LOGFLARE_URL}/logs?source_name=postgres.logs", - }, - }, - }); - }), - ), - ); - - it.live("shares OIDC material across JWT-consuming workloads", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-auth-shared-", - }); - let fetches = 0; - const state = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { workos: { enabled: true, issuer_url: "https://issuer.example" } }, - }, - }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ - stateRoot: root, - stackId, - fetchJson: (url) => { - fetches += 1; - return Effect.succeed( - url.endsWith("openid-configuration") - ? { jwks_uri: "https://issuer.example/keys" } - : { keys: [{ kty: "RSA", n: "n", e: "AQAB" }] }, - ); - }, - }); - const restMaterial = yield* owner.resolve(state, "rest:rest"); - const cachedRestMaterial = yield* owner.resolve(state, "rest:rest"); - const realtimeMaterial = yield* owner.resolve(state, "realtime:realtime"); - expect(fetches).toBe(2); - expect(cachedRestMaterial.auth?.jwks).toBe(restMaterial.auth?.jwks); - expect(restMaterial.auth?.jwks).toBe(realtimeMaterial.auth?.jwks); - expect(restMaterial.analytics).toBeUndefined(); - expect(realtimeMaterial.analytics).toBeUndefined(); - const changedState = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - third_party: { workos: { enabled: true, issuer_url: "https://other.example" } }, - }, - }, - }, - }); - yield* owner.resolve(changedState, "rest:rest"); - expect(fetches).toBe(4); - }), - ), - ); - - it.live("resolves the configured Analytics service-account file without copying it", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-analytics-" }); - yield* fs.writeFileString(path.join(root, "gcp.json"), "{}"); - const state = yield* compiledState(root, { - capabilities: { analytics: { settings: { gcp_jwt_path: "gcp.json" } } }, - }); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const material = yield* owner.resolve(state, "analytics:analytics"); - expect(material.analytics?.gcpJwtPath).toBe( - path.join(yield* fs.realPath(root), "gcp.json"), - ); - }), - ), - ); - - it.live("does not resolve disabled capability inputs", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-disabled-", - }); - const base = yield* compiledState(root); - if (base.definition === undefined) return yield* Effect.die("compiled definition missing"); - const definition = base.definition; - const state: PersistedStackState = { - ...base, - definition: { - ...definition, - capabilities: { - ...definition.capabilities, - analytics: { - ...definition.capabilities.analytics, - enabled: false, - settings: { - ...definition.capabilities.analytics.settings, - gcp_jwt_path: "missing.json", - }, - }, - functions: { - ...definition.capabilities.functions, - enabled: false, - settings: { - ...definition.capabilities.functions.settings, - edge_runtime: { - policy: null, - deno_version: null, - verify_jwt_default: null, - import_map_default: null, - secrets: { SUPABASE_RESERVED: { slot: "missing" } }, - }, - }, - }, - }, - }, - }; - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const material = yield* owner.resolve(state, "analytics:analytics"); - expect(material.analytics).toBeUndefined(); - expect(material.functions).toBeUndefined(); - }), - ), - ); - - it.live("returns live Auth template mappings and rejects URL id collisions", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "runtime-input-templates-" }); - yield* fs.writeFileString(path.join(root, "confirm.html"), "confirm"); - const state = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - email: { - template: { confirm: { content_path: "confirm.html" } }, - }, - }, - }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const canonicalRoot = yield* fs.realPath(root); - const templates = yield* owner.resolveAuthTemplates(state); - expect(templates).toEqual([ - { - id: "confirm", - path: "confirm.html", - canonicalPath: path.join(canonicalRoot, "confirm.html"), - extension: ".html", - }, - ]); - const collisionState = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - email: { - template: { welcome_notification: { content_path: "confirm.html" } }, - notification: { welcome: { enabled: true, content_path: "confirm.html" } }, - }, - }, - }, - }, - }); - const failed = yield* owner.resolveAuthTemplates(collisionState).pipe(Effect.exit); - expect(errorOf(failed)?.message).toContain("Duplicate Auth email template id"); - yield* fs.writeFileString(path.join(root, "welcome"), "welcome"); - const extensionCollisionState = yield* compiledState(root, { - capabilities: { - auth: { - settings: { - email: { - template: { - welcome: { content_path: "confirm.html" }, - "welcome.html": { content_path: "welcome" }, - }, - }, - }, - }, - }, - }); - const extensionCollision = yield* owner - .resolveAuthTemplates(extensionCollisionState) - .pipe(Effect.exit); - expect(errorOf(extensionCollision)?.message).toContain("Duplicate Auth email URL"); - }), - ), - ); - - it.live("validates Functions Edge Runtime secrets and returns real names", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "runtime-input-functions-", - }); - const state = yield* compiledState(root, { - capabilities: { - functions: { - enabled: true, - settings: { - edge_runtime: { secrets: { API_TOKEN: Redacted.make("actual-value") } }, - }, - }, - }, - }); - const owner = yield* makeRuntimeInputOwner({ stateRoot: root, stackId }); - const material = yield* owner.resolve(state, "functions:edge-runtime"); - expect(material.functions?.secrets).toEqual({ API_TOKEN: "actual-value" }); - const invalid = yield* compiledState(root, { - capabilities: { - functions: { - enabled: true, - settings: { - edge_runtime: { secrets: { SUPABASE_TOKEN: Redacted.make("value") } }, - }, - }, - }, - }); - const failed = yield* owner.resolve(invalid, "functions:edge-runtime").pipe(Effect.exit); - expect(errorOf(failed)?.message).toContain("secret name is reserved"); - }), - ), - ); -}); diff --git a/packages/stack/src/runtime/schema-init.integration.test.ts b/packages/stack/src/runtime/schema-init.integration.test.ts deleted file mode 100644 index de025a20b3..0000000000 --- a/packages/stack/src/runtime/schema-init.integration.test.ts +++ /dev/null @@ -1,753 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - ConfigProvider, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Redacted, - Schema, - Sink, - Stream, -} from "effect"; -import { ChildProcessSpawner } from "effect/unstable/process"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- capture env-file contents before the scoped workspace is removed -import { readFileSync } from "node:fs"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { deriveStackId } from "../identity/Identity.ts"; -import { RequiresActivatedProcessError } from "../public/Errors.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import { STACK_STATE_FORMAT } from "../state/StackState.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { defaultRuntimeEnvironment } from "../supervisor/Launcher.ts"; -import type { RuntimeArtifactPreparer } from "../preparation/RuntimeArtifacts.ts"; -import type { - ContainerContainerSpec, - ContainerEngine, - ContainerNetworkSpec, - ContainerResource, - ContainerVolumeSpec, -} from "./ContainerEngine.ts"; -import { schemaInitContainerName } from "./ContainerRuntime.ts"; -import { schemaInitWorkloads } from "./SchemaInit.ts"; - -interface FakeContainerState { - resources: Array; - calls: Array; - createdSpecs: Array; - envFiles: Map; - nextId: number; -} - -const fakeContainerEngine = (state: FakeContainerState): ContainerEngine => { - const id = (prefix: string): string => `${prefix}-${state.nextId++}`; - const find = (resourceId: string): ContainerResource | undefined => - state.resources.find((resource) => resource.id === resourceId); - return { - kind: "docker", - preflight: Effect.succeed({ host: "host.docker.internal" }), - probe: Effect.void, - inspectImage: () => Effect.succeed({ present: true }), - pullImage: () => Effect.void, - listResources: () => - Effect.sync(() => { - state.calls.push("list-resources"); - return [...state.resources]; - }), - createNetwork: (spec: ContainerNetworkSpec) => - Effect.sync(() => { - state.calls.push("create-network"); - const resource: ContainerResource = { - id: id("network"), - name: spec.name, - kind: "network", - labels: spec.labels, - }; - state.resources.push(resource); - return resource; - }), - removeNetwork: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`remove-network:${resourceId}`); - state.resources = state.resources.filter( - (resource) => resource.id !== resourceId && resource.name !== resourceId, - ); - }), - createVolume: (spec: ContainerVolumeSpec) => - Effect.sync(() => { - const resource: ContainerResource = { - id: id("volume"), - name: spec.name, - kind: "volume", - labels: spec.labels, - }; - state.resources.push(resource); - return resource; - }), - removeVolume: (resourceId: string) => - Effect.sync(() => { - state.resources = state.resources.filter((resource) => resource.id !== resourceId); - }), - createContainer: (spec: ContainerContainerSpec) => - Effect.sync(() => { - state.calls.push("create-container"); - if (spec.envFile !== undefined) - state.envFiles.set(spec.envFile, readFileSync(spec.envFile, "utf8")); - state.createdSpecs.push(spec); - const resource: ContainerResource = { - id: id("container"), - name: spec.name, - kind: spec.role, - labels: spec.labels, - state: "created", - }; - state.resources.push(resource); - return resource; - }), - copyToContainer: () => Effect.void, - startContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`start:${resourceId}`); - const resource = find(resourceId); - if (resource !== undefined) - state.resources = state.resources.map((entry) => - entry.id === resourceId ? { ...entry, state: "running" } : entry, - ); - }), - waitContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`wait:${resourceId}`); - return 0; - }), - stopContainer: () => Effect.void, - removeContainer: (resourceId: string) => - Effect.sync(() => { - state.calls.push(`remove:${resourceId}`); - state.resources = state.resources.filter((resource) => resource.id !== resourceId); - }), - streamLogs: () => Stream.empty, - }; -}; - -const fakePreparer: RuntimeArtifactPreparer = { - prepare: (_runtime, workload: PlannedWorkload) => - Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: "1", - outcome: "cached", - artifactRoot: "/tmp/schema-init-artifact", - executablePath: "bin/prepare", - image: `example/${workload.capability}:1`, - }), -}; - -const liveStackId = StackIdSchema.make("b".repeat(64)); -const password = Redacted.make("s3cret"); -const jwtSecret = Redacted.make("jwt-secret-value-that-is-long-enough"); -const nativeLaunchEnvSchema = Schema.Struct({ - executable: Schema.optionalKey(Schema.String), - env: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), -}); - -const envFromFile = (text: string): Record => - Object.fromEntries( - text - .split("\n") - .filter((line) => line.includes("=")) - .map((line) => { - const index = line.indexOf("="); - return [line.slice(0, index), line.slice(index + 1)]; - }), - ); - -describe("schemaInit", () => { - it.live("runs container one-shots with distinct names and empty publications", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-live-" }); - const state: FakeContainerState = { - resources: [ - { - id: "net-live", - name: "supabase-live-network", - kind: "network", - labels: { - stackId: liveStackId, - ownerSessionId: "owner-session", - role: "network", - }, - }, - ], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["auth"], - { - kind: "live", - stackId: liveStackId, - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, - ); - expect(state.createdSpecs).toHaveLength(1); - const spec = state.createdSpecs[0]; - expect(spec).toBeDefined(); - if (spec === undefined) return; - expect(spec.name.endsWith("-schema-init")).toBe(true); - expect(spec.publications).toEqual([]); - expect(spec.network).toBe("net-live"); - expect(spec.entrypoint).toBe("/usr/local/bin/auth"); - expect(spec.command).toEqual(["migrate"]); - expect(spec.envFile).toBeDefined(); - if (spec.envFile === undefined) return; - const env = envFromFile(state.envFiles.get(spec.envFile) ?? ""); - expect(env.GOTRUE_DB_DATABASE_URL).toContain("@supabase-database:5432/postgres"); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("joins the ephemeral Postgres network and dials supabase-database:5432", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-eph-" }); - const state: FakeContainerState = { - resources: [], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["auth"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - networkId: "net-eph", - }, - { - containerEngine: fakeContainerEngine(state), - artifactPreparer: fakePreparer, - platform: "linux", - }, - ); - expect(state.calls).not.toContain("create-network"); - const spec = state.createdSpecs[0]; - expect(spec).toBeDefined(); - if (spec === undefined) return; - expect(spec.name).toBe( - schemaInitContainerName({ - stackId: spec.labels.stackId, - workloadId: "auth:auth", - }), - ); - expect(spec.network).toBe("net-eph"); - expect(spec.extraHosts).toEqual(["host.docker.internal:host-gateway"]); - expect(spec.envFile).toBeDefined(); - if (spec.envFile === undefined) return; - const env = envFromFile(state.envFiles.get(spec.envFile) ?? ""); - expect(env.GOTRUE_DB_DATABASE_URL).toContain("@supabase-database:5432/postgres"); - expect(env.GOTRUE_DB_DATABASE_URL).toContain("supabase_auth_admin"); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live( - "rewrites ephemeral docker endpoints to the published URL without a cluster network", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "schema-init-eph-rewrite-", - }); - const state: FakeContainerState = { - resources: [], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["auth"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { - containerEngine: fakeContainerEngine(state), - artifactPreparer: fakePreparer, - platform: "linux", - }, - ); - expect(state.calls).toContain("create-network"); - const spec = state.createdSpecs[0]; - expect(spec).toBeDefined(); - if (spec === undefined) return; - expect(spec.network).not.toBe("net-eph"); - expect(spec.extraHosts).toEqual(["host.docker.internal:host-gateway"]); - expect(spec.envFile).toBeDefined(); - if (spec.envFile === undefined) return; - const env = envFromFile(state.envFiles.get(spec.envFile) ?? ""); - expect(env.GOTRUE_DB_DATABASE_URL).toContain("@host.docker.internal:54322/postgres"); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("removes a schema-init network by name if create is interrupted", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "schema-init-net-interrupt-", - }); - const state: FakeContainerState = { - resources: [], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - const created = yield* Deferred.make(); - const engine = fakeContainerEngine(state); - const hanging: typeof engine = { - ...engine, - createNetwork: (spec) => - Effect.gen(function* () { - yield* engine.createNetwork(spec); - yield* Deferred.succeed(created, undefined); - return yield* Effect.never; - }), - }; - const fiber = yield* Effect.forkChild( - schemaInitWorkloads( - ["auth"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { - containerEngine: hanging, - artifactPreparer: fakePreparer, - platform: "linux", - }, - ), - ); - yield* Deferred.await(created); - yield* Fiber.interrupt(fiber); - expect(state.resources.filter((resource) => resource.kind === "network")).toEqual([]); - expect(state.calls.some((call) => call.startsWith("remove-network:"))).toBe(true); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("resolves pooler env without an activated pooler process", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-pooler-" }); - const state: FakeContainerState = { - resources: [ - { - id: "net-live", - name: "supabase-live-network", - kind: "network", - labels: { - stackId: liveStackId, - ownerSessionId: "owner-session", - role: "network", - }, - }, - ], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["pooler"], - { - kind: "live", - stackId: liveStackId, - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, - ); - expect(state.createdSpecs).toHaveLength(2); - expect(state.createdSpecs.map((spec) => spec.entrypoint)).toEqual([ - "/app/bin/prepare", - "/app/bin/provision-tenant", - ]); - const envFile = state.createdSpecs[0]?.envFile; - expect(envFile).toBeDefined(); - if (envFile === undefined) return; - const env = envFromFile(state.envFiles.get(envFile) ?? ""); - expect(env.DATABASE_URL).toContain("@supabase-database:5432/_supabase"); - expect(env.POSTGRES_HOST).toBe("supabase-database"); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("tags docker analytics as requiring an activated process", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-analytics-" }); - const state: FakeContainerState = { - resources: [], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - const exit = yield* schemaInitWorkloads( - ["analytics"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, - ).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (!Exit.isFailure(exit)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); - expect(error).toBeInstanceOf(RequiresActivatedProcessError); - if (!(error instanceof RequiresActivatedProcessError)) return; - expect(error.capability).toBe("analytics"); - expect(state.createdSpecs).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("skips a disabled capability without creating a one-shot", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-disabled-" }); - const state: FakeContainerState = { - resources: [], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["analytics"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - analytics: { enabled: false }, - }, - }, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, - ); - expect(state.createdSpecs).toEqual([]); - expect(state.calls.filter((call) => call === "create-container")).toEqual([]); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("schema-inits the trio when Studio is on and analytics is off", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-trio-" }); - const state: FakeContainerState = { - resources: [], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["auth", "storage", "realtime"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: { - capabilities: { - studio: { enabled: true }, - analytics: { enabled: false }, - }, - }, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, - ); - const workloads = new Set(state.createdSpecs.map((spec) => spec.labels.workloadId)); - expect(workloads.has("auth:auth")).toBe(true); - expect(workloads.has("storage:storage")).toBe(true); - expect(workloads.has("realtime:realtime")).toBe(true); - expect(workloads.has("studio:studio")).toBe(false); - expect(workloads.has("mail:mail")).toBe(false); - expect(workloads.has("functions:edge-runtime")).toBe(false); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("omits SEED_SELF_HOST from native realtime schema-init", () => { - const recorded: Array<{ readonly executable: string; readonly env: Record }> = - []; - const decoder = new TextDecoder(); - const spawner = ChildProcessSpawner.make(() => - Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(999_999), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: (fd) => - fd === 4 - ? Sink.forEach((chunk: Uint8Array) => - Effect.sync(() => { - const decoded = Schema.decodeOption( - Schema.fromJsonString(nativeLaunchEnvSchema), - )(decoder.decode(chunk)); - if (Option.isNone(decoded)) return; - recorded.push({ - executable: decoded.value.executable ?? "", - env: { ...decoded.value.env }, - }); - }), - ) - : Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ), - ); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-native-" }); - yield* schemaInitWorkloads( - ["realtime"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "native" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { artifactPreparer: fakePreparer }, - ); - expect(recorded).toHaveLength(1); - expect(recorded[0]?.executable.endsWith("bin/prepare")).toBe(true); - expect(recorded[0]?.env.SEED_SELF_HOST).toBeUndefined(); - expect(recorded[0]?.env.APP_NAME).toBe("realtime"); - expect(recorded[0]?.env.GEN_RPC_TCP_SERVER_PORT).toBe("5369"); - expect(recorded[0]?.env.GEN_RPC_SOCKET_IP).toBe("127.0.0.1"); - }), - ).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.provide(NodeServices.layer), - ); - }); - - it.live("includes native schema-init stderr on a one-shot failure", () => { - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const reservedPassword = "s3cret@x"; - const encodedPassword = encodeURIComponent(reservedPassword); - let dbEncKey = ""; - const spawner = ChildProcessSpawner.make(() => - Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(999_998), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(1)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: Stream.suspend(() => - Stream.fromIterable([ - encoder.encode( - `migrate failed password=${reservedPassword} encoded=${encodedPassword} enc=${dbEncKey}\n`, - ), - ]), - ), - all: Stream.empty, - getInputFd: (fd) => - fd === 4 - ? Sink.forEach((chunk: Uint8Array) => - Effect.sync(() => { - const decoded = Schema.decodeOption( - Schema.fromJsonString(nativeLaunchEnvSchema), - )(decoder.decode(chunk)); - if (Option.isNone(decoded)) return; - dbEncKey = decoded.value.env?.DB_ENC_KEY ?? ""; - }), - ) - : Sink.drain, - getOutputFd: () => Stream.empty, - unref: Effect.succeed(Effect.void), - }), - ), - ); - return Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const projectRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "schema-init-native-fail-", - }); - const exit = yield* schemaInitWorkloads( - ["realtime"], - { - kind: "ephemeral", - projectRoot, - runtime: { kind: "native" }, - config: {}, - databaseUrl: `postgresql://postgres:${encodedPassword}@127.0.0.1:54322/postgres`, - secrets: { databasePassword: Redacted.make(reservedPassword), jwtSecret }, - }, - { artifactPreparer: fakePreparer }, - ).pipe(Effect.exit); - expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) { - const pretty = Cause.pretty(exit.cause); - expect(pretty).toContain("stderr: migrate failed"); - expect(pretty).toContain("[REDACTED]"); - expect(pretty).not.toContain(reservedPassword); - expect(pretty).not.toContain(encodedPassword); - expect(dbEncKey.length).toBeGreaterThan(0); - expect(pretty).not.toContain(dbEncKey); - } - }), - ).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.provide(NodeServices.layer), - ); - }); - - it.live("reuses persisted Realtime secrets for live docker schema-init", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const home = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-home-" }); - const projectRoot = yield* fs.makeTempDirectoryScoped({ prefix: "schema-init-live-rt-" }); - const knownEncKey = "live-db-enc-key1"; - const knownKeyBase = "live-secret-key-base-value-that-is-long-enough"; - yield* Effect.gen(function* () { - const env = yield* defaultRuntimeEnvironment; - const identity = { - projectRoot, - branchContext: "main", - stackName: "default", - }; - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - yield* store.initialize(stackId, { - format: STACK_STATE_FORMAT, - identity, - runtime: { kind: "container", engine: "docker" }, - desiredLifecycle: "running", - ports: [], - privatePorts: [], - secrets: { - "secret:realtime.settings.db_enc_key": { - policy: "managed", - value: knownEncKey, - }, - "secret:realtime.settings.secret_key_base": { - policy: "managed", - value: knownKeyBase, - }, - "secret:functions.environment.DOGFOOD_SMTP": { - policy: "passthrough", - value: "smtp-pass", - }, - }, - }); - const state: FakeContainerState = { - resources: [ - { - id: "net-live", - name: "supabase-live-network", - kind: "network", - labels: { - stackId, - ownerSessionId: "owner-session", - role: "network", - }, - }, - ], - calls: [], - createdSpecs: [], - envFiles: new Map(), - nextId: 1, - }; - yield* schemaInitWorkloads( - ["realtime"], - { - kind: "live", - stackId, - projectRoot, - runtime: { kind: "container", engine: "docker" }, - config: {}, - databaseUrl: "postgresql://postgres:s3cret@127.0.0.1:54322/postgres", - secrets: { databasePassword: password, jwtSecret }, - }, - { containerEngine: fakeContainerEngine(state), artifactPreparer: fakePreparer }, - ); - const spec = state.createdSpecs[0]; - expect(spec).toBeDefined(); - if (spec === undefined) return; - expect(spec.envFile).toBeDefined(); - if (spec.envFile === undefined) return; - const launched = envFromFile(state.envFiles.get(spec.envFile) ?? ""); - expect(launched.SEED_SELF_HOST).toBe("true"); - expect(launched.DB_ENC_KEY).toBe(knownEncKey); - expect(launched.SECRET_KEY_BASE).toBe(knownKeyBase); - }).pipe( - Effect.provide(ConfigProvider.layer(ConfigProvider.fromUnknown({ SUPABASE_HOME: home }))), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/runtime/schema-init.unit.test.ts b/packages/stack/src/runtime/schema-init.unit.test.ts deleted file mode 100644 index 99f14a74f5..0000000000 --- a/packages/stack/src/runtime/schema-init.unit.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { - parseSchemaInitDatabaseUrl, - rewriteDatabaseEnvironment, - schemaInitArtifactIdentity, - schemaInitHostGatewayExtraHosts, -} from "./SchemaInit.ts"; - -describe("parseSchemaInitDatabaseUrl", () => { - it("keeps host, port, password, and database name", () => { - const parsed = parseSchemaInitDatabaseUrl( - "postgresql://supabase_auth_admin:s3cret%40x@127.0.0.1:54322/_supabase", - ); - expect(parsed).toEqual({ - host: "127.0.0.1", - port: 54322, - password: "s3cret@x", - database: "_supabase", - }); - }); - - it("rejects a non-postgres URL", () => { - expect(parseSchemaInitDatabaseUrl("https://example.test/postgres")).toBeUndefined(); - }); -}); - -describe("rewriteDatabaseEnvironment", () => { - it("rewrites host, port, and password without changing user or database name", () => { - const rewritten = rewriteDatabaseEnvironment( - { - DB_HOST: "supabase-database", - DB_PORT: "5432", - DB_PASSWORD: "old", - GOTRUE_DB_DATABASE_URL: - "postgresql://supabase_auth_admin:old@supabase-database:5432/postgres", - DATABASE_URL: "ecto://supabase_admin:old@supabase-database:5432/_supabase", - API_EXTERNAL_URL: "http://127.0.0.1:54321", - }, - { host: "host.docker.internal", port: 54322, password: "fresh" }, - ); - expect(rewritten.DB_HOST).toBe("host.docker.internal"); - expect(rewritten.DB_PORT).toBe("54322"); - expect(rewritten.DB_PASSWORD).toBe("fresh"); - expect(rewritten.GOTRUE_DB_DATABASE_URL).toBe( - "postgresql://supabase_auth_admin:fresh@host.docker.internal:54322/postgres", - ); - expect(rewritten.DATABASE_URL).toBe( - "ecto://supabase_admin:fresh@host.docker.internal:54322/_supabase", - ); - expect(rewritten.API_EXTERNAL_URL).toBe("http://127.0.0.1:54321"); - }); -}); - -describe("schemaInitHostGatewayExtraHosts", () => { - it("adds host-gateway only on Linux Engine for host.docker.internal", () => { - expect(schemaInitHostGatewayExtraHosts("linux", "host.docker.internal")).toEqual([ - "host.docker.internal:host-gateway", - ]); - expect(schemaInitHostGatewayExtraHosts("darwin", "host.docker.internal")).toEqual([]); - expect(schemaInitHostGatewayExtraHosts("linux", "127.0.0.1")).toEqual([]); - }); -}); - -describe("schemaInitArtifactIdentity", () => { - it("returns a version:image pin and rejects unknown releases", () => { - const identity = schemaInitArtifactIdentity("auth"); - expect(identity).toMatch(/^v.+:/); - expect(schemaInitArtifactIdentity("auth", "not-a-catalog-release")).toBeUndefined(); - }); -}); diff --git a/packages/stack/src/runtime/workload-runtime.integration.test.ts b/packages/stack/src/runtime/workload-runtime.integration.test.ts deleted file mode 100644 index 2e48ba1000..0000000000 --- a/packages/stack/src/runtime/workload-runtime.integration.test.ts +++ /dev/null @@ -1,1449 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect, Exit, FileSystem, Redacted, Schema } from "effect"; -import { catalogReleaseFor, WORKLOAD_CATALOG } from "../model/WorkloadCatalog.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { deriveStackId } from "../identity/Identity.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { makePortCoordinator, type ListenerIntents } from "../state/PortCoordinator.ts"; -import { bindHeldPort, bindHostListener } from "../supervisor/HostListener.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { - containerResolutionFor, - FUNCTIONS_BOOTSTRAP_CONTAINER_PATH, - FUNCTIONS_CONTAINER_ROOT, - privateBindingIntentsFor, - resolveContainerResolutionFor, - runtimeSpecFor, - validateWorkloadRuntimeInputs, -} from "./WorkloadRuntimeSpec.ts"; - -const disabledListenerIntents: ListenerIntents = { - api: { enabled: false, address: "127.0.0.1", port: "automatic" }, - database: { enabled: false, address: "127.0.0.1", port: "automatic" }, - pooler: { enabled: false, address: "127.0.0.1", port: "automatic" }, - studio: { enabled: false, address: "127.0.0.1", port: "automatic" }, - mailUi: { enabled: false, address: "127.0.0.1", port: "automatic" }, - smtp: { enabled: false, address: "127.0.0.1", port: "automatic" }, - pop3: { enabled: false, address: "127.0.0.1", port: "automatic" }, - functionsInspector: { enabled: false, address: "127.0.0.1", port: "automatic" }, -}; - -const state: PersistedStackState = { - format: "supabase-stack-state-v1", - identity: { - projectRoot: "/tmp/supabase-runtime-spec", - branchContext: "ordinary-workspace", - stackName: "runtime-spec", - }, - runtime: { kind: "native" }, - desiredLifecycle: "running", - ports: [ - { field: "database", port: 55432, intent: "exact" }, - { field: "api", port: 54321, intent: "exact" }, - ], - privatePorts: [ - { workloadId: "database:database", binding: "primary", port: 30_001 }, - { workloadId: "rest:rest", binding: "primary", port: 30_002 }, - { workloadId: "rest:rest", binding: "admin", port: 30_015 }, - { workloadId: "auth:auth", binding: "primary", port: 30_003 }, - { workloadId: "realtime:realtime", binding: "primary", port: 30_004 }, - { workloadId: "realtime:realtime", binding: "rpc", port: 30_019 }, - { workloadId: "storage:storage", binding: "primary", port: 30_005 }, - { workloadId: "storage:imgproxy", binding: "primary", port: 30_006 }, - { workloadId: "functions:edge-runtime", binding: "primary", port: 30_007 }, - { workloadId: "functions:edge-runtime", binding: "inspector", port: 30_018 }, - { workloadId: "studio:studio", binding: "primary", port: 30_008 }, - { workloadId: "studio:pgmeta", binding: "primary", port: 30_009 }, - { workloadId: "mail:mail", binding: "ui", port: 30_010 }, - { workloadId: "mail:mail", binding: "smtp", port: 30_011 }, - { workloadId: "mail:mail", binding: "pop3", port: 30_012 }, - { workloadId: "analytics:analytics", binding: "primary", port: 30_013 }, - { workloadId: "analytics:vector", binding: "primary", port: 30_014 }, - { workloadId: "pooler:pooler", binding: "primary", port: 30_016 }, - { workloadId: "pooler:pooler", binding: "admin", port: 30_017 }, - ], - secrets: { - "secret:database.internal.password": { policy: "managed", value: "postgres" }, - "secret:auth.settings.jwt_secret": { policy: "managed", value: "symmetric-secret" }, - "secret:auth.settings.publishable_key": { - policy: "managed", - value: "sb_publishable_test", - }, - "secret:auth.settings.secret_key": { policy: "managed", value: "sb_secret_test" }, - "secret:realtime.settings.db_enc_key": { policy: "managed", value: "realtime-db-key" }, - "secret:realtime.settings.secret_key_base": { - policy: "managed", - value: "realtime-secret-base", - }, - }, -}; - -const planned = (id: string): PlannedWorkload => { - const release = catalogReleaseFor(id); - if (release === undefined) throw new Error(`Missing test catalog release: ${id}`); - const separator = id.indexOf(":"); - const capability = CAPABILITY_NAMES.find((name) => name === id.slice(0, separator)); - if (capability === undefined) throw new Error(`Missing test capability: ${id}`); - return { - id, - capability, - dependencies: [], - readiness: {}, - artifacts: { - native: { kind: "native", release: release.version }, - container: { kind: "container", image: release.containerImage }, - }, - selected: { kind: "native", release: release.version }, - }; -}; - -describe("workload runtime catalog", () => { - it.live("consumes persisted runtime defaults without rebuilding them", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured: PersistedStackState = { ...state, definition: compiled.definition }; - const rest = planned("rest:rest"); - expect(runtimeSpecFor(rest)?.env(configured, rest, 3000)).toMatchObject({ - PGRST_DB_SCHEMAS: "public,graphql_public", - PGRST_DB_EXTRA_SEARCH_PATH: "public,extensions", - PGRST_DB_MAX_ROWS: "1000", - }); - const auth = planned("auth:auth"); - expect(runtimeSpecFor(auth)?.env(configured, auth, 3000)).toMatchObject({ - GOTRUE_SITE_URL: "http://127.0.0.1:3000", - GOTRUE_JWT_EXP: "3600", - GOTRUE_SMTP_ADMIN_EMAIL: "admin@email.com", - GOTRUE_SMTP_SENDER_NAME: "Admin", - }); - const storage = planned("storage:storage"); - expect(runtimeSpecFor(storage)?.env(configured, storage, 5000)).toMatchObject({ - FILE_SIZE_LIMIT: "52428800", - ENABLE_IMAGE_TRANSFORMATION: "true", - S3_PROTOCOL_ENABLED: "true", - S3_PROTOCOL_ACCESS_KEY_ID: "625729a08b95bf1b7ff351a663f3a23c", - STORAGE_S3_REGION: "local", - }); - const realtime = planned("realtime:realtime"); - expect(runtimeSpecFor(realtime)?.env(configured, realtime, 4000)).toMatchObject({ - MAX_HEADER_LENGTH: "4096", - GEN_RPC_TCP_SERVER_PORT: "30019", - GEN_RPC_TCP_CLIENT_PORT: "30019", - GEN_RPC_SOCKET_IP: "127.0.0.1", - }); - const functions = planned("functions:edge-runtime"); - expect(runtimeSpecFor(functions)?.env(configured, functions, 9000)).toMatchObject({ - SUPABASE_INTERNAL_FUNCTIONS_ROOT: `${state.identity.projectRoot}/supabase/functions`, - EDGE_RUNTIME_POLICY: "per_worker", - EDGE_RUNTIME_DENO_VERSION: "2", - }); - expect(runtimeSpecFor(functions)?.args(configured, functions, 9000)).toContain( - "--policy=per_worker", - ); - - const bigquery = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { capabilities: { analytics: { settings: { backend: "bigquery" } } } }, - }).pipe(Effect.provide(NodeServices.layer)); - const analytics = planned("analytics:analytics"); - expect( - runtimeSpecFor(analytics)?.env( - { ...state, definition: bigquery.definition }, - analytics, - 4000, - ), - ).toMatchObject({ - GOOGLE_PROJECT_ID: "local", - GOOGLE_PROJECT_NUMBER: "0", - }); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("derives one closed private binding intent for every planned workload binding", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - }).pipe(Effect.provide(NodeServices.layer)); - const intents = privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }); - expect(intents).toContainEqual({ workloadId: "database:database", binding: "primary" }); - expect(intents).toContainEqual({ workloadId: "mail:mail", binding: "ui" }); - expect(intents).toContainEqual({ workloadId: "mail:mail", binding: "smtp" }); - expect(intents).toContainEqual({ workloadId: "mail:mail", binding: "pop3" }); - expect(intents).toContainEqual({ workloadId: "rest:rest", binding: "admin" }); - expect(intents).toContainEqual({ workloadId: "realtime:realtime", binding: "rpc" }); - expect(intents.filter(({ workloadId }) => workloadId === "mail:mail")).toHaveLength(3); - expect( - intents.every(({ binding }) => - ["primary", "admin", "ui", "smtp", "pop3", "inspector", "rpc"].includes(binding), - ), - ).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("assigns distinct native realtime rpc ports to two stacks", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - }).pipe(Effect.provide(NodeServices.layer)); - const intents = privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }); - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-realtime-rpc-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const coordinator = makePortCoordinator({ - stateRoot: root, - store, - bindHost: bindHostListener, - bindPrivate: (address, port) => bindHeldPort(address, port, "private-binding"), - }); - const ports: number[] = []; - for (const stackName of ["first", "second"] as const) { - const identity = { ...state.identity, projectRoot: root, stackName }; - const stackId = yield* deriveStackId(identity); - yield* store.initialize(stackId, { - ...state, - identity, - desiredLifecycle: "running", - ports: [], - privatePorts: [], - }); - const reservation = yield* coordinator.acquire(stackId, disabledListenerIntents, intents); - const rpc = reservation.privateAssignments.find( - ({ workloadId, binding }) => workloadId === "realtime:realtime" && binding === "rpc", - ); - if (rpc === undefined) throw new Error("Missing realtime rpc assignment"); - ports.push(rpc.port); - } - expect(ports).toHaveLength(2); - expect(ports[0]).not.toBe(ports[1]); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("omits realtime rpc from Docker publications", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "container", engine: "docker" }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured: PersistedStackState = { - ...state, - definition: compiled.definition, - runtime: { kind: "container", engine: "docker" }, - }; - expect( - privateBindingIntentsFor(compiled.executionPlan, configured).some( - ({ workloadId, binding }) => workloadId === "realtime:realtime" && binding === "rpc", - ), - ).toBe(false); - expect( - containerResolutionFor(configured, planned("realtime:realtime"))?.publications, - ).not.toContainEqual(expect.objectContaining({ containerPort: 5369 })); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("starts the Edge Runtime inspector on its private binding", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { functions: { settings: { inspector: { mode: "run" } } } }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const functions = planned("functions:edge-runtime"); - const configured: PersistedStackState = { - ...state, - definition: compiled.definition, - }; - const spec = runtimeSpecFor(functions); - expect(spec?.args(configured, functions, 30_007)).toContain("--inspect=127.0.0.1:30018"); - expect(spec?.privateEndpoint(configured, "inspector", "native")).toEqual({ - host: "127.0.0.1", - port: 30_018, - }); - expect(spec?.containerArgs(configured, functions, 9000)).toContain("--inspect=0.0.0.0:9229"); - expect(containerResolutionFor(configured, functions)?.publications).toContainEqual({ - address: "127.0.0.1", - hostPort: 30_018, - containerPort: 9229, - }); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("does not reserve disabled pooler or unconfigured inspector bindings", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { capabilities: { pooler: { enabled: false } } }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured = { ...state, definition: compiled.definition }; - const intents = privateBindingIntentsFor(compiled.executionPlan, configured); - expect(intents.some(({ workloadId }) => workloadId === "pooler:pooler")).toBe(false); - expect( - intents.some( - ({ workloadId, binding }) => - workloadId === "functions:edge-runtime" && binding === "inspector", - ), - ).toBe(false); - expect( - containerResolutionFor(configured, planned("functions:edge-runtime"))?.publications, - ).not.toContainEqual(expect.objectContaining({ containerPort: 9229 })); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("defaults an enabled inspector listener to run mode", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { listeners: { functionsInspector: { enabled: true, port: 9223 } } }, - }).pipe(Effect.provide(NodeServices.layer)); - const functions = planned("functions:edge-runtime"); - const configured: PersistedStackState = { - ...state, - definition: compiled.definition, - ports: [...state.ports, { field: "functionsInspector", port: 9223, intent: "exact" }], - }; - expect(runtimeSpecFor(functions)?.args(configured, functions, 30_007)).toContain( - "--inspect=127.0.0.1:30018", - ); - const retainedDisabledPort: PersistedStackState = { - ...configured, - definition: { - ...compiled.definition, - listeners: { - ...compiled.definition.listeners, - functionsInspector: { - ...compiled.definition.listeners.functionsInspector, - enabled: false, - }, - }, - }, - ports: [...state.ports, { field: "functionsInspector", port: 9223, intent: "exact" }], - }; - expect( - runtimeSpecFor(functions)?.args(retainedDisabledPort, functions, 30_007), - ).not.toContain("--inspect=127.0.0.1:30018"); - - const mainOnly = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { capabilities: { functions: { settings: { inspector: { main: true } } } } }, - }).pipe(Effect.provide(NodeServices.layer)); - expect( - runtimeSpecFor(functions)?.args( - { ...configured, definition: mainOnly.definition }, - functions, - 30_007, - ), - ).toEqual([ - "start", - expect.any(String), - "--port=30007", - "--policy=per_worker", - "--inspect=127.0.0.1:30018", - "--inspect-main", - ]); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("compiles pgmeta's primary port before the Vector companion", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { - analytics: { enabled: true, settings: {} }, - studio: { enabled: true }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const relevant = privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }).filter( - ({ workloadId }) => workloadId === "studio:pgmeta" || workloadId === "analytics:vector", - ); - expect(relevant).toEqual([ - { workloadId: "studio:pgmeta", binding: "primary" }, - { workloadId: "analytics:vector", binding: "primary" }, - ]); - - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-pgmeta-ports-" }); - const identity = { - ...state.identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(stackId, { - ...state, - identity, - desiredLifecycle: "running", - ports: [], - privatePorts: [], - }); - const reservation = yield* makePortCoordinator({ - stateRoot: root, - store, - bindHost: bindHostListener, - bindPrivate: (address, port) => bindHeldPort(address, port, "private-binding"), - }).acquire( - stackId, - disabledListenerIntents, - privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }), - ); - const pgmetaPrimary = reservation.privateAssignments.find( - ({ workloadId, binding }) => workloadId === "studio:pgmeta" && binding === "primary", - ); - const vectorPrimary = reservation.privateAssignments.find( - ({ workloadId, binding }) => workloadId === "analytics:vector" && binding === "primary", - ); - if (pgmetaPrimary === undefined || vectorPrimary === undefined) - throw new Error("Compiled plan did not reserve pgmeta and Vector bindings"); - expect(vectorPrimary.port).not.toBe(pgmetaPrimary.port); - const pgmetaResolution = containerResolutionFor( - { ...state, privatePorts: reservation.privateAssignments }, - planned("studio:pgmeta"), - ); - expect(pgmetaResolution?.publications).toEqual([ - { address: "127.0.0.1", hostPort: pgmetaPrimary.port, containerPort: 8080 }, - ]); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it("provides private command, environment and readiness metadata for every workload", () => { - for (const [index, id] of Object.keys(WORKLOAD_CATALOG).entries()) { - const workload = planned(id); - const spec = runtimeSpecFor(workload); - expect(spec).toBeDefined(); - if (spec === undefined) continue; - const port = 30_000 + index; - expect(spec.containerPort).toBeGreaterThan(0); - expect(spec.readiness.protocol).toMatch(/http|tcp/u); - expect(spec.args(state, workload, port)).toBeInstanceOf(Array); - expect(typeof spec.cwd(state, workload)).toBe("string"); - expect(spec.privateEndpoint(state, spec.readiness.binding)).toEqual({ - host: "127.0.0.1", - port: expect.any(Number), - }); - expect(spec.privateEndpoint(state, spec.readiness.binding, "container")).toEqual({ - host: WORKLOAD_CATALOG[id]?.containerAlias, - port: spec.bindings[spec.readiness.binding]?.containerPort, - }); - expect(containerResolutionFor(state, workload)?.networkAliases).toEqual([ - WORKLOAD_CATALOG[id]?.containerAlias, - ]); - } - }); - - it.live("uses durable binding assignments for native endpoints and container publications", () => - Effect.gen(function* () { - const mail = planned("mail:mail"); - const mailResolution = containerResolutionFor(state, mail); - expect(mailResolution?.publications).toEqual([ - { address: "127.0.0.1", hostPort: 30010, containerPort: 8025 }, - { address: "127.0.0.1", hostPort: 30011, containerPort: 1025 }, - { address: "127.0.0.1", hostPort: 30012, containerPort: 1110 }, - ]); - expect(runtimeSpecFor(mail)?.env(state, mail, 30010)).toMatchObject({ - MP_UI_BIND_ADDR: "127.0.0.1:30010", - MP_SMTP_BIND_ADDR: "127.0.0.1:30011", - MP_POP3_BIND_ADDR: "127.0.0.1:30012", - }); - const realtime = planned("realtime:realtime"); - expect(runtimeSpecFor(realtime)?.env(state, realtime, 32000).PORT).toBe("32000"); - const secondState: PersistedStackState = { - ...state, - privatePorts: state.privatePorts.map((assignment) => ({ - ...assignment, - port: assignment.port + 1000, - })), - }; - expect( - runtimeSpecFor(planned("rest:rest"))?.env(secondState, planned("rest:rest"), 3000), - ).toMatchObject({ - PGRST_DB_URI: expect.stringContaining("@127.0.0.1:31001"), - }); - expect( - runtimeSpecFor(planned("storage:storage"))?.env( - secondState, - planned("storage:storage"), - 5000, - ).IMGPROXY_URL, - ).toBe("http://127.0.0.1:31006"); - const missing = { - ...state, - privatePorts: state.privatePorts.filter( - ({ workloadId, binding }) => !(workloadId === mail.id && binding === "smtp"), - ), - }; - const failed = yield* resolveContainerResolutionFor(missing, mail).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("uses owner-resolved StackId data paths for native persistence", () => - Effect.sync(() => { - const database = planned("database:database"); - const storage = planned("storage:storage"); - const databaseSpec = runtimeSpecFor(database); - const storageSpec = runtimeSpecFor(storage); - expect( - databaseSpec?.env(state, database, 5432, "native", { - database: { dataPath: "/state/stack/data/database" }, - }).PGDATA, - ).toBe("/state/stack/data/database"); - expect( - storageSpec?.env(state, storage, 5000, "native", { - storage: { dataPath: "/state/stack/data/storage" }, - }).FILE_STORAGE_BACKEND_PATH, - ).toBe("/state/stack/data/storage"); - expect( - storageSpec?.env(state, storage, 5000, "container", { - storage: { dataPath: "/ignored/native/path" }, - }).FILE_STORAGE_BACKEND_PATH, - ).toBe("/mnt"); - expect( - runtimeSpecFor(planned("storage:imgproxy"))?.env( - state, - planned("storage:imgproxy"), - 5001, - "native", - { - storage: { dataPath: "/state/stack/data/storage" }, - }, - ).IMGPROXY_LOCAL_FILESYSTEM_ROOT, - ).toBe("/"); - expect( - runtimeSpecFor(planned("storage:imgproxy"))?.env( - state, - planned("storage:imgproxy"), - 5001, - "container", - ).IMGPROXY_LOCAL_FILESYSTEM_ROOT, - ).toBe("/"); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it("resolves service-owned startup processes and container init contracts", () => { - const root = "/tmp/slim artifact"; - const auth = planned("auth:auth"); - expect(runtimeSpecFor(auth)?.nativeStartupProcesses(root, state, auth, 9999)).toEqual([ - { executable: `${root}/bin/auth`, args: ["migrate"], cwd: root }, - ]); - const storage = planned("storage:storage"); - expect(runtimeSpecFor(storage)?.nativeStartupProcesses(root, state, storage, 5000)).toEqual([ - { executable: `${root}/bin/prepare`, args: [], cwd: root }, - ]); - const realtime = planned("realtime:realtime"); - expect(runtimeSpecFor(realtime)?.nativeStartupProcesses(root, state, realtime, 4000)).toEqual([ - { executable: `${root}/bin/prepare`, args: [], cwd: root }, - ]); - const analytics = planned("analytics:analytics"); - expect(runtimeSpecFor(analytics)?.nativeStartupProcesses(root, state, analytics, 4000)).toEqual( - [{ executable: `${root}/bin/prepare`, args: [], cwd: root }], - ); - const pooler = planned("pooler:pooler"); - expect(runtimeSpecFor(pooler)?.nativeStartupProcesses(root, state, pooler, 6543)).toEqual([ - { executable: `${root}/bin/prepare`, args: [], cwd: root }, - { executable: `${root}/bin/provision-tenant`, args: [], cwd: root }, - ]); - expect(runtimeSpecFor(pooler)?.env(state, pooler, 6543, "container")).toMatchObject({ - POSTGRES_HOST: "supabase-database", - POSTGRES_PORT: "5432", - POSTGRES_PASSWORD: "postgres", - }); - - expect(containerResolutionFor(state, auth)?.command).toEqual([]); - expect(containerResolutionFor(state, auth)?.startup).toEqual([ - { entrypoint: "/usr/local/bin/auth", command: ["migrate"] }, - ]); - expect(containerResolutionFor(state, storage)?.command).toEqual([]); - expect(containerResolutionFor(state, storage)?.startup).toEqual([ - { entrypoint: "/slim-runtime/bin/prepare", command: [] }, - ]); - expect(containerResolutionFor(state, realtime)?.entrypoint).toBe("/usr/bin/tini"); - expect(containerResolutionFor(state, realtime)?.command).toEqual([ - "-s", - "-g", - "--", - "/app/bin/server", - ]); - expect(containerResolutionFor(state, realtime)?.startup).toEqual([ - { entrypoint: "/app/bin/prepare", command: [] }, - ]); - expect(containerResolutionFor(state, analytics)?.command).toEqual([]); - expect(containerResolutionFor(state, pooler)?.entrypoint).toBe("/usr/bin/tini"); - expect(containerResolutionFor(state, pooler)?.command).toEqual([ - "-s", - "-g", - "--", - "/app/bin/server", - ]); - expect(containerResolutionFor(state, pooler)?.startup).toEqual([ - { entrypoint: "/app/bin/prepare", command: [] }, - { entrypoint: "/app/bin/provision-tenant", command: [] }, - ]); - }); - - const compileNestedConfiguredState = () => - compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { - rest: { - settings: { - schemas: ["private"], - extra_search_path: ["extensions"], - external_url: "https://api.example", - }, - }, - storage: { - settings: { - image_transformation: { enabled: true }, - s3_protocol: { enabled: false }, - }, - }, - functions: { - settings: { - edge_runtime: { - policy: "oneshot", - deno_version: 1, - verify_jwt_default: false, - import_map_default: "shared-deno.json", - }, - inspector: { mode: "brk", main: true }, - functions: { hello: { verify_jwt: false } }, - }, - }, - studio: { settings: { api_url: "https://studio.example" } }, - pooler: { enabled: true, settings: { pool_mode: "session", max_client_conn: 250 } }, - auth: { - settings: { - site_url: "https://example.test", - additional_redirect_urls: ["https://example.test/callback"], - jwt_issuer: "https://issuer.example", - enable_signup: false, - minimum_password_length: 12, - password_requirements: "letters_digits", - email: { - double_confirm_changes: true, - secure_password_change: false, - template: { - confirmation: { - content_path: "templates/confirmation.html", - subject: "Confirm", - }, - }, - notification: { - password_recovery: { - enabled: true, - content_path: "templates/recovery.html", - subject: "Reset", - }, - }, - }, - sms: { - enable_signup: true, - twilio: { enabled: true, account_sid: "AC123", message_service_sid: "MG123" }, - twilio_verify: { enabled: true, account_sid: "VA123" }, - test_otp: { "+33123456789": "123456" }, - }, - mfa: { phone: { otp_length: 8 } }, - }, - }, - analytics: { - settings: { - backend: "bigquery", - gcp_project_id: "project-42", - gcp_project_number: "42", - gcp_jwt_path: "secrets/gcp.json", - }, - }, - realtime: { settings: { ip_version: "IPv6" } }, - }, - }, - }).pipe( - Effect.map((compiled): PersistedStackState => ({ - ...state, - definition: compiled.definition, - secrets: { - ...state.secrets, - "secret:analytics.settings.api_key": { policy: "passthrough", value: "api-key" }, - }, - })), - Effect.provide(NodeServices.layer), - ); - - it.live("maps Rest and dependent database endpoints from nested settings", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const rest = planned("rest:rest"); - expect(runtimeSpecFor(rest)?.env(configured, rest, 3000, "native")).toMatchObject({ - PGRST_DB_SCHEMAS: "private", - PGRST_DB_EXTRA_SEARCH_PATH: "extensions", - PGRST_DB_URI: expect.stringContaining("@127.0.0.1:30001"), - PGRST_ADMIN_SERVER_PORT: "30015", - PGRST_OPENAPI_SERVER_PROXY_URI: "https://api.example", - }); - expect(runtimeSpecFor(rest)?.env(configured, rest, 3000, "container")).toMatchObject({ - PGRST_DB_URI: expect.stringContaining("@supabase-database:5432"), - PGRST_ADMIN_SERVER_PORT: "3001", - }); - for (const { id, key } of [ - { id: "realtime:realtime", key: "DB_HOST" }, - { id: "studio:pgmeta", key: "PG_META_DB_HOST" }, - { id: "analytics:analytics", key: "DB_HOSTNAME" }, - ]) { - expect( - runtimeSpecFor(planned(id))?.env(configured, planned(id), 4000, "container")[key], - ).toBe("supabase-database"); - } - }), - ); - - it.live("maps Storage capability settings and persistence paths", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const storage = planned("storage:storage"); - expect(runtimeSpecFor(storage)?.env(configured, storage, 5000)).toMatchObject({ - ENABLE_IMAGE_TRANSFORMATION: "true", - S3_PROTOCOL_ENABLED: "false", - FILE_SIZE_LIMIT: "52428800", - VECTOR_ENABLED: "true", - VECTOR_BUCKET_PROVIDER: "pgvector", - VECTOR_STORE_MIGRATIONS_ENABLED: "true", - VECTOR_DATABASE_URL: expect.stringContaining("postgres:postgres@127.0.0.1"), - }); - expect( - runtimeSpecFor(storage)?.env(configured, storage, 5000, "container").IMGPROXY_URL, - ).toBe("http://supabase-imgproxy:5001"); - }), - ); - - it.live.each([ - { input: "50MiB", expected: "52428800" }, - { input: "1.5KB", expected: "1500" }, - { input: "2 GiB", expected: "2147483648" }, - ])("normalizes Storage file size $input for the runtime", ({ input, expected }) => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { capabilities: { storage: { settings: { file_size_limit: input } } } }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured: PersistedStackState = { ...state, definition: compiled.definition }; - const storage = planned("storage:storage"); - - expect(runtimeSpecFor(storage)?.env(configured, storage, 5000).FILE_SIZE_LIMIT).toBe( - expected, - ); - }), - ); - - it.live("maps Auth capability settings and template requirements", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const auth = planned("auth:auth"); - const authSpec = runtimeSpecFor(auth); - const authEnvironment = authSpec?.env(configured, auth, 9999); - const authTemplateEnvironment = authSpec?.env(configured, auth, 9999, "native", { - auth: { templateBaseUrl: "http://supabase-gateway:8088" }, - }); - const authEnvironmentWithKeys = authSpec?.env(configured, auth, 9999, "container", { - auth: { jwtKeys: '[{"kty":"EC"}]' }, - }); - expect(authEnvironment).toMatchObject({ - GOTRUE_SITE_URL: "https://example.test", - GOTRUE_URI_ALLOW_LIST: "https://example.test/callback", - GOTRUE_DISABLE_SIGNUP: "true", - GOTRUE_PASSWORD_MIN_LENGTH: "12", - GOTRUE_PASSWORD_REQUIRED_CHARACTERS: - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ:0123456789", - GOTRUE_SMS_PROVIDER: "twilio", - GOTRUE_SMS_TEST_OTP: "+33123456789:123456", - GOTRUE_SMS_OTP_LENGTH: "6", - GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: "true", - GOTRUE_SECURITY_UPDATE_PASSWORD_REQUIRE_REAUTHENTICATION: "false", - GOTRUE_SMTP_HOST: "127.0.0.1", - GOTRUE_SMTP_PORT: "30011", - GOTRUE_JWT_ISSUER: "https://issuer.example", - GOTRUE_SMS_TWILIO_ACCOUNT_SID: "AC123", - GOTRUE_SMS_TWILIO_VERIFY_ACCOUNT_SID: "VA123", - }); - expect(authEnvironment?.GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI).toBeUndefined(); - expect(authEnvironmentWithKeys?.GOTRUE_JWT_KEYS).toBe('[{"kty":"EC"}]'); - expect(authTemplateEnvironment).toMatchObject({ - GOTRUE_MAILER_TEMPLATES_CONFIRMATION: - "http://supabase-gateway:8088/email/confirmation.html", - GOTRUE_MAILER_SUBJECTS_CONFIRMATION: "Confirm", - GOTRUE_MAILER_NOTIFICATIONS_PASSWORD_RECOVERY_ENABLED: "true", - GOTRUE_MAILER_TEMPLATES_PASSWORD_RECOVERY_NOTIFICATION: - "http://supabase-gateway:8088/email/password_recovery_notification.html", - GOTRUE_MAILER_SUBJECTS_PASSWORD_RECOVERY_NOTIFICATION: "Reset", - }); - yield* validateWorkloadRuntimeInputs(configured, auth, { - auth: { templateBaseUrl: "http://supabase-gateway:8088" }, - }); - const missingTemplateBase = yield* validateWorkloadRuntimeInputs(configured, auth).pipe( - Effect.exit, - ); - expect(Exit.isFailure(missingTemplateBase)).toBe(true); - }), - ); - - it.live("maps Realtime readiness and JWT material from nested settings", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const realtime = planned("realtime:realtime"); - expect( - runtimeSpecFor(realtime)?.env(configured, realtime, 4000, "container", { - auth: { jwks: '{"keys":[]}' }, - }), - ).toMatchObject({ - ERL_AFLAGS: "-proto_dist inet6_tcp", - API_JWT_JWKS: '{"keys":[]}', - }); - expect(runtimeSpecFor(realtime)?.readiness).toMatchObject({ path: "/healthcheck" }); - expect(runtimeSpecFor(realtime)?.readiness.headers).toBeUndefined(); - }), - ); - - it.live("maps Analytics credentials and mount inputs", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const analytics = planned("analytics:analytics"); - const analyticsSpec = runtimeSpecFor(analytics); - expect( - analyticsSpec?.env(configured, analytics, 4000, "native", { - analytics: { gcpJwtPath: "/tmp/gcp.json" }, - }), - ).toMatchObject({ - GOOGLE_PROJECT_ID: "project-42", - GOOGLE_PROJECT_NUMBER: "42", - GOOGLE_APPLICATION_CREDENTIALS: "/tmp/gcp.json", - LOGFLARE_PRIVATE_ACCESS_TOKEN: "api-key", - }); - expect(analyticsSpec?.env(configured, analytics, 4000, "container")).toMatchObject({ - LOGFLARE_PRIVATE_ACCESS_TOKEN: "api-key", - }); - expect(analyticsSpec?.env(configured, analytics, 4000, "native")).not.toHaveProperty( - "LOGFLARE_PUBLIC_ACCESS_TOKEN", - ); - expect( - analyticsSpec?.containerMounts?.(configured, analytics, { - analytics: { gcpJwtPath: "/tmp/gcp.json" }, - }), - ).toEqual([ - { - source: "/tmp/gcp.json", - target: "/opt/app/rel/logflare/bin/gcloud.json", - readOnly: true, - }, - ]); - expect(analyticsSpec?.args(configured, analytics, 4000)).toEqual(["start"]); - expect(containerResolutionFor(configured, analytics)?.command).toEqual([]); - }), - ); - - it.live("maps Pooler native and container publications", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const pooler = planned("pooler:pooler"); - const poolerSpec = runtimeSpecFor(pooler); - expect(poolerSpec?.env(configured, pooler, 30016)).toMatchObject({ - POOL_MODE: "session", - MAX_CLIENT_CONN: "250", - TENANT_ID: "pooler-dev", - PORT: "30017", - PROXY_PORT_SESSION: "30016", - PROXY_PORT_TRANSACTION: "6543", - }); - expect(poolerSpec?.env(configured, pooler, 30016, "container")).toMatchObject({ - PROXY_PORT_SESSION: "5432", - PROXY_PORT_TRANSACTION: "6543", - }); - expect(containerResolutionFor(configured, pooler)?.publications).toEqual([ - { address: "127.0.0.1", hostPort: 30016, containerPort: 5432 }, - { address: "127.0.0.1", hostPort: 30017, containerPort: 4000 }, - ]); - }), - ); - - it.live("builds the Functions Edge Runtime launch contract", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const functions = planned("functions:edge-runtime"); - const environment = runtimeSpecFor(functions)?.env(configured, functions, 9000, "native"); - const functionsConfig = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( - environment?.SUPABASE_INTERNAL_FUNCTIONS_CONFIG ?? "{}", - ); - expect(functionsConfig).toMatchObject({ - $default: { verify_jwt: false, import_map_root: "shared-deno.json" }, - }); - expect(environment).toMatchObject({ SUPABASE_URL: "http://127.0.0.1:54321" }); - expect( - runtimeSpecFor(functions)?.env(configured, functions, 9000, "container", { - hostRoute: { host: "host.docker.internal" }, - }), - ).toMatchObject({ SUPABASE_URL: "http://host.docker.internal:54321" }); - const resolution = containerResolutionFor(configured, functions, { - hostRoute: { host: "host.docker.internal" }, - }); - expect(resolution?.command).toEqual( - expect.arrayContaining([ - "--main-service=" + FUNCTIONS_BOOTSTRAP_CONTAINER_PATH, - "--inspect-brk=0.0.0.0:9229", - "--inspect-main", - ]), - ); - expect(resolution).toMatchObject({ - env: { - EDGE_RUNTIME_POLICY: "oneshot", - EDGE_RUNTIME_DENO_VERSION: "1", - INSPECTOR_MODE: "brk", - INSPECTOR_MAIN: "true", - }, - mounts: [ - { - source: state.identity.projectRoot + "/supabase/functions", - target: FUNCTIONS_CONTAINER_ROOT, - readOnly: true, - }, - ], - }); - expect( - Object.keys(resolution?.env ?? {}).some((key) => key.startsWith("FUNCTIONS_FUNCTIONS_")), - ).toBe(false); - expect(resolution?.env.EDGE_RUNTIME_PORT).toBe("9000"); - }), - ); - - it.live("resolves Functions bootstrap and native process paths", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const functions = planned("functions:edge-runtime"); - const bootstrapResolution = containerResolutionFor(configured, functions, { - functions: { bootstrapPath: "/tmp/functions/4/main.ts" }, - }); - expect(bootstrapResolution?.bootstrap).toEqual({ - source: "/tmp/functions/4/main.ts", - destination: "/root", - }); - expect( - runtimeSpecFor(functions)?.nativeProcess( - "/tmp/edge-artifact", - configured, - functions, - 9000, - { - functions: { bootstrapPath: "/tmp/functions/4/main.ts" }, - }, - ), - ).toMatchObject({ - args: expect.arrayContaining(["--main-service=."]), - cwd: "/tmp/functions/4", - }); - expect( - runtimeSpecFor(functions)?.nativeProcess("/tmp/edge-artifact", configured, functions, 9000), - ).toMatchObject({ - args: expect.arrayContaining(["--main-service=."]), - cwd: state.identity.projectRoot + "/supabase/functions", - }); - }), - ); - - it.live("builds Studio's native and container launch paths", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const studio = planned("studio:studio"); - expect( - runtimeSpecFor(studio)?.env(configured, studio, 3000, "container", { - hostRoute: { host: "host.docker.internal" }, - }), - ).toMatchObject({ - SUPABASE_URL: "http://host.docker.internal:54321", - STUDIO_PG_META_URL: "http://supabase-pgmeta:8080", - LOGFLARE_URL: "http://supabase-analytics:4000", - EDGE_FUNCTIONS_MANAGEMENT_FOLDER: FUNCTIONS_CONTAINER_ROOT, - }); - expect(runtimeSpecFor(studio)?.env(configured, studio, 3000, "native")).toMatchObject({ - SUPABASE_URL: "https://studio.example", - SUPABASE_PUBLIC_URL: "http://127.0.0.1:54321", - }); - expect(runtimeSpecFor(studio)?.containerMounts?.(configured, studio)).toEqual([ - { - source: state.identity.projectRoot + "/supabase/functions", - target: FUNCTIONS_CONTAINER_ROOT, - readOnly: true, - }, - ]); - expect( - runtimeSpecFor(studio)?.nativeProcess("/tmp/native-artifact", configured, studio, 3000), - ).toEqual({ - executable: "/tmp/native-artifact/bin/studio", - args: [], - cwd: state.identity.projectRoot, - }); - }), - ); - - it.live("builds Vector and database native/container launch paths", () => - Effect.gen(function* () { - const configured = yield* compileNestedConfiguredState(); - const vector = planned("analytics:vector"); - expect( - runtimeSpecFor(vector)?.nativeProcess("/tmp/native-artifact", configured, vector, 9001), - ).toEqual({ - executable: "/tmp/native-artifact/bin/vector", - args: ["--config", "/tmp/native-artifact/share/doc/vector/config/vector.yaml"], - cwd: state.identity.projectRoot, - }); - expect( - runtimeSpecFor(vector)?.nativeProcess("/tmp/native-artifact", configured, vector, 9001, { - analytics: { vectorConfigPath: "/tmp/vector.yaml" }, - }), - ).toEqual({ - executable: "/tmp/native-artifact/bin/vector", - args: ["--config", "/tmp/vector.yaml"], - cwd: state.identity.projectRoot, - }); - expect(runtimeSpecFor(vector)?.env(configured, vector, 30014, "native")).toMatchObject({ - VECTOR_API_ADDRESS: "127.0.0.1:30014", - LOGFLARE_URL: "http://127.0.0.1:30013", - LOGFLARE_PRIVATE_ACCESS_TOKEN: "api-key", - }); - expect(runtimeSpecFor(vector)?.env(configured, vector, 9001, "container")).toMatchObject({ - VECTOR_API_ADDRESS: "0.0.0.0:9001", - LOGFLARE_URL: "http://supabase-analytics:4000", - LOGFLARE_PRIVATE_ACCESS_TOKEN: "api-key", - }); - expect(containerResolutionFor(configured, vector)).toMatchObject({ - command: [], - mounts: [], - }); - expect( - containerResolutionFor(configured, vector, { - analytics: { vectorConfigPath: "/tmp/vector.yaml" }, - }), - ).toMatchObject({ - command: ["--config", "/etc/vector/vector.yaml"], - mounts: [{ source: "/tmp/vector.yaml", target: "/etc/vector/vector.yaml", readOnly: true }], - }); - const database = planned("database:database"); - expect( - runtimeSpecFor(database)?.nativeProcess("/tmp/native-artifact", configured, database, 5432), - ).toMatchObject({ - gracefulStopSignal: "SIGINT", - gracefulStopTimeout: "15 seconds", - }); - }), - ); - - it.live("selects PostgREST symmetric and resolved-JWKS credentials", () => - Effect.gen(function* () { - const rest = runtimeSpecFor(planned("rest:rest")); - expect(rest?.env(state, planned("rest:rest"), 3000).PGRST_JWT_SECRET).toBe( - "symmetric-secret", - ); - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { security: { jwt: { signing: { kind: "jwks-file", path: "jwt.json" } } } }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured: PersistedStackState = { ...state, definition: compiled.definition }; - expect( - rest?.env(configured, planned("rest:rest"), 3000, "native", { - auth: { jwks: '{"keys":[]}' }, - }).PGRST_JWT_SECRET, - ).toBe('{"keys":[]}'); - const failed = yield* validateWorkloadRuntimeInputs(configured, planned("rest:rest")).pipe( - Effect.exit, - ); - expect(Exit.isFailure(failed)).toBe(true); - const unresolved = yield* resolveContainerResolutionFor( - configured, - planned("rest:rest"), - ).pipe(Effect.exit); - expect(Exit.isFailure(unresolved)).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("emits only enabled Auth provider fields and gates phone MFA options", () => - Effect.gen(function* () { - const disabledCompiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - }).pipe(Effect.provide(NodeServices.layer)); - const disabledState: PersistedStackState = { - ...state, - definition: disabledCompiled.definition, - }; - const disabled = runtimeSpecFor(planned("auth:auth"))?.env( - disabledState, - planned("auth:auth"), - 9999, - ); - expect(disabled?.GOTRUE_EXTERNAL_GOOGLE_ENABLED).toBe("false"); - expect(disabled?.GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI).toBeUndefined(); - expect(disabled?.GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID).toBeUndefined(); - expect(disabled?.GOTRUE_MFA_PHONE_ENROLL_ENABLED).toBe("false"); - expect(disabled?.GOTRUE_MFA_PHONE_VERIFY_ENABLED).toBe("false"); - expect(disabled?.GOTRUE_MFA_PHONE_OTP_LENGTH).toBeUndefined(); - - const enabledCompiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { - settings: { - external: { - google: { - enabled: true, - client_id: "google-client", - url: "https://accounts.google.test", - }, - }, - mfa: { phone: { enroll_enabled: true, otp_length: 8 } }, - }, - }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const enabledState: PersistedStackState = { - ...state, - definition: enabledCompiled.definition, - }; - const enabled = runtimeSpecFor(planned("auth:auth"))?.env( - enabledState, - planned("auth:auth"), - 9999, - ); - expect(enabled).toMatchObject({ - GOTRUE_EXTERNAL_GOOGLE_ENABLED: "true", - GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID: "google-client", - GOTRUE_EXTERNAL_GOOGLE_URL: "https://accounts.google.test", - GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI: expect.stringContaining("/callback"), - GOTRUE_MFA_PHONE_ENROLL_ENABLED: "true", - GOTRUE_MFA_PHONE_OTP_LENGTH: "8", - GOTRUE_MFA_PHONE_TEMPLATE: "Your code is {{ .Code }}", - GOTRUE_MFA_PHONE_MAX_FREQUENCY: "5s", - }); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("maps persisted Storage S3 credentials and omits vector settings when disabled", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { - storage: { - settings: { - s3_protocol: { - enabled: true, - region: "eu-west-1", - access_key_id: "access-42", - secret_access_key: Redacted.make("secret-42"), - }, - vector: { enabled: false }, - }, - }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured: PersistedStackState = { - ...state, - definition: compiled.definition, - secrets: { - ...state.secrets, - "secret:storage.settings.s3_protocol.secret_access_key": { - policy: "managed", - value: "secret-42", - }, - }, - }; - const env = runtimeSpecFor(planned("storage:storage"))?.env( - configured, - planned("storage:storage"), - 5000, - ); - expect(env).toMatchObject({ - S3_PROTOCOL_ENABLED: "true", - S3_PROTOCOL_ACCESS_KEY_ID: "access-42", - S3_PROTOCOL_ACCESS_KEY_SECRET: "secret-42", - STORAGE_S3_REGION: "eu-west-1", - }); - expect(env).not.toHaveProperty("VECTOR_ENABLED"); - expect(env).not.toHaveProperty("VECTOR_DATABASE_URL"); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it("uses Auth's local JWT secret for every internal JWT consumer", () => { - const consumers = [ - ["auth:auth", "GOTRUE_JWT_SECRET"], - ["realtime:realtime", "API_JWT_SECRET"], - ["storage:storage", "AUTH_JWT_SECRET"], - ["storage:storage", "PGRST_JWT_SECRET"], - ["pooler:pooler", "API_JWT_SECRET"], - ["pooler:pooler", "METRICS_JWT_SECRET"], - ["functions:edge-runtime", "SUPABASE_INTERNAL_JWT_SECRET"], - ] as const; - for (const [id, key] of consumers) { - const workload = planned(id); - const spec = runtimeSpecFor(workload); - expect(spec?.env(state, workload, 3000)[key]).toBe("symmetric-secret"); - } - }); - - it("uses managed per-stack Realtime encryption keys", () => { - const workload = planned("realtime:realtime"); - expect(runtimeSpecFor(workload)?.env(state, workload, 3000)).toMatchObject({ - DB_ENC_KEY: "realtime-db-key", - SECRET_KEY_BASE: "realtime-secret-base", - }); - }); - - it.live("keeps Auth's local JWT secret alongside resolved JWKS material", () => - Effect.gen(function* () { - const compileWith = (config: Parameters[0]["config"]) => - compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config, - }).pipe(Effect.provide(NodeServices.layer)); - const jwks = '{"keys":[{"kty":"EC"}]}'; - const jwksCompiled = yield* compileWith({ - security: { jwt: { signing: { kind: "jwks-file", path: "jwt.json" } } }, - }); - const jwksState: PersistedStackState = { - ...state, - definition: jwksCompiled.definition, - }; - const internal = [ - ["auth:auth", "GOTRUE_JWT_SECRET"], - ["realtime:realtime", "API_JWT_SECRET"], - ["realtime:realtime", "METRICS_JWT_SECRET"], - ["storage:storage", "AUTH_JWT_SECRET"], - ["storage:storage", "PGRST_JWT_SECRET"], - ["pooler:pooler", "API_JWT_SECRET"], - ["pooler:pooler", "METRICS_JWT_SECRET"], - ["functions:edge-runtime", "SUPABASE_INTERNAL_JWT_SECRET"], - ] as const; - for (const [id, key] of internal) { - const workload = planned(id); - const spec = runtimeSpecFor(workload); - expect( - spec?.env(jwksState, workload, 3000, "native", { - auth: { jwtKeys: '[{"kty":"EC"}]', jwks }, - })[key], - ).toBe("symmetric-secret"); - } - expect( - runtimeSpecFor(planned("rest:rest"))?.env(jwksState, planned("rest:rest"), 3000, "native", { - auth: { jwtKeys: '[{"kty":"EC"}]', jwks }, - }).PGRST_JWT_SECRET, - ).toBe(jwks); - - const thirdPartyCompiled = yield* compileWith({ - capabilities: { - auth: { - settings: { - third_party: { firebase: { enabled: true, project_id: "project-42" } }, - }, - }, - }, - }); - const thirdPartyState: PersistedStackState = { - ...state, - definition: thirdPartyCompiled.definition, - }; - expect( - runtimeSpecFor(planned("realtime:realtime"))?.env( - thirdPartyState, - planned("realtime:realtime"), - 3000, - "native", - { auth: { jwks } }, - ).API_JWT_SECRET, - ).toBe("symmetric-secret"); - expect( - runtimeSpecFor(planned("realtime:realtime"))?.env( - thirdPartyState, - planned("realtime:realtime"), - 3000, - "native", - { auth: { jwks } }, - ).API_JWT_JWKS, - ).toBe(jwks); - expect( - runtimeSpecFor(planned("storage:storage"))?.env( - thirdPartyState, - planned("storage:storage"), - 3000, - "native", - { auth: { jwks } }, - ).AUTH_JWT_SECRET, - ).toBe("symmetric-secret"); - expect( - runtimeSpecFor(planned("pooler:pooler"))?.env( - thirdPartyState, - planned("pooler:pooler"), - 3000, - "native", - { auth: { jwks } }, - ).METRICS_JWT_SECRET, - ).toBe("symmetric-secret"); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("requires resolved JWKS material for an enabled third-party provider", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { - settings: { - third_party: { firebase: { enabled: true, project_id: "project-42" } }, - }, - }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const configured: PersistedStackState = { ...state, definition: compiled.definition }; - const failed = yield* validateWorkloadRuntimeInputs(configured, planned("rest:rest")).pipe( - Effect.exit, - ); - expect(Exit.isFailure(failed)).toBe(true); - const valid = yield* validateWorkloadRuntimeInputs(configured, planned("rest:rest"), { - auth: { jwks: '{"keys":[]}' }, - }).pipe(Effect.exit); - expect(Exit.isSuccess(valid)).toBe(true); - }).pipe(Effect.provide(NodeServices.layer)), - ); - - it.live("passes Edge Runtime the JWT material used by functions serve", () => - Effect.gen(function* () { - const functions = planned("functions:edge-runtime"); - const functionSpec = runtimeSpecFor(functions); - expect(functionSpec).toBeDefined(); - if (functionSpec === undefined) return; - const symmetricDefault = containerResolutionFor(state, functions); - expect(symmetricDefault?.env).toMatchObject({ - SUPABASE_INTERNAL_JWT_SECRET: "symmetric-secret", - SUPABASE_INTERNAL_PUBLISHABLE_KEY: "sb_publishable_test", - SUPABASE_INTERNAL_SECRET_KEY: "sb_secret_test", - SUPABASE_INTERNAL_HOST_PORT: "54321", - SUPABASE_JWKS: '{"keys":[]}', - }); - expect( - functionSpec?.containerArgs(state, functions, functionSpec.containerPort), - ).not.toContain("sb_publishable_test"); - expect( - functionSpec?.containerArgs(state, functions, functionSpec.containerPort), - ).not.toContain("sb_secret_test"); - expect(functionSpec?.args(state, functions, functionSpec.containerPort)).not.toContain( - "sb_publishable_test", - ); - expect(functionSpec?.args(state, functions, functionSpec.containerPort)).not.toContain( - "sb_secret_test", - ); - const withoutApiAssignment: PersistedStackState = { - ...state, - ports: state.ports.filter((assignment) => assignment.field !== "api"), - }; - expect(containerResolutionFor(withoutApiAssignment, functions)?.env).not.toHaveProperty( - "SUPABASE_INTERNAL_HOST_PORT", - ); - const symmetric = containerResolutionFor(state, functions, { - auth: { jwks: '{"keys":[{"kty":"EC"}]}' }, - functions: { - secrets: { - APP_SECRET: "value", - EMPTY_SECRET: "", - EDGE_RUNTIME_PORT: "secret-collision", - SUPABASE_INTERNAL_JWT_SECRET: "forbidden-jwt", - SUPABASE_INTERNAL_PUBLISHABLE_KEY: "forbidden-publishable", - SUPABASE_INTERNAL_SECRET_KEY: "forbidden-secret", - SUPABASE_INTERNAL_HOST_PORT: "forbidden-port", - SUPABASE_JWKS: "forbidden-jwks", - }, - }, - }); - expect(symmetric?.env).toMatchObject({ - SUPABASE_INTERNAL_JWT_SECRET: "symmetric-secret", - SUPABASE_INTERNAL_PUBLISHABLE_KEY: "sb_publishable_test", - SUPABASE_INTERNAL_SECRET_KEY: "sb_secret_test", - SUPABASE_INTERNAL_HOST_PORT: "54321", - SUPABASE_JWKS: '{"keys":[{"kty":"EC"}]}', - APP_SECRET: "value", - EMPTY_SECRET: "", - }); - expect(symmetric?.env.EDGE_RUNTIME_PORT).toBe("9000"); - - const jwksCompiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { security: { jwt: { signing: { kind: "jwks-file", path: "jwt.json" } } } }, - }).pipe(Effect.provide(NodeServices.layer)); - const jwksState: PersistedStackState = { ...state, definition: jwksCompiled.definition }; - const jwksResolution = yield* resolveContainerResolutionFor(jwksState, functions, { - auth: { jwtKeys: '[{"kty":"EC"}]', jwks: '{"keys":[{"kty":"EC"}]}' }, - }); - expect(jwksResolution?.env).toMatchObject({ - SUPABASE_JWKS: '{"keys":[{"kty":"EC"}]}', - SUPABASE_INTERNAL_JWT_SECRET: "symmetric-secret", - }); - - const thirdPartyCompiled = yield* compileStack({ - projectRoot: state.identity.projectRoot, - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { - settings: { - third_party: { firebase: { enabled: true, project_id: "project-42" } }, - }, - }, - }, - }, - }).pipe(Effect.provide(NodeServices.layer)); - const thirdPartyState: PersistedStackState = { - ...state, - definition: thirdPartyCompiled.definition, - }; - const thirdPartyResolution = yield* resolveContainerResolutionFor( - thirdPartyState, - functions, - { auth: { jwks: '{"keys":[{"kty":"EC"}]}' } }, - ); - expect(thirdPartyResolution?.env.SUPABASE_JWKS).toBe('{"keys":[{"kty":"EC"}]}'); - expect(thirdPartyResolution?.env.SUPABASE_INTERNAL_JWT_SECRET).toBe("symmetric-secret"); - }).pipe(Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts new file mode 100644 index 0000000000..93273d6c30 --- /dev/null +++ b/packages/stack/src/services/Analytics.ts @@ -0,0 +1,53 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { databaseConnection } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + databaseUrl: Schema.String, + backend: Schema.optionalKey(Schema.Literal("postgres")), + apiKey: Schema.optionalKey(Schema.String), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("analytics", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "analytics", + executable: "bin/logflare", + ports: { http: 4000 }, + healthPath: "/health", + env: (creation, endpoints, container) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const db = yield* databaseConnection(creation.config.databaseUrl); + return { + DATABASE_URL: creation.config.databaseUrl, + ...(http === undefined + ? {} + : { PORT: String(http.port), PHX_HTTP_PORT: String(http.port) }), + DB_DATABASE: "_supabase", + DB_SCHEMA: "_analytics", + DB_HOSTNAME: db.host, + DB_PORT: db.port, + DB_USERNAME: db.username ?? "supabase_admin", + DB_PASSWORD: db.password ?? "postgres", + LOGFLARE_SUPABASE_MODE: "true", + LOGFLARE_SINGLE_TENANT: "true", + ...(container ? {} : { LOGFLARE_GRPC_PORT: "0" }), + ...(creation.config.apiKey === undefined + ? {} + : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), + POSTGRES_BACKEND_URL: creation.config.databaseUrl, + POSTGRES_BACKEND_SCHEMA: "_analytics", + }; + }), + args: () => Effect.succeed(["start"]), + mounts: () => Effect.succeed([]), + startup: [{ args: [], nativeExecutable: "prepare", skipInContainer: true }], +}); diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts new file mode 100644 index 0000000000..f22ee4eb57 --- /dev/null +++ b/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts @@ -0,0 +1,134 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Redacted } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { makeService } from "../Service.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; +import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; +import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live( + "serves Analytics, Vector, and Imgproxy with their real endpoints", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-optional-data-" }); + yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); + const secret = "catalog-optional-data-secret-with-at-least-32-chars"; + const databaseRecipe = yield* makeServiceRecipe( + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + }, + dockerOptions(root), + ); + const database = yield* makeService(databaseRecipe.definition, { + id: "database", + config: databaseRecipe.creation, + }); + yield* database.start; + yield* database.ready; + const databaseRelay = yield* makeDockerTcpRelay(databaseRecipe.endpoint("sql")); + const databaseUrl = `postgresql://supabase_admin:postgres@${databaseRelay.host}:${databaseRelay.port}/postgres`; + + const analyticsRecipe = yield* makeServiceRecipe( + { + service: "analytics", + config: { databaseUrl, backend: "postgres", apiKey: "catalog-analytics" }, + }, + dockerOptions(root), + ); + const analytics = yield* makeService(analyticsRecipe.definition, { + id: "analytics", + config: analyticsRecipe.creation, + }); + yield* analytics.start; + yield* analytics.ready; + const analyticsEndpoint = yield* analyticsRecipe.endpoint("http"); + const analyticsRelay = yield* makeDockerHttpRelay(analyticsRecipe.endpoint("http")); + const analyticsResponse = yield* client.execute( + HttpClientRequest.get( + `http://${analyticsEndpoint.host}:${analyticsEndpoint.port}/health`, + ), + ); + expect(analyticsResponse.status).toBe(200); + + const vectorRecipe = yield* makeServiceRecipe( + { + service: "vector", + config: { + analyticsUrl: `http://${analyticsRelay.host}:${analyticsRelay.port}`, + apiKey: "catalog-analytics", + configPath: `${root}/vector.yaml`, + }, + }, + dockerOptions(root), + ); + yield* fs.writeFileString( + `${root}/vector.yaml`, + "sources:\n dummy:\n type: demo_logs\n format: syslog\n interval: 60\n" + + "sinks:\n print:\n type: console\n inputs: [dummy]\n encoding:\n codec: json\n" + + "api:\n enabled: true\n address: 0.0.0.0:9001\n", + ); + const vector = yield* makeService(vectorRecipe.definition, { + id: "vector", + config: vectorRecipe.creation, + }); + yield* vector.start; + yield* vector.ready; + const vectorEndpoint = yield* vectorRecipe.endpoint("http"); + const vectorResponse = yield* client.execute( + HttpClientRequest.get(`http://${vectorEndpoint.host}:${vectorEndpoint.port}/health`), + ); + expect(vectorResponse.status).toBe(200); + + const imageRoot = `${root}/images`; + yield* fs.makeDirectory(imageRoot, { recursive: true }); + const imgproxyRecipe = yield* makeServiceRecipe( + { service: "imgproxy", config: { filePath: imageRoot } }, + dockerOptions(root), + ); + const imgproxy = yield* makeService(imgproxyRecipe.definition, { + id: "imgproxy", + config: imgproxyRecipe.creation, + }); + yield* imgproxy.start; + yield* imgproxy.ready; + const imgproxyEndpoint = yield* imgproxyRecipe.endpoint("http"); + const imgproxyResponse = yield* client.execute( + HttpClientRequest.get( + `http://${imgproxyEndpoint.host}:${imgproxyEndpoint.port}/health`, + ), + ); + expect(imgproxyResponse.status).toBe(200); + + yield* imgproxy.stop; + yield* vector.stop; + yield* analytics.stop; + yield* database.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); +}); diff --git a/packages/stack/src/services/Auth.ts b/packages/stack/src/services/Auth.ts new file mode 100644 index 0000000000..19e7d679ae --- /dev/null +++ b/packages/stack/src/services/Auth.ts @@ -0,0 +1,118 @@ +import { Effect, Schema } from "effect"; +import { ServiceError } from "../Service.ts"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { localJwtSecret } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +import { Settings, settingsEnvironment } from "./AuthSettings.ts"; + +const Smtp = Schema.Struct({ + host: Schema.String, + port: Schema.Finite, + user: Schema.String, + pass: Schema.String, + adminEmail: Schema.String, + senderName: Schema.optionalKey(Schema.String), +}); + +export const Config = Schema.Struct({ + databaseUrl: Schema.String, + siteUrl: Schema.optionalKey(Schema.String), + externalApiUrl: Schema.optionalKey(Schema.String), + jwtSecret: Schema.optionalKey(Schema.String), + jwtExpiry: Schema.optionalKey(Schema.Finite), + disableSignup: Schema.optionalKey(Schema.Boolean), + smtpUrl: Schema.optionalKey(Schema.String), + smtpAdminEmail: Schema.optionalKey(Schema.String), + smtpSenderName: Schema.optionalKey(Schema.String), + settings: Schema.optionalKey(Settings), + smtp: Schema.optionalKey(Smtp), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("auth", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +const smtpEnvironment = Effect.fn("Auth.smtpEnvironment")((value: string) => + Effect.try({ + try: () => { + const url = new URL(value); + return { + GOTRUE_SMTP_HOST: url.hostname, + GOTRUE_SMTP_PORT: url.port || "25", + GOTRUE_SMTP_ADMIN_EMAIL: "noreply@example.com", + ...(url.username === "" ? {} : { GOTRUE_SMTP_USER: decodeURIComponent(url.username) }), + ...(url.password === "" ? {} : { GOTRUE_SMTP_PASS: decodeURIComponent(url.password) }), + }; + }, + catch: (cause) => new ServiceError({ operation: "launch", message: String(cause), cause }), + }), +); + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "auth", + executable: "bin/auth", + ports: { http: 9999 }, + healthPath: "/health", + env: (creation, endpoints, container) => { + const http = endpoints.get("http"); + const base = { + GOTRUE_API_HOST: container ? "0.0.0.0" : "127.0.0.1", + GOTRUE_DB_DATABASE_URL: creation.config.databaseUrl, + DATABASE_URL: creation.config.databaseUrl, + GOTRUE_DB_DRIVER: "postgres", + GOTRUE_SITE_URL: creation.config.siteUrl ?? "http://localhost:3000", + GOTRUE_JWT_SECRET: creation.config.jwtSecret ?? localJwtSecret, + GOTRUE_JWT_AUD: "authenticated", + GOTRUE_JWT_ADMIN_ROLES: "service_role", + GOTRUE_JWT_DEFAULT_GROUP_NAME: "authenticated", + GOTRUE_MAILER_AUTOCONFIRM: "true", + GOTRUE_DISABLE_SIGNUP: String(creation.config.disableSignup ?? false), + ...(http === undefined ? {} : { GOTRUE_API_PORT: String(http.port) }), + API_EXTERNAL_URL: + creation.config.externalApiUrl ?? creation.config.siteUrl ?? "http://localhost:3000", + ...(creation.config.jwtExpiry === undefined + ? {} + : { GOTRUE_JWT_EXP: String(creation.config.jwtExpiry) }), + ...settingsEnvironment( + creation.config.settings, + creation.config.externalApiUrl ?? creation.config.siteUrl ?? "http://localhost:3000", + ), + }; + const smtp = creation.config.smtp; + if (smtp !== undefined) { + return Effect.succeed({ + ...base, + GOTRUE_SMTP_HOST: smtp.host, + GOTRUE_SMTP_PORT: String(smtp.port), + GOTRUE_SMTP_USER: smtp.user, + GOTRUE_SMTP_PASS: smtp.pass, + GOTRUE_SMTP_ADMIN_EMAIL: smtp.adminEmail, + ...(smtp.senderName === undefined ? {} : { GOTRUE_SMTP_SENDER_NAME: smtp.senderName }), + }); + } + return creation.config.smtpUrl === undefined + ? Effect.succeed(base) + : smtpEnvironment(creation.config.smtpUrl).pipe( + Effect.map((value) => ({ + ...base, + ...value, + ...(creation.config.smtpAdminEmail === undefined + ? {} + : { GOTRUE_SMTP_ADMIN_EMAIL: creation.config.smtpAdminEmail }), + ...(creation.config.smtpSenderName === undefined + ? {} + : { GOTRUE_SMTP_SENDER_NAME: creation.config.smtpSenderName }), + })), + ); + }, + args: () => Effect.succeed([]), + mounts: () => Effect.succeed([]), + startup: [ + { args: ["migrate"], nativeExecutable: "auth", containerEntrypoint: "/usr/local/bin/auth" }, + ], +}); diff --git a/packages/stack/src/services/AuthSettings.ts b/packages/stack/src/services/AuthSettings.ts new file mode 100644 index 0000000000..a9069a3f99 --- /dev/null +++ b/packages/stack/src/services/AuthSettings.ts @@ -0,0 +1,279 @@ +import { Schema } from "effect"; + +const Hook = Schema.Struct({ + enabled: Schema.Boolean, + uri: Schema.optionalKey(Schema.String), + secrets: Schema.optionalKey(Schema.String), +}); + +const Factor = Schema.Struct({ + enroll_enabled: Schema.Boolean, + verify_enabled: Schema.Boolean, +}); + +const External = Schema.Struct({ + enabled: Schema.Boolean, + client_id: Schema.String, + secret: Schema.optionalKey(Schema.String), + url: Schema.String, + redirect_uri: Schema.String, + skip_nonce_check: Schema.Boolean, + email_optional: Schema.Boolean, +}); + +const Twilio = Schema.Struct({ + enabled: Schema.Boolean, + account_sid: Schema.optionalKey(Schema.String), + auth_token: Schema.optionalKey(Schema.String), + message_service_sid: Schema.optionalKey(Schema.String), + content_sid: Schema.optionalKey(Schema.String), +}); + +const Messagebird = Schema.Struct({ + enabled: Schema.Boolean, + access_key: Schema.optionalKey(Schema.String), + originator: Schema.optionalKey(Schema.String), +}); + +const Textlocal = Schema.Struct({ + enabled: Schema.Boolean, + api_key: Schema.optionalKey(Schema.String), + sender: Schema.optionalKey(Schema.String), +}); + +const Vonage = Schema.Struct({ + enabled: Schema.Boolean, + api_key: Schema.optionalKey(Schema.String), + api_secret: Schema.optionalKey(Schema.String), + from: Schema.optionalKey(Schema.String), +}); + +const Template = Schema.Struct({ + subject: Schema.optionalKey(Schema.String), +}); + +const Notification = Schema.Struct({ + enabled: Schema.Boolean, + subject: Schema.optionalKey(Schema.String), +}); + +const Email = Schema.Struct({ + enable_signup: Schema.Boolean, + double_confirm_changes: Schema.Boolean, + enable_confirmations: Schema.Boolean, + secure_password_change: Schema.Boolean, + max_frequency: Schema.String, + otp_length: Schema.Finite, + otp_expiry: Schema.Finite, + template: Schema.Record(Schema.String, Template), + notification: Schema.Record(Schema.String, Notification), +}); + +const Sms = Schema.Struct({ + enable_signup: Schema.Boolean, + enable_confirmations: Schema.Boolean, + max_frequency: Schema.String, + otp_length: Schema.Finite, + otp_expiry: Schema.Finite, + template: Schema.String, + test_otp: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), + twilio: Twilio, + twilio_verify: Twilio, + messagebird: Messagebird, + textlocal: Textlocal, + vonage: Vonage, +}); + +const Mfa = Schema.Struct({ + totp: Factor, + web_authn: Factor, + phone: Schema.Struct({ + ...Factor.fields, + otp_length: Schema.Finite, + template: Schema.String, + max_frequency: Schema.String, + }), + max_enrolled_factors: Schema.Finite, +}); + +const RateLimit = Schema.Struct({ + email_sent: Schema.optionalKey(Schema.Finite), + sms_sent: Schema.optionalKey(Schema.Finite), + anonymous_users: Schema.optionalKey(Schema.Finite), + token_refresh: Schema.optionalKey(Schema.Finite), + sign_in_sign_ups: Schema.optionalKey(Schema.Finite), + token_verifications: Schema.optionalKey(Schema.Finite), + web3: Schema.optionalKey(Schema.Finite), +}); + +const Captcha = Schema.Struct({ + enabled: Schema.Boolean, + provider: Schema.optionalKey(Schema.String), + secret: Schema.optionalKey(Schema.String), +}); + +const Sessions = Schema.Struct({ + timebox: Schema.optionalKey(Schema.String), + inactivity_timeout: Schema.optionalKey(Schema.String), +}); + +const Web3 = Schema.Struct({ + solana: Schema.Struct({ enabled: Schema.Boolean }), + ethereum: Schema.Struct({ enabled: Schema.Boolean }), +}); + +const OAuthServer = Schema.Struct({ + enabled: Schema.Boolean, + authorization_url_path: Schema.String, + allow_dynamic_registration: Schema.Boolean, +}); + +/** Authentication policy passed to the Auth process. */ +export const Settings = Schema.Struct({ + additionalRedirectUrls: Schema.optionalKey(Schema.String), + enableRefreshTokenRotation: Schema.optionalKey(Schema.Boolean), + refreshTokenReuseInterval: Schema.optionalKey(Schema.Finite), + enableManualLinking: Schema.optionalKey(Schema.Boolean), + enableAnonymousSignIns: Schema.optionalKey(Schema.Boolean), + minimumPasswordLength: Schema.optionalKey(Schema.Finite), + passwordRequirements: Schema.optionalKey(Schema.String), + jwtIssuer: Schema.optionalKey(Schema.String), + rateLimit: Schema.optionalKey(RateLimit), + email: Schema.optionalKey(Email), + sms: Schema.optionalKey(Sms), + captcha: Schema.optionalKey(Captcha), + hooks: Schema.optionalKey(Schema.Record(Schema.String, Hook)), + mfa: Schema.optionalKey(Mfa), + sessions: Schema.optionalKey(Sessions), + external: Schema.optionalKey(Schema.Record(Schema.String, External)), + web3: Schema.optionalKey(Web3), + oauthServer: Schema.optionalKey(OAuthServer), +}); +export interface Settings extends Schema.Schema.Type {} + +/** Converts Auth policy into the upstream process configuration. */ +export const settingsEnvironment = ( + settings: Settings | undefined, + externalUrl: string, +): Record => { + if (settings === undefined) return {}; + const env: Record = {}; + const put = (name: string, value: string | boolean | number | undefined) => { + if (value !== undefined) env[name] = String(value); + }; + put("GOTRUE_URI_ALLOW_LIST", settings.additionalRedirectUrls); + put("GOTRUE_SECURITY_REFRESH_TOKEN_ROTATION_ENABLED", settings.enableRefreshTokenRotation); + put("GOTRUE_SECURITY_REFRESH_TOKEN_REUSE_INTERVAL", settings.refreshTokenReuseInterval); + put("GOTRUE_SECURITY_MANUAL_LINKING_ENABLED", settings.enableManualLinking); + put("GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED", settings.enableAnonymousSignIns); + put("GOTRUE_PASSWORD_MIN_LENGTH", settings.minimumPasswordLength); + put("GOTRUE_PASSWORD_REQUIRED_CHARACTERS", settings.passwordRequirements); + put("GOTRUE_JWT_ISSUER", settings.jwtIssuer); + if (settings.email !== undefined) { + const email = settings.email; + put("GOTRUE_EXTERNAL_EMAIL_ENABLED", email.enable_signup); + put("GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED", email.double_confirm_changes); + put("GOTRUE_MAILER_AUTOCONFIRM", !email.enable_confirmations); + put("GOTRUE_SECURITY_UPDATE_PASSWORD_REQUIRE_REAUTHENTICATION", email.secure_password_change); + put("GOTRUE_MAILER_OTP_LENGTH", email.otp_length); + put("GOTRUE_MAILER_OTP_EXP", email.otp_expiry); + put("GOTRUE_SMTP_MAX_FREQUENCY", email.max_frequency); + for (const [name, template] of Object.entries(email.template)) + put(`GOTRUE_MAILER_SUBJECTS_${name.toUpperCase()}`, template.subject); + for (const [name, notification] of Object.entries(email.notification)) { + put(`GOTRUE_MAILER_NOTIFICATIONS_${name.toUpperCase()}_ENABLED`, notification.enabled); + put(`GOTRUE_MAILER_SUBJECTS_${name.toUpperCase()}_NOTIFICATION`, notification.subject); + } + } + if (settings.rateLimit !== undefined) { + const limits = settings.rateLimit; + put("GOTRUE_RATE_LIMIT_EMAIL_SENT", limits.email_sent); + put("GOTRUE_RATE_LIMIT_SMS_SENT", limits.sms_sent); + put("GOTRUE_RATE_LIMIT_ANONYMOUS_USERS", limits.anonymous_users); + put("GOTRUE_RATE_LIMIT_TOKEN_REFRESH", limits.token_refresh); + put("GOTRUE_RATE_LIMIT_OTP", limits.sign_in_sign_ups); + put("GOTRUE_RATE_LIMIT_VERIFY", limits.token_verifications); + put("GOTRUE_RATE_LIMIT_WEB3", limits.web3); + } + if (settings.sms !== undefined) { + const sms = settings.sms; + put("GOTRUE_EXTERNAL_PHONE_ENABLED", sms.enable_signup); + put("GOTRUE_SMS_AUTOCONFIRM", !sms.enable_confirmations); + put("GOTRUE_SMS_MAX_FREQUENCY", sms.max_frequency); + put("GOTRUE_SMS_OTP_LENGTH", sms.otp_length); + put("GOTRUE_SMS_OTP_EXP", sms.otp_expiry); + put("GOTRUE_SMS_TEMPLATE", sms.template); + put( + "GOTRUE_SMS_TEST_OTP", + Object.entries(sms.test_otp ?? {}) + .map(([phone, code]) => `${phone}:${code}`) + .join(","), + ); + const providers = { + twilio: sms.twilio, + twilio_verify: sms.twilio_verify, + messagebird: sms.messagebird, + textlocal: sms.textlocal, + vonage: sms.vonage, + }; + for (const [name, provider] of Object.entries(providers)) { + if (!provider.enabled) continue; + put("GOTRUE_SMS_PROVIDER", name); + for (const [key, value] of Object.entries(provider)) + if (key !== "enabled") put(`GOTRUE_SMS_${name.toUpperCase()}_${key.toUpperCase()}`, value); + break; + } + } + if (settings.captcha !== undefined) { + put("GOTRUE_SECURITY_CAPTCHA_ENABLED", settings.captcha.enabled); + put("GOTRUE_SECURITY_CAPTCHA_PROVIDER", settings.captcha.provider); + put("GOTRUE_SECURITY_CAPTCHA_SECRET", settings.captcha.secret); + } + for (const [name, hook] of Object.entries(settings.hooks ?? {})) { + put(`GOTRUE_HOOK_${name.toUpperCase()}_ENABLED`, hook.enabled); + if (hook.enabled) { + put(`GOTRUE_HOOK_${name.toUpperCase()}_URI`, hook.uri); + put(`GOTRUE_HOOK_${name.toUpperCase()}_SECRETS`, hook.secrets); + } + } + if (settings.mfa !== undefined) { + const mfa = settings.mfa; + for (const [name, factor] of Object.entries({ + totp: mfa.totp, + phone: mfa.phone, + web_authn: mfa.web_authn, + })) { + put(`GOTRUE_MFA_${name.toUpperCase()}_ENROLL_ENABLED`, factor.enroll_enabled); + put(`GOTRUE_MFA_${name.toUpperCase()}_VERIFY_ENABLED`, factor.verify_enabled); + } + put("GOTRUE_MFA_MAX_ENROLLED_FACTORS", mfa.max_enrolled_factors); + put("GOTRUE_MFA_PHONE_TEMPLATE", mfa.phone.template); + put("GOTRUE_MFA_PHONE_OTP_LENGTH", mfa.phone.otp_length); + put("GOTRUE_MFA_PHONE_MAX_FREQUENCY", mfa.phone.max_frequency); + } + put("GOTRUE_SESSIONS_TIMEBOX", settings.sessions?.timebox); + put("GOTRUE_SESSIONS_INACTIVITY_TIMEOUT", settings.sessions?.inactivity_timeout); + for (const [name, provider] of Object.entries(settings.external ?? {})) { + const prefix = `GOTRUE_EXTERNAL_${name.toUpperCase()}`; + put(`${prefix}_ENABLED`, provider.enabled); + if (!provider.enabled) continue; + put(`${prefix}_CLIENT_ID`, provider.client_id); + put(`${prefix}_SECRET`, provider.secret); + put(`${prefix}_URL`, provider.url || undefined); + put( + `${prefix}_REDIRECT_URI`, + provider.redirect_uri || `${externalUrl.replace(/\/+$/u, "")}/callback`, + ); + put(`${prefix}_SKIP_NONCE_CHECK`, provider.skip_nonce_check); + put(`${prefix}_EMAIL_OPTIONAL`, provider.email_optional); + } + put("GOTRUE_EXTERNAL_WEB3_SOLANA_ENABLED", settings.web3?.solana.enabled); + put("GOTRUE_EXTERNAL_WEB3_ETHEREUM_ENABLED", settings.web3?.ethereum.enabled); + put("GOTRUE_OAUTH_SERVER_ENABLED", settings.oauthServer?.enabled); + put("GOTRUE_OAUTH_SERVER_AUTHORIZATION_PATH", settings.oauthServer?.authorization_url_path); + put( + "GOTRUE_OAUTH_SERVER_ALLOW_DYNAMIC_REGISTRATION", + settings.oauthServer?.allow_dynamic_registration, + ); + return env; +}; diff --git a/packages/stack/src/services/AuthStorage.integration.test.ts b/packages/stack/src/services/AuthStorage.integration.test.ts new file mode 100644 index 0000000000..bc16f5a77a --- /dev/null +++ b/packages/stack/src/services/AuthStorage.integration.test.ts @@ -0,0 +1,195 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Redacted, Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { SignJWT } from "jose"; +import { makeService } from "../Service.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; +import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; +import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live( + "migrates Auth and serves Storage against the owned database", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-auth-storage-" }); + yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); + const secret = "catalog-auth-storage-secret-with-at-least-32-chars"; + const databaseRecipe = yield* makeServiceRecipe( + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + }, + dockerOptions(root), + ); + const database = yield* makeService(databaseRecipe.definition, { + id: "database", + config: databaseRecipe.creation, + }); + yield* database.start; + yield* database.ready; + const databaseRelay = yield* makeDockerTcpRelay(databaseRecipe.endpoint("sql")); + const databaseUrl = `postgresql://supabase_admin:postgres@${databaseRelay.host}:${databaseRelay.port}/postgres`; + const authRecipe = yield* makeServiceRecipe( + { + service: "auth", + config: { databaseUrl, jwtSecret: secret, jwtExpiry: 3600 }, + }, + dockerOptions(root), + ); + const auth = yield* makeService(authRecipe.definition, { + id: "auth", + config: authRecipe.creation, + }); + yield* auth.start; + yield* auth.ready; + const authEndpoint = yield* authRecipe.endpoint("http"); + const email = "catalog@example.test"; + const password = "catalog-password-123"; + const signupRequest = yield* HttpClientRequest.bodyJson({ email, password })( + HttpClientRequest.post(`http://${authEndpoint.host}:${authEndpoint.port}/signup`), + ); + const signup = yield* client.execute(signupRequest); + expect(signup.status).toBe(200); + const signupPayload = yield* signup.json; + const accessToken = yield* Schema.decodeUnknownEffect( + Schema.Struct({ access_token: Schema.String }), + )(signupPayload); + expect(accessToken.access_token.length).toBeGreaterThan(0); + const loginRequest = yield* HttpClientRequest.bodyJson({ + email, + password, + })( + HttpClientRequest.post( + `http://${authEndpoint.host}:${authEndpoint.port}/token?grant_type=password`, + ), + ); + const login = yield* client.execute(loginRequest); + const loginBody = yield* login.text; + expect(login.status, loginBody).toBe(200); + const loginToken = yield* Schema.decodeEffect( + Schema.fromJsonString(Schema.Struct({ access_token: Schema.String })), + )(loginBody); + expect(loginToken.access_token.length).toBeGreaterThan(0); + + const storageRoot = `${root}/storage`; + yield* fs.makeDirectory(storageRoot, { recursive: true }); + const imgproxyRecipe = yield* makeServiceRecipe( + { service: "imgproxy", config: { filePath: storageRoot } }, + dockerOptions(root), + ); + const imgproxy = yield* makeService(imgproxyRecipe.definition, { + id: "imgproxy", + config: imgproxyRecipe.creation, + }); + yield* imgproxy.start; + yield* imgproxy.ready; + const imgproxyRelay = yield* makeDockerHttpRelay(imgproxyRecipe.endpoint("http")); + const storageRecipe = yield* makeServiceRecipe( + { + service: "storage", + config: { + databaseUrl, + filePath: storageRoot, + jwtSecret: secret, + imgproxyUrl: `http://${imgproxyRelay.host}:${imgproxyRelay.port}`, + }, + }, + dockerOptions(root), + ); + const storage = yield* makeService(storageRecipe.definition, { + id: "storage", + config: storageRecipe.creation, + }); + yield* storage.start; + yield* storage.ready; + const storageEndpoint = yield* storageRecipe.endpoint("http"); + const serviceToken = yield* Effect.tryPromise(() => + new SignJWT({ role: "service_role" }) + .setProtectedHeader({ alg: "HS256", typ: "JWT" }) + .setSubject("catalog-service") + .setIssuedAt() + .setExpirationTime("1h") + .sign(new TextEncoder().encode(secret)), + ); + const bucketRequest = yield* HttpClientRequest.bodyJson({ name: "catalog" })( + HttpClientRequest.post( + `http://${storageEndpoint.host}:${storageEndpoint.port}/bucket`, + ).pipe( + HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), + HttpClientRequest.setHeader("apikey", serviceToken), + ), + ); + const bucket = yield* client.execute(bucketRequest); + expect(bucket.status).toBe(200); + const uploadRequest = HttpClientRequest.bodyText( + HttpClientRequest.post( + `http://${storageEndpoint.host}:${storageEndpoint.port}/object/catalog/hello.txt`, + ).pipe( + HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), + HttpClientRequest.setHeader("apikey", serviceToken), + ), + "catalog-storage", + "text/plain", + ); + const upload = yield* client.execute(uploadRequest); + expect(upload.status).toBe(200); + const imageBytes = Uint8Array.from( + atob( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + ), + (value) => value.charCodeAt(0), + ); + const imageRequest = HttpClientRequest.bodyUint8Array( + HttpClientRequest.post( + `http://${storageEndpoint.host}:${storageEndpoint.port}/object/catalog/source.png`, + ).pipe( + HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), + HttpClientRequest.setHeader("apikey", serviceToken), + ), + imageBytes, + "image/png", + ); + const imageUpload = yield* client.execute(imageRequest); + expect(imageUpload.status).toBe(200); + const transformedImage = yield* client.execute( + HttpClientRequest.get( + `http://${storageEndpoint.host}:${storageEndpoint.port}/render/image/authenticated/catalog/source.png?width=1&height=1`, + ).pipe( + HttpClientRequest.setHeader("Authorization", `Bearer ${serviceToken}`), + HttpClientRequest.setHeader("apikey", serviceToken), + ), + ); + expect(transformedImage.status).toBe(200); + expect(transformedImage.headers["content-type"]).toContain("image/"); + yield* imgproxy.stop; + yield* storage.stop; + yield* auth.stop; + yield* database.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); +}); diff --git a/packages/stack/src/services/Catalog.integration.test.ts b/packages/stack/src/services/Catalog.integration.test.ts new file mode 100644 index 0000000000..b423e54a18 --- /dev/null +++ b/packages/stack/src/services/Catalog.integration.test.ts @@ -0,0 +1,66 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Layer, Redacted } from "effect"; +import { makeServiceRecipe } from "./Catalog.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +describe("service catalog", () => { + it.live("validates endpoint names at the service creation boundary", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-ports-" }); + const result = yield* makeServiceRecipe( + { + service: "rest", + config: { databaseUrl: "postgres://db" }, + endpoints: { smtp: { port: "auto" } }, + }, + options(root), + ).pipe(Effect.exit); + expect(Exit.isFailure(result)).toBe(true); + const misplacedDatabaseVersion = yield* makeServiceRecipe( + { + service: "database", + version: "17", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make("catalog-database-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + }, + options(root), + ).pipe(Effect.exit); + expect(Exit.isFailure(misplacedDatabaseVersion)).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + + it.live("rejects Storage creation without a file path", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-storage-config-" }); + const error = yield* Effect.flip( + makeServiceRecipe( + { + service: "storage", + config: { databaseUrl: "postgres://db" }, + }, + options(root), + ), + ); + expect(error.operation).toBe("config"); + expect(error.message).toContain("Invalid service creation"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts new file mode 100644 index 0000000000..a7823c4266 --- /dev/null +++ b/packages/stack/src/services/Catalog.ts @@ -0,0 +1,346 @@ +import { Crypto, Effect, FileSystem, Path, Ref, Schema, Stream } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import { HttpClient } from "effect/unstable/http"; +import { makeContainerRuntime } from "../runtime/Container.ts"; +import { ServiceError, type ServiceDefinition } from "../Service.ts"; +import { + makeDatabase, + DatabaseConfig, + DatabaseCreation, + DatabaseEndpoints, + type DatabaseComponent, +} from "./Database.ts"; +import * as Analytics from "./Analytics.ts"; +import * as Auth from "./Auth.ts"; +import * as Functions from "./Functions.ts"; +import * as Imgproxy from "./Imgproxy.ts"; +import * as Mail from "./Mail.ts"; +import * as Pgmeta from "./Pgmeta.ts"; +import * as Pooler from "./Pooler.ts"; +import * as Realtime from "./Realtime.ts"; +import * as Rest from "./Rest.ts"; +import * as Storage from "./Storage.ts"; +import * as Studio from "./Studio.ts"; +import * as Vector from "./Vector.ts"; +import { + CatalogError, + type CatalogOptions, + type CatalogRecipe as RecipeCatalogRecipe, + type ProcessRecipeResult, +} from "./Recipe.ts"; +import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; +import type { ServiceKind } from "../Artifacts.ts"; +import type { ServiceInstanceContext } from "../Service.ts"; + +export type { CatalogLog } from "./Recipe.ts"; + +const endpointSchemas = [ + ["database", DatabaseEndpoints], + ["rest", Rest.Endpoints], + ["auth", Auth.Endpoints], + ["realtime", Realtime.Endpoints], + ["storage", Storage.Endpoints], + ["imgproxy", Imgproxy.Endpoints], + ["functions", Functions.Endpoints], + ["studio", Studio.Endpoints], + ["pgmeta", Pgmeta.Endpoints], + ["mail", Mail.Endpoints], + ["analytics", Analytics.Endpoints], + ["vector", Vector.Endpoints], + ["pooler", Pooler.Endpoints], +] as const; + +export const allowedEndpointNames = (kind: ServiceKind): ReadonlyArray => { + const schema = endpointSchemas.find(([service]) => service === kind)?.[1]; + return schema === undefined ? [] : Object.keys(schema.fields); +}; + +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const validateEndpointNames = (input: unknown): CatalogError | undefined => { + if (!isRecord(input) || typeof input.service !== "string" || !isRecord(input.endpoints)) + return undefined; + const entry = endpointSchemas.find(([service]) => service === input.service); + if (entry === undefined) return undefined; + const invalid = Object.keys(input.endpoints).find( + (name) => !Object.keys(entry[1].fields).includes(name), + ); + return invalid === undefined + ? undefined + : new CatalogError({ + operation: "config", + message: `Endpoint ${invalid} is not supported by service ${input.service}`, + }); +}; + +export const ServiceCreation = Schema.Union([ + DatabaseCreation, + Rest.Creation, + Auth.Creation, + Realtime.Creation, + Storage.Creation, + Imgproxy.Creation, + Functions.Creation, + Studio.Creation, + Pgmeta.Creation, + Mail.Creation, + Analytics.Creation, + Vector.Creation, + Pooler.Creation, +]); +export type ServiceCreation = Schema.Schema.Type; +export const serviceSchemas = { + database: DatabaseConfig, + rest: Rest.Config, + auth: Auth.Config, + realtime: Realtime.Config, + storage: Storage.Config, + imgproxy: Imgproxy.Config, + functions: Functions.Config, + studio: Studio.Config, + pgmeta: Pgmeta.Config, + mail: Mail.Config, + analytics: Analytics.Config, + vector: Vector.Config, + pooler: Pooler.Config, +} as const; + +const serviceError = (operation: string, cause: unknown) => + new ServiceError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const widen = ( + isCreation: (value: unknown) => value is C, + definition: ServiceDefinition, +): ServiceDefinition => ({ + prepare: (candidate) => + isCreation(candidate) + ? (definition.prepare?.(candidate) ?? Effect.void) + : Effect.fail(serviceError("prepare", "Service kind cannot change during restart")), + launch: (context) => + isCreation(context.config) + ? definition.launch({ ...context, config: context.config }) + : Effect.fail(serviceError("launch", "Service kind cannot change during restart")), + removeData: (context) => + isCreation(context.config) + ? definition.removeData({ ...context, config: context.config }) + : Effect.fail(serviceError("destroy", "Service kind cannot change during restart")), +}); + +const catalogRecipe = ( + creation: C, + result: ProcessRecipeResult, + isCreation: (value: unknown) => value is C, +): RecipeCatalogRecipe => ({ + creation, + definition: widen(isCreation, result.definition), + endpoint: (name) => + Ref.get(result.endpoints).pipe( + Effect.flatMap((endpoints) => { + const endpoint = endpoints.get(name); + return endpoint === undefined + ? Effect.fail( + new CatalogError({ + operation: "endpoint", + message: `Endpoint ${name} is not running`, + service: creation.service, + }), + ) + : Effect.succeed(endpoint); + }), + ), + logs: result.logs, +}); + +const databaseRecipe = ( + creation: Schema.Schema.Type, + component: DatabaseComponent, +): CatalogRecipe => ({ + creation, + definition: { + prepare: (candidate) => + candidate.service === "database" + ? (component.definition.prepare?.(candidate.config) ?? Effect.void) + : Effect.fail(serviceError("prepare", "Service kind cannot change during restart")), + launch: (context) => + context.config.service === "database" + ? component.definition.launch({ ...context, config: context.config.config }) + : Effect.fail(serviceError("launch", "Service kind cannot change during restart")), + removeData: (context) => + context.config.service === "database" + ? component.definition.removeData({ ...context, config: context.config.config }) + : Effect.fail(serviceError("destroy", "Service kind cannot change during restart")), + }, + resetDatabaseData: (context) => + context.config.service === "database" + ? component + .resetData({ ...context, config: context.config.config }) + .pipe( + Effect.mapError( + (cause) => new CatalogError({ operation: "reset", message: cause.message, cause }), + ), + ) + : Effect.fail( + new CatalogError({ operation: "reset", message: "Service kind cannot change" }), + ), + endpoint: (name) => + name === "sql" + ? component.endpoint.pipe( + Effect.mapError( + (cause) => + new CatalogError({ + operation: "endpoint", + message: cause.message, + service: "database", + cause, + }), + ), + ) + : Effect.fail( + new CatalogError({ + operation: "endpoint", + message: `Unknown endpoint ${name}`, + service: "database", + }), + ), + logs: component.logs.pipe( + Stream.mapError( + (cause) => + new CatalogError({ operation: "logs", message: cause.message, service: "database", cause }), + ), + ), +}); + +export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( + (input: unknown, options: CatalogOptions) => + Effect.gen(function* () { + const endpointError = validateEndpointNames(input); + if (endpointError !== undefined) return yield* endpointError; + if (isRecord(input) && input.service === "database" && input.version !== undefined) + return yield* new CatalogError({ + operation: "config", + message: "Database artifact version belongs in config.version", + service: "database", + }); + const creation = yield* Schema.decodeUnknownEffect(ServiceCreation)(input).pipe( + Effect.mapError( + (cause) => + new CatalogError({ operation: "config", message: "Invalid service creation", cause }), + ), + ); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const client = yield* HttpClient.HttpClient; + if (creation.service === "database") { + const component = yield* makeDatabase({ + stackId: options.stackId, + instanceId: options.instanceId, + root: options.root, + cacheRoot: options.cacheRoot, + runtime: options.runtime, + }).pipe( + Effect.mapError( + (cause) => + new CatalogError({ + operation: "database", + message: cause.message, + service: "database", + cause, + }), + ), + ); + return databaseRecipe(creation, component); + } + const container = + options.runtime === "native" + ? undefined + : yield* makeContainerRuntime({ engine: options.runtime }); + const deps: ProcessDependencies = { fs, path, crypto, client, spawner, container }; + switch (creation.service) { + case "rest": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Rest.makeSpec()), + Schema.is(Rest.Creation), + ); + case "auth": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Auth.makeSpec()), + Schema.is(Auth.Creation), + ); + case "realtime": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Realtime.makeSpec()), + Schema.is(Realtime.Creation), + ); + case "storage": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Storage.makeSpec()), + Schema.is(Storage.Creation), + ); + case "imgproxy": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Imgproxy.makeSpec()), + Schema.is(Imgproxy.Creation), + ); + case "functions": + return catalogRecipe( + creation, + yield* Functions.makeRecipe(creation, options, deps), + Schema.is(Functions.Creation), + ); + case "studio": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Studio.makeSpec()), + Schema.is(Studio.Creation), + ); + case "pgmeta": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Pgmeta.makeSpec()), + Schema.is(Pgmeta.Creation), + ); + case "mail": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Mail.makeSpec()), + Schema.is(Mail.Creation), + ); + case "analytics": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Analytics.makeSpec()), + Schema.is(Analytics.Creation), + ); + case "vector": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Vector.makeSpec()), + Schema.is(Vector.Creation), + ); + case "pooler": + return catalogRecipe( + creation, + yield* makeProcessRecipe(creation, options, deps, Pooler.makeSpec()), + Schema.is(Pooler.Creation), + ); + } + }), +); + +export type CatalogRecipe = RecipeCatalogRecipe & { + readonly resetDatabaseData?: ( + context: ServiceInstanceContext, + ) => Effect.Effect; +}; diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts new file mode 100644 index 0000000000..f89afc496f --- /dev/null +++ b/packages/stack/src/services/Database.integration.test.ts @@ -0,0 +1,384 @@ +import { PgClient } from "@effect/sql-pg"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Deferred, Effect, FileSystem, Layer, Path, Redacted, Ref, Stream } from "effect"; +import { tmpdir } from "node:os"; +import { makeService } from "../Service.ts"; +import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; + +const config: DatabaseConfig = { + version: "17", + databasePassword: Redacted.make("supabase-test-password"), + jwtSecret: Redacted.make("supabase-test-jwt-secret"), + jwtExpiry: 3600, + rootKey: Redacted.make("a".repeat(64)), +}; + +const artifactCacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + +const query = ( + endpoint: BackendEndpoint, + password: Redacted.Redacted, + statement: string, + database = "postgres", +) => + Effect.scoped( + Effect.gen(function* () { + const host = endpoint.kind === "unix" ? endpoint.path : endpoint.host; + const services = yield* Layer.build( + PgClient.layer({ + host, + port: endpoint.port, + database, + username: "supabase_admin", + password, + }), + ); + return yield* Context.get(services, PgClient.PgClient).unsafe(statement); + }), + ); + +describe("database component", { timeout: 180_000 }, () => { + for (const target of [ + { runtime: "native", version: "17" }, + { runtime: "docker", version: "15" }, + { runtime: "docker", version: "17" }, + ] as const) + it.live( + `preserves default encryption keys and Vault secrets after ${target.runtime} PostgreSQL ${target.version} recreation`, + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-default-key-" }); + const recipe = yield* makeDatabase({ + stackId: "default-key-test", + instanceId: "database", + root, + cacheRoot: artifactCacheRoot, + runtime: target.runtime, + }); + const defaults: DatabaseConfig = { + healthTimeoutMs: 120_000, + version: target.version, + databasePassword: config.databasePassword, + jwtSecret: config.jwtSecret, + jwtExpiry: config.jwtExpiry, + }; + const service = yield* makeService(recipe.definition, { + id: "database", + config: defaults, + }); + yield* service.start; + yield* service.ready; + const endpoint = yield* recipe.endpoint; + yield* query( + endpoint, + defaults.databasePassword, + "CREATE EXTENSION IF NOT EXISTS pgsodium; CREATE EXTENSION IF NOT EXISTS supabase_vault", + ); + const derivation = "SELECT encode(pgsodium.derive_key(1), 'hex') AS key"; + const originalKey = yield* query(endpoint, defaults.databasePassword, derivation); + yield* query( + endpoint, + defaults.databasePassword, + "SELECT vault.create_secret('preserved-value', 'persistence-probe')", + ); + expect( + yield* query( + endpoint, + defaults.databasePassword, + "SELECT decrypted_secret FROM vault.decrypted_secrets WHERE name = 'persistence-probe'", + ), + ).toEqual([{ decrypted_secret: "preserved-value" }]); + yield* service.stop; + yield* service.start; + yield* service.ready; + const restarted = yield* recipe.endpoint; + expect(yield* query(restarted, defaults.databasePassword, derivation)).toEqual( + originalKey, + ); + expect( + yield* query( + restarted, + defaults.databasePassword, + "SELECT decrypted_secret FROM vault.decrypted_secrets WHERE name = 'persistence-probe'", + ), + ).toEqual([{ decrypted_secret: "preserved-value" }]); + if (target.runtime === "native") { + const second = yield* makeDatabase({ + stackId: "default-key-test", + instanceId: "database-second", + root, + cacheRoot: artifactCacheRoot, + runtime: target.runtime, + }); + const secondService = yield* makeService(second.definition, { + id: "database-second", + config: defaults, + }); + yield* secondService.start; + yield* secondService.ready; + const secondEndpoint = yield* second.endpoint; + yield* query( + secondEndpoint, + defaults.databasePassword, + "CREATE EXTENSION IF NOT EXISTS pgsodium", + ); + expect( + yield* query(secondEndpoint, defaults.databasePassword, derivation), + ).not.toEqual(originalKey); + yield* secondService.destroy; + } + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + + it.live( + "persists SQL data across exact-session stop and reopen, isolates instances, and validates restart before stopping", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-" }); + const cacheRoot = artifactCacheRoot; + const first = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "first", + root, + cacheRoot, + runtime: "native", + }); + const loggedConfig: DatabaseConfig = { ...config, settings: { log_statement: "all" } }; + const service = yield* makeService(first.definition, { + id: "database:first", + config: loggedConfig, + }); + const observed = yield* Ref.make(""); + const caughtUp = yield* Deferred.make(); + yield* first.logs.pipe( + Stream.runForEach(({ bytes }) => + Ref.updateAndGet(observed, (text) => text + new TextDecoder().decode(bytes)).pipe( + Effect.flatMap((text) => + text.includes("bootstrap-logs-settled") + ? Deferred.succeed(caughtUp, undefined).pipe(Effect.asVoid) + : Effect.void, + ), + ), + ), + Effect.forkScoped({ startImmediately: true }), + ); + + yield* service.start; + yield* service.ready; + let firstEndpoint = yield* first.endpoint; + expect(firstEndpoint.kind).toBe("unix"); + yield* query( + firstEndpoint, + config.databasePassword, + "CREATE EXTENSION IF NOT EXISTS pgsodium", + ); + const derivation = "SELECT encode(pgsodium.derive_key(1), 'hex') AS key"; + const firstKey = yield* query(firstEndpoint, config.databasePassword, derivation); + expect(firstKey).toEqual([{ key: expect.stringMatching(/^[a-f0-9]{64}$/u) }]); + yield* query( + firstEndpoint, + config.databasePassword, + "ALTER ROLE supabase_admin SET log_statement = 'all'; ALTER ROLE supabase_admin SET log_min_duration_statement = 0", + ); + yield* service.restart(loggedConfig); + yield* service.ready; + firstEndpoint = yield* first.endpoint; + yield* query(firstEndpoint, config.databasePassword, "SELECT 'bootstrap-logs-settled'"); + yield* Deferred.await(caughtUp).pipe(Effect.timeout("5 seconds")); + expect(yield* Ref.get(observed)).not.toContain(Redacted.value(config.databasePassword)); + expect(yield* Ref.get(observed)).not.toContain(Redacted.value(config.jwtSecret)); + yield* query( + firstEndpoint, + config.databasePassword, + "CREATE TABLE IF NOT EXISTS phase_one (value text NOT NULL)", + ); + yield* query( + firstEndpoint, + config.databasePassword, + "INSERT INTO phase_one(value) VALUES ('persisted')", + ); + + yield* service + .restart({ ...config, settings: { listen_addresses: "*" } }) + .pipe(Effect.flip); + expect((yield* service.get).lifecycle).toBe("running"); + yield* service.restart({ ...config, version: "unsupported" }).pipe(Effect.flip); + expect((yield* service.get).lifecycle).toBe("running"); + const mismatch = yield* service.restart({ ...config, version: "15" }).pipe(Effect.flip); + expect(mismatch.message).toContain("does not match"); + expect((yield* service.get).lifecycle).toBe("running"); + yield* service.stop; + yield* fs.remove(path.join(root, "first", ".supabase-database-ready.json")); + const incomplete = yield* service.restart({ ...config, version: "15" }).pipe(Effect.flip); + expect(incomplete.message).toContain("major does not match"); + + const reopened = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "first", + root, + cacheRoot, + runtime: "native", + }); + const reopenedService = yield* makeService(reopened.definition, { + id: "database:first-reopened", + config, + }); + yield* reopenedService.start; + yield* reopenedService.ready; + const reopenedEndpoint = yield* reopened.endpoint; + expect(reopenedEndpoint.kind).toBe("unix"); + expect(yield* query(reopenedEndpoint, config.databasePassword, derivation)).toEqual( + firstKey, + ); + const rows = yield* query( + reopenedEndpoint, + config.databasePassword, + "SELECT value FROM phase_one", + ); + expect(rows).toEqual([{ value: "persisted" }]); + + const secondRoot = path.join(root, "second-root"); + const second = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "second", + root: secondRoot, + cacheRoot, + runtime: "native", + }); + const secondService = yield* makeService(second.definition, { + id: "database:second", + config, + }); + yield* secondService.start; + yield* secondService.ready; + const secondEndpoint = yield* second.endpoint; + expect(secondEndpoint.kind).toBe("unix"); + yield* query( + secondEndpoint, + config.databasePassword, + "CREATE EXTENSION IF NOT EXISTS pgsodium", + ); + expect(yield* query(secondEndpoint, config.databasePassword, derivation)).toEqual( + firstKey, + ); + if ( + firstEndpoint.kind === "unix" && + reopenedEndpoint.kind === "unix" && + secondEndpoint.kind === "unix" + ) { + expect(secondEndpoint.path).not.toBe(reopenedEndpoint.path); + } + yield* query( + secondEndpoint, + config.databasePassword, + "CREATE TABLE isolated (value text NOT NULL)", + ); + + yield* secondService.stop; + yield* reopenedService.destroy; + yield* secondService.destroy; + expect(yield* fs.exists(path.join(root, "first"))).toBe(false); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + + for (const version of ["15", "17"]) + it.live(`PostgreSQL ${version} persists container data and reconciles credentials`, () => + Effect.scoped( + Effect.gen(function* () { + const databaseConfig: DatabaseConfig = { ...config, version }; + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-container-" }); + const cacheRoot = artifactCacheRoot; + const database = yield* makeDatabase({ + stackId: "stack-integration-container", + instanceId: "database", + root, + cacheRoot, + runtime: "docker", + }); + const service = yield* makeService(database.definition, { + id: "database:container", + config: databaseConfig, + }); + yield* service.start; + yield* service.ready; + const endpoint = yield* database.endpoint; + expect(endpoint.kind).toBe("tcp"); + yield* query( + endpoint, + databaseConfig.databasePassword, + "CREATE EXTENSION IF NOT EXISTS pgsodium", + ); + const deriveKey = "SELECT encode(pgsodium.derive_key(1), 'hex') AS key"; + const originalKey = yield* query(endpoint, databaseConfig.databasePassword, deriveKey); + const schemas = yield* query( + endpoint, + databaseConfig.databasePassword, + "SELECT schema_name FROM information_schema.schemata WHERE schema_name IN ('_analytics', '_supavisor') ORDER BY schema_name", + "_supabase", + ); + expect(schemas).toEqual([{ schema_name: "_analytics" }, { schema_name: "_supavisor" }]); + yield* query( + endpoint, + databaseConfig.databasePassword, + "CREATE TABLE persisted (value text NOT NULL)", + ); + yield* query( + endpoint, + databaseConfig.databasePassword, + "INSERT INTO persisted(value) VALUES ('docker')", + ); + const rotated = Redacted.make("supabase-rotated-password"); + yield* service.restart({ + ...databaseConfig, + databasePassword: rotated, + rootKey: Redacted.make("b".repeat(64)), + }); + yield* service.ready; + const rotatedEndpoint = yield* database.endpoint; + expect(yield* query(rotatedEndpoint, rotated, deriveKey)).not.toEqual(originalKey); + const persisted = yield* query(rotatedEndpoint, rotated, "SELECT value FROM persisted"); + expect(persisted).toEqual([{ value: "docker" }]); + yield* service + .restart({ ...databaseConfig, databasePassword: rotated, settings: { port: 9999 } }) + .pipe(Effect.flip); + expect((yield* service.get).lifecycle).toBe("running"); + yield* service.stop; + const reopened = yield* makeDatabase({ + stackId: "stack-integration-container", + instanceId: "database", + root, + cacheRoot, + runtime: "docker", + }); + const replacementPassword = Redacted.make("reopened-target-password"); + const reopenedService = yield* makeService(reopened.definition, { + id: "database:reopened", + config: { ...databaseConfig, databasePassword: replacementPassword }, + }); + yield* reopenedService.start; + yield* reopenedService.ready; + expect(yield* query(yield* reopened.endpoint, replacementPassword, deriveKey)).toEqual( + originalKey, + ); + expect( + yield* query( + yield* reopened.endpoint, + replacementPassword, + "SELECT value FROM persisted", + ), + ).toEqual([{ value: "docker" }]); + yield* reopenedService.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts new file mode 100644 index 0000000000..620d0ee525 --- /dev/null +++ b/packages/stack/src/services/Database.ts @@ -0,0 +1,818 @@ +import { PgClient } from "@effect/sql-pg"; +import { + Context, + Crypto, + Data, + Effect, + Exit, + FileSystem, + Layer, + Path, + PubSub, + Redacted, + Ref, + Schedule, + Schema, + Scope, + Stream, +} from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import { HttpClient } from "effect/unstable/http"; +import { + prepareNativeArtifact, + postgresVersion, + resolveArtifact, + type PreparedNativeArtifact, +} from "../Artifacts.ts"; +import { + makeContainerRuntime, + type ContainerProcess, + type ContainerRuntime, +} from "../runtime/Container.ts"; +import { DatabaseBootstrapError, runDatabaseBootstrap } from "../runtime/DatabaseBootstrap.ts"; +import { + ensureInternalDatabase, + makeDatabaseSessionFromSqlClient, +} from "../runtime/PostgresDatabaseSession.ts"; +import { + ServiceError, + ServiceLaunchError, + type RuntimeSession, + type ServiceDefinition, + type ServiceInstanceContext, +} from "../Service.ts"; +import { + defaultNativeProcessLauncher, + spawnNativeProcess, + type NativeProcess, +} from "../runtime/NativeProcess.ts"; +import type { StackId } from "../identity/StackId.ts"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; + +export const DatabaseConfig = Schema.Struct({ + version: Schema.String, + databasePassword: Schema.Redacted(Schema.String), + jwtSecret: Schema.Redacted(Schema.String), + jwtExpiry: Schema.Finite, + healthTimeoutMs: Schema.optionalKey(Schema.Finite), + rootKey: Schema.optionalKey(Schema.Redacted(Schema.String)), + settings: Schema.optionalKey( + Schema.Record(Schema.String, Schema.Union([Schema.String, Schema.Finite, Schema.Boolean])), + ), +}); + +const DatabaseReadyMarker = Schema.Struct({ + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + profile: Schema.Literal("supabase"), +}); + +export interface DatabaseConfig extends Schema.Schema.Type {} + +export const DatabaseEndpoints = Schema.Struct({ sql: Schema.optionalKey(EndpointIntent) }); +export interface DatabaseEndpoints extends Schema.Schema.Type {} +export const DatabaseCreation = serviceCreation("database", DatabaseConfig, DatabaseEndpoints); +export interface DatabaseCreation extends Schema.Schema.Type {} + +export type DatabaseRuntime = "native" | "docker" | "podman"; + +export type BackendEndpoint = + | { readonly kind: "unix"; readonly path: string; readonly port: 5432 } + | { readonly kind: "tcp"; readonly host: "127.0.0.1"; readonly port: number }; + +interface DatabaseLog { + readonly stream: "stdout" | "stderr"; + readonly bytes: Uint8Array; +} + +export class DatabaseError extends Data.TaggedError("DatabaseError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface DatabaseOptions { + readonly stackId: StackId | string; + readonly instanceId: string; + readonly root: string; + readonly cacheRoot: string; + readonly runtime: DatabaseRuntime; +} + +export interface DatabaseComponent { + readonly definition: ServiceDefinition; + readonly resetData: ( + context: ServiceInstanceContext, + ) => Effect.Effect; + readonly endpoint: Effect.Effect; + readonly logs: Stream.Stream; +} + +const errorFor = (operation: string, cause: unknown): ServiceError => + cause instanceof ServiceError + ? cause + : new ServiceError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const databaseError = (operation: string, cause: unknown): DatabaseError => + new DatabaseError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const processExit = ( + exitCode: Effect.Effect, +): Effect.Effect> => + exitCode.pipe( + Effect.flatMap((code) => + Number(code) === 0 + ? Effect.void + : Effect.fail( + new ServiceError({ + operation: "exit", + message: `PostgreSQL exited with code ${String(code)}`, + }), + ), + ), + Effect.mapError((cause) => errorFor("exit", cause)), + Effect.exit, + ); + +const reconcileContainerPassword = Effect.fn("Database.reconcileContainerPassword")( + ( + engine: "docker" | "podman", + id: string, + password: Redacted.Redacted, + spawner: ChildProcessSpawnerService["Service"], + ) => + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn( + ChildProcess.make( + engine, + [ + "exec", + "-i", + id, + "/opt/postgres/bin/psql", + "-h", + "/tmp", + "-p", + "5432", + "-U", + "supabase_admin", + "-d", + "postgres", + "-X", + "-v", + "ON_ERROR_STOP=1", + ], + { stdin: "pipe" }, + ), + ); + const statement = `BEGIN; SET LOCAL log_statement = 'none'; SET LOCAL log_min_error_statement = 'panic'; SET LOCAL log_min_duration_statement = -1; SET LOCAL log_min_duration_sample = -1; SET LOCAL standard_conforming_strings = on; ALTER ROLE supabase_admin PASSWORD '${Redacted.value(password).replaceAll("'", "''")}'; COMMIT;`; + const [, , , code] = yield* Effect.all( + [ + Stream.make(new TextEncoder().encode(statement)).pipe(Stream.run(child.stdin)), + child.stdout.pipe(Stream.runDrain), + child.stderr.pipe(Stream.runDrain), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor("health", "Local database credential setup has not succeeded"); + }), + ).pipe( + Effect.mapError(() => errorFor("health", "Local database credential setup failed")), + Effect.retry(Schedule.spaced("250 millis")), + ), +); + +const health = Effect.fn("Database.health")(( + endpoint: BackendEndpoint, + config: DatabaseConfig, + reconcile: Effect.Effect, + context: { + readonly fs: FileSystem.FileSystem; + readonly instanceRoot: string; + readonly version: string; + readonly runtime: DatabaseRuntime; + }, +): Effect.Effect => { + const host = endpoint.kind === "unix" ? endpoint.path : endpoint.host; + const probe = Effect.scoped( + Effect.gen(function* () { + const layer = yield* Layer.build( + PgClient.layer({ + host, + port: endpoint.port, + database: "postgres", + username: "supabase_admin", + password: config.databasePassword, + connectTimeout: "2 seconds", + }), + ); + const client = Context.get(layer, PgClient.PgClient); + yield* client.unsafe("SELECT 1"); + }), + ); + const retryProbe = probe.pipe(Effect.retry(Schedule.spaced("250 millis"))); + return reconcile.pipe( + Effect.andThen(retryProbe), + Effect.andThen( + Effect.scoped( + Effect.gen(function* () { + const layer = yield* Layer.build( + PgClient.layer({ + host, + port: endpoint.port, + database: "postgres", + username: "supabase_admin", + password: config.databasePassword, + connectTimeout: "2 seconds", + }), + ); + const client = Context.get(layer, PgClient.PgClient); + const session = makeDatabaseSessionFromSqlClient(client); + const openInternal = Effect.gen(function* () { + const internalLayer = yield* Layer.build( + PgClient.layer({ + host, + port: endpoint.port, + database: "_supabase", + username: "supabase_admin", + password: config.databasePassword, + connectTimeout: "2 seconds", + }), + ); + return makeDatabaseSessionFromSqlClient(Context.get(internalLayer, PgClient.PgClient)); + }).pipe( + Effect.mapError( + (cause) => + new DatabaseBootstrapError({ + message: "Unable to connect to the internal database", + cause, + }), + ), + ); + yield* ensureInternalDatabase(session, openInternal).pipe( + Effect.mapError((cause) => errorFor("bootstrap", cause)), + ); + yield* runDatabaseBootstrap(session, { + databasePassword: config.databasePassword, + jwtSecret: config.jwtSecret, + jwtExpiry: config.jwtExpiry, + }).pipe(Effect.mapError((cause) => errorFor("bootstrap", cause))); + const readyMarker = yield* Schema.encodeEffect( + Schema.fromJsonString(DatabaseReadyMarker), + )({ + version: context.version, + runtime: context.runtime, + profile: "supabase", + }).pipe(Effect.mapError((cause) => errorFor("bootstrap", cause))); + const stage = yield* context.fs.makeTempDirectoryScoped({ + directory: context.instanceRoot, + prefix: ".ready-", + }); + yield* context.fs.writeFileString(`${stage}/marker`, readyMarker, { mode: 0o600 }); + yield* context.fs.rename( + `${stage}/marker`, + `${context.instanceRoot}/.supabase-database-ready.json`, + ); + }), + ), + ), + Effect.timeoutOrElse({ + duration: config.healthTimeoutMs ?? 60_000, + orElse: () => Effect.fail(errorFor("health", "Database readiness timed out")), + }), + Effect.mapError((cause) => errorFor("health", cause)), + ); +}); + +const publishLogs = Effect.fn("Database.publishLogs")(( + process: { + readonly stdout: Stream.Stream; + readonly stderr: Stream.Stream; + }, + logs: PubSub.PubSub, + scope: Scope.Closeable, +): Effect.Effect => { + const drain = (stream: Stream.Stream, name: DatabaseLog["stream"]) => + stream.pipe( + Stream.runForEach((bytes) => PubSub.publish(logs, { stream: name, bytes })), + Effect.catch((cause) => Effect.logError(cause)), + ); + return Effect.all( + [ + Effect.forkIn(drain(process.stdout, "stdout"), scope), + Effect.forkIn(drain(process.stderr, "stderr"), scope), + ], + { concurrency: "unbounded", discard: true }, + ); +}); + +const runtimeFromContainer = (process: ContainerProcess): RuntimeSession => ({ + health: Effect.void, + exit: processExit(process.exitCode), + stop: process.stop.pipe(Effect.mapError((cause) => errorFor("stop", cause))), + remove: process.remove.pipe(Effect.mapError((cause) => errorFor("remove", cause))), +}); + +const ensureOwnedRoot = Effect.fn("Database.ensureOwnedRoot")(( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + stackId: string, + instanceId: string, +): Effect.Effect => { + const ownerFile = path.join(root, ".supabase-database-owner.json"); + const marker = JSON.stringify({ stackId, instanceId }); + return Effect.gen(function* () { + yield* fs + .makeDirectory(root, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => databaseError("data", cause))); + const present = yield* fs + .exists(ownerFile) + .pipe(Effect.mapError((cause) => databaseError("data", cause))); + if (present) { + const existing = yield* fs + .readFileString(ownerFile) + .pipe(Effect.mapError((cause) => databaseError("data", cause))); + if (existing !== marker) + return yield* databaseError("data", "Database root belongs to another instance"); + } else { + const entries = yield* fs + .readDirectory(root) + .pipe(Effect.mapError((cause) => databaseError("data", cause))); + if (entries.length > 0) + return yield* databaseError("data", "Database root is non-empty and unmarked"); + yield* fs + .writeFileString(ownerFile, marker, { mode: 0o600, flag: "wx" }) + .pipe(Effect.mapError((cause) => databaseError("data", cause))); + } + }); +}); + +const removeOwnedRoot = Effect.fn("Database.removeOwnedRoot")(( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + stackId: string, + instanceId: string, + removeData: Effect.Effect, +): Effect.Effect => { + const ownerFile = path.join(root, ".supabase-database-owner.json"); + const marker = JSON.stringify({ stackId, instanceId }); + return Effect.gen(function* () { + const present = yield* fs + .exists(ownerFile) + .pipe(Effect.mapError((cause) => errorFor("destroy", cause))); + if (!present) { + if (yield* fs.exists(root).pipe(Effect.mapError((cause) => errorFor("destroy", cause)))) + return yield* errorFor("destroy", "Database root is unmarked"); + return; + } + const existing = yield* fs + .readFileString(ownerFile) + .pipe(Effect.mapError((cause) => errorFor("destroy", cause))); + if (existing !== marker) + return yield* errorFor("destroy", "Database root belongs to another instance"); + yield* removeData; + yield* fs + .remove(root, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => errorFor("destroy", cause))); + }); +}); + +const nativeProcess = ( + artifact: PreparedNativeArtifact, + config: DatabaseConfig, + dataPath: string, + socketPath: string, + rootKeyPath: string, + settings: ReadonlyArray, + context: ServiceInstanceContext, + stackId: string, + instanceId: string, + spawner: ChildProcessSpawnerService["Service"], +): Effect.Effect => + spawnNativeProcess( + { + executable: artifact.executable, + args: [ + "-D", + dataPath, + "-p", + "5432", + "-c", + "listen_addresses=", + "-c", + `unix_socket_directories=${socketPath}`, + ...settings, + ], + env: { + PGDATA: dataPath, + PGSODIUM_KEY_FILE: rootKeyPath, + POSTGRES_USER: "supabase_admin", + POSTGRES_DB: "postgres", + POSTGRES_PASSWORD: Redacted.value(config.databasePassword), + }, + gracefulStopSignal: "SIGINT", + gracefulStopTimeout: "15 seconds", + }, + defaultNativeProcessLauncher(), + { stackId, workloadId: instanceId }, + ).pipe( + Scope.provide(context.scope), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.mapError((cause) => errorFor("launch", cause)), + ); + +/** Creates a database component backed by one native or container PostgreSQL session. */ +export const makeDatabase = ( + options: DatabaseOptions, +): Effect.Effect< + DatabaseComponent, + DatabaseError, + | FileSystem.FileSystem + | Path.Path + | Crypto.Crypto + | ChildProcessSpawner.ChildProcessSpawner + | HttpClient.HttpClient + | Scope.Scope +> => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const client = yield* HttpClient.HttpClient; + const logs = yield* PubSub.sliding(256); + const endpoint = yield* Ref.make(undefined); + const prepared = yield* Ref.make>(new Map()); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(String(options.stackId))) + return yield* databaseError("identity", "Invalid stack id"); + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(options.instanceId)) + return yield* databaseError("identity", "Invalid instance id"); + const instanceRoot = path.join(options.root, options.instanceId); + yield* ensureOwnedRoot(fs, path, instanceRoot, String(options.stackId), options.instanceId); + const container: ContainerRuntime | undefined = + options.runtime === "native" + ? undefined + : yield* makeContainerRuntime({ engine: options.runtime }); + + const dataCommand = Effect.fn("Database.containerFiles")( + function* (version: string, args: ReadonlyArray) { + if (container === undefined) return ""; + const artifact = yield* resolveArtifact({ + service: "database", + version: postgresVersion(version), + }); + return yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* container.launchTool({ + image: artifact.image, + stackId: String(options.stackId), + instanceId: options.instanceId, + entrypoint: "/usr/bin/busybox", + args, + env: {}, + mounts: [{ source: instanceRoot, target: "/instance", readOnly: false }], + }); + return yield* Effect.acquireUseRelease( + Effect.succeed(child), + (child) => + Effect.gen(function* () { + const [, stdout, , code] = yield* Effect.all( + [ + Stream.empty.pipe(Stream.run(child.stdin)), + child.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (text, chunk) => (text + chunk).slice(-65536), + ), + ), + child.stderr.pipe(Stream.runDrain), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (code !== 0) + return yield* errorFor("data", "Container filesystem operation failed"); + return stdout; + }), + (child) => child.stop.pipe(Effect.andThen(child.remove)), + ); + }), + ); + }, + Effect.mapError((cause) => errorFor("data", cause)), + ); + + const prepareArtifact = Effect.fn("Database.prepareArtifact")(( + input: DatabaseConfig, + ): Effect.Effect => { + const version = postgresVersion(input.version); + if (!Number.isInteger(input.jwtExpiry) || input.jwtExpiry <= 0) + return Effect.fail( + new ServiceError({ + operation: "prepare", + message: "jwtExpiry must be a positive integer", + }), + ); + const forbidden = new Set([ + "listen_addresses", + "unix_socket_directories", + "data_directory", + "config_file", + "hba_file", + "external_pid_file", + "port", + ]); + if (Object.keys(input.settings ?? {}).some((key) => forbidden.has(key.toLowerCase()))) + return Effect.fail( + new ServiceError({ + operation: "prepare", + message: "Database settings override managed isolation", + }), + ); + if (options.runtime === "native") + return prepareNativeArtifact({ service: "database", version }, options.cacheRoot).pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(HttpClient.HttpClient, client), + Effect.tap((artifact) => + Ref.update(prepared, (map) => new Map(map).set(version, artifact)), + ), + Effect.asVoid, + Effect.mapError((cause) => errorFor("prepare", cause)), + ); + const selectedContainer = container; + if (selectedContainer === undefined) + return Effect.fail( + new ServiceError({ operation: "prepare", message: "Container runtime is unavailable" }), + ); + return resolveArtifact({ service: "database", version }).pipe( + Effect.flatMap((artifact) => selectedContainer.prepare(artifact.image)), + Effect.mapError((cause) => errorFor("prepare", cause)), + ); + }); + + const prepare = Effect.fn("Database.prepare")( + function* (input: DatabaseConfig) { + const markerPath = path.join(instanceRoot, ".supabase-database-ready.json"); + const hasMarker = yield* fs.exists(markerPath); + if (hasMarker) { + const marker = yield* fs + .readFileString(markerPath) + .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(DatabaseReadyMarker)))); + if ( + marker.version.split(".")[0] !== postgresVersion(input.version).split(".")[0] || + marker.runtime !== options.runtime + ) + return yield* errorFor( + "prepare", + "Initialized database artifact/runtime does not match the requested configuration", + ); + } + const versionPath = path.join(instanceRoot, "data", "PG_VERSION"); + if (!hasMarker && options.runtime === "native" && (yield* fs.exists(versionPath))) { + const initialized = (yield* fs.readFileString(versionPath)).trim(); + if (initialized !== postgresVersion(input.version).split(".")[0]) + return yield* errorFor( + "prepare", + "Initialized PostgreSQL major does not match the requested configuration", + ); + } + yield* prepareArtifact(input); + if (!hasMarker && container !== undefined) { + const initialized = (yield* dataCommand(input.version, [ + "sh", + "-c", + "if [ -f /instance/data/PG_VERSION ]; then /usr/bin/busybox cat /instance/data/PG_VERSION; fi", + ])).trim(); + if (initialized !== "" && initialized !== postgresVersion(input.version).split(".")[0]) + return yield* errorFor( + "prepare", + "Initialized PostgreSQL major does not match the requested configuration", + ); + } + }, + Effect.mapError((cause) => errorFor("prepare", cause)), + ); + + const launch = Effect.fn("Database.launch")( + ( + context: ServiceInstanceContext, + ): Effect.Effect => + Effect.gen(function* () { + const config = { ...context.config, version: postgresVersion(context.config.version) }; + const dataPath = path.join(instanceRoot, "data"); + yield* fs + .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + const rootKeyPath = + config.rootKey === undefined ? undefined : path.join(instanceRoot, "pgsodium_root.key"); + if (rootKeyPath !== undefined && config.rootKey !== undefined) + yield* fs + .writeFileString(rootKeyPath, Redacted.value(config.rootKey), { + mode: options.runtime === "native" ? 0o600 : 0o644, + }) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + const settings = Object.entries(config.settings ?? {}).flatMap(([key, value]) => [ + "-c", + `${key}=${String(value)}`, + ]); + if (options.runtime === "native") { + // PostgreSQL limits Unix socket paths to 103 bytes, independently of the user's state root. + const socketPath = yield* Effect.acquireRelease( + fs.makeTempDirectory({ directory: "/tmp", prefix: "supabase-pg-" }), + (directory) => + fs + .remove(directory, { recursive: true, force: true }) + .pipe(Effect.catch((cause) => Effect.logError(cause))), + ).pipe( + Scope.provide(context.scope), + Effect.mapError((cause) => errorFor("launch", cause)), + ); + const artifact = (yield* Ref.get(prepared)).get(config.version); + if (artifact === undefined) + return yield* errorFor("launch", `Artifact ${config.version} was not prepared`); + const process = yield* nativeProcess( + artifact, + config, + dataPath, + socketPath, + rootKeyPath ?? path.join(dataPath, "pgsodium_root.key"), + settings, + context, + String(options.stackId), + options.instanceId, + spawner, + ); + const selectedEndpoint: BackendEndpoint = { + kind: "unix", + path: socketPath, + port: 5432, + }; + yield* Ref.set(endpoint, selectedEndpoint); + yield* publishLogs(process, logs, context.scope); + return { + health: health(selectedEndpoint, config, Effect.void, { + fs, + instanceRoot, + version: config.version, + runtime: options.runtime, + }), + exit: processExit(process.exitCode), + stop: process.kill.pipe(Effect.mapError((cause) => errorFor("stop", cause))), + remove: fs.remove(socketPath, { recursive: true, force: true }).pipe( + Effect.mapError((cause) => errorFor("remove", cause)), + Effect.tap(() => Ref.set(endpoint, undefined)), + ), + } satisfies RuntimeSession; + } + const image = yield* resolveArtifact({ + service: "database", + version: config.version, + }).pipe(Effect.mapError((cause) => errorFor("launch", cause))); + const selectedContainer = container; + if (selectedContainer === undefined) + return yield* errorFor("launch", "Container runtime is unavailable"); + yield* dataCommand(config.version, ["chown", "100:101", "/instance/data"]); + const launched = yield* selectedContainer + .launch({ + image: image.image, + stackId: String(options.stackId), + instanceId: options.instanceId, + env: { + PGDATA: "/var/lib/postgresql/data", + PGSODIUM_KEY_FILE: + rootKeyPath === undefined + ? "/var/lib/postgresql/data/pgsodium_root.key" + : "/etc/postgresql-custom/pgsodium_root.key", + POSTGRES_USER: "supabase_admin", + POSTGRES_DB: "postgres", + POSTGRES_PASSWORD: Redacted.value(config.databasePassword), + }, + args: ["-p", "5432", "-c", "listen_addresses=*", ...settings], + mounts: [ + { source: dataPath, target: "/var/lib/postgresql/data", readOnly: false }, + ...(rootKeyPath === undefined + ? [] + : [ + { + source: rootKeyPath, + target: "/etc/postgresql-custom/pgsodium_root.key", + readOnly: true, + }, + ]), + ], + ports: [5432], + }) + .pipe( + Effect.catchTag("ContainerLaunchError", ({ failure, process }) => + Effect.fail( + new ServiceLaunchError({ + failure: errorFor("launch", failure), + runtime: runtimeFromContainer(process), + }), + ), + ), + Effect.mapError((cause) => + cause instanceof ServiceLaunchError ? cause : errorFor("launch", cause), + ), + Scope.provide(context.scope), + ); + const port = launched.ports[5432]; + if (port === undefined) + return yield* errorFor("launch", "Container did not publish PostgreSQL"); + const selectedEndpoint: BackendEndpoint = { kind: "tcp", host: "127.0.0.1", port }; + yield* Ref.set(endpoint, selectedEndpoint); + yield* publishLogs(launched, logs, context.scope); + return { + ...runtimeFromContainer(launched), + health: health( + selectedEndpoint, + config, + reconcileContainerPassword( + options.runtime, + launched.id, + config.databasePassword, + spawner, + ), + { + fs, + instanceRoot, + version: config.version, + runtime: options.runtime, + }, + ), + remove: runtimeFromContainer(launched).remove.pipe( + Effect.tap(() => Ref.set(endpoint, undefined)), + ), + } satisfies RuntimeSession; + }), + ); + + const definition: ServiceDefinition = { + prepare, + launch, + removeData: (context) => + removeOwnedRoot( + fs, + path, + instanceRoot, + String(options.stackId), + options.instanceId, + Effect.gen(function* () { + if (container === undefined || !(yield* fs.exists(path.join(instanceRoot, "data")))) + return; + const artifact = yield* resolveArtifact({ + service: "database", + version: postgresVersion(context.config.version), + }); + yield* container.prepare(artifact.image); + yield* dataCommand(context.config.version, ["rm", "-rf", "/instance/data"]); + }).pipe(Effect.mapError((cause) => errorFor("destroy", cause))), + ), + }; + const resetData = Effect.fn("Database.resetData")( + (context: ServiceInstanceContext) => + definition + .removeData(context) + .pipe( + Effect.andThen( + ensureOwnedRoot( + fs, + path, + instanceRoot, + String(options.stackId), + options.instanceId, + ).pipe(Effect.mapError((cause) => errorFor("reset", cause))), + ), + ), + ); + return { + definition, + resetData, + endpoint: Ref.get(endpoint).pipe( + Effect.flatMap((value) => + value === undefined + ? Effect.fail(databaseError("endpoint", "Database is not running")) + : Effect.succeed(value), + ), + ), + logs: Stream.fromPubSub(logs), + } satisfies DatabaseComponent; + }); diff --git a/packages/stack/src/services/DatabaseSnapshot.integration.test.ts b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts new file mode 100644 index 0000000000..328e9925eb --- /dev/null +++ b/packages/stack/src/services/DatabaseSnapshot.integration.test.ts @@ -0,0 +1,349 @@ +import { PgClient } from "@effect/sql-pg"; +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Data, Effect, FileSystem, Layer, Path, Redacted, Schema, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { makeService, ServiceError } from "../Service.ts"; +import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; +import { SnapshotDescriptor, makeDatabaseSnapshots } from "./DatabaseSnapshot.ts"; +import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; + +class SnapshotTestError extends Data.TaggedError("SnapshotTestError")<{ + readonly message: string; +}> {} + +const config: DatabaseConfig = { + version: "17", + databasePassword: Redacted.make("supabase-snapshot-source-password"), + jwtSecret: Redacted.make("supabase-snapshot-jwt-secret"), + jwtExpiry: 3600, +}; + +const query = (endpoint: BackendEndpoint, password: Redacted.Redacted, statement: string) => + Effect.scoped( + Effect.gen(function* () { + const host = endpoint.kind === "unix" ? endpoint.path : endpoint.host; + const services = yield* Layer.build( + PgClient.layer({ + host, + port: endpoint.port, + database: "postgres", + username: "supabase_admin", + password, + }), + ); + return yield* Context.get(services, PgClient.PgClient).unsafe(statement); + }), + ); + +const storageError = (cause: unknown) => + cause instanceof ServiceError + ? cause + : new ServiceError({ + operation: "storage", + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const Marker = Schema.fromJsonString( + Schema.Struct({ + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + profile: Schema.Literal("supabase"), + }), +); + +const writeMarker = ( + fs: FileSystem.FileSystem, + root: string, + runtime: "native" | "docker" | "podman", + version = "17.6.1.173", +) => + Effect.gen(function* () { + const marker = yield* Schema.encodeEffect(Marker)({ version, runtime, profile: "supabase" }); + yield* fs.writeFileString(`${root}/.supabase-database-ready.json`, marker, { mode: 0o600 }); + }); + +const prepareRoot = ( + fs: FileSystem.FileSystem, + runtime: "native" | "docker" | "podman", + prefix: string, +) => + Effect.gen(function* () { + const root = yield* fs.makeTempDirectoryScoped({ prefix }); + yield* fs.makeDirectory(`${root}/data`, { recursive: true }); + yield* fs.writeFileString(`${root}/data/PG_VERSION`, "17\n"); + yield* writeMarker(fs, root, runtime); + return root; + }); + +const tar = (args: ReadonlyArray) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn(ChildProcess.make("tar", args, { stdin: "ignore" })); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.mkString), + child.stderr.pipe(Stream.decodeText, Stream.mkString), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) return yield* new SnapshotTestError({ message: stderr }); + return stdout; + }), + ); + +for (const runtime of ["native", "docker"] as const) { + it.live( + `exports and restores SQL data through the ${runtime} database service`, + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: `snapshot-${runtime}-` }); + if (runtime === "docker") yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); + const source = yield* makeDatabase({ + stackId: `snapshot-${runtime}`, + instanceId: "source", + root, + cacheRoot: "/tmp/supabase-stack-artifacts", + runtime, + }); + const sourceService = yield* makeService(source.definition, { + id: `database:${runtime}:source`, + config, + }); + yield* sourceService.start; + yield* sourceService.ready; + yield* query( + yield* source.endpoint, + config.databasePassword, + "CREATE TABLE snapshot_data (value text NOT NULL); INSERT INTO snapshot_data VALUES ('restored-row')", + ); + yield* sourceService.stop; + + const sourceRoot = path.join(root, "source"); + const sourceSnapshots = yield* makeDatabaseSnapshots({ + instanceRoot: sourceRoot, + runtime, + version: config.version, + stackId: `snapshot-${runtime}`, + instanceId: "source", + }); + const archive = path.join(root, `${runtime}.tar`); + yield* sourceService.storage( + sourceSnapshots + .exportSnapshot({ destination: archive }) + .pipe(Effect.mapError(storageError)), + ); + + const target = yield* makeDatabase({ + stackId: `snapshot-${runtime}`, + instanceId: "target", + root, + cacheRoot: "/tmp/supabase-stack-artifacts", + runtime, + }); + const targetPassword = Redacted.make(`snapshot-${runtime}-target-password`); + const targetService = yield* makeService(target.definition, { + id: `database:${runtime}:target`, + config: { ...config, databasePassword: targetPassword }, + }); + const targetRoot = path.join(root, "target"); + if (runtime === "native") yield* fs.makeDirectory(path.join(targetRoot, "data")); + const targetSnapshots = yield* makeDatabaseSnapshots({ + instanceRoot: targetRoot, + runtime, + version: config.version, + stackId: `snapshot-${runtime}`, + instanceId: "target", + }); + yield* targetService.storage( + targetSnapshots + .restoreSnapshot({ source: archive }) + .pipe(Effect.mapError(storageError)), + ); + yield* targetService.start; + yield* targetService.ready; + const rows = yield* query( + yield* target.endpoint, + targetPassword, + "SELECT value FROM snapshot_data", + ); + expect(rows).toEqual([{ value: "restored-row" }]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 180_000 }, + ); +} + +it.live("rejects nonempty targets, incompatible versions, and unsafe members", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const sourceRoot = yield* prepareRoot(fs, "native", "snapshot-invalid-source-"); + const destinationRoot = yield* fs.makeTempDirectoryScoped({ prefix: "snapshot-invalid-" }); + const source = yield* makeDatabaseSnapshots({ + instanceRoot: sourceRoot, + runtime: "native", + version: "17", + stackId: "stack", + instanceId: "source", + }); + const archive = `${destinationRoot}/baseline.tar`; + yield* source.exportSnapshot({ destination: archive }); + + const nonempty = yield* makeDatabaseSnapshots({ + instanceRoot: yield* prepareRoot(fs, "native", "snapshot-nonempty-"), + runtime: "native", + version: "17", + stackId: "stack", + instanceId: "nonempty", + }); + const nonemptyFailure = yield* nonempty + .restoreSnapshot({ source: archive }) + .pipe(Effect.flip); + expect(nonemptyFailure.operation).toBe("restore"); + + const incompatibleRoot = yield* prepareRoot(fs, "native", "snapshot-version-"); + yield* fs.remove(`${incompatibleRoot}/data/PG_VERSION`); + yield* writeMarker(fs, incompatibleRoot, "native", "15.14.1.173"); + const incompatible = yield* makeDatabaseSnapshots({ + instanceRoot: incompatibleRoot, + runtime: "native", + version: "15", + stackId: "stack", + instanceId: "version", + }); + const versionFailure = yield* incompatible + .restoreSnapshot({ source: archive }) + .pipe(Effect.flip); + expect(versionFailure.operation).toBe("descriptor"); + + const unsafeSource = yield* prepareRoot(fs, "native", "snapshot-unsafe-source-"); + yield* fs.symlink("/tmp", `${unsafeSource}/data/unsafe-link`); + const unsafe = yield* makeDatabaseSnapshots({ + instanceRoot: unsafeSource, + runtime: "native", + version: "17", + stackId: "stack", + instanceId: "unsafe", + }); + const unsafeArchive = `${destinationRoot}/unsafe.tar`; + yield* unsafe.exportSnapshot({ destination: unsafeArchive }); + const unsafeTarget = yield* prepareRoot(fs, "native", "snapshot-unsafe-target-"); + yield* fs.remove(`${unsafeTarget}/data/PG_VERSION`); + const unsafeRestore = yield* makeDatabaseSnapshots({ + instanceRoot: unsafeTarget, + runtime: "native", + version: "17", + stackId: "stack", + instanceId: "unsafe-target", + }); + const unsafeFailure = yield* unsafeRestore + .restoreSnapshot({ source: unsafeArchive }) + .pipe(Effect.flip); + expect(unsafeFailure.operation).toBe("validate"); + + const descriptor = yield* tar(["-xOf", archive, "metadata/descriptor.json"]); + const malformedRoot = yield* fs.makeTempDirectoryScoped({ prefix: "snapshot-malformed-" }); + yield* fs.makeDirectory(`${malformedRoot}/data`, { recursive: true }); + yield* fs.makeDirectory(`${malformedRoot}/metadata`, { recursive: true }); + yield* fs.writeFileString(`${malformedRoot}/metadata/descriptor.json`, descriptor); + const malformedArchive = `${destinationRoot}/missing-pg-version.tar`; + yield* tar(["-cf", malformedArchive, "-C", malformedRoot, "data", "metadata"]); + const malformedTarget = yield* fs.makeTempDirectoryScoped({ + prefix: "snapshot-malformed-target-", + }); + yield* fs.makeDirectory(`${malformedTarget}/data`, { recursive: true }); + const malformedStore = yield* makeDatabaseSnapshots({ + instanceRoot: malformedTarget, + runtime: "native", + version: "17", + stackId: "stack", + instanceId: "malformed", + }); + const malformedFailure = yield* malformedStore + .restoreSnapshot({ source: malformedArchive }) + .pipe(Effect.flip); + expect(malformedFailure.operation).toBe("validate"); + expect(yield* fs.readDirectory(`${malformedTarget}/data`)).toEqual([]); + + yield* fs.writeFileString(`${malformedRoot}/data/PG_VERSION`, "16\n"); + const wrongVersionArchive = `${destinationRoot}/wrong-pg-version.tar`; + yield* tar(["-cf", wrongVersionArchive, "-C", malformedRoot, "data", "metadata"]); + const wrongVersionTarget = yield* fs.makeTempDirectoryScoped({ + prefix: "snapshot-wrong-version-target-", + }); + yield* fs.makeDirectory(`${wrongVersionTarget}/data`, { recursive: true }); + const wrongVersionStore = yield* makeDatabaseSnapshots({ + instanceRoot: wrongVersionTarget, + runtime: "native", + version: "17", + stackId: "stack", + instanceId: "wrong-version", + }); + const wrongVersionFailure = yield* wrongVersionStore + .restoreSnapshot({ source: wrongVersionArchive }) + .pipe(Effect.flip); + expect(wrongVersionFailure.operation).toBe("validate"); + expect(yield* fs.readDirectory(`${wrongVersionTarget}/data`)).toEqual([]); + + const dockerRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "snapshot-docker-unsafe-source-", + }); + yield* fs.makeDirectory(`${dockerRoot}/data`, { recursive: true }); + yield* fs.makeDirectory(`${dockerRoot}/metadata`, { recursive: true }); + yield* fs.writeFileString(`${dockerRoot}/data/PG_VERSION`, "17\n"); + yield* fs.symlink("/tmp", `${dockerRoot}/data/unsafe-link`); + const dockerDescriptor = yield* Schema.encodeEffect( + Schema.fromJsonString(SnapshotDescriptor), + )({ + format: "supabase-database-snapshot-v1", + version: "17.6.1.173", + runtime: "docker", + platform: process.platform, + arch: process.arch, + profile: "supabase", + }); + yield* fs.writeFileString(`${dockerRoot}/metadata/descriptor.json`, dockerDescriptor); + const dockerArchive = `${destinationRoot}/unsafe-docker.tar`; + yield* tar(["-cf", dockerArchive, "-C", dockerRoot, "data", "metadata"]); + const dockerTarget = yield* fs.makeTempDirectoryScoped({ + prefix: "snapshot-docker-unsafe-target-", + }); + yield* fs.makeDirectory(`${dockerTarget}/data`, { recursive: true }); + const dockerStore = yield* makeDatabaseSnapshots({ + instanceRoot: dockerTarget, + runtime: "docker", + version: "17", + stackId: "stack", + instanceId: "docker-unsafe", + }); + const dockerFailure = yield* dockerStore + .restoreSnapshot({ source: dockerArchive }) + .pipe(Effect.flip); + expect(dockerFailure.operation).toBe("validate"); + + const dockerNonemptyRoot = yield* fs.makeTempDirectoryScoped({ + prefix: "snapshot-docker-nonempty-target-", + }); + yield* fs.makeDirectory(`${dockerNonemptyRoot}/data`, { recursive: true }); + yield* fs.writeFileString(`${dockerNonemptyRoot}/data/sentinel`, "occupied\n"); + const dockerNonemptyStore = yield* makeDatabaseSnapshots({ + instanceRoot: dockerNonemptyRoot, + runtime: "docker", + version: "17", + stackId: "stack", + instanceId: "docker-nonempty", + }); + const dockerNonemptyFailure = yield* dockerNonemptyStore + .restoreSnapshot({ source: archive }) + .pipe(Effect.flip); + expect(dockerNonemptyFailure.operation).toBe("restore"); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts new file mode 100644 index 0000000000..6f512a54c7 --- /dev/null +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -0,0 +1,554 @@ +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { Crypto, Data, Effect, FileSystem, Path, Schema, Stream } from "effect"; +import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import type { DatabaseRuntime } from "./Database.ts"; +import { makeContainerRuntime, type ContainerRuntime } from "../runtime/Container.ts"; + +export const SnapshotDescriptor = Schema.Struct({ + format: Schema.Literal("supabase-database-snapshot-v1"), + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + platform: Schema.String, + arch: Schema.String, + profile: Schema.Literal("supabase"), +}); +export interface SnapshotDescriptor extends Schema.Schema.Type {} + +export class DatabaseSnapshotError extends Data.TaggedError("DatabaseSnapshotError")<{ + readonly operation: string; + readonly message: string; + readonly cause?: unknown; +}> {} + +export interface DatabaseSnapshot { + readonly descriptor: SnapshotDescriptor; + readonly destination: string; +} + +const ReadyMarker = Schema.Struct({ + version: Schema.String, + runtime: Schema.Literals(["native", "docker", "podman"]), + profile: Schema.Literal("supabase"), +}); + +const errorFor = (operation: string, cause: unknown) => + cause instanceof DatabaseSnapshotError + ? cause + : new DatabaseSnapshotError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const safeRelative = (name: string) => { + if (name.startsWith("/") || name.includes("\\")) return false; + const parts = name.split("/"); + return parts.every((part) => part !== "" && part !== "." && part !== ".."); +}; + +export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function* (options: { + readonly instanceRoot: string; + readonly runtime: DatabaseRuntime; + readonly version: string; + readonly stackId: string; + readonly instanceId: string; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const version = postgresVersion(options.version); + const container: ContainerRuntime | undefined = + options.runtime === "native" + ? undefined + : yield* makeContainerRuntime({ engine: options.runtime }); + const helperImage = + options.runtime === "native" + ? undefined + : yield* resolveArtifact({ service: "database", version }).pipe( + Effect.mapError((cause) => errorFor("artifact", cause)), + ); + const descriptor: SnapshotDescriptor = { + format: "supabase-database-snapshot-v1", + version, + runtime: options.runtime, + platform: process.platform, + arch: process.arch, + profile: "supabase", + }; + + const commandRaw = Effect.fn("DatabaseSnapshot.command")(function* (args: ReadonlyArray) { + return yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn(ChildProcess.make("tar", args, { stdin: "ignore" })); + const [stdout, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => `${all}${chunk}`.slice(0, 65536), + ), + ), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => `${all}${chunk}`.slice(0, 65536), + ), + ), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor("tar", stderr.trim() || `tar exited with ${String(code)}`); + return stdout; + }), + ); + }); + const command = (args: ReadonlyArray) => + commandRaw(args).pipe(Effect.mapError((cause) => errorFor("tar", cause))); + + const commandLines = Effect.fn("DatabaseSnapshot.commandLines")( + ( + args: ReadonlyArray, + onLine: (line: string) => Effect.Effect, + ) => + Effect.scoped( + Effect.gen(function* () { + const child = yield* spawner.spawn(ChildProcess.make("tar", args, { stdin: "ignore" })); + const [, stderr, code] = yield* Effect.all( + [ + child.stdout.pipe(Stream.decodeText, Stream.splitLines, Stream.runForEach(onLine)), + child.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => `${all}${chunk}`.slice(0, 65536), + ), + ), + child.exitCode, + ], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* errorFor("tar", stderr.trim() || `tar exited with ${String(code)}`); + }), + ).pipe( + Effect.mapError((cause) => + cause instanceof DatabaseSnapshotError ? cause : errorFor("tar", cause), + ), + ), + ); + + const toolCommand = Effect.fn("DatabaseSnapshot.toolCommand")( + ( + stage: string, + script: string, + writableInstance = false, + onLine?: (line: string) => Effect.Effect, + ) => + Effect.gen(function* () { + if (container === undefined) + return yield* errorFor("container", "Container runtime missing"); + if (helperImage === undefined) + return yield* errorFor("container", "Container runtime missing"); + yield* container + .prepare(helperImage.image) + .pipe(Effect.mapError((cause) => errorFor("container", cause))); + return yield* Effect.scoped( + Effect.gen(function* () { + const acquire = container + .launchTool({ + image: helperImage.image, + stackId: options.stackId, + instanceId: options.instanceId, + entrypoint: "/usr/bin/busybox", + env: {}, + mounts: [ + { + source: options.instanceRoot, + target: "/instance", + readOnly: !writableInstance, + }, + { source: stage, target: "/stage", readOnly: false }, + ], + args: ["sh", "-c", script], + }) + .pipe( + Effect.catchTag("ContainerLaunchError", ({ failure, process: partial }) => + partial.stop.pipe( + Effect.andThen(partial.remove), + Effect.mapError((cause) => errorFor("container-cleanup", cause)), + Effect.andThen(Effect.fail(errorFor("container", failure))), + ), + ), + ); + return yield* Effect.acquireUseRelease( + acquire, + (owned) => + Effect.all( + [ + onLine === undefined + ? owned.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => `${all}${chunk}`.slice(0, 65536), + ), + ) + : owned.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runForEach(onLine), + Effect.as(""), + ), + owned.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (all, chunk) => `${all}${chunk}`.slice(0, 65536), + ), + ), + owned.exitCode, + ], + { concurrency: "unbounded" }, + ).pipe( + Effect.flatMap(([stdout, stderr, code]) => + Number(code) === 0 + ? Effect.succeed(stdout) + : Effect.fail( + errorFor( + "container", + stderr.trim() || `Helper exited with ${String(code)}`, + ), + ), + ), + ), + (owned) => + owned.stop.pipe( + Effect.andThen(owned.remove), + Effect.mapError((cause) => errorFor("container", cause)), + ), + ); + }), + ); + }).pipe(Effect.mapError((cause) => errorFor("container", cause))), + ); + + const cleanupStage = Effect.fn("DatabaseSnapshot.cleanupStage")((directory: string) => + Effect.gen(function* () { + if ( + options.runtime !== "native" && + ((yield* fs.exists(path.join(directory, "input"))) || + (yield* fs.exists(path.join(directory, "extracted")))) + ) { + yield* toolCommand(directory, "/usr/bin/busybox rm -rf /stage/extracted /stage/input"); + } + yield* fs.remove(directory, { recursive: true, force: true }); + }).pipe(Effect.mapError((cause) => errorFor("cleanup", cause))), + ); + + const readReady = Effect.fn("DatabaseSnapshot.readReady")(() => + Effect.gen(function* () { + const marker = yield* fs.readFileString( + path.join(options.instanceRoot, ".supabase-database-ready.json"), + ); + return yield* Schema.decodeEffect(Schema.fromJsonString(ReadyMarker))(marker); + }).pipe(Effect.mapError((cause) => errorFor("ready", cause))), + ); + + const verifyData = Effect.fn("DatabaseSnapshot.verifyData")(function* () { + const data = path.join(options.instanceRoot, "data"); + if (options.runtime === "native") { + if (!(yield* fs.exists(data).pipe(Effect.mapError((cause) => errorFor("data", cause))))) + return yield* errorFor("data", "Database data directory is absent"); + if ( + yield* fs + .exists(path.join(data, "postmaster.pid")) + .pipe(Effect.mapError((cause) => errorFor("data", cause))) + ) + return yield* errorFor("data", "Database must be stopped before snapshotting"); + if ( + !(yield* fs + .exists(path.join(data, "PG_VERSION")) + .pipe(Effect.mapError((cause) => errorFor("data", cause)))) + ) + return yield* errorFor("data", "Database data directory is not initialized"); + } else { + yield* toolCommand( + options.instanceRoot, + "test -d /instance/data && test -f /instance/data/PG_VERSION && test ! -e /instance/data/postmaster.pid", + ).pipe( + Effect.mapError((cause) => + cause instanceof DatabaseSnapshotError && cause.operation === "container" + ? new DatabaseSnapshotError({ + operation: "data", + message: cause.message, + cause, + }) + : cause, + ), + ); + } + return data; + }); + + const exportSnapshot = Effect.fn("DatabaseSnapshot.export")(function* ({ + destination, + }: { + readonly destination: string; + }) { + const ready = yield* readReady(); + if ( + ready.version !== version || + ready.runtime !== options.runtime || + ready.profile !== "supabase" + ) + return yield* errorFor("ready", "Database readiness marker does not match the instance"); + yield* verifyData(); + if (yield* fs.exists(destination).pipe(Effect.mapError((cause) => errorFor("export", cause)))) + return yield* errorFor("export", "Destination already exists"); + const parent = path.dirname(destination); + const stageToken = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("export", cause)), + ); + return yield* Effect.acquireUseRelease( + fs + .makeTempDirectory({ directory: parent, prefix: `.supabase-snapshot-${stageToken}-` }) + .pipe(Effect.mapError((cause) => errorFor("export", cause))), + (stage) => + Effect.gen(function* () { + const metadata = path.join(stage, "metadata"); + yield* fs + .makeDirectory(metadata, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("stage", cause))); + const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(SnapshotDescriptor))( + descriptor, + ).pipe(Effect.mapError((cause) => errorFor("descriptor", cause))); + yield* fs + .writeFileString(path.join(metadata, "descriptor.json"), encoded, { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("descriptor", cause))); + const archive = path.join(stage, "snapshot.tar"); + if (options.runtime === "native") + yield* command([ + "-cf", + archive, + "-C", + options.instanceRoot, + "data", + "-C", + stage, + "metadata", + ]); + else { + yield* fs + .writeFileString(archive, "", { mode: 0o600 }) + .pipe(Effect.mapError((cause) => errorFor("stage", cause))); + yield* toolCommand( + stage, + "/usr/bin/busybox mkdir -p /stage/input && /usr/bin/busybox cp -R /instance/data /stage/input/data && /usr/bin/busybox cp -R /stage/metadata /stage/input/metadata && /usr/bin/busybox tar -cf /stage/snapshot.tar -C /stage/input data metadata", + ); + } + yield* fs + .link(archive, destination) + .pipe(Effect.mapError((cause) => errorFor("publish", cause))); + yield* fs.remove(archive).pipe(Effect.mapError((cause) => errorFor("publish", cause))); + return { descriptor, destination }; + }), + cleanupStage, + ); + }); + + const restoreSnapshot = Effect.fn("DatabaseSnapshot.restore")(function* ({ + source, + }: { + readonly source: string; + }) { + const data = path.join(options.instanceRoot, "data"); + const dataExists = yield* fs + .exists(data) + .pipe(Effect.mapError((cause) => errorFor("restore", cause))); + if (options.runtime === "native") { + if ( + dataExists && + (yield* fs.readDirectory(data).pipe(Effect.mapError((cause) => errorFor("restore", cause)))) + .length > 0 + ) + return yield* errorFor("restore", "Restore target data directory must be empty"); + } else { + yield* toolCommand( + options.instanceRoot, + 'test ! -e /instance/data || test -z "$(/usr/bin/busybox ls -A /instance/data)"', + true, + ).pipe( + Effect.mapError((cause) => + cause instanceof DatabaseSnapshotError && cause.operation === "container" + ? new DatabaseSnapshotError({ + operation: "restore", + message: cause.message, + cause, + }) + : cause, + ), + ); + } + const stageToken = yield* crypto.randomUUIDv4.pipe( + Effect.mapError((cause) => errorFor("restore", cause)), + ); + return yield* Effect.acquireUseRelease( + fs + .makeTempDirectory({ + directory: options.instanceRoot, + prefix: `.supabase-restore-${stageToken}-`, + }) + .pipe(Effect.mapError((cause) => errorFor("restore", cause))), + (stage) => + Effect.gen(function* () { + const archive = path.join(stage, "source.tar"); + yield* fs + .copyFile(source, archive) + .pipe(Effect.mapError((cause) => errorFor("stage", cause))); + const descriptorText = + options.runtime === "native" + ? yield* command(["-xOf", archive, "metadata/descriptor.json"]) + : yield* toolCommand( + stage, + "/usr/bin/busybox tar -xOf /stage/source.tar metadata/descriptor.json", + ); + const incoming = yield* Schema.decodeEffect(Schema.fromJsonString(SnapshotDescriptor))( + descriptorText, + ).pipe(Effect.mapError((cause) => errorFor("descriptor", cause))); + if ( + incoming.format !== descriptor.format || + incoming.version !== descriptor.version || + incoming.runtime !== descriptor.runtime || + incoming.platform !== descriptor.platform || + incoming.arch !== descriptor.arch || + incoming.profile !== descriptor.profile + ) + return yield* errorFor( + "descriptor", + "Snapshot is incompatible with this database instance", + ); + const validateName = (line: string) => { + const name = line.replace(/\/$/u, ""); + return name.length === 0 || + ((["data", "metadata", "metadata/descriptor.json"].includes(name) || + name.startsWith("data/")) && + safeRelative(name)) + ? Effect.void + : Effect.fail(errorFor("validate", `Unsafe snapshot member: ${name}`)); + }; + if (options.runtime === "native") yield* commandLines(["-tf", archive], validateName); + else + yield* toolCommand( + stage, + "/usr/bin/busybox tar -tf /stage/source.tar", + false, + validateName, + ); + const validateType = (line: string) => + line.length === 0 || line[0] === "-" || line[0] === "d" + ? Effect.void + : Effect.fail(errorFor("validate", "Snapshot contains a non-regular member")); + if (options.runtime === "native") yield* commandLines(["-tvf", archive], validateType); + else + yield* toolCommand( + stage, + "/usr/bin/busybox tar -tvf /stage/source.tar", + false, + validateType, + ); + const extracted = path.join(stage, "extracted"); + yield* fs + .makeDirectory(extracted, { recursive: true }) + .pipe(Effect.mapError((cause) => errorFor("extract", cause))); + if (options.runtime === "native") yield* command(["-xf", archive, "-C", extracted]); + else + yield* toolCommand( + stage, + "/usr/bin/busybox tar -xf /stage/source.tar -C /stage/extracted", + ); + const expectedMajor = version.split(".")[0]; + if (options.runtime === "native") { + const restoredVersion = path.join(extracted, "data", "PG_VERSION"); + if ( + !(yield* fs + .exists(restoredVersion) + .pipe(Effect.mapError((cause) => errorFor("validate", cause)))) + ) + return yield* errorFor("validate", "Snapshot data is not initialized"); + const actualVersion = yield* fs + .readFileString(restoredVersion) + .pipe(Effect.mapError((cause) => errorFor("validate", cause))); + if (actualVersion.trim() !== expectedMajor) + return yield* errorFor( + "validate", + "Snapshot data has an incompatible PostgreSQL major version", + ); + } else { + yield* toolCommand( + stage, + "/usr/bin/busybox test -f /stage/extracted/data/PG_VERSION && /usr/bin/busybox cat /stage/extracted/data/PG_VERSION || /usr/bin/busybox echo __missing_pg_version__", + false, + (line) => + line.trim() === expectedMajor + ? Effect.void + : Effect.fail( + errorFor( + "validate", + "Snapshot data is missing or has an incompatible PostgreSQL major version", + ), + ), + ); + } + if (options.runtime === "native") { + yield* fs + .rename(path.join(extracted, "data"), data) + .pipe(Effect.mapError((cause) => errorFor("publish", cause))); + } else { + const stageName = path.basename(stage); + yield* toolCommand( + stage, + `if test -e /instance/data; then /usr/bin/busybox rmdir /instance/data; fi && /usr/bin/busybox mv /instance/${stageName}/extracted/data /instance/data && /usr/bin/busybox chown -R 100:101 /instance/data && /usr/bin/busybox chmod 700 /instance/data`, + true, + ); + } + const marker = yield* Schema.encodeEffect(Schema.fromJsonString(ReadyMarker))({ + version, + runtime: options.runtime, + profile: "supabase", + }).pipe(Effect.mapError((cause) => errorFor("ready", cause))); + yield* fs + .writeFileString( + path.join(options.instanceRoot, ".supabase-database-ready.json"), + marker, + { mode: 0o600 }, + ) + .pipe( + Effect.mapError((cause) => errorFor("ready", cause)), + Effect.catch((failure: DatabaseSnapshotError) => { + const cleanup = + options.runtime === "native" + ? fs + .remove(data, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => errorFor("cleanup", cause))) + : toolCommand( + options.instanceRoot, + "/usr/bin/busybox rm -rf /instance/data", + true, + ).pipe(Effect.asVoid); + return cleanup.pipe(Effect.andThen(Effect.fail(failure))); + }), + ); + + return { descriptor: incoming, destination: source }; + }), + cleanupStage, + ); + }); + + return { exportSnapshot, restoreSnapshot }; +}); diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts new file mode 100644 index 0000000000..70d12dc7d9 --- /dev/null +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -0,0 +1,208 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Ref, Schema, Stream } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { makeService } from "../Service.ts"; +import { bundleServeMainTemplate } from "../../tests/serve-main-bundler.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live("serves a standalone Functions bootstrap over HTTP", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-functions-" }); + const stackId = "b".repeat(64); + const instanceId = "functions-instance"; + const functionsRoot = root + "/user-functions"; + yield* fs.makeDirectory(functionsRoot + "/hello", { recursive: true }); + yield* fs.writeFileString( + functionsRoot + "/hello/index.ts", + "Deno.serve(() => Response.json({ custom: Deno.env.get('CUSTOM_ENV'), root: Deno.env.get('SUPABASE_INTERNAL_FUNCTIONS_ROOT'), port: Deno.env.get('EDGE_RUNTIME_PORT'), url: Deno.env.get('SUPABASE_URL'), db: Deno.env.get('SUPABASE_DB_URL'), jwt: Deno.env.get('SUPABASE_INTERNAL_JWT_SECRET'), anon: Deno.env.get('SUPABASE_ANON_KEY'), service: Deno.env.get('SUPABASE_SERVICE_ROLE_KEY') }));", + ); + const bootstrap = yield* bundleServeMainTemplate; + const recipe = yield* makeServiceRecipe( + { + service: "functions", + config: { + functionsRoot, + bootstrap, + databaseUrl: "postgres://functions-db", + apiUrl: "http://functions-api", + jwtSecret: "functions-jwt-secret", + env: { + CUSTOM_ENV: "custom-value", + SUPABASE_INTERNAL_FUNCTIONS_ROOT: "overridden-root", + EDGE_RUNTIME_PORT: "overridden-port", + SUPABASE_URL: "overridden-url", + SUPABASE_DB_URL: "overridden-db", + SUPABASE_INTERNAL_JWT_SECRET: "overridden-jwt", + SUPABASE_ANON_KEY: "overridden-anon", + SUPABASE_SERVICE_ROLE_KEY: "overridden-service", + }, + verifyJwt: false, + inspector: true, + }, + }, + { ...dockerOptions(root), stackId, instanceId }, + ); + const instance = yield* makeService(recipe.definition, { + id: instanceId, + config: recipe.creation, + }); + yield* instance.start; + yield* instance.ready; + const endpoint = yield* recipe.endpoint("http"); + const response = yield* client.execute( + HttpClientRequest.get("http://" + endpoint.host + ":" + endpoint.port + "/hello"), + ); + expect(response.status).toBe(200); + expect(yield* response.json).toEqual( + expect.objectContaining({ + custom: "custom-value", + port: "9000", + url: "http://functions-api", + db: "postgres://functions-db", + anon: expect.stringMatching(/^ey/u), + service: expect.stringMatching(/^ey/u), + }), + ); + const inspector = yield* recipe.endpoint("inspector"); + const inspectorResponse = yield* client.execute( + HttpClientRequest.get( + "http://" + inspector.host + ":" + inspector.port + "/json/version", + ), + ); + expect(inspectorResponse.status).toBe(200); + yield* instance.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); + +for (const runtime of ["native", "docker"] as const) { + it.live( + `serves configured function files, secrets and JWT policies in ${runtime}`, + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + // Edge Runtime's Linux sandbox does not expose host paths under /tmp. + const temporaryRoot = `${process.cwd()}/tmp`; + yield* fs.makeDirectory(temporaryRoot, { recursive: true }); + const root = yield* fs.makeTempDirectoryScoped({ + directory: temporaryRoot, + prefix: "functions-configured-", + }); + const filesRoot = `${root}/project`; + const functionsRoot = `${filesRoot}/supabase/functions`; + yield* fs.makeDirectory(`${functionsRoot}/locked`, { recursive: true }); + yield* fs.makeDirectory(`${filesRoot}/source`, { recursive: true }); + yield* fs.writeFileString( + `${filesRoot}/source/main.ts`, + `import {message} from "message"; Deno.serve(async () => Response.json({message, local: Deno.env.get("LOCAL"), shared: Deno.env.get("SHARED"), asset: await Deno.readTextFile(new URL("./asset.txt", import.meta.url))}));`, + ); + yield* fs.writeFileString( + `${filesRoot}/source/message.ts`, + 'export const message = "custom entrypoint";', + ); + yield* fs.writeFileString(`${filesRoot}/source/asset.txt`, "static content"); + yield* fs.writeFileString( + `${filesRoot}/supabase/import_map.json`, + '{"imports":{"message":"../source/message.ts"}}', + ); + yield* fs.writeFileString( + `${functionsRoot}/locked/index.ts`, + 'Deno.serve(() => new Response("locked"));', + ); + const recipe = yield* makeServiceRecipe( + { + service: "functions", + config: { + functionsRoot, + filesRoot, + bootstrap: yield* bundleServeMainTemplate, + jwtSecret: "test-function-jwt-with-at-least-32-characters", + verifyJwt: true, + env: { + SHARED: "shared", + LOCAL: "global", + SUPABASE_INTERNAL_DEBUG: "true", + }, + functions: { + hello: { + verifyJWT: false, + entrypoint: `${filesRoot}/source/main.ts`, + import_map: `${filesRoot}/supabase/import_map.json`, + static_files: [`${filesRoot}/source/*.txt`], + env: { LOCAL: "function" }, + }, + disabled: { enabled: false, entrypoint: `${filesRoot}/source/main.ts` }, + }, + }, + }, + { + ...options(root), + stackId: "c".repeat(64), + instanceId: "configured", + runtime, + cacheRoot: "/tmp/supabase-stack-artifacts", + }, + ); + const logs = yield* Ref.make(""); + yield* recipe.logs.pipe( + Stream.runForEach(({ bytes }) => + Ref.update(logs, (text) => text + new TextDecoder().decode(bytes)), + ), + Effect.forkScoped({ startImmediately: true }), + ); + const instance = yield* makeService(recipe.definition, { + id: "configured", + config: recipe.creation, + }); + yield* instance.start; + yield* instance.ready; + const endpoint = yield* recipe.endpoint("http"); + const base = `http://${endpoint.host}:${endpoint.port}`; + const response = yield* client.execute(HttpClientRequest.get(`${base}/hello`)); + const responseText = yield* response.text; + expect(response.status, `${yield* Ref.get(logs)}\n${responseText}`).toBe(200); + const body = yield* Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ + message: Schema.String, + local: Schema.String, + shared: Schema.String, + asset: Schema.String, + }), + ), + )(responseText); + expect(body).toEqual({ + message: "custom entrypoint", + local: "function", + shared: "shared", + asset: "static content", + }); + expect((yield* client.get(`${base}/disabled`)).status).toBe(404); + expect((yield* client.get(`${base}/locked`)).status).toBe(401); + yield* instance.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); +} diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts new file mode 100644 index 0000000000..c5295883b8 --- /dev/null +++ b/packages/stack/src/services/Functions.ts @@ -0,0 +1,260 @@ +import { Crypto, Effect, FileSystem, Path, Ref, Schema } from "effect"; +import { + makeFunctionsBootstrapOwner, + type FunctionsBootstrapOwner, +} from "../functions/FunctionsBootstrap.ts"; +import { StackIdSchema } from "../identity/StackId.ts"; +import { ServiceError } from "../Service.ts"; +import { serviceJwt } from "./ServiceConfig.ts"; +import { + CatalogError, + EndpointIntent, + serviceCreation, + type CatalogOptions, + type ProcessRecipeResult, +} from "./Recipe.ts"; +import { + makeProcessRecipe, + type ProcessDependencies, + type ProcessRecipeSpec, +} from "./ProcessRecipe.ts"; + +const FunctionSettings = Schema.Struct({ + enabled: Schema.optionalKey(Schema.Boolean), + verifyJWT: Schema.optionalKey(Schema.Boolean), + entrypoint: Schema.optionalKey(Schema.String), + import_map: Schema.optionalKey(Schema.String), + static_files: Schema.optionalKey(Schema.Array(Schema.String)), + env: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), +}); + +export const Config = Schema.Struct({ + functionsRoot: Schema.String, + filesRoot: Schema.optionalKey(Schema.String), + functions: Schema.optionalKey(Schema.Record(Schema.String, FunctionSettings)), + bootstrap: Schema.String, + databaseUrl: Schema.optionalKey(Schema.String), + env: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)), + apiUrl: Schema.optionalKey(Schema.String), + jwtSecret: Schema.optionalKey(Schema.String), + policy: Schema.optionalKey(Schema.String), + verifyJwt: Schema.optionalKey(Schema.Boolean), + inspector: Schema.optionalKey(Schema.Boolean), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ + http: Schema.optionalKey(EndpointIntent), + inspector: Schema.optionalKey(EndpointIntent), +}); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("functions", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +const FunctionsRuntimeConfigJson = Schema.fromJsonString( + Schema.Record(Schema.String, FunctionSettings), +); + +const makeSpec = ( + functionsRoot: Ref.Ref, + bootstrap: FunctionsBootstrapOwner, + path: Path.Path, + fs: FileSystem.FileSystem, +): ProcessRecipeSpec => ({ + service: "functions", + executable: "bin/edge-runtime", + ports: { http: 9000, inspector: 9229 }, + healthPath: "/_internal/health", + enabledPort: (creation, name) => name !== "inspector" || creation.config.inspector === true, + env: (creation, endpoints, container) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const filesRoot = creation.config.filesRoot; + const canonicalFilesRoot = + filesRoot === undefined || container + ? filesRoot + : yield* fs.realPath(filesRoot).pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "launch", + message: "Unable to resolve Functions project directory", + cause, + }), + ), + ); + const runtimePath = (value: string) => + !path.isAbsolute(value) || filesRoot === undefined || canonicalFilesRoot === undefined + ? value + : path.join( + container ? "/__supabase_project" : canonicalFilesRoot, + path.relative(filesRoot, value), + ); + const root = + container && filesRoot === undefined + ? "/__supabase_functions" + : runtimePath(creation.config.functionsRoot); + const functions = Object.fromEntries( + Object.entries(creation.config.functions ?? {}).map(([name, settings]) => [ + name, + { + ...settings, + ...(settings.entrypoint === undefined + ? {} + : { entrypoint: runtimePath(settings.entrypoint) }), + ...(settings.import_map === undefined + ? {} + : { import_map: runtimePath(settings.import_map) }), + ...(settings.static_files === undefined + ? {} + : { static_files: settings.static_files.map(runtimePath) }), + ...(creation.config.verifyJwt === false ? { verifyJWT: false } : {}), + }, + ]), + ); + const jwt = creation.config.jwtSecret; + return { + ...creation.config.env, + ...(http === undefined ? {} : { EDGE_RUNTIME_PORT: String(http.port) }), + SUPABASE_INTERNAL_FUNCTIONS_ROOT: root, + ...(filesRoot === undefined + ? {} + : { SUPABASE_INTERNAL_FUNCTIONS_FILES_ROOT: runtimePath(filesRoot) }), + ...(jwt === undefined + ? {} + : { + SUPABASE_INTERNAL_JWT_SECRET: jwt, + SUPABASE_ANON_KEY: yield* serviceJwt("anon", jwt), + SUPABASE_SERVICE_ROLE_KEY: yield* serviceJwt("service_role", jwt), + }), + ...(creation.config.verifyJwt === undefined && creation.config.functions === undefined + ? {} + : { + SUPABASE_INTERNAL_FUNCTIONS_CONFIG: yield* Schema.encodeEffect( + FunctionsRuntimeConfigJson, + )({ $default: { verifyJWT: creation.config.verifyJwt ?? true }, ...functions }).pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "launch", + message: "Unable to encode Functions runtime configuration", + cause, + }), + ), + ), + }), + ...(creation.config.apiUrl === undefined ? {} : { SUPABASE_URL: creation.config.apiUrl }), + ...(creation.config.databaseUrl === undefined + ? {} + : { SUPABASE_DB_URL: creation.config.databaseUrl }), + ...(creation.config.policy === undefined + ? {} + : { EDGE_RUNTIME_POLICY: creation.config.policy }), + }; + }), + args: (creation, endpoints, context) => + Effect.gen(function* () { + const override = yield* Ref.get(functionsRoot); + const root = context.container + ? override === undefined + ? "/__supabase_functions" + : "/__supabase_bootstrap" + : (override ?? creation.config.functionsRoot); + const http = endpoints.get("http"); + const inspector = endpoints.get("inspector"); + return [ + "start", + `--main-service=${root}`, + ...(http === undefined ? [] : [`--port=${http.port}`]), + ...(creation.config.policy === undefined ? [] : [`--policy=${creation.config.policy}`]), + ...(creation.config.inspector === true && inspector !== undefined + ? [`--inspect=${context.container ? "0.0.0.0" : "127.0.0.1"}:${inspector.port}`] + : []), + ]; + }), + mounts: (creation) => + Effect.gen(function* () { + const override = yield* Ref.get(functionsRoot); + const files = creation.config.filesRoot; + const source = files ?? creation.config.functionsRoot; + const target = files === undefined ? "/__supabase_functions" : "/__supabase_project"; + return [ + { source, target, readOnly: true }, + ...(override === undefined + ? [] + : [{ source: override, target: "/__supabase_bootstrap", readOnly: true }]), + ]; + }), + startup: [], + prepare: (creation) => + bootstrap.write({ content: creation.config.bootstrap }).pipe( + Effect.flatMap((target) => Ref.set(functionsRoot, path.dirname(target))), + Effect.mapError( + (cause) => + new ServiceError({ + operation: "prepare", + message: "Unable to prepare Functions bootstrap", + cause, + }), + ), + ), + removeData: () => + bootstrap.cleanupAll.pipe( + Effect.mapError( + (cause) => + new ServiceError({ + operation: "destroy", + message: "Unable to remove Functions bootstrap", + cause, + }), + ), + ), +}); + +export const makeRecipe = Effect.fn("Functions.makeRecipe")( + ( + creation: Creation, + options: CatalogOptions, + deps: ProcessDependencies, + ): Effect.Effect, CatalogError> => + Effect.gen(function* () { + const functionsRoot = yield* Ref.make(undefined); + const stackId = yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( + Effect.mapError( + (cause) => + new CatalogError({ + operation: "functions", + message: "Invalid stack identity", + service: "functions", + cause, + }), + ), + ); + const bootstrap = yield* makeFunctionsBootstrapOwner({ + root: options.root, + stackId, + instanceId: options.instanceId, + }).pipe( + Effect.provideService(FileSystem.FileSystem, deps.fs), + Effect.provideService(Path.Path, deps.path), + Effect.provideService(Crypto.Crypto, deps.crypto), + Effect.mapError( + (cause) => + new CatalogError({ + operation: "functions", + message: "Unable to create Functions bootstrap owner", + service: "functions", + cause, + }), + ), + ); + return yield* makeProcessRecipe( + creation, + options, + deps, + makeSpec(functionsRoot, bootstrap, deps.path, deps.fs), + ); + }), +); diff --git a/packages/stack/src/services/Imgproxy.ts b/packages/stack/src/services/Imgproxy.ts new file mode 100644 index 0000000000..de0bd405e1 --- /dev/null +++ b/packages/stack/src/services/Imgproxy.ts @@ -0,0 +1,38 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ filePath: Schema.optionalKey(Schema.String) }); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("imgproxy", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "imgproxy", + executable: "bin/imgproxy", + ports: { http: 5001 }, + healthPath: "/health", + env: (_creation, endpoints, container) => { + const http = endpoints.get("http"); + return Effect.succeed({ + ...(http === undefined ? {} : { IMGPROXY_PORT: String(http.port) }), + ...(http === undefined + ? {} + : { IMGPROXY_BIND: `${container ? "0.0.0.0" : "127.0.0.1"}:${http.port}` }), + IMGPROXY_LOCAL_FILESYSTEM_ROOT: "/", + }); + }, + args: () => Effect.succeed([]), + mounts: (creation) => + Effect.succeed( + creation.config.filePath === undefined + ? [] + : [{ source: creation.config.filePath, target: "/mnt", readOnly: true }], + ), + startup: [], +}); diff --git a/packages/stack/src/services/Mail.integration.test.ts b/packages/stack/src/services/Mail.integration.test.ts new file mode 100644 index 0000000000..4727dae9c1 --- /dev/null +++ b/packages/stack/src/services/Mail.integration.test.ts @@ -0,0 +1,47 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { makeService } from "../Service.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live("launches the real Mailpit recipe and serves its HTTP endpoint", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-mail-" }); + const recipe = yield* makeServiceRecipe( + { service: "mail", config: {} }, + dockerOptions(root), + ); + const instance = yield* makeService(recipe.definition, { + id: "mail", + config: recipe.creation, + }); + yield* instance.start; + yield* instance.ready; + const endpoint = yield* recipe.endpoint("http"); + const response = yield* client.execute( + HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}/readyz`), + ); + expect(response.status).toBe(200); + yield* instance.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); diff --git a/packages/stack/src/services/Mail.ts b/packages/stack/src/services/Mail.ts new file mode 100644 index 0000000000..cfa43810b7 --- /dev/null +++ b/packages/stack/src/services/Mail.ts @@ -0,0 +1,54 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ + http: Schema.optionalKey(EndpointIntent), + smtp: Schema.optionalKey(EndpointIntent), + pop3: Schema.optionalKey(EndpointIntent), +}); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("mail", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "mail", + executable: "bin/mailpit", + ports: { http: 8025, smtp: 1025, pop3: 1110 }, + healthPath: "/readyz", + env: (_creation, endpoints, container) => { + const http = endpoints.get("http"); + const smtp = endpoints.get("smtp"); + const pop3 = endpoints.get("pop3"); + return Effect.succeed({ + ...(http === undefined + ? {} + : { + MP_UI_PORT: String(http.port), + MP_UI_BIND_ADDR: `${container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`, + }), + ...(smtp === undefined + ? {} + : { + MP_SMTP_PORT: String(smtp.port), + MP_SMTP_BIND_ADDR: `${container ? "0.0.0.0" : "127.0.0.1"}:${smtp.port}`, + }), + ...(pop3 === undefined + ? {} + : { + MP_POP3_PORT: String(pop3.port), + MP_POP3_BIND_ADDR: `${container ? "0.0.0.0" : "127.0.0.1"}:${pop3.port}`, + }), + // Docker gateway addresses can make SMTP reverse DNS delay Auth recovery. + MP_SMTP_DISABLE_RDNS: "true", + }); + }, + args: () => Effect.succeed([]), + mounts: () => Effect.succeed([]), + startup: [], +}); diff --git a/packages/stack/src/services/Pgmeta.ts b/packages/stack/src/services/Pgmeta.ts new file mode 100644 index 0000000000..87c95452a3 --- /dev/null +++ b/packages/stack/src/services/Pgmeta.ts @@ -0,0 +1,41 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { databaseConnection } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ databaseUrl: Schema.String }); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("pgmeta", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "pgmeta", + executable: "bin/pgmeta", + ports: { http: 8080 }, + healthPath: "/health", + env: (creation, endpoints, container) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const db = yield* databaseConnection(creation.config.databaseUrl); + return { + DATABASE_URL: creation.config.databaseUrl, + PG_META_HOST: container ? "0.0.0.0" : "127.0.0.1", + ...(container ? {} : { PG_META_ADMIN_PORT: "0" }), + ...(http === undefined ? {} : { PG_META_PORT: String(http.port) }), + PG_META_DB_URL: creation.config.databaseUrl, + PG_META_DB_HOST: db.host, + PG_META_DB_PORT: db.port, + PG_META_DB_NAME: db.database, + PG_META_DB_USER: db.username ?? "supabase_admin", + PG_META_DB_PASSWORD: db.password ?? "postgres", + }; + }), + args: () => Effect.succeed([]), + mounts: () => Effect.succeed([]), + startup: [], +}); diff --git a/packages/stack/src/services/Pooler.integration.test.ts b/packages/stack/src/services/Pooler.integration.test.ts new file mode 100644 index 0000000000..8e5aa5e8de --- /dev/null +++ b/packages/stack/src/services/Pooler.integration.test.ts @@ -0,0 +1,101 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Effect, FileSystem, Layer, Redacted } from "effect"; +import { PgClient } from "@effect/sql-pg"; +import { makeService } from "../Service.ts"; +import { ProxyError } from "../Proxy.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; +import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; +import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live( + "serves Pooler SQL against the owned database", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-pooler-" }); + yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); + const secret = "catalog-pooler-secret-with-at-least-32-chars"; + const databaseRecipe = yield* makeServiceRecipe( + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + }, + dockerOptions(root), + ); + const database = yield* makeService(databaseRecipe.definition, { + id: "database", + config: databaseRecipe.creation, + }); + yield* database.start; + yield* database.ready; + const databaseEndpoint = yield* databaseRecipe.endpoint("sql"); + if (databaseEndpoint.kind !== "tcp" || databaseEndpoint.host === undefined) + return yield* new ProxyError({ message: "Docker database did not expose TCP" }); + const databaseRelay = yield* makeDockerTcpRelay(databaseRecipe.endpoint("sql")); + const databaseUrl = `postgresql://supabase_admin:postgres@${databaseRelay.host}:${databaseRelay.port}/_supabase`; + for (const poolMode of ["transaction", "session"] as const) { + const tenant = `catalog-${poolMode}`; + const poolerRecipe = yield* makeServiceRecipe( + { + service: "pooler", + config: { + databaseUrl, + jwtSecret: secret, + tenant, + poolMode, + defaultPoolSize: 7, + maxClientConnections: 42, + }, + }, + dockerOptions(root), + ); + const pooler = yield* makeService(poolerRecipe.definition, { + id: `pooler-${poolMode}`, + config: poolerRecipe.creation, + }); + yield* pooler.start; + yield* pooler.ready; + const sqlEndpoint = yield* poolerRecipe.endpoint("sql"); + if (sqlEndpoint.kind !== "tcp" || sqlEndpoint.host === undefined) + return yield* new ProxyError({ message: "Pooler did not expose TCP" }); + const poolerLayer = yield* Layer.build( + PgClient.layer({ + host: sqlEndpoint.host, + port: sqlEndpoint.port, + database: "postgres", + username: `supabase_admin.${tenant}`, + password: Redacted.make("postgres"), + }), + ); + const sql = Context.get(poolerLayer, PgClient.PgClient); + const rows = yield* sql.unsafe<{ readonly value: number }>("SELECT 1 AS value"); + expect(rows[0]?.value).toBe(1); + yield* pooler.stop; + } + yield* database.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); +}); diff --git a/packages/stack/src/services/Pooler.ts b/packages/stack/src/services/Pooler.ts new file mode 100644 index 0000000000..6e9a1c69c5 --- /dev/null +++ b/packages/stack/src/services/Pooler.ts @@ -0,0 +1,83 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { databaseConnection, localJwtSecret } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + databaseUrl: Schema.String, + jwtSecret: Schema.optionalKey(Schema.String), + tenant: Schema.optionalKey(Schema.String), + defaultPoolSize: Schema.optionalKey(Schema.Finite), + maxClientConnections: Schema.optionalKey(Schema.Finite), + poolMode: Schema.optionalKey(Schema.Literals(["transaction", "session"])), +}); +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ + http: Schema.optionalKey(EndpointIntent), + sql: Schema.optionalKey(EndpointIntent), +}); +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("pooler", Config, Endpoints); +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "pooler", + executable: "bin/server", + ports: { http: 4000, sql: 6543 }, + healthPath: "/api/health", + env: (creation, endpoints, container) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const sql = endpoints.get("sql"); + const db = yield* databaseConnection(creation.config.databaseUrl); + const mode = creation.config.poolMode ?? "transaction"; + return { + DATABASE_URL: creation.config.databaseUrl, + ...(http === undefined ? {} : { PORT: String(http.port) }), + ...(creation.config.tenant === undefined ? {} : { TENANT_ID: creation.config.tenant }), + POSTGRES_HOST: db.host, + POSTGRES_PORT: db.port, + POSTGRES_USER: db.username ?? "supabase_admin", + POSTGRES_PASSWORD: db.password ?? "postgres", + API_JWT_SECRET: creation.config.jwtSecret ?? localJwtSecret, + METRICS_JWT_SECRET: creation.config.jwtSecret ?? localJwtSecret, + REGION: "local", + CLUSTER_POSTGRES: "true", + SECRET_KEY_BASE: localJwtSecret, + VAULT_ENC_KEY: "0123456789abcdef0123456789abcdef", + DEFAULT_POOL_SIZE: String(creation.config.defaultPoolSize ?? 20), + MAX_CLIENT_CONN: String(creation.config.maxClientConnections ?? 100), + POOL_MODE: mode, + // Supavisor advertises Ranch-bound ports, so native instances can use ephemeral internal listeners. + ...(container + ? {} + : { + PROXY_PORT: "0", + SESSION_PROXY_PORTS: "0", + TRANSACTION_PROXY_PORTS: "0", + }), + ...(sql === undefined + ? {} + : { + PROXY_PORT_SESSION: String(container ? 5432 : mode === "session" ? sql.port : 0), + PROXY_PORT_TRANSACTION: String( + container ? 6543 : mode === "transaction" ? sql.port : 0, + ), + }), + }; + }), + args: (_creation, _endpoints, context) => + Effect.succeed(context.container ? ["-s", "-g", "--", "/app/bin/server"] : ["start"]), + mounts: () => Effect.succeed([]), + containerPort: (creation, name, port) => + name === "sql" && creation.config.poolMode === "session" ? 5432 : port, + containerEntrypoint: () => "/usr/bin/tini", + startup: [ + { args: [], nativeExecutable: "prepare", containerEntrypoint: "/app/bin/prepare" }, + { + args: [], + nativeExecutable: "provision-tenant", + containerEntrypoint: "/app/bin/provision-tenant", + }, + ], +}); diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts new file mode 100644 index 0000000000..88785432db --- /dev/null +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -0,0 +1,489 @@ +import { + Cause, + Crypto, + Effect, + Exit, + FileSystem, + Path, + PubSub, + Ref, + Schedule, + Scope, + Stream, +} from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; +import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import * as Net from "node:net"; +import { prepareNativeArtifact, resolveArtifact, type ServiceKind } from "../Artifacts.ts"; +import { + type ContainerError, + type ContainerProcess, + type ContainerRuntime, +} from "../runtime/Container.ts"; +import { + defaultNativeProcessLauncher, + type NativeProcess, + type NativeProcessError, + spawnNativeProcess, +} from "../runtime/NativeProcess.ts"; +import { ServiceError, ServiceLaunchError, type RuntimeSession } from "../Service.ts"; +import { + type CatalogLog, + CatalogError, + type CatalogOptions, + type ProcessRecipeResult, + type RecipeCreation, + type ServiceEndpoint, +} from "./Recipe.ts"; + +interface RecipeMount { + readonly source: string; + readonly target: string; + readonly readOnly: boolean; +} + +interface StartupProcess { + readonly args: ReadonlyArray; + readonly nativeExecutable?: string; + readonly containerEntrypoint?: string; + readonly skipInContainer?: boolean; +} + +export interface ProcessRecipeSpec> { + readonly service: C["service"]; + readonly executable: string; + readonly ports: Readonly>; + readonly healthPath: string; + readonly args: ( + creation: C, + endpoints: ReadonlyMap, + context: { readonly container: boolean; readonly artifactRoot?: string }, + ) => Effect.Effect, ServiceError>; + readonly env: ( + creation: C, + endpoints: ReadonlyMap, + container: boolean, + ) => Effect.Effect>, ServiceError>; + readonly mounts: ( + creation: C, + context: { readonly container: boolean }, + ) => Effect.Effect, ServiceError>; + readonly startup: ReadonlyArray; + readonly enabledPort?: (creation: C, name: string) => boolean; + readonly containerPort?: (creation: C, name: string, port: number) => number; + readonly containerEntrypoint?: (creation: C) => string | undefined; + readonly prepare?: (creation: C) => Effect.Effect; + readonly removeData?: (creation: C) => Effect.Effect; +} + +export interface ProcessDependencies { + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; + readonly crypto: Crypto.Crypto; + readonly client: HttpClient.HttpClient; + readonly spawner: ChildProcessSpawnerService["Service"]; + readonly container: ContainerRuntime | undefined; +} + +const serviceError = (operation: string, cause: unknown): ServiceError => + cause instanceof ServiceError + ? cause + : new ServiceError({ + operation, + message: Cause.isTimeoutError(cause) + ? `Service ${operation} timed out` + : cause instanceof Error + ? cause.message + : String(cause), + cause, + }); + +const catalogError = (operation: string, message: string, service?: ServiceKind, cause?: unknown) => + new CatalogError({ + operation, + message, + ...(service === undefined ? {} : { service }), + ...(cause === undefined ? {} : { cause }), + }); + +const processExit = ( + exitCode: Effect.Effect, +): Effect.Effect> => + exitCode.pipe( + Effect.flatMap((code) => + Number(code) === 0 + ? Effect.void + : Effect.fail( + new ServiceError({ operation: "exit", message: `Process exited with ${code}` }), + ), + ), + Effect.mapError((cause) => serviceError("exit", cause)), + Effect.exit, + ); + +const runtimeFromContainer = (process: ContainerProcess): RuntimeSession => ({ + health: Effect.void, + exit: processExit(process.exitCode), + stop: process.stop.pipe(Effect.mapError((cause) => serviceError("stop", cause))), + remove: process.remove.pipe(Effect.mapError((cause) => serviceError("remove", cause))), +}); + +const reserveNativePort = Effect.fn("ProcessRecipe.reserveNativePort")( + (requested: number): Effect.Effect => + Effect.scoped( + Effect.gen(function* () { + const allocation = yield* Effect.acquireRelease( + Effect.callback<{ readonly port: number; readonly server: Net.Server }, CatalogError>( + (resume) => { + const server = Net.createServer(); + const onError = (cause: Error) => + resume( + Effect.fail( + catalogError( + "launch", + "Unable to reserve native service port", + undefined, + cause, + ), + ), + ); + server.once("error", onError); + server.listen({ host: "127.0.0.1", port: requested }, () => { + const address = server.address(); + if (address === null || typeof address === "string") { + onError(new Error("Native service port reservation returned no address")); + } else { + resume(Effect.succeed({ port: address.port, server })); + } + }); + return Effect.sync(() => { + server.off("error", onError); + if (server.listening) server.close(); + }); + }, + ), + ({ server }) => + Effect.callback((resume) => { + server.close(() => resume(Effect.void)); + return Effect.void; + }), + ); + return allocation.port; + }), + ), +); + +const publishLogs = Effect.fn("ProcessRecipe.publishLogs")(( + process: { + readonly stdout: Stream.Stream; + readonly stderr: Stream.Stream; + }, + logs: PubSub.PubSub, + scope: Scope.Closeable, +): Effect.Effect => { + const drain = (stream: Stream.Stream, name: CatalogLog["stream"]) => + stream.pipe( + Stream.runForEach((bytes) => PubSub.publish(logs, { stream: name, bytes })), + Effect.catch((cause) => Effect.logError(cause)), + Effect.asVoid, + ); + return Effect.all( + [ + Effect.forkIn(drain(process.stdout, "stdout"), scope), + Effect.forkIn(drain(process.stderr, "stderr"), scope), + ], + { concurrency: "unbounded", discard: true }, + ); +}); + +const awaitStartup = Effect.fn("ProcessRecipe.awaitStartup")( + ( + process: { + readonly stdout: Stream.Stream; + readonly stderr: Stream.Stream; + readonly exitCode: Effect.Effect; + }, + logs: PubSub.PubSub, + ): Effect.Effect, ServiceError> => + Effect.gen(function* () { + const [stdout, stderr, exitCode] = yield* Effect.all( + [ + process.stdout.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (text, chunk) => text + chunk, + ), + ), + process.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (text, chunk) => text + chunk, + ), + ), + process.exitCode, + ], + { concurrency: "unbounded" }, + ).pipe(Effect.mapError((cause) => serviceError("launch", cause))); + if (stdout.length > 0) + yield* PubSub.publish(logs, { + stream: "stdout", + bytes: new TextEncoder().encode(stdout), + }); + if (stderr.length > 0) + yield* PubSub.publish(logs, { + stream: "stderr", + bytes: new TextEncoder().encode(stderr), + }); + return { code: Number(exitCode), stderr }; + }).pipe( + Effect.timeout("60 seconds"), + Effect.mapError((cause) => serviceError("launch", cause)), + ), +); + +const readiness = Effect.fn("ProcessRecipe.readiness")( + ( + client: HttpClient.HttpClient, + endpoint: ServiceEndpoint, + path: string, + ): Effect.Effect => + client.execute(HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}${path}`)).pipe( + Effect.flatMap((response) => + (response.status >= 200 && response.status < 300) || response.status === 401 + ? Effect.void + : Effect.fail( + new ServiceError({ operation: "health", message: `HTTP ${response.status}` }), + ), + ), + Effect.retry({ schedule: Schedule.spaced("250 millis") }), + Effect.timeout("60 seconds"), + Effect.mapError((cause) => serviceError("health", cause)), + Effect.asVoid, + ), +); + +export const makeProcessRecipe = >( + creation: C, + options: CatalogOptions, + deps: ProcessDependencies, + spec: ProcessRecipeSpec, +): Effect.Effect> => + Effect.gen(function* () { + const prepared = yield* Ref.make(undefined); + const preparedRoot = yield* Ref.make(undefined); + const endpoints = yield* Ref.make>(new Map()); + const logs = yield* PubSub.sliding(256); + + const prepare = Effect.fn("ProcessRecipe.prepare")(function* (candidate: C) { + if (spec.prepare !== undefined) yield* spec.prepare(candidate); + const resolved = yield* resolveArtifact({ + service: candidate.service, + version: candidate.version, + }).pipe(Effect.mapError((cause) => serviceError("prepare", cause))); + if (options.runtime === "native") { + const artifact = yield* prepareNativeArtifact( + { service: candidate.service, version: candidate.version }, + options.cacheRoot, + options.platform, + ).pipe( + Effect.provideService(FileSystem.FileSystem, deps.fs), + Effect.provideService(Path.Path, deps.path), + Effect.provideService(Crypto.Crypto, deps.crypto), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), + Effect.provideService(HttpClient.HttpClient, deps.client), + Effect.mapError((cause) => serviceError("prepare", cause)), + ); + yield* Ref.set(prepared, artifact.executable); + yield* Ref.set(preparedRoot, artifact.root); + } else { + if (deps.container === undefined) + return yield* serviceError("prepare", "Container runtime unavailable"); + yield* deps.container + .prepare(resolved.image) + .pipe(Effect.mapError((cause) => serviceError("prepare", cause))); + } + }); + + const launch = Effect.fn("ProcessRecipe.launch")(function* (context: { + readonly id: string; + readonly config: C; + readonly scope: Scope.Closeable; + }) { + const desired = new Map(); + for (const [name, port] of Object.entries(spec.ports)) { + if (spec.enabledPort !== undefined && !spec.enabledPort(context.config, name)) continue; + desired.set(name, { + kind: "tcp", + host: "127.0.0.1", + port: + options.runtime === "native" + ? yield* reserveNativePort(0).pipe( + Effect.mapError((cause) => serviceError("launch", cause)), + ) + : port, + }); + } + if (options.runtime === "native") { + const executable = yield* Ref.get(prepared); + const artifactRoot = yield* Ref.get(preparedRoot); + if (executable === undefined) + return yield* serviceError("launch", "Artifact was not prepared"); + if (artifactRoot === undefined) + return yield* serviceError("launch", "Artifact root was not prepared"); + for (const [index, process] of spec.startup.entries()) { + const startupProcess = yield* spawnNativeProcess( + { + executable: `${artifactRoot}/bin/${process.nativeExecutable ?? "prepare"}`, + args: process.args, + env: yield* spec.env(context.config, desired, false), + cwd: artifactRoot, + }, + defaultNativeProcessLauncher(), + { + stackId: String(options.stackId), + workloadId: `${context.id}-${context.config.service}-startup-${index}`, + }, + ).pipe( + Scope.provide(context.scope), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), + Effect.mapError((cause) => serviceError("launch", cause)), + ); + const result = yield* awaitStartup(startupProcess, logs); + if (result.code !== 0) + return yield* serviceError( + "launch", + `${context.config.service} startup exited with ${result.code}: ${result.stderr.trim()}`, + ); + } + const native: NativeProcess = yield* spawnNativeProcess( + { + executable, + args: yield* spec.args(context.config, desired, { container: false, artifactRoot }), + env: yield* spec.env(context.config, desired, false), + gracefulStopSignal: "SIGTERM", + gracefulStopTimeout: "5 seconds", + }, + defaultNativeProcessLauncher(), + { stackId: String(options.stackId), workloadId: context.id }, + ).pipe( + Scope.provide(context.scope), + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), + Effect.mapError((cause) => serviceError("launch", cause)), + ); + yield* Ref.set(endpoints, desired); + yield* publishLogs(native, logs, context.scope); + const ready = desired.get("http"); + return { + health: + ready === undefined + ? Effect.fail(serviceError("health", "Recipe has no HTTP readiness endpoint")) + : readiness(deps.client, ready, spec.healthPath), + exit: processExit(native.exitCode), + stop: native.kill.pipe(Effect.mapError((cause) => serviceError("stop", cause))), + remove: Ref.set(endpoints, new Map()), + } satisfies RuntimeSession; + } + if (deps.container === undefined) + return yield* serviceError("launch", "Container runtime unavailable"); + const resolved = yield* resolveArtifact({ + service: context.config.service, + version: context.config.version, + }).pipe(Effect.mapError((cause) => serviceError("launch", cause))); + const containerDesired = new Map(); + for (const [name, endpoint] of desired) { + const port = + spec.containerPort === undefined + ? endpoint.port + : spec.containerPort(context.config, name, endpoint.port); + containerDesired.set(name, { ...endpoint, port }); + } + for (const process of spec.startup) { + if (process.skipInContainer === true) continue; + const startupProcess = yield* deps.container + .launchTool({ + image: resolved.image, + stackId: options.stackId, + instanceId: options.instanceId, + env: yield* spec.env(context.config, containerDesired, true), + entrypoint: process.containerEntrypoint, + args: process.args, + mounts: yield* spec.mounts(context.config, { container: true }), + }) + .pipe( + Effect.mapError((cause) => serviceError("launch", cause)), + Scope.provide(context.scope), + ); + const result = yield* awaitStartup(startupProcess, logs); + yield* startupProcess.remove.pipe( + Effect.mapError((cause) => serviceError("launch", cause)), + ); + if (result.code !== 0) + return yield* serviceError( + "launch", + `${context.config.service} startup exited with ${result.code}: ${result.stderr.trim()}`, + ); + } + const launched = yield* deps.container + .launch({ + image: resolved.image, + stackId: options.stackId, + instanceId: options.instanceId, + env: yield* spec.env(context.config, containerDesired, true), + entrypoint: spec.containerEntrypoint?.(context.config), + args: yield* spec.args(context.config, containerDesired, { container: true }), + mounts: yield* spec.mounts(context.config, { container: true }), + ports: [...containerDesired.values()].map((endpoint) => endpoint.port), + }) + .pipe( + Effect.catchTag("ContainerLaunchError", ({ failure, process }) => + Effect.fail( + new ServiceLaunchError({ + failure: serviceError("launch", failure), + runtime: runtimeFromContainer(process), + }), + ), + ), + Effect.mapError((cause) => + cause instanceof ServiceLaunchError ? cause : serviceError("launch", cause), + ), + Scope.provide(context.scope), + ); + const selected = new Map(); + for (const [name, endpoint] of containerDesired) { + const published = launched.ports[endpoint.port]; + if (published === undefined) + return yield* serviceError("launch", `Container did not publish ${name}`); + selected.set(name, { kind: "tcp", host: "127.0.0.1", port: published }); + } + yield* Ref.set(endpoints, selected); + yield* publishLogs(launched, logs, context.scope); + const runtime = runtimeFromContainer(launched); + const ready = selected.get("http"); + return { + ...runtime, + health: + ready === undefined + ? Effect.fail(serviceError("health", "Recipe has no HTTP readiness endpoint")) + : readiness(deps.client, ready, spec.healthPath), + remove: runtime.remove.pipe(Effect.tap(() => Ref.set(endpoints, new Map()))), + } satisfies RuntimeSession; + }); + + return { + definition: { + prepare, + launch, + removeData: (context) => + (spec.removeData?.(context.config) ?? Effect.void).pipe( + Effect.andThen(Ref.set(endpoints, new Map())), + ), + }, + endpoints, + logs: Stream.fromPubSub(logs).pipe( + Stream.mapError((cause) => catalogError("logs", String(cause), creation.service)), + ), + }; + }); diff --git a/packages/stack/src/services/Realtime.integration.test.ts b/packages/stack/src/services/Realtime.integration.test.ts new file mode 100644 index 0000000000..9b04f65925 --- /dev/null +++ b/packages/stack/src/services/Realtime.integration.test.ts @@ -0,0 +1,126 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Redacted } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { makeService } from "../Service.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; +import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; +import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live( + "serves Realtime, pg-meta, and Studio against the owned database", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-optional-api-" }); + yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); + const secret = "catalog-optional-api-secret-with-at-least-32-chars"; + const databaseRecipe = yield* makeServiceRecipe( + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + }, + dockerOptions(root), + ); + const database = yield* makeService(databaseRecipe.definition, { + id: "database", + config: databaseRecipe.creation, + }); + yield* database.start; + yield* database.ready; + const databaseRelay = yield* makeDockerTcpRelay(databaseRecipe.endpoint("sql")); + const databaseUrl = `postgresql://supabase_admin:postgres@${databaseRelay.host}:${databaseRelay.port}/postgres`; + + const realtimeRecipe = yield* makeServiceRecipe( + { service: "realtime", config: { databaseUrl, jwtSecret: secret } }, + dockerOptions(root), + ); + const realtime = yield* makeService(realtimeRecipe.definition, { + id: "realtime", + config: realtimeRecipe.creation, + }); + yield* realtime.start; + yield* realtime.ready; + const realtimeEndpoint = yield* realtimeRecipe.endpoint("http"); + const realtimeResponse = yield* client.execute( + HttpClientRequest.get( + `http://${realtimeEndpoint.host}:${realtimeEndpoint.port}/healthcheck`, + ), + ); + expect(realtimeResponse.status).toBe(200); + + const pgmetaRecipe = yield* makeServiceRecipe( + { service: "pgmeta", config: { databaseUrl } }, + dockerOptions(root), + ); + const pgmeta = yield* makeService(pgmetaRecipe.definition, { + id: "pgmeta", + config: pgmetaRecipe.creation, + }); + yield* pgmeta.start; + yield* pgmeta.ready; + const pgmetaEndpoint = yield* pgmetaRecipe.endpoint("http"); + const pgmetaRelay = yield* makeDockerHttpRelay(pgmetaRecipe.endpoint("http")); + const schemasResponse = yield* client.execute( + HttpClientRequest.get(`http://${pgmetaEndpoint.host}:${pgmetaEndpoint.port}/schemas`), + ); + expect(schemasResponse.status).toBe(200); + expect(yield* schemasResponse.text).toContain("public"); + + const studioRecipe = yield* makeServiceRecipe( + { + service: "studio", + config: { + pgmetaUrl: `http://${pgmetaRelay.host}:${pgmetaRelay.port}`, + analyticsApiKey: "catalog-analytics-key", + apiUrl: "http://localhost:8000", + publicApiUrl: "http://localhost:8000", + jwtSecret: secret, + }, + }, + dockerOptions(root), + ); + const studio = yield* makeService(studioRecipe.definition, { + id: "studio", + config: studioRecipe.creation, + }); + yield* studio.start; + yield* studio.ready; + const studioEndpoint = yield* studioRecipe.endpoint("http"); + const profileResponse = yield* client.execute( + HttpClientRequest.get( + `http://${studioEndpoint.host}:${studioEndpoint.port}/api/platform/profile`, + ), + ); + expect(profileResponse.status).toBe(200); + + yield* studio.stop; + yield* pgmeta.stop; + yield* realtime.stop; + yield* database.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); +}); diff --git a/packages/stack/src/services/Realtime.ts b/packages/stack/src/services/Realtime.ts new file mode 100644 index 0000000000..2c20ca83fc --- /dev/null +++ b/packages/stack/src/services/Realtime.ts @@ -0,0 +1,72 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { databaseConnection, localJwtSecret } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + databaseUrl: Schema.String, + jwtSecret: Schema.optionalKey(Schema.String), + dbEncryptionKey: Schema.optionalKey(Schema.String), + secretKeyBase: Schema.optionalKey(Schema.String), + ipVersion: Schema.optionalKey(Schema.Literals(["IPv4", "IPv6"])), + maxHeaderLength: Schema.optionalKey(Schema.Finite), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ + http: Schema.optionalKey(EndpointIntent), + rpc: Schema.optionalKey(EndpointIntent), +}); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("realtime", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "realtime", + executable: "bin/server", + ports: { http: 4000, rpc: 5369 }, + healthPath: "/healthcheck", + env: (creation, endpoints, container) => + Effect.gen(function* () { + const db = yield* databaseConnection(creation.config.databaseUrl); + const http = endpoints.get("http"); + const rpc = endpoints.get("rpc"); + const jwt = creation.config.jwtSecret ?? localJwtSecret; + return { + DATABASE_URL: creation.config.databaseUrl, + ...(http === undefined ? {} : { PORT: String(http.port) }), + DB_URL: creation.config.databaseUrl, + DB_HOST: db.host, + DB_PORT: db.port, + DB_USER: db.username ?? "supabase_admin", + DB_PASSWORD: db.password ?? "postgres", + DB_NAME: db.database, + DB_AFTER_CONNECT_QUERY: "SET search_path TO _realtime", + API_JWT_SECRET: jwt, + METRICS_JWT_SECRET: jwt, + DB_ENC_KEY: creation.config.dbEncryptionKey ?? "0123456789abcdef", + SECRET_KEY_BASE: creation.config.secretKeyBase ?? localJwtSecret, + DNS_NODES: "''", + APP_NAME: "realtime", + SEED_SELF_HOST: "true", + MAX_HEADER_LENGTH: String(creation.config.maxHeaderLength ?? 4096), + ERL_AFLAGS: + creation.config.ipVersion === "IPv6" ? "-proto_dist inet6_tcp" : "-proto_dist inet_tcp", + RUN_JANITOR: "true", + ...(rpc === undefined + ? {} + : { + GEN_RPC_TCP_SERVER_PORT: String(rpc.port), + GEN_RPC_TCP_CLIENT_PORT: String(rpc.port), + GEN_RPC_SOCKET_IP: container ? "0.0.0.0" : "127.0.0.1", + }), + }; + }), + args: (_creation, _endpoints, context) => + Effect.succeed(context.container ? ["-s", "-g", "--", "/app/bin/server"] : []), + mounts: () => Effect.succeed([]), + startup: [{ args: [], containerEntrypoint: "/app/bin/prepare" }], + containerEntrypoint: () => "/usr/bin/tini", +}); diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts new file mode 100644 index 0000000000..6e900c71c5 --- /dev/null +++ b/packages/stack/src/services/Recipe.ts @@ -0,0 +1,76 @@ +import { Data, Schema } from "effect"; +import type { Stream } from "effect"; +import type { Effect, Ref } from "effect"; +import type { ServiceKind } from "../Artifacts.ts"; +import type { ServiceDefinition } from "../Service.ts"; + +type CatalogRuntime = "native" | "docker" | "podman"; + +export class CatalogError extends Data.TaggedError("CatalogError")<{ + readonly operation: string; + readonly message: string; + readonly service?: ServiceKind; + readonly cause?: unknown; +}> {} + +export interface ServiceEndpoint { + readonly kind: "tcp" | "unix"; + readonly host?: "127.0.0.1"; + readonly path?: string; + readonly port: number; +} + +export const EndpointIntent = Schema.Struct({ + port: Schema.Union([Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)), Schema.Literal("auto")]), +}); +export interface EndpointIntent extends Schema.Schema.Type {} + +export const serviceCreation = < + K extends ServiceKind, + C extends Schema.Schema, + E extends Schema.Schema, +>( + service: K, + config: C, + endpoints: E, +) => + Schema.Struct({ + service: Schema.Literal(service), + version: Schema.optionalKey(Schema.String), + config, + endpoints: Schema.optionalKey(endpoints), + }); + +export interface CatalogOptions { + readonly stackId: string; + readonly instanceId: string; + readonly root: string; + readonly cacheRoot: string; + readonly runtime: CatalogRuntime; + readonly platform?: { readonly os: string; readonly arch: string }; +} + +export interface CatalogLog { + readonly stream: "stdout" | "stderr"; + readonly bytes: Uint8Array; +} + +export interface CatalogRecipe { + readonly creation: C; + readonly definition: ServiceDefinition; + readonly endpoint: (name: string) => Effect.Effect; + readonly logs: Stream.Stream; +} + +export interface RecipeCreation { + readonly service: K; + readonly version?: string; + readonly config: C; + readonly endpoints?: unknown; +} + +export interface ProcessRecipeResult { + readonly definition: ServiceDefinition; + readonly endpoints: Ref.Ref>; + readonly logs: Stream.Stream; +} diff --git a/packages/stack/src/services/Rest.integration.test.ts b/packages/stack/src/services/Rest.integration.test.ts new file mode 100644 index 0000000000..cf72eae675 --- /dev/null +++ b/packages/stack/src/services/Rest.integration.test.ts @@ -0,0 +1,287 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Effect, FileSystem, Layer, Redacted, Ref } from "effect"; +import { PgClient } from "@effect/sql-pg"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { SignJWT } from "jose"; +import { tmpdir } from "node:os"; +import * as Network from "../Network.ts"; +import * as State from "../State.ts"; +import { makeService } from "../Service.ts"; +import { ProxyError } from "../Proxy.ts"; +import { makeServiceRecipe } from "./Catalog.ts"; +import { cleanupDockerRoot } from "../../tests/docker-cleanup.ts"; + +const makeTestState = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); + +const makeTestNetwork = (options: { + readonly stackId: string; + readonly runtime: Network.NetworkRuntime; + readonly state: State.Interface; +}) => + Layer.build( + Network.layer({ stackId: options.stackId, runtime: options.runtime }).pipe( + Layer.provide(Layer.succeed(State.Service, options.state)), + ), + ).pipe(Effect.map((context) => Context.get(context, Network.Service))); + +const options = (root: string) => ({ + stackId: "catalog-test", + instanceId: "instance", + root, + cacheRoot: `${root}/cache`, + runtime: "native" as const, +}); + +const dockerOptions = (root: string) => ({ + ...options(root), + runtime: "docker" as const, +}); + +describe("service catalog", () => { + it.live( + "routes PostgREST through an owned database public endpoint", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-rest-" }); + yield* Effect.addFinalizer(() => cleanupDockerRoot(root)); + const stackId = "catalog-network"; + const state = yield* makeTestState(root + "/state"); + yield* state.save({ + id: stackId, + identity: { projectRoot: root, branchContext: "test", stackName: "catalog" }, + runtime: "docker", + instances: [], + composition: {}, + ports: [], + }); + const network = yield* makeTestNetwork({ stackId, runtime: "docker", state }); + const secret = "catalog-rest-secret-with-at-least-32-chars"; + const databaseRecipe = yield* makeServiceRecipe( + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + }, + dockerOptions(root), + ); + const database = yield* makeService(databaseRecipe.definition, { + id: "database", + config: databaseRecipe.creation, + }); + const databaseActive = yield* Ref.make(true); + const databaseNamespace = yield* network.register({ + id: "database", + endpoints: { + sql: { + protocol: "tcp", + port: "auto", + enabled: Ref.get(databaseActive), + backend: databaseRecipe.endpoint("sql").pipe( + Effect.flatMap((endpoint) => + endpoint.kind === "tcp" && endpoint.host !== undefined + ? Effect.succeed({ host: endpoint.host, port: endpoint.port }) + : Effect.fail(new ProxyError({ message: "Database endpoint is not TCP" })), + ), + Effect.mapError((cause) => + cause instanceof ProxyError + ? cause + : new ProxyError({ message: cause.message, cause }), + ), + ), + }, + }, + }); + yield* databaseNamespace.bind; + const databaseRuntimeAddress = yield* databaseNamespace.address("sql", "runtime"); + const databaseHostAddress = yield* databaseNamespace.address("sql", "host"); + yield* database.start; + yield* database.ready; + const databaseLayer = yield* Layer.build( + PgClient.layer({ + host: databaseHostAddress.host, + port: databaseHostAddress.port, + database: "postgres", + username: "supabase_admin", + password: Redacted.make("postgres"), + }), + ); + const sql = Context.get(databaseLayer, PgClient.PgClient); + yield* sql.unsafe( + "CREATE TABLE IF NOT EXISTS public.catalog_probe (id integer PRIMARY KEY, value text NOT NULL);", + ); + yield* sql.unsafe("GRANT SELECT ON public.catalog_probe TO anon;"); + yield* sql.unsafe("TRUNCATE public.catalog_probe;"); + yield* sql.unsafe("INSERT INTO public.catalog_probe (id, value) VALUES (1, 'catalog');"); + + const restRecipe = yield* makeServiceRecipe( + { + service: "rest", + config: { + databaseUrl: + "postgresql://supabase_admin:postgres@" + + databaseRuntimeAddress.host + + ":" + + databaseRuntimeAddress.port + + "/postgres", + jwtSecret: secret, + anonRole: "anon", + }, + }, + dockerOptions(root), + ); + const rest = yield* makeService(restRecipe.definition, { + id: "rest", + config: restRecipe.creation, + }); + const restActive = yield* Ref.make(true); + const restNamespace = yield* network.register({ + id: "rest", + endpoints: { + http: { + protocol: "http", + port: "auto", + enabled: Ref.get(restActive), + backend: restRecipe.endpoint("http").pipe( + Effect.flatMap((endpoint) => + endpoint.host === undefined + ? Effect.fail(new ProxyError({ message: "REST endpoint has no host" })) + : Effect.succeed({ host: endpoint.host, port: endpoint.port }), + ), + Effect.mapError((cause) => new ProxyError({ message: cause.message, cause })), + ), + }, + }, + }); + yield* restNamespace.bind; + const publicAddress = yield* restNamespace.address("http", "host"); + yield* rest.start; + yield* rest.ready; + const token = yield* Effect.tryPromise(() => + new SignJWT({ role: "anon" }) + .setProtectedHeader({ alg: "HS256", typ: "JWT" }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(new TextEncoder().encode(secret)), + ); + const response = yield* client.execute( + HttpClientRequest.get( + "http://" + publicAddress.host + ":" + publicAddress.port + "/catalog_probe", + ).pipe(HttpClientRequest.setHeader("Authorization", "Bearer " + token)), + ); + const responseBody = yield* response.text; + expect(response.status).toBe(200); + expect(responseBody).toContain("catalog"); + + yield* rest.stop; + yield* Ref.set(restActive, false); + yield* restNamespace.close; + yield* restNamespace.release; + yield* database.stop; + yield* Ref.set(databaseActive, false); + yield* databaseNamespace.close; + yield* databaseNamespace.release; + yield* network.release; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 120_000 }, + ); + + it.live("routes native PostgREST to the owned database Unix socket", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-rest-native-" }); + const stackId = "catalog-native"; + const secret = "catalog-native-secret-with-at-least-32-chars"; + const databaseRecipe = yield* makeServiceRecipe( + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + }, + { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, + ); + const database = yield* makeService(databaseRecipe.definition, { + id: "database", + config: databaseRecipe.creation, + }); + yield* database.start; + yield* database.ready; + const databaseEndpoint = yield* databaseRecipe.endpoint("sql"); + if (databaseEndpoint.kind !== "unix" || databaseEndpoint.path === undefined) + return yield* new ProxyError({ message: "Native database did not expose a Unix socket" }); + const databaseLayer = yield* Layer.build( + PgClient.layer({ + host: databaseEndpoint.path, + port: databaseEndpoint.port, + database: "postgres", + username: "supabase_admin", + password: Redacted.make("postgres"), + }), + ); + const sql = Context.get(databaseLayer, PgClient.PgClient); + yield* sql.unsafe( + "CREATE TABLE IF NOT EXISTS public.catalog_native_probe (id integer PRIMARY KEY, value text NOT NULL);", + ); + yield* sql.unsafe("GRANT SELECT ON public.catalog_native_probe TO anon;"); + yield* sql.unsafe("TRUNCATE public.catalog_native_probe;"); + yield* sql.unsafe( + "INSERT INTO public.catalog_native_probe (id, value) VALUES (1, 'catalog-native');", + ); + const restRecipe = yield* makeServiceRecipe( + { + service: "rest", + config: { + databaseUrl: + "postgresql://supabase_admin:postgres@127.0.0.1/postgres?host=" + + encodeURIComponent(databaseEndpoint.path), + jwtSecret: secret, + anonRole: "anon", + }, + }, + { ...options(root), stackId, cacheRoot: `${tmpdir()}/supabase-stack-artifacts` }, + ); + const rest = yield* makeService(restRecipe.definition, { + id: "rest", + config: restRecipe.creation, + }); + yield* rest.start; + yield* rest.ready; + const endpoint = yield* restRecipe.endpoint("http"); + const token = yield* Effect.tryPromise(() => + new SignJWT({ role: "anon" }) + .setProtectedHeader({ alg: "HS256", typ: "JWT" }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(new TextEncoder().encode(secret)), + ); + const response = yield* client.execute( + HttpClientRequest.get( + "http://" + endpoint.host + ":" + endpoint.port + "/catalog_native_probe", + ).pipe(HttpClientRequest.setHeader("Authorization", "Bearer " + token)), + ); + const responseBody = yield* response.text; + expect(response.status).toBe(200); + expect(responseBody).toContain("catalog-native"); + yield* rest.stop; + yield* database.stop; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); +}); diff --git a/packages/stack/src/services/Rest.ts b/packages/stack/src/services/Rest.ts new file mode 100644 index 0000000000..f407bb81eb --- /dev/null +++ b/packages/stack/src/services/Rest.ts @@ -0,0 +1,51 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + databaseUrl: Schema.String, + externalApiUrl: Schema.optionalKey(Schema.String), + schemas: Schema.optionalKey(Schema.String), + extraSearchPath: Schema.optionalKey(Schema.String), + anonRole: Schema.optionalKey(Schema.String), + jwtSecret: Schema.optionalKey(Schema.String), + jwks: Schema.optionalKey(Schema.String), + maxRows: Schema.optionalKey(Schema.Finite), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("rest", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "rest", + executable: "bin/postgrest", + ports: { http: 3000 }, + healthPath: "/", + env: (creation, endpoints) => { + const http = endpoints.get("http"); + const jwtSecret = creation.config.jwtSecret ?? creation.config.jwks; + return Effect.succeed({ + DATABASE_URL: creation.config.databaseUrl, + PGRST_DB_URI: creation.config.databaseUrl, + ...(http === undefined ? {} : { PGRST_SERVER_PORT: String(http.port) }), + PGRST_DB_SCHEMAS: creation.config.schemas ?? "public,graphql_public", + ...(creation.config.extraSearchPath === undefined + ? {} + : { PGRST_DB_EXTRA_SEARCH_PATH: creation.config.extraSearchPath }), + PGRST_DB_ANON_ROLE: creation.config.anonRole ?? "anon", + PGRST_DB_MAX_ROWS: String(creation.config.maxRows ?? 1000), + ...(jwtSecret === undefined ? {} : { PGRST_JWT_SECRET: jwtSecret }), + ...(creation.config.externalApiUrl === undefined + ? {} + : { PGRST_OPENAPI_SERVER_PROXY_URI: creation.config.externalApiUrl }), + }); + }, + args: () => Effect.succeed([]), + mounts: () => Effect.succeed([]), + startup: [], +}); diff --git a/packages/stack/src/services/ServiceConfig.ts b/packages/stack/src/services/ServiceConfig.ts new file mode 100644 index 0000000000..25615e0aca --- /dev/null +++ b/packages/stack/src/services/ServiceConfig.ts @@ -0,0 +1,52 @@ +import { Effect } from "effect"; +import { SignJWT } from "jose"; +import { ServiceError } from "../Service.ts"; + +const serviceError = (operation: string, cause: unknown): ServiceError => + cause instanceof ServiceError + ? cause + : new ServiceError({ + operation, + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +export const localJwtSecret = "supabase-local-development-jwt-secret"; + +export const serviceJwt = Effect.fn("ServiceConfig.serviceJwt")( + (role: string, secret: string): Effect.Effect => + Effect.tryPromise(() => + new SignJWT({ role, aud: "authenticated" }) + .setProtectedHeader({ alg: "HS256", typ: "JWT" }) + .setIssuedAt() + .sign(new TextEncoder().encode(secret)), + ).pipe(Effect.mapError((cause) => serviceError("launch", cause))), +); + +export const databaseConnection = Effect.fn("ServiceConfig.databaseConnection")( + ( + value: string, + ): Effect.Effect< + Readonly<{ + readonly host: string; + readonly port: string; + readonly database: string; + readonly username: string | undefined; + readonly password: string | undefined; + }>, + ServiceError + > => + Effect.try({ + try: () => { + const url = new URL(value); + return { + host: url.hostname || "127.0.0.1", + port: url.port || "5432", + database: url.pathname.replace(/^\//u, "") || "postgres", + username: url.username === "" ? undefined : decodeURIComponent(url.username), + password: url.password === "" ? undefined : decodeURIComponent(url.password), + }; + }, + catch: (cause) => serviceError("launch", cause), + }), +); diff --git a/packages/stack/src/services/Storage.ts b/packages/stack/src/services/Storage.ts new file mode 100644 index 0000000000..2837d86d65 --- /dev/null +++ b/packages/stack/src/services/Storage.ts @@ -0,0 +1,92 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { databaseConnection, localJwtSecret, serviceJwt } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + databaseUrl: Schema.String, + filePath: Schema.String, + jwtSecret: Schema.optionalKey(Schema.String), + imgproxyUrl: Schema.optionalKey(Schema.String), + fileSizeLimit: Schema.optionalKey(Schema.String), + s3ProtocolEnabled: Schema.optionalKey(Schema.Boolean), + vectorEnabled: Schema.optionalKey(Schema.Boolean), + vectorDatabaseUrl: Schema.optionalKey(Schema.String), + vectorMaxBuckets: Schema.optionalKey(Schema.Finite), + vectorMaxIndexes: Schema.optionalKey(Schema.Finite), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("storage", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "storage", + executable: "bin/storage", + ports: { http: 5000 }, + healthPath: "/status", + env: (creation, endpoints, container) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const db = yield* databaseConnection(creation.config.databaseUrl); + const jwt = creation.config.jwtSecret ?? localJwtSecret; + const anon = yield* serviceJwt("anon", jwt); + const service = yield* serviceJwt("service_role", jwt); + const filePath = container ? "/mnt" : creation.config.filePath; + return { + DATABASE_URL: creation.config.databaseUrl, + ...(http === undefined ? {} : { STORAGE_PORT: String(http.port), PORT: String(http.port) }), + ANON_KEY: anon, + SERVICE_KEY: service, + AUTH_JWT_SECRET: jwt, + PGRST_JWT_SECRET: jwt, + TENANT_ID: "stub", + REGION: "local", + GLOBAL_S3_BUCKET: "stub", + STORAGE_BACKEND: "file", + DB_HOST: db.host, + DB_PORT: db.port, + DB_USER: db.username ?? "supabase_admin", + DB_PASSWORD: db.password ?? "postgres", + DB_NAME: db.database, + FILE_STORAGE_BACKEND_PATH: filePath, + STORAGE_FILE_BACKEND_PATH: filePath, + ...(creation.config.s3ProtocolEnabled === undefined + ? {} + : { S3_PROTOCOL_ENABLED: String(creation.config.s3ProtocolEnabled) }), + ...(creation.config.vectorEnabled === undefined + ? {} + : { VECTOR_ENABLED: String(creation.config.vectorEnabled) }), + ...(creation.config.vectorEnabled !== true + ? {} + : { + VECTOR_BUCKET_PROVIDER: "pgvector", + VECTOR_STORE_MIGRATIONS_ENABLED: "true", + VECTOR_DATABASE_URL: creation.config.vectorDatabaseUrl ?? creation.config.databaseUrl, + }), + ...(creation.config.vectorMaxBuckets === undefined + ? {} + : { VECTOR_MAX_BUCKETS: String(creation.config.vectorMaxBuckets) }), + ...(creation.config.vectorMaxIndexes === undefined + ? {} + : { VECTOR_MAX_INDEXES: String(creation.config.vectorMaxIndexes) }), + ...(creation.config.fileSizeLimit === undefined + ? {} + : { FILE_SIZE_LIMIT: creation.config.fileSizeLimit }), + ...(creation.config.imgproxyUrl === undefined + ? {} + : { + IMGPROXY_URL: creation.config.imgproxyUrl, + ENABLE_IMAGE_TRANSFORMATION: "true", + }), + }; + }), + args: () => Effect.succeed([]), + mounts: (creation, _context) => + Effect.succeed([{ source: creation.config.filePath, target: "/mnt", readOnly: false }]), + startup: [{ args: [], containerEntrypoint: "/slim-runtime/bin/prepare" }], +}); diff --git a/packages/stack/src/services/Studio.ts b/packages/stack/src/services/Studio.ts new file mode 100644 index 0000000000..408eccb5b5 --- /dev/null +++ b/packages/stack/src/services/Studio.ts @@ -0,0 +1,78 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { serviceJwt } from "./ServiceConfig.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + functionsRoot: Schema.optionalKey(Schema.String), + pgmetaUrl: Schema.optionalKey(Schema.String), + analyticsUrl: Schema.optionalKey(Schema.String), + analyticsApiKey: Schema.optionalKey(Schema.String), + functionsUrl: Schema.optionalKey(Schema.String), + apiUrl: Schema.optionalKey(Schema.String), + publicApiUrl: Schema.optionalKey(Schema.String), + jwtSecret: Schema.optionalKey(Schema.String), + openaiApiKey: Schema.optionalKey(Schema.String), +}); + +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); + +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("studio", Config, Endpoints); + +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "studio", + executable: "bin/studio", + ports: { http: 3000 }, + healthPath: "/api/platform/profile", + env: (creation, endpoints, container) => + Effect.gen(function* () { + const http = endpoints.get("http"); + const jwt = creation.config.jwtSecret; + const values: Record = { + ...(http === undefined ? {} : { PORT: String(http.port) }), + HOSTNAME: container ? "0.0.0.0" : "127.0.0.1", + }; + if (creation.config.pgmetaUrl !== undefined) + values.STUDIO_PG_META_URL = creation.config.pgmetaUrl; + if (creation.config.analyticsUrl !== undefined) + values.LOGFLARE_URL = creation.config.analyticsUrl; + if (creation.config.analyticsApiKey !== undefined) { + values.LOGFLARE_PRIVATE_ACCESS_TOKEN = creation.config.analyticsApiKey; + values.NEXT_PUBLIC_ENABLE_LOGS = "true"; + } + if (jwt !== undefined) { + values.SUPABASE_ANON_KEY = yield* serviceJwt("anon", jwt); + values.SUPABASE_SERVICE_KEY = yield* serviceJwt("service_role", jwt); + } + if (creation.config.apiUrl !== undefined) values.SUPABASE_URL = creation.config.apiUrl; + if (creation.config.publicApiUrl !== undefined) + values.SUPABASE_PUBLIC_URL = creation.config.publicApiUrl; + if (creation.config.openaiApiKey !== undefined) + values.OPENAI_API_KEY = creation.config.openaiApiKey; + if (creation.config.functionsRoot !== undefined) + values.EDGE_FUNCTIONS_MANAGEMENT_FOLDER = container + ? "/__supabase_functions" + : creation.config.functionsRoot; + if (creation.config.functionsUrl !== undefined) + values.EDGE_FUNCTIONS_URL = creation.config.functionsUrl; + return values; + }), + args: () => Effect.succeed([]), + mounts: (creation) => + Effect.succeed( + creation.config.functionsRoot === undefined + ? [] + : [ + { + source: creation.config.functionsRoot, + target: "/__supabase_functions", + readOnly: true, + }, + ], + ), + startup: [], +}); diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts new file mode 100644 index 0000000000..ef97a4fd21 --- /dev/null +++ b/packages/stack/src/services/Vector.ts @@ -0,0 +1,62 @@ +import { Effect, Schema } from "effect"; +import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; + +export const Config = Schema.Struct({ + analyticsUrl: Schema.String, + apiKey: Schema.optionalKey(Schema.String), + configPath: Schema.optionalKey(Schema.String), +}); +export interface Config extends Schema.Schema.Type {} +export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); +export interface Endpoints extends Schema.Schema.Type {} +export const Creation = serviceCreation("vector", Config, Endpoints); +export interface Creation extends Schema.Schema.Type {} + +export const makeSpec = (): ProcessRecipeSpec => ({ + service: "vector", + executable: "bin/vector", + ports: { http: 9001 }, + healthPath: "/health", + env: (creation, endpoints, container) => { + const http = endpoints.get("http"); + return Effect.succeed({ + ...(http === undefined + ? {} + : { + VECTOR_API_ADDRESS: `${container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`, + VECTOR_API_PORT: String(http.port), + }), + LOGFLARE_URL: creation.config.analyticsUrl, + ...(creation.config.apiKey === undefined + ? {} + : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), + }); + }, + args: (creation, _endpoints, context) => { + if (context.container && creation.config.configPath !== undefined) + return Effect.succeed(["--config", "/etc/vector/vector.yaml"]); + if (!context.container) { + const configPath = + creation.config.configPath ?? + (context.artifactRoot === undefined + ? "share/doc/vector/config/vector.yaml" + : `${context.artifactRoot}/share/doc/vector/config/vector.yaml`); + return Effect.succeed(["--config", configPath]); + } + return Effect.succeed([]); + }, + mounts: (creation) => + Effect.succeed( + creation.config.configPath === undefined + ? [] + : [ + { + source: creation.config.configPath, + target: "/etc/vector/vector.yaml", + readOnly: true, + }, + ], + ), + startup: [], +}); diff --git a/packages/stack/src/shutdown.integration.test.ts b/packages/stack/src/shutdown.integration.test.ts new file mode 100644 index 0000000000..4a383e0a62 --- /dev/null +++ b/packages/stack/src/shutdown.integration.test.ts @@ -0,0 +1,141 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { Context, Data, Effect, FileSystem, Layer, Option, Schema, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { fileURLToPath } from "node:url"; +import * as PromiseStack from "./index.ts"; +import { HostEndpoint } from "./HostProcess.ts"; +import { StackErrorSchema } from "./Rpc.ts"; +import * as State from "./State.ts"; +import { open as openEffect } from "./effect.ts"; + +const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); +class FixtureError extends Data.TaggedError("ShutdownFixtureError")<{ readonly message: string }> {} +const fixturePath = fileURLToPath(new URL("../tests/host-process-fixture.ts", import.meta.url)); + +const waitForEndpoint = (child: ChildProcessSpawner.ChildProcessHandle) => + child.getOutputFd(3).pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runHead, + Effect.timeoutOrElse({ + duration: "10 seconds", + orElse: () => Effect.fail(new FixtureError({ message: "owner fixture readiness timed out" })), + }), + Effect.flatMap( + Option.match({ + onNone: () => + Effect.fail(new FixtureError({ message: "owner fixture exited before readiness" })), + onSome: (line) => + Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ type: Schema.Literal("ready"), endpoint: HostEndpoint }), + ), + )(line).pipe( + Effect.map((ready) => ready.endpoint), + Effect.mapError((cause) => new FixtureError({ message: String(cause) })), + ), + }), + ), + ); + +const withHeldOwner = ( + root: string, + run: (options: { + readonly stateRoot: string; + readonly cacheRoot: string; + readonly id: string; + }) => Effect.Effect, +) => + Effect.gen(function* () { + const stateRoot = `${root}/state`; + const cacheRoot = `${root}/cache`; + const id = "shutdown-held"; + const context = yield* Layer.build(State.layer({ root: stateRoot })); + const state = Context.get(context, State.Service); + yield* state.save({ + id, + runtime: "native", + identity: { projectRoot: root, branchContext: "main", stackName: "shutdown-held" }, + instances: [], + composition: { members: [], dependencies: [] }, + ports: [], + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* Effect.acquireUseRelease( + spawner.spawn( + ChildProcess.make(process.execPath, [fixturePath, stateRoot, cacheRoot, id, "rpc-held"], { + cwd: process.cwd(), + detached: true, + stdin: "ignore", + stdout: "ignore", + stderr: "inherit", + additionalFds: { + fd3: { type: "output" }, + fd4: { type: "input" }, + }, + forceKillAfter: "2 seconds", + }), + ), + (owner) => + Effect.gen(function* () { + const endpoint = yield* waitForEndpoint(owner); + expect(endpoint.pid).toBe(Number(owner.pid)); + return yield* run({ stateRoot, cacheRoot, id }); + }), + (owner) => + Effect.gen(function* () { + yield* Stream.run(Stream.succeed(new Uint8Array([1])), owner.getInputFd(4)); + yield* owner.exitCode; + }), + ); + }); + +it.live("Effect stop reports shutdown-exit when the acknowledged owner remains alive", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shutdown-effect-" }); + const error = yield* withHeldOwner(root, ({ stateRoot, cacheRoot, id }) => + Effect.gen(function* () { + const stack = yield* openEffect({ + stateRoot, + cacheRoot, + id, + }); + return yield* Effect.flip(stack.stop); + }), + ); + expect(Schema.is(StackErrorSchema)(error)).toBe(true); + if (!Schema.is(StackErrorSchema)(error)) + return yield* Effect.die("unexpected shutdown error"); + expect(error.operation).toBe("shutdown-exit"); + expect(error.message).toContain("shutdown acknowledgement"); + }), + ).pipe(Effect.provide(layer)), +); + +it.live("Promise destroy waits for owner disappearance after the RPC acknowledgement", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shutdown-promise-" }); + const error = yield* withHeldOwner(root, ({ stateRoot, cacheRoot, id }) => + Effect.acquireUseRelease( + Effect.tryPromise(() => PromiseStack.open({ stateRoot, cacheRoot, id })), + (stack) => + Effect.tryPromise(() => stack.destroy()).pipe( + Effect.flip, + Effect.map((failure) => failure.cause), + ), + (stack) => Effect.tryPromise(() => stack.close()), + ), + ); + expect(Schema.is(StackErrorSchema)(error)).toBe(true); + if (!Schema.is(StackErrorSchema)(error)) + return yield* Effect.die("unexpected shutdown error"); + expect(error).toMatchObject({ operation: "shutdown-exit" }); + expect(error.message).toContain("shutdown acknowledgement"); + }), + ).pipe(Effect.provide(layer)), +); diff --git a/packages/stack/src/state/MaterializedSettings.ts b/packages/stack/src/state/MaterializedSettings.ts deleted file mode 100644 index 7f993d221f..0000000000 --- a/packages/stack/src/state/MaterializedSettings.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { CAPABILITY_NAMES, type CapabilityName } from "../public/Capability.ts"; -import type { PersistedStackState } from "./StackState.ts"; -import { Effect, Schema } from "effect"; -import { StackStateInvalidError } from "../public/Errors.ts"; - -/** A plain object in the persisted, materialized settings document. */ -export const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null && !Array.isArray(value); - -/** Returns one capability's persisted settings without applying defaults. */ -export const settingsFor = (state: PersistedStackState, capability: CapabilityName): unknown => - state.definition?.capabilities[capability].settings; - -/** Resolves a persisted secret slot to the value supplied to a workload. */ -export const secret = (state: PersistedStackState, slot: string): string => - state.secrets[slot]?.value ?? ""; - -/** Converts a materialized setting leaf into the string form expected by workloads. */ -export const settingValue = (state: PersistedStackState, value: unknown): string => { - if (value === null || value === undefined) return ""; - if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") - return String(value); - if (Array.isArray(value)) return value.map((entry) => settingValue(state, entry)).join(","); - if (isRecord(value) && typeof value.slot === "string" && Object.keys(value).length === 1) - return secret(state, value.slot); - return Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))(value); -}; - -/** Flattens nested materialized settings into environment-style keys. */ -export const flattenSettings = ( - state: PersistedStackState, - value: unknown, - prefix: string, - out: Record, -): void => { - if (value === null || value === undefined) return; - if (isRecord(value) && typeof value.slot === "string" && Object.keys(value).length === 1) { - out[prefix] = secret(state, value.slot); - return; - } - if (Array.isArray(value)) { - out[prefix] = value.map((entry) => settingValue(state, entry)).join(","); - return; - } - if (isRecord(value)) { - for (const [key, entry] of Object.entries(value)) { - const normalized = key.replace(/[^A-Za-z0-9]+/gu, "_").toUpperCase(); - flattenSettings( - state, - entry, - prefix.length === 0 ? normalized : `${prefix}_${normalized}`, - out, - ); - } - return; - } - out[prefix] = settingValue(state, value); -}; - -/** Reads a dotted path from one capability's materialized settings. */ -export const valueAt = ( - state: PersistedStackState, - capability: CapabilityName, - path: string, -): string => { - let current: unknown = settingsFor(state, capability); - for (const segment of path.split(".")) { - if (!isRecord(current)) return ""; - current = current[segment]; - } - return settingValue(state, current); -}; - -/** Ensures every materialized secret slot is present before runtime mutation. */ -export const validateMaterializedSecrets = ( - state: PersistedStackState, - capability?: CapabilityName, -): Effect.Effect => { - const visit = (value: unknown, path: string): StackStateInvalidError | undefined => { - if (isRecord(value) && typeof value.slot === "string" && Object.keys(value).length === 1) { - return state.secrets[value.slot] === undefined - ? new StackStateInvalidError({ - message: `Missing materialized secret slot ${value.slot} at ${path}`, - slot: value.slot, - }) - : undefined; - } - if (Array.isArray(value)) { - for (const [index, entry] of value.entries()) { - const failure = visit(entry, `${path}[${index}]`); - if (failure !== undefined) return failure; - } - } else if (isRecord(value)) { - for (const [key, entry] of Object.entries(value)) { - const failure = visit(entry, path.length === 0 ? key : `${path}.${key}`); - if (failure !== undefined) return failure; - } - } - return undefined; - }; - const names = capability === undefined ? CAPABILITY_NAMES : [capability]; - for (const name of names) { - const settings = state.definition?.capabilities[name]?.settings; - const failure = visit(settings, `${name}.settings`); - if (failure !== undefined) return Effect.fail(failure); - } - return Effect.void; -}; diff --git a/packages/stack/src/state/Ownership.ts b/packages/stack/src/state/Ownership.ts deleted file mode 100644 index e4b7baf4e3..0000000000 --- a/packages/stack/src/state/Ownership.ts +++ /dev/null @@ -1,598 +0,0 @@ -import { - Context, - Data, - Effect, - Exit, - FileSystem, - Option, - Path, - Predicate, - Schema, - Scope, -} from "effect"; -import { NodeSocketServer } from "@effect/platform-node"; -import * as Socket from "effect/unstable/socket/Socket"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; -import { resolveStackPaths } from "./Paths.ts"; -import { StackOwnershipConflictError, StackStateInvalidError } from "../public/Errors.ts"; -import { OwnerSessionIdSchema } from "../control/MaintenanceProtocol.ts"; -import { NetworkPortSchema } from "../public/Status.ts"; - -/** Bump this format string so owner metadata version mismatches fail closed. */ -const OWNERSHIP_FORMAT = "supabase-stack-owner-v1" as const; -export const OWNER_LOCK_FORMAT = "supabase-stack-lease-v1" as const; - -interface UnixControlEndpoint { - readonly kind: "unix"; - readonly path: string; -} - -interface WindowsControlEndpoint { - readonly kind: "pipe"; - readonly name: string; -} - -export type ControlEndpoint = UnixControlEndpoint | WindowsControlEndpoint; - -/** Host details are supplied by the process composition boundary. */ -export interface StackRuntimeEnvironmentValue { - readonly stateRoot: string; - /** Optional shared immutable artifact cache; defaults to `/artifacts`. */ - readonly artifactCacheRoot?: string; - /** A short local IPC directory (for example `/tmp`). */ - readonly tempRoot: string; - readonly platform: "posix" | "windows"; - readonly supervisorCommand?: string; - readonly supervisorEntrypoint?: string; -} - -export class StackRuntimeEnvironment extends Context.Service< - StackRuntimeEnvironment, - StackRuntimeEnvironmentValue ->()("@supabase/stack/StackRuntimeEnvironment") {} - -// stackId and endpoint are derived from the identity directory and lease port. -// Keep them in the in-memory lease value for callers, but do not duplicate them -// in the durable control document. -const OwnerMetadataDiskSchema = Schema.Struct({ - format: Schema.Literal(OWNERSHIP_FORMAT), - ownerSessionId: OwnerSessionIdSchema, - leasePort: NetworkPortSchema, - rpcRelease: Schema.String, -}); -export type OwnerMetadata = Schema.Schema.Type & { - readonly stackId: StackId; - readonly endpoint: ControlEndpoint; -}; - -const OwnerLockSchema = Schema.Struct({ - format: Schema.Literal(OWNER_LOCK_FORMAT), - token: Schema.String, - port: NetworkPortSchema, -}); -export type OwnerLock = Schema.Schema.Type; -export class LeaseSlotTaken extends Data.TaggedError("LeaseSlotTaken")<{}> {} - -export interface OwnerLease { - readonly metadata: OwnerMetadata; - readonly lockPath: string; - readonly metadataPath: string; - /** Releases only this session's exact metadata and atomic owner lock. */ - readonly release: Effect.Effect; -} - -const ownerLockFrom = (value: unknown): Effect.Effect => - Schema.decodeUnknownEffect(OwnerLockSchema)(value, { onExcessProperty: "error" }).pipe( - Effect.mapError((error) => stateError(`Invalid owner lock: ${String(error)}`)), - Effect.flatMap((lock) => - lock.token.length > 0 - ? Effect.succeed(lock) - : Effect.fail(stateError("Invalid owner lock lease")), - ), - ); - -export const readOwnerLock = ( - fs: FileSystem.FileSystem, - lockPath: string, -): Effect.Effect => - fs.readFileString(lockPath).pipe( - Effect.flatMap((text) => - Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text).pipe( - Effect.mapError((error) => stateError(`Unable to parse owner lock: ${String(error)}`)), - Effect.flatMap(ownerLockFrom), - ), - ), - Effect.catchTag("PlatformError", (error) => - Predicate.isTagged(error.reason, "NotFound") - ? Effect.undefined - : Effect.fail(stateError(`Unable to read owner lock: ${error.message}`)), - ), - ); - -export interface HeldPortLease { - readonly port: number; - readonly close: Effect.Effect; -} - -export const acquirePortLease = ( - port: number, -): Effect.Effect => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const scope = yield* Scope.make(); - const close = Scope.close(scope, Exit.void); - return yield* Effect.gen(function* () { - const server = yield* restore( - NodeSocketServer.make({ host: "127.0.0.1", port, allowHalfOpen: false }).pipe( - Effect.provideService(Scope.Scope, scope), - Effect.mapError((error) => { - const cause = error.reason.cause; - const code = - typeof cause === "object" && - cause !== null && - "code" in cause && - typeof cause.code === "string" - ? cause.code - : undefined; - return new StackStateInvalidError({ - message: `Unable to acquire owner lease: ${String(cause)}`, - code, - cause, - }); - }), - ), - ); - if (!Predicate.isTagged(server.address, "TcpAddress")) - return yield* stateError("Owner lease did not expose a bound port"); - yield* Effect.forkIn( - server.run((socket) => - socket - .run(() => Effect.void, { - onOpen: Effect.scoped( - Effect.gen(function* () { - const write = yield* socket.writer; - yield* write(new Socket.CloseEvent()); - }), - ).pipe(Effect.ignore), - }) - .pipe(Effect.ignore), - ), - scope, - ); - return { port: server.address.port, close }; - }).pipe(Effect.onExit((exit) => (Exit.isFailure(exit) ? close : Effect.void))); - }), - ); - -const stateError = (message: string) => new StackStateInvalidError({ message }); - -const decodeStackId = (value: string): Effect.Effect => - Schema.decodeEffect(StackIdSchema)(value).pipe( - Effect.mapError((error) => stateError(`Invalid StackId: ${String(error)}`)), - ); - -const metadataFrom = ( - value: unknown, - stackId: StackId, - environment: Pick, -): Effect.Effect => - Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.Unknown))(value).pipe( - Effect.map(({ stackId: _stackId, endpoint: _endpoint, ...disk }) => disk), - Effect.flatMap((disk) => - Schema.decodeUnknownEffect(OwnerMetadataDiskSchema)(disk, { onExcessProperty: "error" }), - ), - Effect.mapError((error) => stateError(`Invalid owner metadata: ${String(error)}`)), - Effect.map((metadata) => ({ - ...metadata, - stackId, - endpoint: controlEndpointFor(stackId, environment, metadata.leasePort), - })), - ); - -/** - * Computes the one local endpoint for an identity. The complete digest is used - * so two identities can never alias. The caller supplies a short IPC root; - * no project path is embedded in the endpoint. - */ -export const controlEndpointFor = ( - stackId: StackId | string, - environment: Pick, - leasePort: number, -): ControlEndpoint => { - const root = environment.tempRoot.replace(/[\\/]+$/, ""); - // Encode the complete digest compactly so the private directory and socket - // remain below the Unix domain socket path limit even with a long temp root. - const token = Buffer.from(String(stackId), "hex").toString("base64url"); - if (environment.platform === "windows") - return { kind: "pipe", name: `\\\\.\\pipe\\supabase-stack-${token}-${leasePort}` }; - return { kind: "unix", path: `${root}/supabase-stack-${token}-${leasePort}/control.sock` }; -}; - -/** Reads and validates complete owner metadata without probing or mutating. */ -export const readOwnerMetadata = ( - stateRoot: string, - stackId: StackId | string, - environment: Pick, -): Effect.Effect< - OwnerMetadata | undefined, - StackStateInvalidError, - FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const validId = yield* decodeStackId(String(stackId)); - const paths = yield* resolveStackPaths({ stateRoot, stackId: validId }).pipe( - Effect.mapError((error) => stateError(String(error))), - ); - const raw = yield* fs.readFileString(paths.controlMetadata).pipe( - Effect.map(Option.some), - Effect.catchTag("PlatformError", (error) => - Predicate.isTagged(error.reason, "NotFound") - ? Effect.succeed(Option.none()) - : Effect.fail(stateError(`Unable to read owner metadata: ${error.message}`)), - ), - ); - if (Option.isNone(raw)) return undefined; - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( - raw.value, - ).pipe( - Effect.mapError((error) => stateError(`Unable to parse owner metadata: ${String(error)}`)), - ); - return yield* metadataFrom(parsed, validId, environment); - }); - -export const ownerLockExists = ( - stateRoot: string, - stackId: StackId | string, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const validId = yield* decodeStackId(String(stackId)); - const paths = yield* resolveStackPaths({ stateRoot, stackId: validId }).pipe( - Effect.mapError((error) => stateError(String(error))), - ); - return yield* fs - .exists(path.join(paths.runtime, "owner.lock")) - .pipe( - Effect.mapError((error) => stateError(`Unable to inspect owner lock: ${error.message}`)), - ); - }); - -/** - * Publishes metadata through a sibling temporary file and same-directory - * rename. Readers therefore observe either no document or one complete JSON - * document, never a partially written owner record. - */ -export const publishOwnership = ( - lease: OwnerLease, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(OwnerMetadataDiskSchema))( - lease.metadata, - ).pipe( - Effect.mapError((error) => stateError(`Unable to encode owner metadata: ${String(error)}`)), - ); - const temporary = `${lease.metadataPath}.${lease.metadata.ownerSessionId}.tmp`; - const existing = yield* fs - .exists(lease.metadataPath) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect existing owner metadata: ${error.message}`), - ), - ); - if (existing) - return yield* stateError("Owner metadata already exists; explicit recovery is required"); - const publish = Effect.gen(function* () { - yield* fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)); - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs - .open(temporary, { flag: "wx", mode: 0o600 }) - .pipe( - Effect.mapError((error) => - stateError(`Unable to open owner metadata: ${error.message}`), - ), - ); - yield* file - .writeAll(new TextEncoder().encode(encoded)) - .pipe( - Effect.mapError((error) => - stateError(`Unable to write owner metadata: ${error.message}`), - ), - ); - yield* file.sync.pipe( - Effect.mapError((error) => - stateError(`Unable to sync owner metadata: ${error.message}`), - ), - ); - }), - ); - yield* fs - .rename(temporary, lease.metadataPath) - .pipe( - Effect.mapError((error) => - stateError(`Unable to publish owner metadata: ${error.message}`), - ), - ); - yield* fs - .chmod(lease.metadataPath, 0o600) - .pipe( - Effect.mapError((error) => - stateError(`Unable to secure owner metadata: ${error.message}`), - ), - ); - }); - yield* publish.pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - }); - -export const removeLeaseIfHeld = ( - fs: FileSystem.FileSystem, - lockPath: string, - token: string, -): Effect.Effect => - readOwnerLock(fs, lockPath).pipe( - Effect.flatMap((owner) => - owner?.token === token ? fs.remove(lockPath, { force: true }) : Effect.void, - ), - Effect.catchTags({ - PlatformError: () => Effect.void, - StackStateInvalidError: () => Effect.void, - }), - ); - -export const writeLockTemp = ( - fs: FileSystem.FileSystem, - path: Path.Path, - lockPath: string, - lock: OwnerLock, -): Effect.Effect => - Effect.gen(function* () { - const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(OwnerLockSchema))(lock).pipe( - Effect.mapError((error) => stateError(`Unable to encode owner lock: ${String(error)}`)), - ); - const temporary = path.join(path.dirname(lockPath), `.owner.${lock.token}.tmp`); - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs - .open(temporary, { flag: "wx", mode: 0o600 }) - .pipe( - Effect.mapError((error) => stateError(`Unable to open owner lock: ${error.message}`)), - ); - yield* file - .writeAll(new TextEncoder().encode(encoded)) - .pipe( - Effect.mapError((error) => stateError(`Unable to write owner lock: ${error.message}`)), - ); - yield* file.sync.pipe( - Effect.mapError((error) => stateError(`Unable to sync owner lock: ${error.message}`)), - ); - }), - ); - return temporary; - }).pipe( - Effect.onExit((exit) => - Exit.isFailure(exit) - ? fs - .remove(path.join(path.dirname(lockPath), `.owner.${lock.token}.tmp`), { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)) - : Effect.void, - ), - ); - -export const installLease = (options: { - readonly fs: FileSystem.FileSystem; - readonly lockPath: string; - readonly temporary: string; - readonly observed: OwnerLock | undefined; -}): Effect.Effect => - Effect.gen(function* () { - if (options.observed !== undefined) { - yield* verifyObservedLease(options.fs, options.lockPath, options.observed); - yield* options.fs - .remove(options.lockPath, { force: false }) - .pipe( - Effect.mapError((error) => - Predicate.isTagged(error.reason, "NotFound") - ? new LeaseSlotTaken() - : stateError(`Unable to remove stale owner lock: ${error.message}`), - ), - ); - } - yield* options.fs - .link(options.temporary, options.lockPath) - .pipe( - Effect.mapError((error) => - Predicate.isTagged(error.reason, "AlreadyExists") - ? new LeaseSlotTaken() - : stateError(`Unable to publish owner lock: ${error.message}`), - ), - ); - }); - -/** Revalidates the exact stale record immediately before destructive recovery. */ -export const verifyObservedLease = ( - fs: FileSystem.FileSystem, - lockPath: string, - observed: OwnerLock, -): Effect.Effect => - readOwnerLock(fs, lockPath).pipe( - Effect.flatMap((current) => - current !== undefined && - current.format === observed.format && - current.token === observed.token && - current.port === observed.port - ? Effect.void - : Effect.fail(new LeaseSlotTaken()), - ), - ); - -/** - * Acquires an OS-held loopback lease. The canonical lock is atomically installed - * by hard-linking a uniquely created temporary record, and the listener remains bound for the - * lifetime of the lease. A valid lock is reclaimable only when its recorded - * port can be bound again. - */ -export const acquireOwnership = (options: { - readonly stateRoot: string; - readonly stackId: StackId | string; - readonly ownerSessionId: string; - readonly rpcRelease: string; - readonly environment: Pick; -}): Effect.Effect< - OwnerLease, - StackOwnershipConflictError | StackStateInvalidError, - FileSystem.FileSystem | Path.Path | Scope.Scope -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const validId = yield* decodeStackId(String(options.stackId)); - const path = yield* Path.Path; - const paths = yield* resolveStackPaths({ stateRoot: options.stateRoot, stackId: validId }).pipe( - Effect.mapError((error) => stateError(String(error))), - ); - const lockPath = path.join(paths.runtime, "owner.lock"); - const scope = yield* Scope.Scope; - return yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - yield* restore(fs.makeDirectory(paths.runtime, { recursive: true, mode: 0o700 })).pipe( - Effect.mapError((error) => - stateError(`Unable to create owner directory: ${error.message}`), - ), - ); - - const existing = yield* readOwnerLock(fs, lockPath); - const metadataExists = yield* fs - .exists(paths.controlMetadata) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect stale owner metadata: ${error.message}`), - ), - ); - if (existing === undefined && metadataExists) - return yield* new StackOwnershipConflictError({ - message: "Owner metadata exists without a lease lock; refusing recovery", - stackId: validId, - }); - const held = yield* acquirePortLease(existing?.port ?? 0).pipe( - Effect.mapError((error) => - existing !== undefined && error["code"] === "EADDRINUSE" - ? new StackOwnershipConflictError({ - message: "A Supervisor already owns this stack", - stackId: validId, - }) - : error, - ), - ); - const metadata: OwnerMetadata = { - format: OWNERSHIP_FORMAT, - stackId: validId, - ownerSessionId: options.ownerSessionId, - leasePort: held.port, - endpoint: controlEndpointFor(validId, options.environment, held.port), - rpcRelease: options.rpcRelease, - }; - const acquire = Effect.gen(function* () { - const port = held.port; - const lock: OwnerLock = { - format: OWNER_LOCK_FORMAT, - token: options.ownerSessionId, - port, - }; - const temporary = yield* writeLockTemp(fs, path, lockPath, lock); - const installed = Effect.gen(function* () { - // Revalidate the exact observed stale record before deleting any - // metadata or socket. This closes the recovery crash window. - if (existing !== undefined) - yield* verifyObservedLease(fs, lockPath, existing).pipe( - Effect.mapError((error) => - Predicate.isTagged(error, "LeaseSlotTaken") - ? new StackOwnershipConflictError({ - message: "A Supervisor already owns this stack", - stackId: validId, - }) - : error, - ), - ); - // The old control document can only be removed after the stale - // lease has been revalidated; the canonical lock remains until install. - if (existing !== undefined) { - yield* fs - .remove(paths.controlMetadata, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)); - } - yield* installLease({ - fs, - lockPath, - temporary, - observed: existing, - }).pipe( - Effect.mapError((error) => - Predicate.isTagged(error, "LeaseSlotTaken") - ? new StackOwnershipConflictError({ - message: "A Supervisor already owns this stack", - stackId: validId, - }) - : error, - ), - ); - // Only the process that atomically installed the canonical lock has - // lease authority to remove a stale Unix control socket. A loser - // must not unlink the winner's newly bound endpoint. - if (options.environment.platform === "posix" && metadata.endpoint.kind === "unix") - yield* fs - .remove(metadata.endpoint.path, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)); - }); - yield* installed.pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - const release = Effect.suspend(() => - fs.readFileString(paths.controlMetadata).pipe( - Effect.flatMap((text) => - Schema.decodeEffect(Schema.fromJsonString(OwnerMetadataDiskSchema))(text).pipe( - Effect.flatMap((current) => - current.ownerSessionId === options.ownerSessionId - ? fs.remove(paths.controlMetadata, { force: true }).pipe( - Effect.catchTag("PlatformError", () => Effect.void), - Effect.andThen(removeLeaseIfHeld(fs, lockPath, options.ownerSessionId)), - ) - : Effect.void, - ), - ), - ), - Effect.catchTag("PlatformError", () => - removeLeaseIfHeld(fs, lockPath, options.ownerSessionId), - ), - Effect.ignore, - Effect.andThen(held.close), - ), - ); - const lease = { - metadata, - lockPath, - metadataPath: paths.controlMetadata, - release, - } satisfies OwnerLease; - yield* Scope.addFinalizer(scope, Effect.uninterruptible(release)); - return lease; - }); - return yield* acquire.pipe( - Effect.onExit((exit) => (Exit.isFailure(exit) ? held.close : Effect.void)), - ); - }), - ); - }); diff --git a/packages/stack/src/state/Paths.ts b/packages/stack/src/state/Paths.ts deleted file mode 100644 index c50184019a..0000000000 --- a/packages/stack/src/state/Paths.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Effect, Path, Schema } from "effect"; -import { InvalidProjectRootError, InvalidStackIdentityError } from "../public/Errors.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; - -export interface StackPaths { - /** The exact `/` directory. */ - readonly stackRoot: string; - /** Durable state document; temporary replacements are siblings of this path. */ - readonly stateDocument: string; - readonly data: string; - readonly logs: string; - readonly runtime: string; - /** Control metadata owned by the identity's supervisor. */ - readonly controlMetadata: string; -} - -export interface ResolveStackPathsOptions { - readonly stateRoot: string; - readonly stackId: StackId; -} - -/** - * Resolves all durable and runtime paths under one validated StackId. - * - * This function only computes names. It does not create directories or touch - * the filesystem, so callers can use it from read-only discovery paths. - */ -export const resolveStackPaths = ( - options: ResolveStackPathsOptions, -): Effect.Effect => - Effect.gen(function* () { - const path = yield* Path.Path; - if (options.stateRoot.trim().length === 0) { - return yield* new InvalidProjectRootError({ - stateRoot: options.stateRoot, - message: "The stack state root must not be blank", - }); - } - - const stackId = yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( - Effect.mapError( - (error) => - new InvalidStackIdentityError({ - stackId: options.stackId, - message: `Invalid StackId: ${String(error)}`, - }), - ), - ); - const stateRoot = path.resolve(options.stateRoot); - const stackRoot = path.join(stateRoot, stackId); - const stateDocument = path.join(stackRoot, "state.json"); - return { - stackRoot, - stateDocument, - data: path.join(stackRoot, "data"), - logs: path.join(stackRoot, "logs"), - runtime: path.join(stackRoot, "runtime"), - controlMetadata: path.join(stackRoot, "control.json"), - }; - }); diff --git a/packages/stack/src/state/PortCoordinator.ts b/packages/stack/src/state/PortCoordinator.ts deleted file mode 100644 index a2edfb1e5b..0000000000 --- a/packages/stack/src/state/PortCoordinator.ts +++ /dev/null @@ -1,419 +0,0 @@ -import { Crypto, Effect, Exit, FileSystem, Path, Scope, Schema } from "effect"; -import { NetworkPortSchema, PORT_FIELDS, type PortField } from "../public/Status.ts"; -import { - InvalidProjectRootError, - PortAllocationError, - PortUnavailableError, - StackStateFormatUnsupportedError, - StackStateInvalidError, -} from "../public/Errors.ts"; -import type { - HostPortAssignment, - PersistedStackState, - PrivatePortAssignment, -} from "./StackState.ts"; -import { privateBindingKey } from "./StackState.ts"; -import { - isMissingStateRemnantError, - type StackStateStore, - withRegistryLock, -} from "./StackStateStore.ts"; -import type { HeldPort, HostListener } from "../supervisor/HostListener.ts"; - -interface ListenerIntent { - readonly enabled: boolean; - readonly address: string; - readonly port: "automatic" | number; -} - -export type ListenerIntents = Readonly>; - -interface PrivatePortIntent { - readonly workloadId: string; - readonly binding: string; -} - -export interface PortReservation { - readonly assignments: Readonly>>; - readonly privateAssignments: ReadonlyArray; - readonly hostListeners: ReadonlyArray; -} - -export interface PortCoordinatorOptions { - readonly stateRoot: string; - readonly store: StackStateStore; - readonly bindHost: ( - address: string, - port: number, - field: PortField, - ) => Effect.Effect; - readonly bindPrivate: ( - address: string, - port: number, - binding: string, - ) => Effect.Effect; -} - -export interface PortCoordinator { - readonly acquire: ( - stackId: string, - listenerIntents: ListenerIntents, - privateBindings: ReadonlyArray, - ) => Effect.Effect< - PortReservation, - | PortAllocationError - | PortUnavailableError - | StackStateInvalidError - | StackStateFormatUnsupportedError - | InvalidProjectRootError, - Scope.Scope | Crypto.Crypto | FileSystem.FileSystem | Path.Path - >; -} - -const fields: ReadonlyArray = PORT_FIELDS; -const PORT_MIN = 20_000; -const PORT_MAX = 32_767; -const PORT_POOL_SIZE = PORT_MAX - PORT_MIN + 1; -const PORT_STRIDE = 257; -const MAX_FRESH_BIND_FAILURES = 64; -const idPattern = /^[0-9a-f]{64}$/; - -const assignmentMap = (assignments: ReadonlyArray) => - new Map(assignments.map((assignment) => [assignment.field, assignment])); -const validPort = (port: number): boolean => Schema.is(NetworkPortSchema)(port); -const unavailable = ( - port: number, - field: string, - message = `Port ${port} for ${field} is unavailable`, -) => new PortUnavailableError({ port, field, message }); -const allocation = (field: string, message: string, cause?: unknown) => - new PortAllocationError({ field, message, ...(cause === undefined ? {} : { cause }) }); - -const readAuthoritativeStates = (options: PortCoordinatorOptions) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = path.resolve(options.stateRoot); - const exists = yield* fs - .exists(root) - .pipe(Effect.mapError((error) => new StackStateInvalidError({ message: error.message }))); - if (!exists) return []; - const entries = yield* fs - .readDirectory(root) - .pipe(Effect.mapError((error) => new StackStateInvalidError({ message: error.message }))); - const ids = entries.filter((entry) => idPattern.test(entry)); - const values = yield* Effect.forEach(ids, (id) => - options.store.read(id).pipe( - Effect.mapError((error) => - error instanceof StackStateFormatUnsupportedError - ? new StackStateFormatUnsupportedError({ - ...error, - message: `Unable to read sibling stack state ${id}: ${error.message}`, - }) - : error instanceof StackStateInvalidError - ? new StackStateInvalidError({ - ...error, - message: `Unable to read sibling stack state ${id}: ${error.message}`, - path: path.join(root, id, "state.json"), - }) - : error, - ), - Effect.catchIf(isMissingStateRemnantError, () => Effect.void), - Effect.map((state) => (state === undefined ? undefined : { stackId: id, state })), - ), - ); - return values.filter( - (entry): entry is { readonly stackId: string; readonly state: PersistedStackState } => - entry !== undefined, - ); - }); - -type ForeignPublicOwner = { - readonly stackId: string; - readonly field: string; - readonly intent: "automatic" | "exact"; - readonly lifecycle: PersistedStackState["desiredLifecycle"]; -}; -type ForeignPrivateOwner = { - readonly stackId: string; - readonly field: string; -}; -const ownerText = (owner: ForeignPublicOwner | ForeignPrivateOwner): string => - `${owner.stackId} (${owner.field})`; - -const nativeCode = (cause: unknown): string | undefined => { - if (typeof cause !== "object" || cause === null || !Reflect.has(cause, "code")) return undefined; - const code = Reflect.get(cause, "code"); - return typeof code === "string" ? code : undefined; -}; -const retryable = (error: PortUnavailableError): boolean => { - const code = nativeCode(error.cause); - return code === "EADDRINUSE" || code === "EACCES"; -}; - -export const makePortCoordinator = (options: PortCoordinatorOptions): PortCoordinator => ({ - acquire: (stackId, listenerIntents, privateBindings) => - withRegistryLock( - options.stateRoot, - Effect.gen(function* () { - const current = yield* options.store.read(stackId); - if (current === undefined) - return yield* new StackStateInvalidError({ - message: "Cannot acquire ports for an unconfigured stack", - }); - if (current.desiredLifecycle !== "running") - return yield* new StackStateInvalidError({ - message: "Port acquisition requires desiredLifecycle=running", - }); - - const publicOwners = new Map>(); - const privateOwners = new Map(); - for (const entry of yield* readAuthoritativeStates(options)) { - if (entry.stackId === stackId) continue; - for (const assignment of entry.state.ports) { - const owner: ForeignPublicOwner = { - stackId: entry.stackId, - field: assignment.field, - intent: assignment.intent, - lifecycle: entry.state.desiredLifecycle, - }; - publicOwners.set(assignment.port, [ - ...(publicOwners.get(assignment.port) ?? []), - owner, - ]); - } - for (const assignment of entry.state.privatePorts) - privateOwners.set(assignment.port, { - stackId: entry.stackId, - field: `${assignment.workloadId}:${assignment.binding}`, - }); - } - - const existingPublic = assignmentMap(current.ports); - const existingPrivate = new Map( - current.privatePorts.map((entry) => [privateBindingKey(entry), entry]), - ); - const retainedPublic = new Map(); - const retainedPrivate = new Map(); - const hardClaims = new Map(); - const occupied = new Set(); - const claim = (port: number, field: string): PortAllocationError | undefined => { - const previous = hardClaims.get(port); - if (previous !== undefined) - return allocation(field, `Port ${port} is claimed by both ${previous} and ${field}`); - hardClaims.set(port, field); - occupied.add(port); - return undefined; - }; - const foreignConflict = (port: number, field: string): PortUnavailableError | undefined => { - const privateOwner = privateOwners.get(port); - if (privateOwner !== undefined) - return unavailable( - port, - field, - `Port ${port} for ${field} is reserved by ${ownerText(privateOwner)}`, - ); - const owners = publicOwners.get(port); - const conflict = owners?.find( - (owner) => owner.intent === "automatic" || owner.lifecycle === "running", - ); - if (conflict !== undefined) - return unavailable( - port, - field, - `Port ${port} for ${field} is reserved by ${ownerText(conflict)}`, - ); - return undefined; - }; - - // Preseed every own retained assignment before allocating any fresh field. - for (const field of fields) { - const intent = listenerIntents[field]; - const prior = existingPublic.get(field); - if (!intent.enabled || intent.port !== "automatic" || prior?.intent !== "automatic") - continue; - if (!validPort(prior.port)) return yield* unavailable(prior.port, field); - const foreign = foreignConflict(prior.port, field); - if (foreign !== undefined) return yield* foreign; - const duplicate = claim(prior.port, field); - if (duplicate !== undefined) return yield* duplicate; - retainedPublic.set(field, prior); - } - const requestedPrivate = new Map(); - for (const intent of privateBindings) { - const key = privateBindingKey(intent); - if (intent.workloadId.length === 0 || intent.binding.length === 0) - return yield* allocation( - `${intent.workloadId}:${intent.binding}`, - "Private workload binding is invalid", - ); - if (requestedPrivate.has(key)) - return yield* allocation( - `${intent.workloadId}:${intent.binding}`, - "Duplicate private workload binding", - ); - requestedPrivate.set(key, intent); - const prior = existingPrivate.get(key); - if (prior === undefined) continue; - const label = `${intent.workloadId}:${intent.binding}`; - if (!validPort(prior.port)) return yield* unavailable(prior.port, label); - const foreign = foreignConflict(prior.port, label); - if (foreign !== undefined) return yield* foreign; - const duplicate = claim(prior.port, label); - if (duplicate !== undefined) return yield* duplicate; - retainedPrivate.set(key, prior); - } - const exactAssignments = new Map(); - for (const field of fields) { - const intent = listenerIntents[field]; - if (!intent.enabled || intent.port === "automatic") continue; - if (!validPort(intent.port)) return yield* unavailable(intent.port, field); - const foreign = foreignConflict(intent.port, field); - if (foreign !== undefined) return yield* foreign; - const duplicate = claim(intent.port, field); - if (duplicate !== undefined) return yield* duplicate; - exactAssignments.set(field, { field, port: intent.port, intent: "exact" }); - } - for (const port of publicOwners.keys()) occupied.add(port); - for (const port of privateOwners.keys()) occupied.add(port); - - const crypto = yield* Crypto.Crypto; - const randomStart = yield* crypto.randomIntBetween(0, PORT_POOL_SIZE - 1); - let offset = 0; - const allocateFresh = ( - field: string, - bind: (port: number) => Effect.Effect, - ): Effect.Effect< - { readonly port: number; readonly value: A }, - PortAllocationError | PortUnavailableError, - Scope.Scope - > => - Effect.gen(function* () { - let failures = 0; - while (true) { - if (offset >= PORT_POOL_SIZE) - return yield* allocation( - field, - "No automatic port is available in the shared 20000-32767 pool", - ); - const port = PORT_MIN + ((randomStart + offset * PORT_STRIDE) % PORT_POOL_SIZE); - offset += 1; - if (occupied.has(port)) continue; - const result = yield* bind(port).pipe( - Effect.map((value) => ({ ok: true as const, value })), - Effect.catchTag("PortUnavailableError", (error) => - retryable(error) - ? Effect.succeed({ ok: false as const, error }) - : Effect.fail(error), - ), - ); - if (result.ok) { - occupied.add(port); - return { port, value: result.value }; - } - failures += 1; - if (failures >= MAX_FRESH_BIND_FAILURES) - return yield* allocation( - field, - `No automatic port is available after ${MAX_FRESH_BIND_FAILURES} bind failures`, - result.error.cause, - ); - } - }); - - const parentScope = yield* Scope.Scope; - const result = yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const attemptScope = yield* Scope.fork(parentScope, "sequential"); - return yield* Effect.gen(function* () { - const acquired = yield* restore( - Effect.gen(function* () { - const privateScope = yield* Scope.fork(attemptScope, "sequential"); - const assignments: HostPortAssignment[] = []; - const byField: Partial> = {}; - const listeners: HostListener[] = []; - for (const field of fields) { - const intent = listenerIntents[field]; - if (!intent.enabled) continue; - const retained = retainedPublic.get(field); - const exact = exactAssignments.get(field); - const assignment = retained ?? exact; - if (assignment !== undefined) { - const listener = yield* options - .bindHost(intent.address, assignment.port, field) - .pipe(Effect.provideService(Scope.Scope, attemptScope)); - assignments.push(assignment); - byField[field] = assignment; - listeners.push(listener); - occupied.add(assignment.port); - continue; - } - const fresh = yield* allocateFresh(field, (port) => - options - .bindHost(intent.address, port, field) - .pipe(Effect.provideService(Scope.Scope, attemptScope)), - ); - const assignmentFresh: HostPortAssignment = { - field, - port: fresh.port, - intent: "automatic", - }; - assignments.push(assignmentFresh); - byField[field] = assignmentFresh; - listeners.push(fresh.value); - } - - const privateAssignments: PrivatePortAssignment[] = []; - for (const intent of requestedPrivate.values()) { - const key = privateBindingKey(intent); - const label = `${intent.workloadId}:${intent.binding}`; - const retained = retainedPrivate.get(key); - if (retained !== undefined) { - const held = yield* options - .bindPrivate("127.0.0.1", retained.port, label) - .pipe(Effect.provideService(Scope.Scope, privateScope)); - privateAssignments.push({ ...retained, port: held.port }); - occupied.add(retained.port); - continue; - } - const fresh = yield* allocateFresh(label, (port) => - options - .bindPrivate("127.0.0.1", port, label) - .pipe(Effect.provideService(Scope.Scope, privateScope)), - ); - privateAssignments.push({ - workloadId: intent.workloadId, - binding: intent.binding, - port: fresh.port, - }); - } - const next: PersistedStackState = { - ...current, - ports: assignments, - privatePorts: privateAssignments, - }; - return { - privateScope, - next, - reservation: { - assignments: byField, - privateAssignments, - hostListeners: listeners, - }, - }; - }), - ); - yield* options.store.replaceUnlocked(stackId, acquired.next); - yield* Scope.close(acquired.privateScope, Exit.void); - return acquired.reservation; - }).pipe( - Effect.onExit((exit) => - Exit.isSuccess(exit) ? Effect.void : Scope.close(attemptScope, exit), - ), - ); - }), - ); - return result; - }), - ), -}); diff --git a/packages/stack/src/state/SecretStore.ts b/packages/stack/src/state/SecretStore.ts deleted file mode 100644 index b344817fb4..0000000000 --- a/packages/stack/src/state/SecretStore.ts +++ /dev/null @@ -1,539 +0,0 @@ -import { Clock, Crypto, Effect, FileSystem, Path, Redacted, Schema } from "effect"; -// Effect Crypto does not expose the HMAC, private-key import, or asymmetric signing operations used here. -import { createHmac, createPrivateKey, createSign } from "node:crypto"; -import { - InvalidJwtSigningMaterialError, - StackMustBeStoppedError, - StackSecretMismatchError, -} from "../public/Errors.ts"; -import type { DesiredStackLifecycle } from "../public/Status.ts"; -import type { PersistedSecretValues } from "./StackState.ts"; - -export const AUTH_PUBLISHABLE_KEY_SLOT = "secret:auth.settings.publishable_key"; -export const AUTH_SECRET_KEY_SLOT = "secret:auth.settings.secret_key"; -export const AUTH_ANON_KEY_SLOT = "secret:auth.settings.anon_key"; -export const AUTH_SERVICE_ROLE_KEY_SLOT = "secret:auth.settings.service_role_key"; -export const AUTH_JWT_SECRET_SLOT = "secret:auth.settings.jwt_secret"; -export const DATABASE_INTERNAL_PASSWORD_SLOT = "secret:database.internal.password"; - -type SecretPolicy = "managed" | "passthrough"; - -export type SecretJwtSigning = - | { readonly kind: "symmetric" } - | { readonly kind: "jwks-file"; readonly projectRoot: string; readonly path: string }; - -/** Lifecycle-only metadata for generating omitted managed credentials. */ -export type SecretGenerator = - | { readonly kind: "publishable-key" } - | { readonly kind: "secret-key" } - | { readonly kind: "jwt-secret" } - | { readonly kind: "random-base64url"; readonly bytes: number } - | { - readonly kind: "jwt-token"; - readonly role: "anon" | "service_role"; - readonly signing: SecretJwtSigning; - }; - -export interface SecretDeclaration { - readonly slot: string; - readonly policy: SecretPolicy; - readonly value?: Redacted.Redacted; - readonly generator?: SecretGenerator; -} - -export interface SecretCandidate { - /** Complete pass-through declarations plus any managed slots supplied by the caller. */ - readonly declarations: ReadonlyArray; -} - -export interface ResolvedSecrets { - readonly persisted: PersistedSecretValues; -} - -const secretMismatch = (slot: string, message: string) => - new StackSecretMismatchError({ slot, message }); - -const lifecycleChange = (slot?: string) => - new StackMustBeStoppedError({ - ...(slot === undefined ? {} : { slot }), - message: "Pass-through secrets may only change while the stack is stopped", - }); - -const readSecret = (value: Redacted.Redacted): string => { - const unredacted = Redacted.value(value); - return typeof unredacted === "string" ? unredacted : String(unredacted); -}; -const validSlot = (slot: string): boolean => /^[A-Za-z0-9_.:/-]+$/.test(slot); - -const JWT_ISSUER = "supabase-demo"; -const JWT_HMAC_EXPIRY = 1_983_812_996; -const JWT_ASYMMETRIC_EXPIRY_SECONDS = 60 * 60 * 24 * 365 * 10; -const BASE64URL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - -const JwtHeaderSchema = Schema.Struct({ - alg: Schema.String, - kid: Schema.optionalKey(Schema.String), - typ: Schema.Literal("JWT"), -}); -type JwtHeader = Schema.Schema.Type; -const JwtPayloadSchema = Schema.Struct({ - iss: Schema.Literal(JWT_ISSUER), - role: Schema.Literals(["anon", "service_role"] as const), - exp: Schema.Finite, -}); -type JwtPayload = Schema.Schema.Type; -const JwkSchema = Schema.Struct({ - alg: Schema.optionalKey(Schema.String), - crv: Schema.optionalKey(Schema.String), - d: Schema.optionalKey(Schema.String), - dp: Schema.optionalKey(Schema.String), - dq: Schema.optionalKey(Schema.String), - e: Schema.optionalKey(Schema.String), - kty: Schema.String, - kid: Schema.optionalKey(Schema.String), - n: Schema.optionalKey(Schema.String), - p: Schema.optionalKey(Schema.String), - q: Schema.optionalKey(Schema.String), - qi: Schema.optionalKey(Schema.String), - use: Schema.optionalKey(Schema.String), - key_ops: Schema.optionalKey(Schema.Array(Schema.String)), - ext: Schema.optionalKey(Schema.Boolean), - x: Schema.optionalKey(Schema.String), - y: Schema.optionalKey(Schema.String), -}); -const JwkFileSchema = Schema.Union([Schema.Array(JwkSchema), JwkSchema]); -const encodeJwtHeader = (value: JwtHeader) => - Schema.encodeEffect(Schema.fromJsonString(JwtHeaderSchema))(value); -const encodeJwtPayload = (value: JwtPayload) => - Schema.encodeEffect(Schema.fromJsonString(JwtPayloadSchema))(value); - -export const base64UrlEncode = (bytes: Uint8Array): string => { - let output = ""; - for (let index = 0; index < bytes.length; index += 3) { - const first = bytes[index] ?? 0; - const second = bytes[index + 1]; - const third = bytes[index + 2]; - output += BASE64URL[first >> 2] ?? ""; - output += BASE64URL[((first & 3) << 4) | ((second ?? 0) >> 4)] ?? ""; - if (second !== undefined) output += BASE64URL[((second & 15) << 2) | ((third ?? 0) >> 6)] ?? ""; - if (third !== undefined) output += BASE64URL[third & 63] ?? ""; - } - return output; -}; - -const base64UrlEncodeText = (value: string): string => - base64UrlEncode(new TextEncoder().encode(value)); - -/** Redacts known exact values without attempting to infer transformed/derived secrets. */ -export const redactKnownSecrets = (message: string, known: Iterable): string => { - let result = message; - const secrets = [...new Set([...known].filter((secret) => secret.length > 0))].sort( - (left, right) => right.length - left.length, - ); - for (const secret of secrets) result = result.split(secret).join("[REDACTED]"); - return result; -}; - -const generatedSecret = (crypto: Crypto.Crypto) => - crypto.randomUUIDv4.pipe( - Effect.mapError((error) => - secretMismatch("managed", `Unable to generate managed secret: ${error.message}`), - ), - ); - -const randomEncoded = ( - crypto: Crypto.Crypto, - bytes: number, - slot: string, -): Effect.Effect => - crypto.randomBytes(bytes).pipe( - Effect.map(base64UrlEncode), - Effect.mapError((error) => - secretMismatch(slot, `Unable to generate managed secret: ${error.message}`), - ), - ); - -export interface SigningJwk { - readonly kty: "EC" | "RSA"; - readonly alg: "ES256" | "RS256"; - readonly d: string; - readonly kid?: string; - readonly use?: string; - readonly key_ops?: Array; - readonly ext?: boolean; - readonly crv?: string; - readonly x?: string; - readonly y?: string; - readonly n?: string; - readonly e?: string; - readonly p?: string; - readonly q?: string; - readonly dp?: string; - readonly dq?: string; - readonly qi?: string; -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - -const stringField = (record: Record, key: string): string | undefined => { - const value = record[key]; - return value === undefined ? undefined : typeof value === "string" ? value : undefined; -}; - -const booleanField = (record: Record, key: string): boolean | undefined => { - const value = record[key]; - return value === undefined ? undefined : typeof value === "boolean" ? value : undefined; -}; - -const stringArrayField = ( - record: Record, - key: string, -): Array | undefined => { - const value = record[key]; - return value === undefined - ? undefined - : Array.isArray(value) && value.every((entry) => typeof entry === "string") - ? value - : undefined; -}; - -const privateSigningJwk = (value: unknown): SigningJwk | undefined => { - if (!isRecord(value)) return undefined; - const kty = value.kty; - const alg = value.alg; - const d = stringField(value, "d"); - if (typeof d !== "string") return undefined; - const kid = stringField(value, "kid"); - const use = stringField(value, "use"); - const keyOps = stringArrayField(value, "key_ops"); - const ext = booleanField(value, "ext"); - if (use !== undefined && use !== "sig") return undefined; - if (keyOps !== undefined && !keyOps.includes("sign")) return undefined; - if (kty === "EC" && alg === "ES256") { - const crv = stringField(value, "crv"); - const x = stringField(value, "x"); - const y = stringField(value, "y"); - if (crv !== "P-256" || x === undefined || y === undefined) return undefined; - return { - kty, - alg, - d, - ...(kid === undefined ? {} : { kid }), - ...(use === undefined ? {} : { use }), - ...(keyOps === undefined ? {} : { key_ops: keyOps }), - ...(ext === undefined ? {} : { ext }), - crv, - x, - y, - }; - } - if (kty === "RSA" && alg === "RS256") { - const n = stringField(value, "n"); - const e = stringField(value, "e"); - const p = stringField(value, "p"); - const q = stringField(value, "q"); - if (n === undefined || e === undefined || p === undefined || q === undefined) return undefined; - const dp = stringField(value, "dp"); - const dq = stringField(value, "dq"); - const qi = stringField(value, "qi"); - return { - kty, - alg, - d, - ...(kid === undefined ? {} : { kid }), - ...(use === undefined ? {} : { use }), - ...(keyOps === undefined ? {} : { key_ops: keyOps }), - ...(ext === undefined ? {} : { ext }), - n, - e, - p, - q, - ...(dp === undefined ? {} : { dp }), - ...(dq === undefined ? {} : { dq }), - ...(qi === undefined ? {} : { qi }), - }; - } - return undefined; -}; - -const invalidSigningMaterial = (message = "Unable to resolve JWT signing material") => - new InvalidJwtSigningMaterialError({ message }); - -const readSigningJwks = ( - signing: Extract, -): Effect.Effect< - ReadonlyArray, - InvalidJwtSigningMaterialError, - FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { - const path = yield* Path.Path; - const fs = yield* FileSystem.FileSystem; - const projectRoot = path.resolve(signing.projectRoot); - const candidate = path.resolve(projectRoot, signing.path); - const relative = path.relative(projectRoot, candidate); - if (relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) - return yield* invalidSigningMaterial("JWT signing key file must be inside project root"); - const canonicalRoot = yield* fs - .realPath(projectRoot) - .pipe(Effect.mapError(() => invalidSigningMaterial())); - const canonicalCandidate = yield* fs - .realPath(candidate) - .pipe(Effect.mapError(() => invalidSigningMaterial())); - const canonicalRelative = path.relative(canonicalRoot, canonicalCandidate); - if ( - canonicalRelative === ".." || - canonicalRelative.startsWith(`..${path.sep}`) || - path.isAbsolute(canonicalRelative) - ) - return yield* invalidSigningMaterial("JWT signing key file must be inside project root"); - const raw = yield* fs - .readFileString(canonicalCandidate) - .pipe(Effect.mapError(() => invalidSigningMaterial())); - const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(JwkFileSchema))(raw).pipe( - Effect.mapError(() => invalidSigningMaterial()), - ); - const entries = Array.isArray(parsed) ? parsed : [parsed]; - const keys: Array = []; - for (const entry of entries) { - const key = privateSigningJwk(entry); - if (key === undefined) return yield* invalidSigningMaterial(); - yield* Effect.try({ - try: () => { - createPrivateKey({ key, format: "jwk" }); - }, - catch: () => invalidSigningMaterial(), - }); - keys.push(key); - } - if (keys.length === 0) return yield* invalidSigningMaterial(); - return keys; - }); - -const readSigningJwk = ( - signing: Extract, -): Effect.Effect => - readSigningJwks(signing).pipe(Effect.map((keys) => keys[0]!)); - -export interface ResolvedSigningKeyMaterial { - /** The private signing keys as a JSON array for GoTrue's signing-key input. */ - readonly privateKeysJson: string; - /** Public-only JWKs suitable for JWT verification. */ - readonly publicKeys: ReadonlyArray>>; - /** Public-only JWKs wrapped in a JWKS document. */ - readonly publicJwksJson: string; -} - -const publicSigningJwk = (key: SigningJwk): Readonly> => ({ - kty: key.kty, - ...(key.kid === undefined ? {} : { kid: key.kid }), - use: "sig", - key_ops: ["verify"], - alg: key.alg, - ...(key.ext === undefined ? {} : { ext: key.ext }), - ...(key.kty === "RSA" ? { n: key.n, e: key.e } : { crv: key.crv, x: key.x, y: key.y }), -}); - -/** Reads every valid private key from a contained JWKS file and derives public material. */ -export const resolveSigningKeyMaterial = ( - signing: Extract, -): Effect.Effect< - ResolvedSigningKeyMaterial, - InvalidJwtSigningMaterialError, - FileSystem.FileSystem | Path.Path -> => - readSigningJwks(signing).pipe( - Effect.map((keys) => { - const publicKeys = keys.map(publicSigningJwk); - return { - privateKeysJson: JSON.stringify(keys), - publicKeys, - publicJwksJson: JSON.stringify({ keys: publicKeys }), - }; - }), - ); - -const signWithJwk = ( - jwk: SigningJwk, - payload: string, -): Effect.Effect => - Effect.gen(function* () { - const header: JwtHeader = - jwk.kid === undefined - ? { alg: jwk.alg, typ: "JWT" } - : { alg: jwk.alg, kid: jwk.kid, typ: "JWT" }; - const headerJson = yield* encodeJwtHeader(header).pipe( - Effect.mapError(() => invalidSigningMaterial()), - ); - const data = `${base64UrlEncodeText(headerJson)}.${base64UrlEncodeText(payload)}`; - return yield* Effect.try({ - try: () => { - const privateKey = createPrivateKey({ key: jwk, format: "jwk" }); - const signer = createSign(jwk.alg === "RS256" ? "RSA-SHA256" : "sha256").update(data); - const signature = - jwk.alg === "RS256" - ? signer.end().sign(privateKey) - : signer.end().sign({ key: privateKey, dsaEncoding: "ieee-p1363" }); - return `${data}.${base64UrlEncode(signature)}`; - }, - catch: () => invalidSigningMaterial(), - }); - }); - -const generateJwtToken = ( - generator: Extract, - jwtSecret: string | undefined, -): Effect.Effect => { - const payloadFor = (expiry: number): JwtPayload => ({ - iss: JWT_ISSUER, - role: generator.role, - exp: expiry, - }); - if (generator.signing.kind === "symmetric") { - if (jwtSecret === undefined || jwtSecret.length === 0) - return Effect.fail(invalidSigningMaterial()); - return Effect.gen(function* () { - const header = base64UrlEncodeText( - yield* encodeJwtHeader({ alg: "HS256", typ: "JWT" }).pipe( - Effect.mapError(() => invalidSigningMaterial()), - ), - ); - const payload = base64UrlEncodeText( - yield* encodeJwtPayload(payloadFor(JWT_HMAC_EXPIRY)).pipe( - Effect.mapError(() => invalidSigningMaterial()), - ), - ); - const data = `${header}.${payload}`; - return yield* Effect.try({ - try: () => `${data}.${createHmac("sha256", jwtSecret).update(data).digest("base64url")}`, - catch: () => invalidSigningMaterial(), - }); - }); - } - return readSigningJwk(generator.signing).pipe( - Effect.flatMap((jwk) => - Clock.currentTimeMillis.pipe( - Effect.flatMap((nowMillis) => - encodeJwtPayload( - payloadFor(Math.floor(nowMillis / 1000) + JWT_ASYMMETRIC_EXPIRY_SECONDS), - ).pipe( - Effect.mapError(() => invalidSigningMaterial()), - Effect.flatMap((payload) => signWithJwk(jwk, payload)), - ), - ), - ), - ), - ); -}; - -const generatedFor = ( - declaration: SecretDeclaration, - crypto: Crypto.Crypto, - resolved: Record, -): Effect.Effect< - string, - StackSecretMismatchError | InvalidJwtSigningMaterialError, - FileSystem.FileSystem | Path.Path -> => { - const generator = declaration.generator; - if (generator === undefined) return generatedSecret(crypto); - switch (generator.kind) { - case "publishable-key": - return randomEncoded(crypto, 24, declaration.slot).pipe( - Effect.map((value) => `sb_publishable_${value}`), - ); - case "secret-key": - return randomEncoded(crypto, 24, declaration.slot).pipe( - Effect.map((value) => `sb_secret_${value}`), - ); - case "jwt-secret": - return randomEncoded(crypto, 32, declaration.slot); - case "random-base64url": - return randomEncoded(crypto, generator.bytes, declaration.slot); - case "jwt-token": - return generateJwtToken(generator, resolved[AUTH_JWT_SECRET_SLOT]?.value); - } -}; - -/** - * Resolves managed and pass-through secret declarations against dedicated persisted values. - * Managed omission reuses/generates exactly once; pass-through declarations are complete and - * can change only at a stopped lifecycle boundary. - */ -export const resolveSecrets = ( - candidate: SecretCandidate, - persisted: PersistedSecretValues | undefined, - lifecycle: DesiredStackLifecycle, -): Effect.Effect< - ResolvedSecrets, - StackSecretMismatchError | StackMustBeStoppedError | InvalidJwtSigningMaterialError, - Crypto.Crypto | FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { - const crypto = yield* Crypto.Crypto; - const isUnconfigured = persisted === undefined; - const canMutatePassthrough = lifecycle === "stopped" || lifecycle === "unconfigured"; - const previous = persisted ?? {}; - const declarations = new Map(); - for (const declaration of candidate.declarations) { - if (!validSlot(declaration.slot)) - return yield* secretMismatch("unknown", "Secret slot name is invalid"); - if (declarations.has(declaration.slot)) - return yield* secretMismatch(declaration.slot, "Duplicate secret declaration"); - declarations.set(declaration.slot, declaration); - } - - const resolved: Record = {}; - const deferred: Array<[string, SecretDeclaration]> = []; - for (const [slot, declaration] of declarations) { - const old = previous[slot]; - if (old !== undefined && old.policy !== declaration.policy) { - return yield* secretMismatch(slot, "Secret policy cannot change for an existing slot"); - } - if (declaration.policy === "managed") { - const supplied = - declaration.value === undefined ? undefined : readSecret(declaration.value); - if (old !== undefined) { - if (supplied !== undefined && supplied !== old.value) - return yield* secretMismatch( - slot, - "Supplied managed secret differs from persisted value", - ); - resolved[slot] = old; - } else { - if (supplied === undefined && declaration.generator?.kind === "jwt-token") { - deferred.push([slot, declaration]); - continue; - } - const value = supplied ?? (yield* generatedFor(declaration, crypto, resolved)); - resolved[slot] = { policy: "managed", value }; - } - } else { - if (declaration.value === undefined) - return yield* secretMismatch(slot, "Pass-through secret declarations require a value"); - const value = readSecret(declaration.value); - if ( - !canMutatePassthrough && - ((old === undefined && !isUnconfigured) || (old !== undefined && old.value !== value)) - ) - return yield* lifecycleChange(slot); - resolved[slot] = { policy: "passthrough", value }; - } - } - - for (const [slot, declaration] of deferred) { - const value = yield* generatedFor(declaration, crypto, resolved); - resolved[slot] = { policy: "managed", value }; - } - - for (const [slot, old] of Object.entries(previous)) { - if (old.policy === "managed") { - resolved[slot] ??= old; - } else if (!declarations.has(slot)) { - if (!canMutatePassthrough) return yield* lifecycleChange(slot); - } - } - - return { persisted: resolved }; - }); diff --git a/packages/stack/src/state/StackState.ts b/packages/stack/src/state/StackState.ts deleted file mode 100644 index cc4d146aa7..0000000000 --- a/packages/stack/src/state/StackState.ts +++ /dev/null @@ -1,349 +0,0 @@ -import { Effect, Redacted, Schema, SchemaAST, SchemaGetter, SchemaIssue } from "effect"; -import type { StackIdentity } from "../identity/Identity.ts"; -import { - AuthModule, - DatabaseModule, - FunctionsModule, - MailModule, - PoolerModule, - RealtimeModule, - RestModule, - StorageModule, - StudioModule, - AnalyticsModule, - AuthSettingsSchema, - DatabaseSettingsSchema, - FunctionsSettingsSchema, - MailSettingsSchema, - PoolerSettingsSchema, - RealtimeSettingsSchema, - RestSettingsSchema, - StorageSettingsSchema, - StudioSettingsSchema, - AnalyticsSettingsSchema, -} from "../model/capabilities/index.ts"; -import type { StackDefinition } from "../model/Compiler.ts"; -import type { CapabilityModule } from "../model/CapabilityModule.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import { StackRuntimeSchema } from "../public/Runtime.ts"; -import { DesiredStackLifecycleSchema, NetworkPortSchema, PORT_FIELDS } from "../public/Status.ts"; - -export const STACK_STATE_FORMAT = "supabase-stack-state-v1" as const; - -const PersistedStackIdentitySchema = Schema.Struct({ - projectRoot: Schema.String, - branchContext: Schema.String, - stackName: Schema.String, -}); -export type PersistedStackIdentity = Schema.Schema.Type; - -const HostPortAssignmentSchema = Schema.Struct({ - field: Schema.Literals(PORT_FIELDS), - port: NetworkPortSchema, - intent: Schema.Literals(["automatic", "exact"] as const), -}); -export type HostPortAssignment = Schema.Schema.Type; - -/** A durable loopback endpoint used by the host gateway to reach one workload. */ -const PrivatePortAssignmentSchema = Schema.Struct({ - workloadId: Schema.String.check(Schema.isNonEmpty()), - binding: Schema.String.check(Schema.isNonEmpty()), - port: NetworkPortSchema, -}); -export type PrivatePortAssignment = Schema.Schema.Type; - -/** Stable identity for one durable private workload binding. */ -export const privateBindingKey = ( - assignment: Pick, -): string => `${assignment.workloadId}\u0000${assignment.binding}`; - -const PersistedSecretEntrySchema = Schema.Struct({ - policy: Schema.Literals(["managed", "passthrough"] as const), - value: Schema.String, -}); -/** Secret slots are dynamic because function environment names are user-defined. */ -const PersistedSecretValuesSchema = Schema.Record( - Schema.String.check(Schema.isPattern(/^[A-Za-z0-9_.:/-]+$/)), - PersistedSecretEntrySchema, -); -export type PersistedSecretValues = Schema.Schema.Type; - -const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && - value !== null && - !Array.isArray(value) && - !Redacted.isRedacted(value); - -const hasExactKeys = ( - value: unknown, - keys: ReadonlyArray, -): value is Record => { - if (!isRecord(value)) return false; - const actual = Object.keys(value).sort(); - const expected = [...keys].sort(); - return actual.length === expected.length && expected.every((key, index) => actual[index] === key); -}; - -const isSecretSlot = (value: unknown): value is { readonly slot: string } => - hasExactKeys(value, ["slot"]) && typeof value.slot === "string" && value.slot.length > 0; - -/** Convert persisted materialized leaves back to the input representation solely for validation. */ -const restoreForValidation = (value: unknown): unknown => { - if (isSecretSlot(value)) return Redacted.make(""); - if (value === null) return undefined; - if (Array.isArray(value)) return value.map(restoreForValidation); - if (isRecord(value)) { - return Object.fromEntries( - Object.entries(value) - .map(([key, entry]) => [key, restoreForValidation(entry)] as const) - .filter(([, entry]) => entry !== undefined), - ); - } - return value; -}; - -const hasCompleteDefaults = (value: unknown, defaults: unknown): boolean => { - if (Array.isArray(defaults)) return Array.isArray(value); - if (!isRecord(defaults)) return value !== undefined; - if (!isRecord(value)) return false; - for (const key of Object.keys(defaults)) { - if (!Object.hasOwn(value, key)) return false; - const defaultValue = defaults[key]; - const actualValue = value[key]; - if (isRecord(defaultValue) && Object.keys(defaultValue).length > 0) { - if (!hasCompleteDefaults(actualValue, defaultValue)) return false; - } - } - return true; -}; - -const capabilityKeys = [ - "enabled", - "activation", - "idleTimeoutSeconds", - "version", - "settings", -] as const; - -const invalid = (message: string): Effect.Effect => - Effect.fail(new SchemaIssue.InvalidValue({ message })); - -const validateDynamicRecords = ( - name: CapabilityName, - settings: unknown, -): Effect.Effect => { - if (!isRecord(settings)) return Effect.void; - if (name === "functions" && isRecord(settings.functions)) { - for (const [slug, value] of Object.entries(settings.functions)) { - if (!/^[a-zA-Z0-9_-]+$/.test(slug)) - return invalid(`Materialized function ${slug} has an invalid name`); - if ( - !hasExactKeys(value, [ - "enabled", - "verify_jwt", - "import_map", - "entrypoint", - "static_files", - "env", - ]) - ) - return invalid(`Materialized function ${slug} is missing a defaulted field`); - if (!isRecord(value.env)) return invalid(`Materialized function ${slug} has invalid env`); - for (const secret of Object.values(value.env)) - if (!isSecretSlot(secret)) return invalid("Function secret must be a slot"); - } - } - if (name === "storage" && isRecord(settings.buckets)) { - for (const [bucket, value] of Object.entries(settings.buckets)) { - if (!hasExactKeys(value, ["public", "file_size_limit", "allowed_mime_types", "objects_path"])) - return invalid(`Materialized bucket ${bucket} is missing a defaulted field`); - } - } - if (name === "auth" && isRecord(settings.email)) { - const template = settings.email.template; - if (isRecord(template)) { - for (const [name, value] of Object.entries(template)) - if (!hasExactKeys(value, ["subject", "content_path"])) - return invalid(`Materialized auth email template ${name} is missing a defaulted field`); - } - const notification = settings.email.notification; - if (isRecord(notification)) { - for (const [name, value] of Object.entries(notification)) - if (!hasExactKeys(value, ["enabled", "subject", "content_path"])) - return invalid( - `Materialized auth email notification ${name} is missing a defaulted field`, - ); - } - } - return Effect.void; -}; - -const validateModuleSettings = ( - module: CapabilityModule, - schema: Schema.Codec, - settings: unknown, - options: SchemaAST.ParseOptions, -): Effect.Effect => { - if (!hasCompleteDefaults(settings, module.defaultSettings)) - return invalid(`Materialized ${module.name} settings are incomplete`); - return Schema.decodeUnknownEffect(schema, { onExcessProperty: "error" })( - restoreForValidation(settings), - options, - ).pipe( - Effect.asVoid, - Effect.mapError((error) => error.issue), - Effect.flatMap(() => validateDynamicRecords(module.name, settings)), - ); -}; - -const validateModuleSettingsByName = ( - name: CapabilityName, - settings: unknown, - options: SchemaAST.ParseOptions, -) => { - switch (name) { - case "database": - return validateModuleSettings(DatabaseModule, DatabaseSettingsSchema, settings, options); - case "rest": - return validateModuleSettings(RestModule, RestSettingsSchema, settings, options); - case "auth": - return validateModuleSettings(AuthModule, AuthSettingsSchema, settings, options); - case "realtime": - return validateModuleSettings(RealtimeModule, RealtimeSettingsSchema, settings, options); - case "storage": - return validateModuleSettings(StorageModule, StorageSettingsSchema, settings, options); - case "functions": - return validateModuleSettings(FunctionsModule, FunctionsSettingsSchema, settings, options); - case "studio": - return validateModuleSettings(StudioModule, StudioSettingsSchema, settings, options); - case "mail": - return validateModuleSettings(MailModule, MailSettingsSchema, settings, options); - case "analytics": - return validateModuleSettings(AnalyticsModule, AnalyticsSettingsSchema, settings, options); - case "pooler": - return validateModuleSettings(PoolerModule, PoolerSettingsSchema, settings, options); - } -}; - -const isDefinitionShape = (input: unknown): input is StackDefinition => { - if (!hasExactKeys(input, ["preparation", "capabilities", "listeners", "security"])) return false; - if (input.preparation !== "background" && input.preparation !== "on-demand") return false; - const capabilities = input.capabilities; - if (!hasExactKeys(capabilities, CAPABILITY_NAMES)) return false; - for (const name of CAPABILITY_NAMES) { - const capability = capabilities[name]; - if (!hasExactKeys(capability, capabilityKeys)) return false; - if (typeof capability.enabled !== "boolean") return false; - if (capability.activation !== "eager" && capability.activation !== "lazy") return false; - if ( - capability.idleTimeoutSeconds !== false && - (typeof capability.idleTimeoutSeconds !== "number" || - !Number.isFinite(capability.idleTimeoutSeconds) || - capability.idleTimeoutSeconds <= 0) - ) - return false; - if (typeof capability.version !== "string" || capability.version.length === 0) return false; - } - const listeners = input.listeners; - if (!hasExactKeys(listeners, PORT_FIELDS)) return false; - for (const field of PORT_FIELDS) { - const listener = listeners[field]; - if (!hasExactKeys(listener, ["enabled", "address", "port"])) return false; - if (typeof listener.enabled !== "boolean" || typeof listener.address !== "string") return false; - if (!(listener.port === "automatic" || Schema.is(NetworkPortSchema)(listener.port))) - return false; - } - const security = input.security; - if (!hasExactKeys(security, ["jwt"]) || !hasExactKeys(security.jwt, ["issuer", "signing"])) - return false; - const jwt = security.jwt; - if (!(jwt.issuer === null || typeof jwt.issuer === "string")) return false; - const signing = jwt.signing; - if (signing === null) return true; - if (!isRecord(signing)) return false; - if (signing.kind === "symmetric") - return hasExactKeys(signing, ["kind", "secret"]) && isSecretSlot(signing.secret); - return ( - hasExactKeys(signing, ["kind", "path"]) && - signing.kind === "jwks-file" && - typeof signing.path === "string" - ); -}; - -/** One closed, exhaustive schema for the compiler's fully materialized definition. */ -const StackDefinitionSchema = Schema.declareConstructor()( - [], - () => (input, ast, options) => - Effect.gen(function* () { - if (!isDefinitionShape(input)) - return yield* invalid("Materialized definition shape is invalid"); - const capabilities = input.capabilities; - for (const name of CAPABILITY_NAMES) { - const capability = capabilities[name]; - yield* validateModuleSettingsByName(name, capability.settings, options); - } - return input; - }), - { title: "Materialized StackDefinition" }, -); - -const PortAssignmentsSchema = Schema.Array(HostPortAssignmentSchema).pipe( - Schema.decode({ - decode: SchemaGetter.checkEffect((assignments) => - Effect.succeed( - new Set(assignments.map(({ field }) => field)).size === assignments.length && - new Set(assignments.map(({ port }) => port)).size === assignments.length - ? undefined - : "Duplicate persisted port field or port", - ), - ), - encode: SchemaGetter.passthrough(), - }), -); - -const PrivatePortAssignmentsSchema = Schema.Array(PrivatePortAssignmentSchema).pipe( - Schema.decode({ - decode: SchemaGetter.checkEffect((assignments) => - Effect.succeed( - new Set(assignments.map(privateBindingKey)).size === assignments.length && - new Set(assignments.map(({ port }) => port)).size === assignments.length - ? undefined - : "Duplicate persisted private binding or port", - ), - ), - encode: SchemaGetter.passthrough(), - }), -); - -const stateShape = Schema.Struct({ - format: Schema.Literal(STACK_STATE_FORMAT), - identity: PersistedStackIdentitySchema, - runtime: StackRuntimeSchema, - desiredLifecycle: DesiredStackLifecycleSchema, - definition: Schema.optional(StackDefinitionSchema), - ports: PortAssignmentsSchema, - privatePorts: PrivatePortAssignmentsSchema, - secrets: PersistedSecretValuesSchema, -}); - -/** Complete durable state. An optional definition is present once the identity is configured. */ -export const PersistedStackStateSchema = stateShape.pipe( - Schema.decode({ - decode: SchemaGetter.checkEffect((state) => - Effect.succeed( - state.ports.some(({ port }) => state.privatePorts.some((entry) => entry.port === port)) - ? "Public and private persisted ports must not overlap" - : undefined, - ), - ), - encode: SchemaGetter.passthrough(), - }), -); -export type PersistedStackState = Schema.Schema.Type; - -export const toPersistedIdentity = (identity: StackIdentity): PersistedStackIdentity => ({ - projectRoot: identity.projectRoot, - branchContext: identity.branchContext, - stackName: identity.stackName, -}); diff --git a/packages/stack/src/state/StackStateStore.ts b/packages/stack/src/state/StackStateStore.ts deleted file mode 100644 index a2b5775925..0000000000 --- a/packages/stack/src/state/StackStateStore.ts +++ /dev/null @@ -1,672 +0,0 @@ -import { - Crypto, - Data, - Effect, - Exit, - FileSystem, - Path, - PlatformError, - Predicate, - Schedule, - Schema, -} from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- Effect FileSystem.remove has no empty-directory-only mode; rmdir makes cleanup fail when remnants are present. -import { rmdir } from "node:fs/promises"; -import { - InvalidProjectRootError, - StackStateFormatUnsupportedError, - StackStateInvalidError, -} from "../public/Errors.ts"; -import { resolveStackPaths, type StackPaths } from "./Paths.ts"; -import { deriveStackId } from "../identity/Identity.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import { - PersistedStackStateSchema, - STACK_STATE_FORMAT, - type PersistedStackState, -} from "./StackState.ts"; -import { - acquirePortLease, - installLease, - readOwnerLock, - removeLeaseIfHeld, - writeLockTemp, - type OwnerLock, - OWNER_LOCK_FORMAT, -} from "./Ownership.ts"; - -class RegistryBusyError extends Data.TaggedError("RegistryBusyError")<{}> {} - -export interface StackStateStore { - /** Reads without taking the registry lock. `undefined` means an unconfigured identity. */ - readonly read: ( - stackId: string, - ) => Effect.Effect< - PersistedStackState | undefined, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >; - /** Initializes one identity atomically, returning the winner's complete state. */ - readonly initialize: ( - stackId: string, - state: PersistedStackState, - ) => Effect.Effect< - PersistedStackState, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >; - /** Replaces one complete state value while holding the registry lock. */ - readonly replace: ( - stackId: string, - state: PersistedStackState, - ) => Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >; - /** Internal transaction primitive for callers that already hold the registry lock. */ - readonly replaceUnlocked: ( - stackId: string, - state: PersistedStackState, - ) => Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >; - /** Explicit, exact-identity destructive cleanup. Ordinary lifecycle paths never call this. */ - readonly cleanup: ( - stackId: string, - ) => Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >; - /** Removes only an empty runtime-only remnant and its now-empty identity root. */ - readonly recoverRuntimeRemnant: ( - stackId: string, - ) => Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - >; -} - -const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null && !Array.isArray(value); - -const withoutKeys = ( - value: Readonly>, - keys: ReadonlyArray, -): Record => { - const result = { ...value }; - for (const key of keys) delete result[key]; - return result; -}; - -/** Restores defaults and drops settings removed from the local model when reading older durable state. */ -const normalizeDurableState = (raw: Readonly>): unknown => { - const identity = isRecord(raw.identity) ? withoutKeys(raw.identity, ["stackId"]) : raw.identity; - const definition = raw.definition; - if (!isRecord(definition) || !isRecord(definition.capabilities)) return { ...raw, identity }; - const capabilities: Record = { ...definition.capabilities }; - for (const [capability, value] of Object.entries(capabilities)) { - if (isRecord(value) && !Object.hasOwn(value, "idleTimeoutSeconds")) - capabilities[capability] = { ...value, idleTimeoutSeconds: false }; - } - const obsolete: ReadonlyArray]> = [ - ["database", ["network_restrictions", "ssl_enforcement", "vault"]], - ["rest", ["auto_expose_new_tables", "tls"]], - ["storage", ["analytics"]], - ["analytics", ["vector_port"]], - ]; - for (const [capability, keys] of obsolete) { - const module = capabilities[capability]; - if (!isRecord(module) || !isRecord(module.settings)) continue; - capabilities[capability] = { ...module, settings: withoutKeys(module.settings, keys) }; - } - return { ...raw, identity, definition: { ...definition, capabilities } }; -}; - -const stateError = (message: string, cause?: unknown) => - new StackStateInvalidError({ message, ...(cause === undefined ? {} : { cause }) }); - -const isNotFoundPlatformError = (error: unknown): error is PlatformError.PlatformError => - error instanceof PlatformError.PlatformError && - error.reason instanceof PlatformError.SystemError && - Predicate.isTagged(error.reason, "NotFound"); - -const MISSING_STATE_REMNANT_CODE = "missing-state-remnant" as const; -const MISSING_STATE_REMNANT_MESSAGE = - "Stack state is missing beside identity-owned remnants; refusing to guess replacement state"; - -/** Identifies an empty identity directory left by an interrupted owner launch. */ -export const isMissingStateRemnantError = (error: unknown): error is StackStateInvalidError => - error instanceof StackStateInvalidError && error.code === MISSING_STATE_REMNANT_CODE; - -const validateId = (stackId: string) => - Schema.decodeEffect(StackIdSchema)(stackId).pipe( - Effect.mapError((error) => stateError(`Invalid StackId: ${String(error)}`)), - ); - -const decodeState = ( - raw: unknown, -): Effect.Effect< - PersistedStackState, - StackStateInvalidError | StackStateFormatUnsupportedError -> => { - if (!isRecord(raw)) return Effect.fail(stateError("Persisted stack state must be an object")); - if (typeof raw.format !== "string") - return Effect.fail(stateError("Persisted stack state format is missing or invalid")); - if (raw.format !== STACK_STATE_FORMAT) { - return Effect.fail( - new StackStateFormatUnsupportedError({ - format: raw.format, - message: `Unsupported stack state format; expected ${STACK_STATE_FORMAT}`, - }), - ); - } - if (!isRecord(raw.secrets)) - return Effect.fail(stateError("Persisted secret values must be a record")); - // Schema.Record doesn't enforce key-format checks while decoding JSON, so validate slot names here. - for (const slot of Object.keys(raw.secrets)) - if (!/^[A-Za-z0-9_.:/-]+$/.test(slot)) - return Effect.fail(stateError(`Persisted secret slot key is invalid: ${slot}`)); - return Schema.decodeUnknownEffect(PersistedStackStateSchema)(normalizeDurableState(raw), { - onExcessProperty: "error", - }).pipe( - Effect.mapError((error) => stateError(`Invalid persisted stack state: ${String(error)}`)), - ); -}; - -const ownerDirectory = (fs: FileSystem.FileSystem, directory: string) => - fs - .makeDirectory(directory, { recursive: true, mode: 0o700 }) - .pipe(Effect.flatMap(() => fs.chmod(directory, 0o700))); - -const validateIdentityForStackId = ( - identity: PersistedStackState["identity"], - stackId: string, -): Effect.Effect => { - return deriveStackId(identity).pipe( - Effect.mapError((error) => - stateError(`Unable to validate persisted identity: ${error.message}`), - ), - Effect.flatMap((derived) => - derived === stackId - ? Effect.void - : Effect.fail(stateError("Persisted identity does not match its StackId directory")), - ), - ); -}; - -const validateStateSchema = ( - state: PersistedStackState, -): Effect.Effect => - Schema.decodeEffect(PersistedStackStateSchema)(state, { - onExcessProperty: "error", - }).pipe( - Effect.mapError((error) => stateError(`Invalid persisted stack state: ${String(error)}`)), - Effect.asVoid, - ); - -const atomicWrite = ( - fs: FileSystem.FileSystem, - path: Path.Path, - crypto: Crypto.Crypto, - statePath: string, - stackRoot: string, - state: PersistedStackState, -): Effect.Effect => - Effect.gen(function* () { - const encoded = yield* Schema.encodeEffect(PersistedStackStateSchema)(state).pipe( - Effect.mapError((error) => - stateError(`Unable to encode persisted stack state: ${String(error)}`), - ), - ); - const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(PersistedStackStateSchema))( - encoded, - ).pipe( - Effect.mapError((error) => - stateError(`Unable to encode persisted stack state JSON: ${String(error)}`), - ), - ); - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((error) => - stateError(`Unable to allocate state temporary name: ${error.message}`), - ), - ); - const temporary = path.join(stackRoot, `.state.${token}.tmp`); - yield* Effect.gen(function* () { - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs - .open(temporary, { flag: "wx", mode: 0o600 }) - .pipe( - Effect.mapError((error) => - stateError(`Unable to open temporary state: ${error.message}`), - ), - ); - yield* file - .writeAll(new TextEncoder().encode(serialized)) - .pipe( - Effect.mapError((error) => - stateError(`Unable to write temporary state: ${error.message}`), - ), - ); - yield* file.sync.pipe( - Effect.mapError((error) => - stateError(`Unable to sync temporary state: ${error.message}`), - ), - ); - }), - ); - yield* fs - .rename(temporary, statePath) - .pipe( - Effect.mapError((error) => - stateError(`Unable to atomically replace state: ${error.message}`), - ), - ); - }).pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - yield* fs - .chmod(statePath, 0o600) - .pipe( - Effect.mapError((error) => stateError(`Unable to secure state document: ${error.message}`)), - ); - // Effect Platform has no portable directory-fsync operation; file sync plus same-directory - // rename is the strongest cross-platform atomicity guarantee available here. - }); - -const validateState = ( - stackId: string, - state: PersistedStackState, -): Effect.Effect => - Effect.gen(function* () { - yield* validateIdentityForStackId(state.identity, stackId); - yield* validateStateSchema(state); - }); - -const persistValidatedState = ( - fs: FileSystem.FileSystem, - path: Path.Path, - crypto: Crypto.Crypto, - paths: StackPaths, - state: PersistedStackState, -): Effect.Effect => - Effect.gen(function* () { - yield* ownerDirectory(fs, paths.stackRoot).pipe( - Effect.mapError((error) => - stateError(`Unable to create stack state directory: ${error.message}`), - ), - ); - yield* atomicWrite(fs, path, crypto, paths.stateDocument, paths.stackRoot, state); - }); - -/** Acquires one short cross-process lock using the same OS-held lease as Supervisor ownership. */ -export const withRegistryLock = ( - stateRoot: string, - action: Effect.Effect, -): Effect.Effect< - A, - E | StackStateInvalidError, - R | FileSystem.FileSystem | Path.Path | Crypto.Crypto -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = path.resolve(stateRoot); - yield* fs - .makeDirectory(root, { recursive: true, mode: 0o700 }) - .pipe( - Effect.mapError((error) => - stateError(`Unable to create stack registry root: ${error.message}`), - ), - ); - yield* fs - .chmod(root, 0o700) - .pipe( - Effect.mapError((error) => - stateError(`Unable to secure stack registry root: ${error.message}`), - ), - ); - const lockPath = path.join(root, ".stack-registry.lock"); - const acquire = Effect.suspend(() => - Effect.gen(function* () { - const existing = yield* readOwnerLock(fs, lockPath); - const held = yield* acquirePortLease(existing?.port ?? 0).pipe( - Effect.mapError((error) => - existing !== undefined && error["code"] === "EADDRINUSE" - ? new RegistryBusyError() - : error, - ), - ); - const install = Effect.gen(function* () { - const token = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((error) => - stateError(`Unable to allocate registry lock token: ${error.message}`), - ), - ); - const lock: OwnerLock = { format: OWNER_LOCK_FORMAT, token, port: held.port }; - const temporary = yield* writeLockTemp(fs, path, lockPath, lock); - yield* installLease({ fs, lockPath, temporary, observed: existing }).pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - ); - return { held, token }; - }); - return yield* install.pipe( - Effect.onExit((exit) => (Exit.isFailure(exit) ? held.close : Effect.void)), - ); - }), - ).pipe( - Effect.mapError((error) => - Predicate.isTagged(error, "LeaseSlotTaken") ? new RegistryBusyError() : error, - ), - Effect.retry({ - while: (error) => Predicate.isTagged(error, "RegistryBusyError"), - // Allow a few seconds for a concurrent owner's port lease and state write to finish. - schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ times: 80 })), - }), - Effect.mapError((error) => - Predicate.isTagged(error, "RegistryBusyError") - ? stateError("Stack registry is busy") - : error, - ), - ); - return yield* Effect.acquireUseRelease( - acquire, - () => action, - ({ held, token }) => removeLeaseIfHeld(fs, lockPath, token).pipe(Effect.andThen(held.close)), - ); - }); - -export const makeStackStateStore = (options: { - readonly stateRoot: string; -}): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - - const pathsFor = (stackId: string) => - Effect.gen(function* () { - const valid = yield* validateId(stackId); - return yield* resolveStackPaths({ stateRoot: options.stateRoot, stackId: valid }).pipe( - Effect.mapError((error) => - error instanceof InvalidProjectRootError - ? error - : stateError(`Unable to resolve stack state paths: ${String(error)}`), - ), - ); - }); - - const inspectMissingState = ( - paths: StackPaths, - ): Effect.Effect => - Effect.gen(function* () { - const rootExists = yield* fs - .exists(paths.stackRoot) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect stack root: ${error.message}`), - ), - ); - if (!rootExists) return undefined; - const entries = yield* fs.readDirectory(paths.stackRoot).pipe( - Effect.catchIf( - (error) => isNotFoundPlatformError(error), - () => Effect.void, - ), - Effect.mapError((error) => - stateError(`Unable to inspect stack remnants: ${error.message}`), - ), - ); - // The stack root may disappear after exists() while a concurrent destroy finishes. - if (entries === undefined) return undefined; - if (entries.length > 0) { - if (entries.length === 1 && entries[0] === "runtime") { - const runtimeInfo = yield* fs.stat(paths.runtime).pipe( - Effect.catchIf( - (error) => isNotFoundPlatformError(error), - () => Effect.void, - ), - Effect.mapError((error) => - stateError(`Unable to inspect runtime remnant: ${error.message}`), - ), - ); - if (runtimeInfo === undefined) return undefined; - if (runtimeInfo.type === "Directory") { - const runtimeEntries = yield* fs.readDirectory(paths.runtime).pipe( - Effect.catchIf( - (error) => isNotFoundPlatformError(error), - () => Effect.void, - ), - Effect.mapError((error) => - stateError(`Unable to inspect runtime remnant: ${error.message}`), - ), - ); - if (runtimeEntries === undefined) return undefined; - if (runtimeEntries.length === 0) - return yield* new StackStateInvalidError({ - message: MISSING_STATE_REMNANT_MESSAGE, - code: MISSING_STATE_REMNANT_CODE, - }); - } - } - return yield* stateError(MISSING_STATE_REMNANT_MESSAGE); - } - return undefined; - }); - - const read = ( - stackId: string, - ): Effect.Effect< - PersistedStackState | undefined, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => - Effect.gen(function* () { - const paths = yield* pathsFor(stackId); - const stateRead = yield* fs.readFileString(paths.stateDocument).pipe( - Effect.map((text) => ({ _tag: "present" as const, text })), - Effect.catch((error) => - isNotFoundPlatformError(error) - ? inspectMissingState(paths).pipe( - Effect.map((state) => ({ _tag: "missing" as const, state })), - ) - : Effect.fail(stateError(`Unable to read state document: ${String(error)}`)), - ), - ); - if (stateRead._tag === "missing") return stateRead.state; - const text = stateRead.text; - const raw = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text).pipe( - Effect.mapError((error) => - stateError(`Unable to parse state document: ${String(error)}`), - ), - ); - const decoded = yield* decodeState(raw); - yield* validateIdentityForStackId(decoded.identity, stackId); - return decoded; - }); - - const initialize = ( - stackId: string, - candidate: PersistedStackState, - ): Effect.Effect< - PersistedStackState, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => - withRegistryLock( - options.stateRoot, - Effect.gen(function* () { - const existing = yield* read(stackId).pipe( - Effect.catchIf(isMissingStateRemnantError, () => - recoverRuntimeRemnantUnlocked(stackId).pipe(Effect.asVoid), - ), - ); - if (existing !== undefined) return existing; - const paths = yield* pathsFor(stackId); - yield* validateState(stackId, candidate); - yield* persistValidatedState(fs, path, crypto, paths, candidate); - return candidate; - }), - ); - - const replaceUnlocked = ( - stackId: string, - next: PersistedStackState, - ): Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => - Effect.gen(function* () { - const paths = yield* pathsFor(stackId); - yield* validateIdentityForStackId(next.identity, stackId); - const current = yield* read(stackId); - if (current === undefined) return yield* stateError("Cannot replace missing stack state"); - yield* validateStateSchema(next); - yield* persistValidatedState(fs, path, crypto, paths, next); - }); - - const replace = ( - stackId: string, - next: PersistedStackState, - ): Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError | StackStateFormatUnsupportedError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => withRegistryLock(options.stateRoot, replaceUnlocked(stackId, next)); - - // fs.remove({ recursive: false }) maps to Node's fs.rm, which refuses to remove directories. - // Native rmdir is used instead so a concurrent child creation fails with ENOTEMPTY rather - // than recursively deleting a newly created lease or control file. - const removeEmptyDirectory = (directory: string, label: string) => - Effect.tryPromise({ - try: () => rmdir(directory), - catch: (error) => - stateError( - `Unable to remove empty ${label} during recovery: ${ - error instanceof Error ? error.message : String(error) - }`, - ), - }); - - const recoverRuntimeRemnantUnlocked = ( - stackId: string, - ): Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => - Effect.gen(function* () { - const paths = yield* pathsFor(stackId); - const stateExists = yield* fs - .exists(paths.stateDocument) - .pipe( - Effect.mapError((error) => - stateError( - `Unable to inspect state document during remnant recovery: ${error.message}`, - ), - ), - ); - if (stateExists) - return yield* stateError("Cannot recover a runtime remnant beside persisted state"); - - const rootExists = yield* fs - .exists(paths.stackRoot) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect stack root during remnant recovery: ${error.message}`), - ), - ); - if (!rootExists) return; - const entries = yield* fs - .readDirectory(paths.stackRoot) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect stack remnants during recovery: ${error.message}`), - ), - ); - if (entries.length === 0) { - yield* removeEmptyDirectory(paths.stackRoot, "stack root"); - return; - } - if (entries.length !== 1 || entries[0] !== "runtime") - return yield* stateError("Stack state is missing beside durable identity remnants"); - - const runtimeInfo = yield* fs - .stat(paths.runtime) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect runtime remnant during recovery: ${error.message}`), - ), - ); - if (runtimeInfo.type !== "Directory") - return yield* stateError("Runtime remnant is not a directory"); - const runtimeEntries = yield* fs - .readDirectory(paths.runtime) - .pipe( - Effect.mapError((error) => - stateError(`Unable to inspect runtime remnant during recovery: ${error.message}`), - ), - ); - if (runtimeEntries.length > 0) - return yield* stateError("Runtime remnant is not empty; refusing recovery"); - - yield* removeEmptyDirectory(paths.runtime, "runtime remnant"); - yield* removeEmptyDirectory(paths.stackRoot, "stack root"); - }); - - const recoverRuntimeRemnant = ( - stackId: string, - ): Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => withRegistryLock(options.stateRoot, recoverRuntimeRemnantUnlocked(stackId)); - - const cleanup = ( - stackId: string, - ): Effect.Effect< - void, - InvalidProjectRootError | StackStateInvalidError, - FileSystem.FileSystem | Path.Path | Crypto.Crypto - > => - withRegistryLock( - options.stateRoot, - Effect.gen(function* () { - const paths = yield* pathsFor(stackId); - yield* fs - .remove(paths.stackRoot, { recursive: true, force: true }) - .pipe( - Effect.mapError((error) => - stateError(`Unable to remove exact stack identity: ${error.message}`), - ), - ); - }), - ); - - return { - read, - initialize, - replace, - replaceUnlocked, - cleanup, - recoverRuntimeRemnant, - } satisfies StackStateStore; - }); - -export { PersistedStackStateSchema } from "./StackState.ts"; -export type { PersistedStackState } from "./StackState.ts"; diff --git a/packages/stack/src/state/ownership.integration.test.ts b/packages/stack/src/state/ownership.integration.test.ts deleted file mode 100644 index 958965a6bb..0000000000 --- a/packages/stack/src/state/ownership.integration.test.ts +++ /dev/null @@ -1,636 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Path, - Ref, - Schema, - Stream, -} from "effect"; -import { ChildProcess } from "effect/unstable/process"; -import { createServer, Socket, type Server } from "node:net"; -import { deriveStackId, type StackIdentity } from "../identity/Identity.ts"; -import { StackOwnershipConflictError, StackStateInvalidError } from "../public/Errors.ts"; -import { - acquirePortLease, - acquireOwnership, - controlEndpointFor, - installLease, - LeaseSlotTaken, - ownerLockExists, - publishOwnership, - readOwnerMetadata, - readOwnerLock, - writeLockTemp, - OWNER_LOCK_FORMAT, - type StackRuntimeEnvironmentValue, -} from "./Ownership.ts"; -import { makeStackStateStore, type PersistedStackState } from "./StackStateStore.ts"; -import { resolveStackPaths } from "./Paths.ts"; - -const bindEphemeral = Effect.callback((resume) => { - const server = createServer((socket) => socket.destroy()); - const onError = (error: Error) => { - server.off("listening", onListening); - resume(Effect.fail(error)); - }; - const onListening = () => { - server.off("error", onError); - resume(Effect.succeed(server)); - }; - server.once("error", onError); - server.once("listening", onListening); - server.listen({ host: "127.0.0.1", port: 0 }); - return Effect.sync(() => { - server.off("error", onError); - server.off("listening", onListening); - if (server.listening) server.close(() => undefined); - }); -}); - -const closeServer = (server: Server) => - Effect.callback((resume) => { - if (!server.listening) { - resume(Effect.void); - return; - } - server.close((error) => - error === undefined ? resume(Effect.void) : resume(Effect.fail(error)), - ); - }); - -const observeRejectedPeer = (port: number) => - Effect.callback((resume) => { - const socket = new Socket(); - let connected = false; - let settled = false; - const finish = (result: Effect.Effect) => { - if (settled) return; - settled = true; - socket.off("connect", onConnect); - socket.off("error", onError); - socket.off("close", onClose); - socket.destroy(); - resume(result); - }; - const onConnect = () => { - connected = true; - }; - const onError = (error: Error) => finish(Effect.fail(error)); - const onClose = () => { - if (connected) finish(Effect.void); - }; - socket.once("connect", onConnect); - socket.once("error", onError); - socket.once("close", onClose); - socket.connect({ host: "127.0.0.1", port }); - return Effect.sync(() => { - settled = true; - socket.off("connect", onConnect); - socket.off("error", onError); - socket.off("close", onClose); - socket.destroy(); - }); - }); - -const jsonText = (value: unknown) => - Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))(value); - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const identity: StackIdentity = { - projectRoot: "/tmp/project", - branchContext: "ordinary-workspace", - stackName: "default", -}; - -const stateFor = (): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, -}); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("stack ownership", () => { - it.live("releases an exact port after a competing lease bind fails", () => - withPlatform( - Effect.gen(function* () { - const blocker = yield* bindEphemeral; - const address = blocker.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("Blocker did not expose a bound port"); - const failed = yield* acquirePortLease(address.port).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - const failure = errorOf(failed); - expect(failure).toBeInstanceOf(StackStateInvalidError); - expect(failure?.code).toBe("EADDRINUSE"); - yield* closeServer(blocker); - const lease = yield* acquirePortLease(address.port); - yield* lease.close; - }), - ), - ); - - it.live("closes accepted lease peers before lease cleanup completes", () => - withPlatform( - Effect.acquireRelease(acquirePortLease(0), (lease) => lease.close).pipe( - Effect.flatMap((lease) => observeRejectedPeer(lease.port).pipe(Effect.timeout("1 second"))), - ), - ), - ); - - it.live("initializes one complete state for concurrent equivalent creators", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-init-" }); - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: root }); - const candidate = stateFor(); - const [first, second] = yield* Effect.all( - [store.initialize(stackId, candidate), store.initialize(stackId, candidate)], - { concurrency: 2 }, - ); - expect(first).toEqual(candidate); - expect(second).toEqual(candidate); - expect(yield* store.read(stackId)).toEqual(candidate); - }), - ), - ); - - it.live("publishes and releases an exact session-fenced owner", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-owner-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const lease = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "session-one", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }); - expect(yield* ownerLockExists(root, stackId)).toBe(true); - expect(lease.metadata.endpoint).toEqual( - controlEndpointFor(stackId, environment, lease.metadata.leasePort), - ); - yield* publishOwnership(lease); - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - const persisted = yield* Schema.decodeEffect( - Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), - )(yield* fs.readFileString(paths.controlMetadata)); - expect(persisted).not.toHaveProperty("stackId"); - expect(persisted).not.toHaveProperty("endpoint"); - expect(yield* readOwnerMetadata(root, stackId, environment)).toEqual(lease.metadata); - - const competing = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "session-two", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }).pipe(Effect.exit); - expect(errorOf(competing)).toBeInstanceOf(StackOwnershipConflictError); - yield* lease.release; - expect(yield* ownerLockExists(root, stackId)).toBe(false); - expect(yield* readOwnerMetadata(root, stackId, environment)).toBeUndefined(); - const after = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "session-three", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }); - yield* after.release; - }), - ), - ); - - it.live("recovers an owner lease after an exact supervisor SIGKILL", () => - withPlatform( - Effect.gen(function* () { - if (process.platform === "win32") return yield* Effect.void; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-crash-" }); - const childIdentity = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(childIdentity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const moduleUrl = new URL("./Ownership.ts", import.meta.url).href; - // bare imports in a `node -e` script resolve from cwd, so run from the package root - const cwd = path.resolve(import.meta.dirname, "../.."); - const script = ` - const { Effect } = await import("effect"); - const { NodeServices } = await import("@effect/platform-node"); - const { acquireOwnership, publishOwnership } = await import(process.env.OWNERSHIP_MODULE); - const environment = JSON.parse(process.env.OWNERSHIP_ENVIRONMENT); - const lease = await Effect.runPromise(Effect.scoped(Effect.gen(function* () { - const value = yield* acquireOwnership({ - stateRoot: environment.stateRoot, - stackId: process.env.OWNERSHIP_STACK_ID, - ownerSessionId: "crashed-owner", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }); - yield* publishOwnership(value); - process.stdout.write("READY\\n"); - yield* Effect.never; - }).pipe(Effect.provide(NodeServices.layer)))); - void lease; - `; - const child = yield* ChildProcess.make( - process.execPath, - ["--input-type=module", "-e", script], - { - cwd, - env: { - OWNERSHIP_MODULE: moduleUrl, - OWNERSHIP_STACK_ID: stackId, - OWNERSHIP_ENVIRONMENT: jsonText(environment), - }, - extendEnv: true, - stdout: "pipe", - stderr: "pipe", - }, - ); - const ready = yield* Deferred.make(); - const stderrText = yield* Ref.make>([]); - const childExitCode = yield* Ref.make>(Option.none()); - const output = yield* child.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - line === "READY" ? Deferred.succeed(ready, undefined).pipe(Effect.asVoid) : Effect.void, - ), - Effect.forkChild({ startImmediately: true }), - ); - const stderr = yield* child.stderr.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => Ref.update(stderrText, (lines) => [...lines, line])), - Effect.forkChild({ startImmediately: true }), - ); - const exitWatcher = yield* child.exitCode.pipe( - Effect.flatMap((code) => Ref.set(childExitCode, Option.some(code))), - Effect.forkChild({ startImmediately: true }), - ); - try { - yield* Deferred.await(ready).pipe( - Effect.timeoutOrElse({ - duration: "15 seconds", - orElse: () => - Effect.gen(function* () { - const lines = yield* Ref.get(stderrText); - const exitCode = yield* Ref.get(childExitCode); - const state = Option.isSome(exitCode) - ? ` (exit code ${String(exitCode.value)})` - : ""; - return yield* new StackStateInvalidError({ - message: `owner child did not become ready${state}${lines.length === 0 ? "" : `: ${lines.join("\\n")}`}`, - }); - }), - }), - ); - expect(yield* ownerLockExists(root, stackId)).toBe(true); - const metadata = yield* readOwnerMetadata(root, stackId, environment); - expect(metadata?.ownerSessionId).toBe("crashed-owner"); - yield* child.kill({ killSignal: "SIGKILL" }); - yield* child.exitCode.pipe(Effect.ignore); - const successor = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "successor-owner", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }); - expect(successor.metadata.endpoint).toEqual( - controlEndpointFor(stackId, environment, successor.metadata.leasePort), - ); - expect(yield* ownerLockExists(root, stackId)).toBe(true); - yield* publishOwnership(successor); - expect((yield* readOwnerMetadata(root, stackId, environment))?.ownerSessionId).toBe( - "successor-owner", - ); - yield* successor.release; - expect(yield* ownerLockExists(root, stackId)).toBe(false); - expect(yield* readOwnerMetadata(root, stackId, environment)).toBeUndefined(); - } finally { - yield* child.kill({ killSignal: "SIGKILL" }).pipe(Effect.ignore); - yield* Fiber.interrupt(output); - yield* Fiber.interrupt(stderr); - yield* Fiber.interrupt(exitWatcher); - } - }), - ), - ); - - it.live("derives control identity and endpoint from the stack directory", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-fence-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const lease = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "session-one", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }); - yield* publishOwnership(lease); - const metadataPath = lease.metadataPath; - const text = yield* fs.readFileString(metadataPath); - const persisted = yield* Schema.decodeEffect( - Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), - )(text); - yield* fs.writeFileString( - metadataPath, - jsonText({ - ...persisted, - stackId: "b".repeat(64), - endpoint: { kind: "unix", path: "/wrong/control.sock" }, - unexpected: true, - }), - ); - const rejected = yield* readOwnerMetadata(root, stackId, environment).pipe(Effect.exit); - expect(errorOf(rejected)).toBeInstanceOf(StackStateInvalidError); - yield* fs.writeFileString( - metadataPath, - jsonText({ - ...persisted, - stackId: "b".repeat(64), - endpoint: { kind: "unix", path: "/wrong/control.sock" }, - }), - ); - expect(yield* readOwnerMetadata(root, stackId, environment)).toEqual(lease.metadata); - yield* lease.release; - expect(yield* fs.exists(metadataPath)).toBe(false); - }), - ), - ); - - it.live("reclaims a valid owner lock after its lease is released", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-reclaim-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - const stale = yield* bindEphemeral; - const address = stale.address(); - if (address === null || typeof address === "string") - return yield* new StackStateInvalidError({ message: "missing lease port" }); - const staleEndpoint = controlEndpointFor(stackId, environment, address.port); - if (staleEndpoint.kind !== "unix") - return yield* new StackStateInvalidError({ message: "expected unix endpoint" }); - yield* fs.makeDirectory(path.dirname(staleEndpoint.path), { recursive: true, mode: 0o700 }); - yield* fs.writeFileString( - path.join(paths.runtime, "owner.lock"), - jsonText({ - format: "supabase-stack-lease-v1", - token: "old-session", - port: address.port, - }), - ); - yield* fs.writeFileString( - paths.controlMetadata, - jsonText({ - format: "supabase-stack-owner-v1", - stackId, - ownerSessionId: "old-session", - leasePort: address.port, - endpoint: staleEndpoint, - rpcRelease: "stack-rpc-v1@0.1.0", - }), - ); - yield* fs.writeFileString(staleEndpoint.path, "stale socket"); - yield* closeServer(stale); - const lease = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "new-session", - rpcRelease: "stack-rpc-v1@0.1.0", - environment, - }); - expect(lease.metadata.ownerSessionId).toBe("new-session"); - expect(yield* fs.exists(paths.controlMetadata)).toBe(false); - expect(yield* fs.exists(staleEndpoint.path)).toBe(false); - yield* lease.release; - }), - ), - ); - - it.live("uses a new control endpoint while a prior lease is still alive", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-live-owner-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - const first = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "old-owner", - rpcRelease: "rpc", - environment, - }); - yield* fs.remove(paths.runtime, { recursive: true }); - const replacement = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "new-owner", - rpcRelease: "rpc", - environment, - }); - expect(replacement.metadata.leasePort).not.toBe(first.metadata.leasePort); - expect(replacement.metadata.endpoint).not.toEqual(first.metadata.endpoint); - yield* first.release; - expect(yield* ownerLockExists(root, stackId)).toBe(true); - yield* replacement.release; - }), - ), - ); - - it.live("serializes concurrent recovery contenders", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-race-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - const stale = yield* bindEphemeral; - const address = stale.address(); - if (address === null || typeof address === "string") - return yield* new StackStateInvalidError({ message: "missing lease port" }); - yield* fs.writeFileString( - path.join(paths.runtime, "owner.lock"), - jsonText({ format: "supabase-stack-lease-v1", token: "old", port: address.port }), - ); - yield* closeServer(stale); - const attempts = yield* Effect.all( - ["one", "two"].map((ownerSessionId) => - acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId, - rpcRelease: "rpc", - environment, - }).pipe(Effect.exit), - ), - { concurrency: 2 }, - ); - const successes = attempts.filter(Exit.isSuccess); - expect(successes).toHaveLength(1); - expect( - attempts - .filter(Exit.isFailure) - .map(errorOf) - .some((error) => error instanceof StackOwnershipConflictError), - ).toBe(true); - if (successes[0] !== undefined) yield* successes[0].value.release; - }), - ), - ); - - it.live("fails closed for a malformed owner lock", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-malformed-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - yield* fs.writeFileString(path.join(paths.runtime, "owner.lock"), "not-json"); - const result = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "new", - rpcRelease: "rpc", - environment, - }).pipe(Effect.exit); - expect(errorOf(result)).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); - - it.live("fails closed when metadata exists without an owner lock", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-metadata-only-" }); - const stackId = yield* deriveStackId(identity); - const environment: StackRuntimeEnvironmentValue = { - stateRoot: root, - tempRoot: root, - platform: "posix", - }; - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - const metadata = { - format: "supabase-stack-owner-v1", - stackId, - ownerSessionId: "orphaned-owner", - leasePort: 45_001, - endpoint: controlEndpointFor(stackId, environment, 45_001), - rpcRelease: "rpc", - }; - yield* fs.writeFileString(paths.controlMetadata, jsonText(metadata)); - const result = yield* acquireOwnership({ - stateRoot: root, - stackId, - ownerSessionId: "new-owner", - rpcRelease: "rpc", - environment, - }).pipe(Effect.exit); - expect(errorOf(result)).toBeInstanceOf(StackOwnershipConflictError); - expect(yield* fs.readFileString(paths.controlMetadata)).toBe(jsonText(metadata)); - expect(yield* fs.exists(paths.runtime + "/owner.lock")).toBe(false); - }), - ), - ); - - it.live("does not replace a fresh lock installed after stale observation", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-race-install-" }); - const stackId = yield* deriveStackId(identity); - const paths = yield* resolveStackPaths({ stateRoot: root, stackId }); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - const old = { format: OWNER_LOCK_FORMAT, token: "old", port: 45_001 } as const; - const fresh = { format: OWNER_LOCK_FORMAT, token: "fresh", port: 45_002 } as const; - const lockPath = path.join(paths.runtime, "owner.lock"); - yield* fs.writeFileString(lockPath, jsonText(old)); - const temporary = yield* writeLockTemp(fs, path, lockPath, { - format: OWNER_LOCK_FORMAT, - token: "candidate", - port: 45_003, - }); - yield* fs.writeFileString(lockPath, jsonText(fresh)); - const result = yield* installLease({ fs, lockPath, temporary, observed: old }).pipe( - Effect.exit, - ); - expect(errorOf(result)).toBeInstanceOf(LeaseSlotTaken); - expect(yield* readOwnerLock(fs, lockPath)).toEqual(fresh); - yield* fs.remove(temporary, { force: true }); - }), - ), - ); -}); diff --git a/packages/stack/src/state/ports.integration.test.ts b/packages/stack/src/state/ports.integration.test.ts deleted file mode 100644 index 611fcbfbc1..0000000000 --- a/packages/stack/src/state/ports.integration.test.ts +++ /dev/null @@ -1,892 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Crypto, - Deferred, - Effect, - Exit, - FileSystem, - Fiber, - Option, - Path, - Schema, - Scope, -} from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- fixture supplies the native HTTP server instance that HostListener and gateways adopt. -import { createServer as createHttpServer } from "node:http"; -import { deriveStackId, type StackIdentity } from "../identity/Identity.ts"; -import { - PortAllocationError, - PortUnavailableError, - StackStateFormatUnsupportedError, - StackStateInvalidError, -} from "../public/Errors.ts"; -import { - makePortCoordinator, - type ListenerIntents, - type PortCoordinatorOptions, -} from "./PortCoordinator.ts"; -import type { HostListener } from "../supervisor/HostListener.ts"; -import { - makeStackStateStore, - PersistedStackStateSchema, - type PersistedStackState, -} from "./StackStateStore.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { bindHeldPort, bindHostListener, checkHostPort } from "../supervisor/HostListener.ts"; -import { withRegistryLock } from "./StackStateStore.ts"; - -const intents = (api: "automatic" | number = "automatic"): ListenerIntents => ({ - api: { enabled: true, address: "127.0.0.1", port: api }, - database: { enabled: false, address: "127.0.0.1", port: "automatic" }, - pooler: { enabled: false, address: "127.0.0.1", port: "automatic" }, - studio: { enabled: false, address: "127.0.0.1", port: "automatic" }, - mailUi: { enabled: false, address: "127.0.0.1", port: "automatic" }, - smtp: { enabled: false, address: "127.0.0.1", port: "automatic" }, - pop3: { enabled: false, address: "127.0.0.1", port: "automatic" }, - functionsInspector: { enabled: false, address: "127.0.0.1", port: "automatic" }, -}); - -const identity = (root: string, stackName: string): StackIdentity => ({ - projectRoot: root, - branchContext: "ordinary-workspace", - stackName, -}); - -const state = ( - id: string, - value: StackIdentity, - ports: PersistedStackState["ports"] = [], - privatePorts: PersistedStackState["privatePorts"] = [], -): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity: value, - runtime: { kind: "native" }, - desiredLifecycle: "running", - ports, - privatePorts, - secrets: {}, -}); - -const fakeListener = ( - address: string, - port: number, - field: HostListener["field"], -): Effect.Effect => - Effect.acquireRelease( - Effect.succeed({ - address, - port, - field, - binding: { kind: "http", server: createHttpServer() }, - connections: { sockets: new Set() }, - close: Effect.void, - } satisfies HostListener), - (listener) => listener.close, - ); - -const coordinatorOptions = ( - store: PortCoordinatorOptions["store"], - root: string, - bindHost = fakeListener, -): PortCoordinatorOptions => ({ - stateRoot: root, - store, - bindHost, - bindPrivate: (_address, port) => Effect.succeed({ port, close: Effect.void }), -}); - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); -describe("port acquisition", () => { - it.live("reads legacy sibling state while allocating ports", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-legacy-state-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const ownIdentity = identity(root, "own"); - const ownId = yield* deriveStackId(ownIdentity); - const siblingIdentity = identity(root, "legacy"); - const siblingId = yield* deriveStackId(siblingIdentity); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: {}, - }); - const sibling = { - ...state(siblingId, siblingIdentity), - definition: compiled.definition, - }; - const encoded = yield* Schema.encodeEffect(PersistedStackStateSchema)(sibling); - const legacy = structuredClone(encoded) as unknown as { - definition: { capabilities: Record }; - }; - for (const capability of Object.values(legacy.definition.capabilities)) - delete capability.idleTimeoutSeconds; - yield* store.initialize(ownId, { - ...state(ownId, ownIdentity), - desiredLifecycle: "running", - }); - yield* fs.makeDirectory(path.join(root, siblingId), { recursive: true }); - yield* fs.writeFileString( - path.join(root, siblingId, "state.json"), - yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(legacy), - ); - - const result = yield* makePortCoordinator(coordinatorOptions(store, root)).acquire( - ownId, - intents(), - [], - ); - const api = result.assignments.api; - expect(api?.port).toBeGreaterThan(0); - expect(yield* store.read(ownId)).toEqual( - expect.objectContaining({ - ports: expect.arrayContaining([ - expect.objectContaining({ field: "api", port: api?.port }), - ]), - }), - ); - }), - ), - ); - - it.live("requires running state and fails closed on an unreadable sibling", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-state-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "guard"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, { ...state(id, value), desiredLifecycle: "stopped" }); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const stopped = yield* coordinator.acquire(id, intents(), []).pipe(Effect.exit); - expect(Exit.isFailure(stopped)).toBe(true); - expect((yield* store.read(id))?.ports).toEqual([]); - const sibling = yield* deriveStackId(identity(root, "broken")); - const siblingRoot = path.join(root, sibling); - yield* fs.makeDirectory(siblingRoot, { recursive: true }); - yield* fs.writeFileString(path.join(siblingRoot, "state.json"), "not-json"); - yield* store.replaceUnlocked(id, { ...state(id, value), desiredLifecycle: "running" }); - const result = yield* coordinator.acquire(id, intents(), []).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( - StackStateInvalidError, - ); - }), - ), - ); - - it.live("ignores an empty runtime-only sibling remnant", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-remnant-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const mainIdentity = identity(root, "main"); - const mainId = yield* deriveStackId(mainIdentity); - const siblingId = yield* deriveStackId(identity(root, "runtime-remnant")); - yield* store.initialize(mainId, state(mainId, mainIdentity)); - yield* fs.makeDirectory(path.join(root, siblingId, "runtime"), { recursive: true }); - const result = yield* makePortCoordinator(coordinatorOptions(store, root)).acquire( - mainId, - intents(), - [], - ); - expect(result.assignments.api?.port).toBeGreaterThan(0); - expect(yield* fs.exists(path.join(root, siblingId, "runtime"))).toBe(true); - expect(yield* fs.exists(path.join(root, siblingId, "state.json"))).toBe(false); - }), - ), - ); - - it.live("preserves unsupported sibling format errors with sibling context", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-format-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const ownIdentity = identity(root, "format-owner"); - const ownId = yield* deriveStackId(ownIdentity); - const siblingId = yield* deriveStackId(identity(root, "unsupported-sibling")); - yield* store.initialize(ownId, state(ownId, ownIdentity)); - const siblingState = { - ...state(siblingId, identity(root, "unsupported-sibling")), - format: "supabase-stack-state-v2", - }; - yield* fs.makeDirectory(path.join(root, siblingId), { recursive: true }); - const encodedSiblingState = yield* Schema.encodeEffect( - Schema.fromJsonString(Schema.Unknown), - )(siblingState); - yield* fs.writeFileString(path.join(root, siblingId, "state.json"), encodedSiblingState); - const result = yield* makePortCoordinator(coordinatorOptions(store, root)) - .acquire(ownId, intents(), []) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - if (!Exit.isFailure(result)) return; - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(StackStateFormatUnsupportedError); - if (!(error instanceof StackStateFormatUnsupportedError)) return; - expect(error.format).toBe("supabase-stack-state-v2"); - expect(error.message).toContain(siblingId); - }), - ), - ); - - it.live("excludes durable sibling claims while allowing stopped exact sharing", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-siblings-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const aIdentity = identity(root, "a"); - const bIdentity = identity(root, "b"); - const a = yield* deriveStackId(aIdentity); - const b = yield* deriveStackId(bIdentity); - yield* store.initialize(a, state(a, aIdentity)); - yield* store.initialize(b, { - ...state(b, bIdentity, [{ field: "api", port: 20_000, intent: "automatic" }]), - desiredLifecycle: "stopped", - }); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const automatic = yield* coordinator.acquire(a, intents(), []); - expect(automatic.assignments.api?.port).not.toBe(20_000); - yield* store.replaceUnlocked(b, { - ...state(b, bIdentity, [{ field: "api", port: 20_000, intent: "exact" }]), - desiredLifecycle: "stopped", - }); - const exact = yield* coordinator.acquire(a, intents(20_000), []); - expect(exact.assignments.api).toEqual({ field: "api", port: 20_000, intent: "exact" }); - yield* store.replaceUnlocked(b, { - ...state(b, bIdentity, [{ field: "api", port: 20_000, intent: "exact" }]), - desiredLifecycle: "running", - }); - const conflict = yield* coordinator.acquire(a, intents(20_000), []).pipe(Effect.exit); - expect(Exit.isFailure(conflict)).toBe(true); - }), - ), - ); - - it.live("rolls back every bound listener when a later public bind fails", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-rollback-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "rollback"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const listeners: HostListener[] = []; - const bindHost = (address: string, port: number, field: HostListener["field"]) => - field === "database" - ? Effect.fail( - new PortUnavailableError({ port, field, message: "injected later bind failure" }), - ) - : bindHostListener(address, port, field).pipe( - Effect.tap((listener) => Effect.sync(() => listeners.push(listener))), - ); - const coordinator = makePortCoordinator(coordinatorOptions(store, root, bindHost)); - const result = yield* coordinator - .acquire( - id, - { - ...intents(), - database: { enabled: true, address: "127.0.0.1", port: "automatic" }, - }, - [], - ) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(listeners).toHaveLength(1); - expect(listeners.every((listener) => listener.binding.server.listening === false)).toBe( - true, - ); - expect((yield* store.read(id))?.ports).toEqual([]); - }), - ), - ); - it.live("retains own automatic ports and excludes every sibling reservation", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const aIdentity = identity(root, "a"); - const bIdentity = identity(root, "b"); - const a = yield* deriveStackId(aIdentity); - const b = yield* deriveStackId(bIdentity); - yield* store.initialize(a, state(a, aIdentity)); - yield* store.initialize( - b, - state(b, bIdentity, [{ field: "api", port: 20_000, intent: "automatic" }]), - ); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const first = yield* coordinator.acquire(a, intents(), []); - const repeat = yield* coordinator.acquire(a, intents(), []); - expect(repeat.assignments).toEqual(first.assignments); - expect(first.assignments.api?.port).not.toBe(20_000); - }), - ), - ); - - it.live("preseeds retained ports before allocating a newly enabled earlier field", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-order-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "order"); - const id = yield* deriveStackId(value); - yield* store.initialize( - id, - state(id, value, [{ field: "database", port: 20_321, intent: "automatic" }]), - ); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const result = yield* coordinator.acquire( - id, - { - ...intents(), - database: { enabled: true, address: "127.0.0.1", port: "automatic" }, - }, - [], - ); - expect(result.assignments.database?.port).toBe(20_321); - expect(result.assignments.api?.port).not.toBe(20_321); - }), - ), - ); - - it.live("lets a stopped exact sibling share only an explicitly requested exact port", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-exact-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const aIdentity = identity(root, "a"); - const bIdentity = identity(root, "b"); - const a = yield* deriveStackId(aIdentity); - const b = yield* deriveStackId(bIdentity); - yield* store.initialize(a, state(a, aIdentity)); - yield* store.initialize(b, { - ...state(b, bIdentity, [{ field: "api", port: 24_000, intent: "exact" }]), - desiredLifecycle: "stopped", - }); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const result = yield* coordinator.acquire(a, intents(24_000), []); - expect(result.assignments.api).toEqual({ field: "api", port: 24_000, intent: "exact" }); - const automatic = yield* coordinator.acquire(a, intents(), []); - expect(automatic.assignments.api?.port).not.toBe(24_000); - }), - ), - ); - - it.live("rejects acquisition for a stopped state without changing assignments", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-stopped-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "stopped"); - const id = yield* deriveStackId(value); - const before = state(id, value, [{ field: "api", port: 24_001, intent: "automatic" }]); - yield* store.initialize(id, { ...before, desiredLifecycle: "stopped" }); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const result = yield* coordinator.acquire(id, intents(), []).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect((yield* store.read(id))?.ports).toEqual(before.ports); - }), - ), - ); - - it.live("keeps durable state unchanged when a public bind fails", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-failure-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "failure"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const bindHost = (_address: string, port: number, field: HostListener["field"]) => - Effect.fail(new PortUnavailableError({ port, field, message: "invalid address" })); - const coordinator = makePortCoordinator(coordinatorOptions(store, root, bindHost)); - const result = yield* coordinator.acquire(id, intents(), []).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect((yield* store.read(id))?.ports).toEqual([]); - }), - ), - ); - - it.live("allocates and retains explicit private binding intents in the shared pool", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-private-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "private"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const bindings = [{ workloadId: "database:database", binding: "primary" }]; - const disabledApi = { - ...intents(), - api: { enabled: false, address: "127.0.0.1", port: "automatic" as const }, - }; - const first = yield* coordinator.acquire(id, disabledApi, bindings); - const second = yield* coordinator.acquire(id, disabledApi, bindings); - expect(second.privateAssignments).toEqual(first.privateAssignments); - expect(first.privateAssignments[0]?.port).toBeGreaterThanOrEqual(20_000); - expect(first.privateAssignments[0]?.port).toBeLessThanOrEqual(32_767); - }), - ), - ); - - it.live("persists distinct private bindings and retains them on reacquisition", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-private-state-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "private-state"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const bindings = [ - { workloadId: "database:database", binding: "primary" }, - { workloadId: "rest:rest", binding: "primary" }, - ]; - const disabled = { - ...intents(), - api: { enabled: false, address: "127.0.0.1", port: "automatic" as const }, - }; - const coordinator = makePortCoordinator({ - ...coordinatorOptions(store, root, bindHostListener), - bindPrivate: (address, port, _binding) => - bindHostListener(address, port, "database").pipe( - Effect.map((listener) => ({ port: listener.port, close: listener.close })), - ), - }); - const first = yield* coordinator.acquire(id, disabled, bindings); - expect(first.privateAssignments).toHaveLength(2); - expect(first.privateAssignments[0]?.port).not.toBe(first.privateAssignments[1]?.port); - expect((yield* store.read(id))?.privatePorts).toEqual(first.privateAssignments); - const second = yield* coordinator.acquire(id, disabled, bindings); - expect(second.privateAssignments).toEqual(first.privateAssignments); - }), - ), - ); - - it.live("rejects a retained private claim already reserved by another stack", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-private-conflict-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const aIdentity = identity(root, "a"); - const bIdentity = identity(root, "b"); - const a = yield* deriveStackId(aIdentity); - const b = yield* deriveStackId(bIdentity); - const assignment = { - workloadId: "database:database", - binding: "primary", - port: 20_101, - } as const; - yield* store.initialize(a, { ...state(a, aIdentity), privatePorts: [assignment] }); - yield* store.initialize(b, { - ...state(b, bIdentity), - privatePorts: [{ ...assignment, workloadId: "rest:rest" }], - }); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - const result = yield* coordinator - .acquire( - a, - { - ...intents(), - api: { enabled: false, address: "127.0.0.1", port: "automatic" }, - }, - [{ workloadId: assignment.workloadId, binding: assignment.binding }], - ) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect((yield* store.read(a))?.privatePorts).toEqual([assignment]); - }), - ), - ); - - it.live("drops a removed private binding after successful acquisition", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-private-remove-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "remove"); - const id = yield* deriveStackId(value); - const previous = { - workloadId: "database:database", - binding: "primary", - port: 20_102, - } as const; - yield* store.initialize(id, { ...state(id, value), privatePorts: [previous] }); - const coordinator = makePortCoordinator(coordinatorOptions(store, root)); - yield* coordinator.acquire( - id, - { - ...intents(), - api: { enabled: false, address: "127.0.0.1", port: "automatic" }, - }, - [], - ); - expect((yield* store.read(id))?.privatePorts).toEqual([]); - }), - ), - ); - - it.live("preseeds exact claims before deterministic fresh allocation", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-preseed-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "preseed"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const crypto = yield* Crypto.Crypto; - const deterministic = { ...crypto, randomIntBetween: () => Effect.succeed(0) }; - const result = yield* makePortCoordinator(coordinatorOptions(store, root)) - .acquire( - id, - { - ...intents(), - database: { enabled: true, address: "127.0.0.1", port: 20_000 }, - }, - [], - ) - .pipe(Effect.provideService(Crypto.Crypto, deterministic)); - expect(result.assignments.database?.port).toBe(20_000); - expect(result.assignments.api?.port).not.toBe(20_000); - }), - ), - ); - - it.live("fails honestly after exactly 64 retryable fresh bind failures", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-bound-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "bound"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - let attempts = 0; - const bindHost = (_address: string, port: number, field: HostListener["field"]) => { - attempts += 1; - return Effect.fail( - new PortUnavailableError({ - port, - field, - message: "occupied", - cause: Object.assign(new Error("occupied"), { code: "EADDRINUSE" }), - }), - ); - }; - const crypto = yield* Crypto.Crypto; - const result = yield* makePortCoordinator(coordinatorOptions(store, root, bindHost)) - .acquire(id, intents(), []) - .pipe( - Effect.exit, - Effect.provideService(Crypto.Crypto, { - ...crypto, - randomIntBetween: () => Effect.succeed(0), - }), - ); - expect(attempts).toBe(64); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( - PortAllocationError, - ); - expect((yield* store.read(id))?.ports).toEqual([]); - }), - ), - ); - - it.live("preserves a nonretryable bind failure without writing state", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-nonretry-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "nonretry"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - let attempts = 0; - const bindHost = (_address: string, port: number, field: HostListener["field"]) => { - attempts += 1; - return Effect.fail( - new PortUnavailableError({ - port, - field, - message: "address unavailable", - cause: Object.assign(new Error("address unavailable"), { code: "EADDRNOTAVAIL" }), - }), - ); - }; - const result = yield* makePortCoordinator(coordinatorOptions(store, root, bindHost)) - .acquire(id, intents(), []) - .pipe(Effect.exit); - expect(attempts).toBe(1); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) - expect(Option.getOrUndefined(Cause.findErrorOption(result.cause))).toBeInstanceOf( - PortUnavailableError, - ); - expect((yield* store.read(id))?.ports).toEqual([]); - }), - ), - ); - - it.live("keeps an earlier successful listener through a later fresh retry", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-later-retry-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "later-retry"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const resources: Array<{ readonly field: string; readonly port: number; closed: boolean }> = - []; - let databaseAttempts = 0; - let apiAttempts = 0; - const bindHost = (address: string, port: number, field: HostListener["field"]) => { - if (field === "api") apiAttempts += 1; - if (field === "database") { - databaseAttempts += 1; - if (databaseAttempts === 1) - return Effect.fail( - new PortUnavailableError({ - port, - field, - message: "occupied", - cause: Object.assign(new Error("occupied"), { code: "EACCES" }), - }), - ); - } - const resource = { field, port, closed: false }; - resources.push(resource); - return Effect.acquireRelease( - Effect.succeed({ - address, - port, - field, - binding: { kind: "http", server: createHttpServer() }, - connections: { sockets: new Set() }, - close: Effect.sync(() => { - resource.closed = true; - }), - } satisfies HostListener), - (listener) => listener.close, - ); - }; - const crypto = yield* Crypto.Crypto; - const result = yield* makePortCoordinator(coordinatorOptions(store, root, bindHost)) - .acquire( - id, - { - ...intents(), - database: { enabled: true, address: "127.0.0.1", port: "automatic" }, - }, - [], - ) - .pipe( - Effect.provideService(Crypto.Crypto, { - ...crypto, - randomIntBetween: () => Effect.succeed(0), - }), - ); - expect(result.assignments.api?.port).toBeGreaterThan(0); - expect(apiAttempts).toBe(1); - expect(databaseAttempts).toBe(2); - expect(resources.filter(({ field }) => field === "api")).toHaveLength(1); - expect(resources.every(({ closed }) => closed === false)).toBe(true); - }), - ), - ); - - it.live("binds resources before one failing state write and rolls them all back", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-write-failure-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "write-failure"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - let replaceCalls = 0; - let publicListener: HostListener | undefined; - let privatePort: number | undefined; - const failingStore: PortCoordinatorOptions["store"] = { - ...store, - replaceUnlocked: (_stackId, next) => - Effect.gen(function* () { - replaceCalls += 1; - expect(publicListener?.binding.server.listening).toBe(true); - const assignment = next.privatePorts[0]; - if (assignment === undefined) return yield* Effect.die("missing private assignment"); - privatePort = assignment.port; - const probe = yield* checkHostPort("127.0.0.1", assignment.port, "database").pipe( - Effect.exit, - ); - expect(Exit.isFailure(probe)).toBe(true); - return yield* new StackStateInvalidError({ message: "injected state write failure" }); - }), - }; - const coordinator = makePortCoordinator({ - ...coordinatorOptions(failingStore, root, (address, port, field) => - bindHostListener(address, port, field).pipe( - Effect.tap((listener) => - Effect.sync(() => { - publicListener = listener; - }), - ), - ), - ), - bindPrivate: (address, port, binding) => bindHeldPort(address, port, binding), - }); - const result = yield* coordinator - .acquire( - id, - { - ...intents(), - database: { enabled: true, address: "127.0.0.1", port: "automatic" }, - }, - [{ workloadId: "database:database", binding: "primary" }], - ) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(replaceCalls).toBe(1); - expect(publicListener?.binding.server.listening).toBe(false); - expect(privatePort).toBeDefined(); - expect((yield* store.read(id))?.ports).toEqual([]); - expect((yield* store.read(id))?.privatePorts).toEqual([]); - }), - ), - ); - - it.live("interrupts a waiting transaction and releases its first bound socket", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-interrupt-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "interrupt"); - const id = yield* deriveStackId(value); - const before = state(id, value, [{ field: "studio", port: 20_103, intent: "automatic" }]); - yield* store.initialize(id, before); - const parentScope = yield* Scope.Scope; - const acquisitionScope = yield* Scope.fork(parentScope, "sequential"); - const waiting = yield* Deferred.make(); - let publicListener: HostListener | undefined; - const coordinator = makePortCoordinator({ - ...coordinatorOptions(store, root, (address, port, field) => - field === "api" - ? bindHostListener(address, port, field).pipe( - Effect.tap((listener) => - Effect.sync(() => { - publicListener = listener; - }), - ), - ) - : Effect.gen(function* () { - yield* Deferred.succeed(waiting, undefined); - return yield* Effect.never; - }), - ), - bindPrivate: (_address, _port, _binding) => Effect.never, - }); - const fiber = yield* Effect.forkChild( - coordinator - .acquire( - id, - { - ...intents(), - database: { enabled: true, address: "127.0.0.1", port: "automatic" }, - }, - [], - ) - .pipe(Effect.provideService(Scope.Scope, acquisitionScope)), - { startImmediately: true }, - ); - yield* Deferred.await(waiting); - yield* Fiber.interrupt(fiber); - expect(publicListener?.binding.server.listening).toBe(false); - expect((yield* store.read(id))?.ports).toEqual(before.ports); - yield* withRegistryLock(root, Effect.succeed(true)); - yield* Scope.close(acquisitionScope, Exit.void); - }), - ), - ); - - it.live("finishes a commit after interruption is requested", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-ports-commit-interrupt-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const value = identity(root, "commit-interrupt"); - const id = yield* deriveStackId(value); - yield* store.initialize(id, state(id, value)); - const commitStarted = yield* Deferred.make(); - const releaseCommit = yield* Deferred.make(); - const committingStore: PortCoordinatorOptions["store"] = { - ...store, - replaceUnlocked: (stackId, next) => - Effect.gen(function* () { - const result = yield* store.replaceUnlocked(stackId, next); - yield* Deferred.succeed(commitStarted, undefined); - yield* Deferred.await(releaseCommit); - return result; - }), - }; - const parentScope = yield* Scope.Scope; - const acquisitionScope = yield* Scope.fork(parentScope, "sequential"); - let publicListener: HostListener | undefined; - const coordinator = makePortCoordinator( - coordinatorOptions(committingStore, root, (address, port, field) => - bindHostListener(address, port, field).pipe( - Effect.tap((listener) => - Effect.sync(() => { - publicListener = listener; - }), - ), - ), - ), - ); - const fiber = yield* Effect.forkChild( - coordinator - .acquire(id, intents(), []) - .pipe(Effect.provideService(Scope.Scope, acquisitionScope)), - { startImmediately: true }, - ); - yield* Deferred.await(commitStarted); - const interrupt = yield* Effect.forkChild(Fiber.interrupt(fiber), { - startImmediately: true, - }); - yield* Deferred.succeed(releaseCommit, undefined); - yield* Fiber.join(fiber).pipe(Effect.exit); - yield* Fiber.join(interrupt); - expect((yield* store.read(id))?.ports).toHaveLength(1); - expect(publicListener?.binding.server.listening).toBe(true); - yield* Scope.close(acquisitionScope, Exit.void); - expect(publicListener?.binding.server.listening).toBe(false); - }), - ), - ); -}); diff --git a/packages/stack/src/state/registry-lease.integration.test.ts b/packages/stack/src/state/registry-lease.integration.test.ts deleted file mode 100644 index f9516b6d9f..0000000000 --- a/packages/stack/src/state/registry-lease.integration.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Deferred, - Effect, - Exit, - FileSystem, - Fiber, - Option, - Path, - Ref, - Stream, -} from "effect"; -import { ChildProcess } from "effect/unstable/process"; -import { StackStateInvalidError } from "../public/Errors.ts"; -import { withRegistryLock } from "./StackStateStore.ts"; - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("cross-process registry lease", () => { - it.live("blocks a competing action, then recovers after an exact child kill", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-registry-lease-" }); - const enteredPath = path.join(root, "child-entered"); - const competingPath = path.join(root, "competing-entered"); - const moduleUrl = new URL("./StackStateStore.ts", import.meta.url).href; - // bare imports in a `node -e` script resolve from cwd, so run from the package root - const cwd = path.resolve(import.meta.dirname, "../.."); - const script = ` - const { Effect, FileSystem } = await import("effect"); - const { NodeServices } = await import("@effect/platform-node"); - const { withRegistryLock } = await import(process.env.REGISTRY_MODULE); - await Effect.runPromise(withRegistryLock(process.env.REGISTRY_ROOT, Effect.gen(function* () { - const filesystem = yield* FileSystem.FileSystem; - yield* filesystem.writeFileString(process.env.ENTERED_PATH, "child"); - process.stdout.write("READY\\n"); - yield* Effect.never; - })).pipe(Effect.provide(NodeServices.layer))); - `; - const child = yield* ChildProcess.make( - process.execPath, - ["--input-type=module", "-e", script], - { - cwd, - env: { REGISTRY_MODULE: moduleUrl, REGISTRY_ROOT: root, ENTERED_PATH: enteredPath }, - extendEnv: true, - stdout: "pipe", - stderr: "pipe", - }, - ); - const ready = yield* Deferred.make(); - const stderrText = yield* Ref.make>([]); - const childExitCode = yield* Ref.make>(Option.none()); - const childOutput = yield* child.stdout.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => - line === "READY" ? Deferred.succeed(ready, undefined).pipe(Effect.asVoid) : Effect.void, - ), - Effect.forkChild({ startImmediately: true }), - ); - const childErrors = yield* child.stderr.pipe( - Stream.decodeText, - Stream.splitLines, - Stream.runForEach((line) => Ref.update(stderrText, (lines) => [...lines, line])), - Effect.forkChild({ startImmediately: true }), - ); - const exitWatcher = yield* child.exitCode.pipe( - Effect.flatMap((code) => Ref.set(childExitCode, Option.some(code))), - Effect.forkChild({ startImmediately: true }), - ); - try { - yield* Deferred.await(ready).pipe( - Effect.timeoutOrElse({ - duration: "15 seconds", - orElse: () => - Effect.gen(function* () { - const errors = yield* Ref.get(stderrText); - const exitCode = yield* Ref.get(childExitCode); - const status = Option.isSome(exitCode) - ? ` (exit code ${String(exitCode.value)})` - : ""; - return yield* new StackStateInvalidError({ - message: `registry child did not become ready${status}${errors.length === 0 ? "" : `: ${errors.join("\\n")}`}`, - }); - }), - }), - ); - expect(yield* fs.exists(enteredPath)).toBe(true); - const competing = yield* withRegistryLock( - root, - fs.writeFileString(competingPath, "competing"), - ).pipe(Effect.exit); - expect(Exit.isFailure(competing)).toBe(true); - const contentionError = errorOf(competing); - expect(contentionError).toBeInstanceOf(StackStateInvalidError); - expect(contentionError?.message).toBe("Stack registry is busy"); - expect(yield* fs.exists(competingPath)).toBe(false); - yield* child.kill({ killSignal: "SIGKILL" }); - yield* child.exitCode.pipe(Effect.ignore); - const recovered = yield* withRegistryLock(root, Effect.succeed("recovered")); - expect(recovered).toBe("recovered"); - } finally { - yield* child.kill({ killSignal: "SIGKILL" }).pipe(Effect.ignore); - yield* child.exitCode.pipe(Effect.ignore); - yield* Fiber.interrupt(childOutput); - yield* Fiber.interrupt(childErrors); - yield* Fiber.interrupt(exitWatcher); - } - }), - ), - ); -}); diff --git a/packages/stack/src/state/secrets.integration.test.ts b/packages/stack/src/state/secrets.integration.test.ts deleted file mode 100644 index 18f32b32f5..0000000000 --- a/packages/stack/src/state/secrets.integration.test.ts +++ /dev/null @@ -1,559 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, FileSystem, Option, Path, Redacted, Schema } from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import { createHmac, generateKeyPairSync, createVerify } from "node:crypto"; -import { - InvalidJwtSigningMaterialError, - StackMustBeStoppedError, - StackSecretMismatchError, -} from "../public/Errors.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { - redactKnownSecrets, - resolveSecrets, - resolveSigningKeyMaterial, - type SecretCandidate, -} from "./SecretStore.ts"; - -const layer = NodeServices.layer; -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); -const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); -const decodeJwtPayload = (text: string) => - Schema.decodeUnknownSync( - Schema.Struct({ iss: Schema.String, role: Schema.String, exp: Schema.Finite }), - )(decodeJson(text)); -const managed = (value?: string): SecretCandidate => ({ - declarations: [ - { - slot: "managed:db", - policy: "managed", - value: value === undefined ? undefined : Redacted.make(value), - }, - ], -}); -const passthrough = (slot: string, value: string): SecretCandidate => ({ - declarations: [{ slot, policy: "passthrough", value: Redacted.make(value) }], -}); -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; -const compilerManagedSlots = [ - "secret:database.internal.password", - "secret:auth.settings.publishable_key", - "secret:auth.settings.secret_key", - "secret:auth.settings.jwt_secret", - "secret:auth.settings.anon_key", - "secret:auth.settings.service_role_key", - "secret:storage.settings.s3_protocol.secret_access_key", - "secret:realtime.settings.db_enc_key", - "secret:realtime.settings.secret_key_base", -] as const; -const compilerCandidate = compileStack({ - projectRoot: "/tmp/project", - runtime: { kind: "native" }, -}).pipe(Effect.map((compiled) => ({ declarations: compiled.secrets }))); -const present = (value: T | undefined, description: string): T => { - expect(value, description).toBeDefined(); - if (value === undefined) throw new Error(`Expected ${description}`); - return value; -}; - -describe("managed and pass-through secrets", () => { - it.live("generates managed omissions once and reuses them", () => - Effect.gen(function* () { - const first = yield* resolveSecrets(managed(), undefined, "stopped"); - const second = yield* resolveSecrets(managed(), first.persisted, "running"); - expect(second.persisted["managed:db"]?.value).toBe(first.persisted["managed:db"]?.value); - expect(second.persisted["managed:db"]?.value).not.toHaveLength(0); - }).pipe(Effect.provide(layer)), - ); - - it.live("generates compiler-required managed slots with artifact-compatible values", () => - Effect.gen(function* () { - const candidate = yield* compilerCandidate; - const resolved = yield* resolveSecrets(candidate, undefined, "stopped"); - for (const slot of compilerManagedSlots) { - expect(resolved.persisted[slot]?.policy).toBe("managed"); - expect(resolved.persisted[slot]?.value).toEqual(expect.any(String)); - } - expect( - present( - resolved.persisted["secret:realtime.settings.db_enc_key"]?.value, - "realtime database encryption key", - ), - ).toMatch(/^[A-Za-z0-9_-]{16}$/); - expect( - present( - resolved.persisted["secret:realtime.settings.secret_key_base"]?.value, - "realtime secret key base", - ), - ).toMatch(/^[A-Za-z0-9_-]{64}$/); - expect( - present( - resolved.persisted["secret:auth.settings.publishable_key"]?.value, - "publishable key", - ), - ).toMatch(/^sb_publishable_[A-Za-z0-9_-]{32,}$/); - expect( - present(resolved.persisted["secret:auth.settings.secret_key"]?.value, "secret key"), - ).toMatch(/^sb_secret_[A-Za-z0-9_-]{32,}$/); - expect( - present(resolved.persisted["secret:auth.settings.jwt_secret"]?.value, "JWT secret"), - ).toMatch(/^[A-Za-z0-9_-]{43,}$/); - for (const slot of [ - "secret:auth.settings.anon_key", - "secret:auth.settings.service_role_key", - ]) { - const token = present(resolved.persisted[slot]?.value, slot); - const tokenParts = token.split("."); - expect(tokenParts).toHaveLength(3); - const payloadText = Buffer.from( - present(tokenParts[1], `${slot} payload`), - "base64url", - ).toString(); - const payload = decodeJwtPayload(payloadText); - expect(payload).toMatchObject({ - iss: "supabase-demo", - role: slot.endsWith("anon_key") ? "anon" : "service_role", - exp: expect.any(Number), - }); - expect(payload.exp).toBeGreaterThan(1_900_000_000); - } - }).pipe(Effect.provide(layer)), - ); - - it.live("reuses generated values for every compiler-managed slot", () => - Effect.gen(function* () { - const candidate = yield* compilerCandidate; - const first = yield* resolveSecrets(candidate, undefined, "stopped"); - const second = yield* resolveSecrets(candidate, first.persisted, "running"); - for (const slot of compilerManagedSlots) { - expect(second.persisted[slot]?.policy).toBe("managed"); - expect(second.persisted[slot]?.value).toBe(first.persisted[slot]?.value); - } - }).pipe(Effect.provide(layer)), - ); - - it.live("generates a fresh managed secret for a different stack", () => - Effect.gen(function* () { - const candidate = yield* compilerCandidate; - const first = yield* resolveSecrets(candidate, undefined, "stopped"); - const other = yield* resolveSecrets(candidate, undefined, "stopped"); - const slot = "secret:storage.settings.s3_protocol.secret_access_key"; - expect(other.persisted[slot]?.value).not.toBe(first.persisted[slot]?.value); - }).pipe(Effect.provide(layer)), - ); - - it.live("generates artifact-compatible pooler managed keys", () => - Effect.gen(function* () { - const compiled = yield* compileStack({ - projectRoot: "/tmp/project", - runtime: { kind: "native" }, - config: { capabilities: { pooler: { enabled: true } } }, - }); - const candidate = { declarations: compiled.secrets }; - const first = yield* resolveSecrets(candidate, undefined, "stopped"); - const second = yield* resolveSecrets(candidate, first.persisted, "running"); - const encryptionKey = first.persisted["secret:pooler.settings.encryption_key"]?.value; - const secretKeyBase = first.persisted["secret:pooler.settings.secret_key_base"]?.value; - expect(encryptionKey).toMatch(/^[A-Za-z0-9_-]{32}$/); - expect(secretKeyBase).toMatch(/^[A-Za-z0-9_-]{64}$/); - expect(second.persisted["secret:pooler.settings.encryption_key"]?.value).toBe(encryptionKey); - expect(second.persisted["secret:pooler.settings.secret_key_base"]?.value).toBe(secretKeyBase); - }).pipe(Effect.provide(layer)), - ); - - it.live("rejects mixed valid and invalid private JWK entries as one file", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-jwks-mixed-" }); - const privateJwk = { - ...generateKeyPairSync("ec", { namedCurve: "prime256v1" }).privateKey.export({ - format: "jwk", - }), - alg: "ES256", - }; - yield* fs.writeFileString( - path.join(root, "keys.json"), - encodeJson([privateJwk, { kty: "EC", alg: "ES256", d: "invalid" }]), - ); - const failed = yield* resolveSigningKeyMaterial({ - kind: "jwks-file", - projectRoot: root, - path: "keys.json", - }).pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(InvalidJwtSigningMaterialError); - }).pipe(Effect.provide(layer)), - ); - - it.live("signs generated API keys with the first private ES256 JWK", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-credentials-" }); - const { privateKey, publicKey } = generateKeyPairSync("ec", { namedCurve: "prime256v1" }); - const privateJwk = { ...privateKey.export({ format: "jwk" }), alg: "ES256", kid: "test-key" }; - yield* fs.writeFileString(path.join(root, "keys.json"), encodeJson([privateJwk])); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { security: { jwt: { signing: { kind: "jwks-file", path: "keys.json" } } } }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const token = present( - resolved.persisted["secret:auth.settings.anon_key"]?.value, - "anon token", - ); - const [header, payload, signature] = token.split("."); - const tokenHeader = present(header, "JWT header"); - const tokenPayload = present(payload, "JWT payload"); - const tokenSignature = present(signature, "JWT signature"); - expect(decodeJson(Buffer.from(tokenHeader, "base64url").toString())).toMatchObject({ - alg: "ES256", - kid: "test-key", - }); - const verifier = createVerify("sha256"); - verifier.update(`${tokenHeader}.${tokenPayload}`); - verifier.end(); - expect( - verifier.verify( - { key: publicKey, dsaEncoding: "ieee-p1363" }, - Buffer.from(tokenSignature, "base64url"), - ), - ).toBe(true); - }).pipe(Effect.provide(layer)), - ); - - it.live("signs generated API keys with a private RS256 JWK", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-credentials-" }); - const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); - const privateJwk = { ...privateKey.export({ format: "jwk" }), alg: "RS256", kid: "rsa-key" }; - yield* fs.writeFileString(path.join(root, "keys.json"), encodeJson([privateJwk])); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { security: { jwt: { signing: { kind: "jwks-file", path: "keys.json" } } } }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const token = present( - resolved.persisted["secret:auth.settings.service_role_key"]?.value, - "service role token", - ); - const [header, payload, signature] = token.split("."); - const tokenHeader = present(header, "JWT header"); - const tokenPayload = present(payload, "JWT payload"); - const tokenSignature = present(signature, "JWT signature"); - expect(decodeJson(Buffer.from(tokenHeader, "base64url").toString())).toMatchObject({ - alg: "RS256", - kid: "rsa-key", - }); - const verifier = createVerify("RSA-SHA256"); - verifier.update(`${tokenHeader}.${tokenPayload}`); - verifier.end(); - expect(verifier.verify(publicKey, Buffer.from(tokenSignature, "base64url"))).toBe(true); - }).pipe(Effect.provide(layer)), - ); - - it.effect("derives asymmetric JWT expiry from the Effect Clock", () => - Effect.gen(function* () { - yield* TestClock.setTime(1_700_000_123_456); - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-credentials-" }); - const { privateKey } = generateKeyPairSync("ec", { namedCurve: "prime256v1" }); - const privateJwk = { ...privateKey.export({ format: "jwk" }), alg: "ES256" }; - yield* fs.writeFileString(path.join(root, "keys.json"), encodeJson([privateJwk])); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { security: { jwt: { signing: { kind: "jwks-file", path: "keys.json" } } } }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const token = present( - resolved.persisted["secret:auth.settings.anon_key"]?.value, - "anon token", - ); - const payloadPart = present(token.split(".")[1], "JWT payload"); - const payloadText = Buffer.from(payloadPart, "base64url").toString(); - const payload = decodeJwtPayload(payloadText); - expect(payload).toEqual({ - iss: "supabase-demo", - role: "anon", - exp: 1_700_000_123 + 60 * 60 * 24 * 365 * 10, - }); - expect(payload).not.toHaveProperty("iat"); - }).pipe(Effect.provide(layer)), - ); - - it.live("fails closed for public-only JWKS material", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-credentials-" }); - const publicJwk = { - ...generateKeyPairSync("ec", { namedCurve: "prime256v1" }).publicKey.export({ - format: "jwk", - }), - alg: "ES256", - }; - yield* fs.writeFileString(path.join(root, "public.json"), encodeJson([publicJwk])); - const publicOnly = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - security: { jwt: { signing: { kind: "jwks-file", path: "public.json" } } }, - }, - }); - const publicExit = yield* resolveSecrets( - { declarations: publicOnly.secrets }, - undefined, - "stopped", - ).pipe(Effect.exit); - expect(errorOf(publicExit)).toBeInstanceOf(InvalidJwtSigningMaterialError); - }).pipe(Effect.provide(layer)), - ); - - it.live("rejects a configured JWKS file that does not exist", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-missing-jwks-" }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { security: { jwt: { signing: { kind: "jwks-file", path: "missing.json" } } } }, - }); - - const failed = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ).pipe(Effect.exit); - - expect(errorOf(failed)).toBeInstanceOf(InvalidJwtSigningMaterialError); - }).pipe(Effect.provide(layer)), - ); - - it.live("rejects JWKS files outside the project root", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-credentials-" }); - const escaping = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - security: { jwt: { signing: { kind: "jwks-file", path: "../outside.json" } } }, - }, - }); - const escapingExit = yield* resolveSecrets( - { declarations: escaping.secrets }, - undefined, - "stopped", - ).pipe(Effect.exit); - expect(errorOf(escapingExit)).toBeInstanceOf(InvalidJwtSigningMaterialError); - expect(errorOf(escapingExit)?.message).toContain( - "JWT signing key file must be inside project root", - ); - - const outside = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-outside-" }); - yield* fs.writeFileString( - path.join(outside, "private.json"), - encodeJson([{ kty: "EC", alg: "ES256", crv: "P-256", d: "d", x: "x", y: "y" }]), - ); - yield* fs.symlink(path.join(outside, "private.json"), path.join(root, "linked.json")); - const symlinkEscape = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - security: { jwt: { signing: { kind: "jwks-file", path: "linked.json" } } }, - }, - }); - const symlinkExit = yield* resolveSecrets( - { declarations: symlinkEscape.secrets }, - undefined, - "stopped", - ).pipe(Effect.exit); - expect(errorOf(symlinkExit)).toBeInstanceOf(InvalidJwtSigningMaterialError); - expect(errorOf(symlinkExit)?.message).toContain( - "JWT signing key file must be inside project root", - ); - }).pipe(Effect.provide(layer)), - ); - - it.live("keeps explicit managed credentials unchanged", () => - Effect.gen(function* () { - const configured = { - publishable: "sb_publishable_explicit", - secret: "sb_secret_explicit", - jwt: "a-secure-jwt-secret-value-that-is-long-enough", - anon: "explicit-anon-token", - service: "explicit-service-token", - }; - const compiled = yield* compileStack({ - projectRoot: "/tmp/project", - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { - settings: { - publishable_key: Redacted.make(configured.publishable), - secret_key: Redacted.make(configured.secret), - jwt_secret: Redacted.make(configured.jwt), - anon_key: Redacted.make(configured.anon), - service_role_key: Redacted.make(configured.service), - }, - }, - }, - }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - expect(resolved.persisted["secret:auth.settings.publishable_key"]?.value).toBe( - configured.publishable, - ); - expect(resolved.persisted["secret:auth.settings.secret_key"]?.value).toBe(configured.secret); - expect(resolved.persisted["secret:auth.settings.jwt_secret"]?.value).toBe(configured.jwt); - expect(resolved.persisted["secret:auth.settings.anon_key"]?.value).toBe(configured.anon); - expect(resolved.persisted["secret:auth.settings.service_role_key"]?.value).toBe( - configured.service, - ); - }).pipe(Effect.provide(layer)), - ); - - it.live("signs generated symmetric API keys with the canonical JWT secret", () => - Effect.gen(function* () { - const jwt = "symmetric-jwt-secret-that-is-long-enough"; - const compiled = yield* compileStack({ - projectRoot: "/tmp/project", - runtime: { kind: "native" }, - config: { - capabilities: { auth: { settings: { jwt_secret: Redacted.make(jwt) } } }, - }, - }); - const resolved = yield* resolveSecrets( - { declarations: compiled.secrets }, - undefined, - "stopped", - ); - const token = present( - resolved.persisted["secret:auth.settings.anon_key"]?.value, - "anon token", - ); - const [header, payload, signature] = token.split("."); - const tokenHeader = present(header, "JWT header"); - const tokenPayload = present(payload, "JWT payload"); - const tokenSignature = present(signature, "JWT signature"); - const expected = createHmac("sha256", jwt) - .update(`${tokenHeader}.${tokenPayload}`) - .digest("base64url"); - expect(tokenSignature).toBe(expected); - }).pipe(Effect.provide(layer)), - ); - - it.live("rejects a conflicting managed value in stopped and running states", () => - Effect.gen(function* () { - const first = yield* resolveSecrets(managed("original"), undefined, "stopped"); - for (const lifecycle of ["stopped", "running"] as const) { - const exit = yield* resolveSecrets(managed("different"), first.persisted, lifecycle).pipe( - Effect.exit, - ); - expect(errorOf(exit)).toBeInstanceOf(StackSecretMismatchError); - } - }).pipe(Effect.provide(layer)), - ); - - it.live("allows pass-through add, replacement, and removal while stopped or unconfigured", () => - Effect.gen(function* () { - const first = yield* resolveSecrets(passthrough("pass:smtp", "old"), undefined, "stopped"); - for (const lifecycle of ["stopped", "unconfigured"] as const) { - const replacement = yield* resolveSecrets( - passthrough("pass:smtp", "new"), - first.persisted, - lifecycle, - ); - expect(replacement.persisted["pass:smtp"]?.value).toBe("new"); - const removed = yield* resolveSecrets( - { declarations: [] }, - replacement.persisted, - lifecycle, - ); - expect(removed.persisted["pass:smtp"]).toBeUndefined(); - } - }).pipe(Effect.provide(layer)), - ); - - it.live( - "rejects pass-through additions, replacements, and removals while running or destroying", - () => - Effect.gen(function* () { - const existing = yield* resolveSecrets( - passthrough("pass:smtp", "old"), - undefined, - "stopped", - ); - for (const lifecycle of ["running", "destroying"] as const) { - const added = yield* resolveSecrets( - passthrough("pass:new", "value"), - existing.persisted, - lifecycle, - ).pipe(Effect.exit); - expect(errorOf(added)).toBeInstanceOf(StackMustBeStoppedError); - const replaced = yield* resolveSecrets( - passthrough("pass:smtp", "new"), - existing.persisted, - lifecycle, - ).pipe(Effect.exit); - expect(errorOf(replaced)).toBeInstanceOf(StackMustBeStoppedError); - const removed = yield* resolveSecrets( - { declarations: [] }, - existing.persisted, - lifecycle, - ).pipe(Effect.exit); - expect(errorOf(removed)).toBeInstanceOf(StackMustBeStoppedError); - const unchanged = yield* resolveSecrets( - passthrough("pass:smtp", "old"), - existing.persisted, - lifecycle, - ); - expect(unchanged.persisted["pass:smtp"]?.value).toBe("old"); - } - }).pipe(Effect.provide(layer)), - ); - - it.live("does not include secret bytes in mismatch errors or redaction output", () => - Effect.gen(function* () { - const first = yield* resolveSecrets(managed("top-secret"), undefined, "stopped"); - const exit = yield* resolveSecrets(managed("other-secret"), first.persisted, "stopped").pipe( - Effect.exit, - ); - const error = errorOf(exit); - expect(String(error)).not.toContain("top-secret"); - expect(String(error)).not.toContain("other-secret"); - }).pipe(Effect.provide(layer)), - ); - - it("redacts overlapping known values longest-first", () => { - expect(redactKnownSecrets("token=abc123; short=abc", ["abc", "abc123", "abc", ""])).toBe( - "token=[REDACTED]; short=[REDACTED]", - ); - }); -}); diff --git a/packages/stack/src/state/state-store.integration.test.ts b/packages/stack/src/state/state-store.integration.test.ts deleted file mode 100644 index 71d098c1c1..0000000000 --- a/packages/stack/src/state/state-store.integration.test.ts +++ /dev/null @@ -1,520 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Path, - Redacted, - Schema, -} from "effect"; -import { compileStack, type StackDefinition } from "../model/Compiler.ts"; -import { deriveStackId } from "../identity/Identity.ts"; -import { StackStateFormatUnsupportedError, StackStateInvalidError } from "../public/Errors.ts"; -import { - makeStackStateStore, - withRegistryLock, - PersistedStackStateSchema, - type PersistedStackState, -} from "./StackStateStore.ts"; -import { removeLeaseIfHeld } from "./Ownership.ts"; - -const layer = NodeServices.layer; -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(layer)); - -const identity = { - projectRoot: "/tmp/project", - branchContext: "ordinary-workspace", - stackName: "default", -} as const; - -const state = (definition?: StackDefinition): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "stopped", - definition, - ports: [], - privatePorts: [], - secrets: {}, -}); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; -const jsonText = (value: unknown) => - Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(value); -const jsonTextSync = (value: unknown) => - Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))(value); - -const completeStateFixture = Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-state-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const stackId = yield* deriveStackId(identity); - const compiled = yield* compileStack({ - projectRoot: "/tmp/project", - runtime: { kind: "native" }, - config: { - preparation: "on-demand", - capabilities: { - auth: { - settings: { - secret_key: Redacted.make("state-secret"), - email: { - template: { confirm: { subject: "Confirm" } }, - notification: { welcome: { enabled: true } }, - }, - }, - }, - functions: { settings: { functions: { hello: { verify_jwt: false } } } }, - }, - }, - }); - const complete = state(compiled.definition); - yield* store.initialize(stackId, complete); - const encoded = yield* Schema.encodeEffect(PersistedStackStateSchema)(complete); - return { fs, path, store, root, stackId, complete, encoded }; -}); - -describe("atomic stack state", () => { - it.live("rejects a competing registry action while the lease is held", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-registry-busy-" }); - const entered = yield* Deferred.make(); - const release = yield* Deferred.make(); - let ran = false; - const first = yield* Effect.forkChild( - withRegistryLock( - root, - Effect.gen(function* () { - yield* Deferred.succeed(entered, undefined); - yield* Deferred.await(release); - }), - ), - { startImmediately: true }, - ); - yield* Deferred.await(entered); - const competing = yield* withRegistryLock( - root, - Effect.sync(() => { - ran = true; - return undefined; - }), - ).pipe(Effect.exit); - expect(Exit.isFailure(competing)).toBe(true); - expect(ran).toBe(false); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(first); - }), - ), - ); - - it.live("does not let a stale registry release remove a successor lock", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-registry-fence-", - }); - const path = yield* Path.Path; - const lockPath = path.join(root, ".stack-registry.lock"); - yield* fs.writeFileString( - lockPath, - jsonTextSync({ format: "supabase-stack-lease-v1", token: "successor", port: 45_678 }), - ); - yield* removeLeaseIfHeld(fs, lockPath, "stale-owner"); - expect(yield* fs.readFileString(lockPath)).toContain("successor"); - }), - ), - ); - - it.live("fails closed immediately for malformed registry state", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-registry-invalid-", - }); - yield* fs.writeFileString(path.join(root, ".stack-registry.lock"), "not-json"); - const result = yield* withRegistryLock(root, Effect.void).pipe(Effect.exit); - expect(errorOf(result)).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); - - it.live("round-trips a compiled complete definition", () => - withPlatform( - Effect.gen(function* () { - const { store, stackId, complete } = yield* completeStateFixture; - expect(complete.definition?.preparation).toBe("on-demand"); - expect(yield* store.read(stackId)).toEqual(complete); - }), - ), - ); - - it.live("normalizes legacy definitions without idle timeout fields", () => - withPlatform( - Effect.gen(function* () { - const { fs, path, store, root, stackId, encoded } = yield* completeStateFixture; - const legacy = structuredClone(encoded) as unknown as { - definition: { capabilities: Record }; - }; - for (const capability of Object.values(legacy.definition.capabilities)) - delete capability.idleTimeoutSeconds; - yield* fs.writeFileString(path.join(root, stackId, "state.json"), jsonTextSync(legacy)); - - const result = yield* store.read(stackId); - if (result?.definition === undefined) return yield* Effect.die("definition missing"); - for (const capability of Object.values(result.definition.capabilities)) - expect(capability.idleTimeoutSeconds).toBe(false); - }), - ), - ); - - it.live("rejects malformed nested state documents", () => - withPlatform( - Effect.gen(function* () { - const { fs, path, store, root, stackId, complete, encoded } = yield* completeStateFixture; - const statePath = path.join(root, stackId, "state.json"); - const persisted = yield* Schema.decodeEffect( - Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), - )(yield* fs.readFileString(statePath)); - expect(persisted).not.toHaveProperty("identity.stackId"); - const nestedUnknown = { - ...encoded, - definition: { - ...encoded.definition, - capabilities: { - ...encoded.definition?.capabilities, - auth: { - ...encoded.definition?.capabilities.auth, - settings: { - ...encoded.definition?.capabilities.auth.settings, - email: { - ...encoded.definition?.capabilities.auth.settings.email, - template: { - ...encoded.definition?.capabilities.auth.settings.email?.template, - confirm: { - ...encoded.definition?.capabilities.auth.settings.email?.template?.confirm, - unknown: true, - }, - }, - }, - }, - }, - }, - }, - }; - yield* fs.writeFileString(statePath, yield* jsonText(nestedUnknown)); - const unknownExit = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(unknownExit)).toBeInstanceOf(StackStateInvalidError); - - const oldSnapshot = { - ...encoded, - identity: { ...encoded.identity, stackId }, - secrets: { preserved: { policy: "managed", value: "secret-value" } }, - definition: { - ...encoded.definition, - capabilities: { - ...encoded.definition?.capabilities, - database: { - ...encoded.definition?.capabilities.database, - settings: { - ...encoded.definition?.capabilities.database.settings, - network_restrictions: { enabled: true, allowed_cidrs: ["10.0.0.0/8"] }, - ssl_enforcement: { enabled: true }, - vault: {}, - }, - }, - rest: { - ...encoded.definition?.capabilities.rest, - settings: { - ...encoded.definition?.capabilities.rest.settings, - auto_expose_new_tables: true, - tls: { enabled: false }, - }, - }, - storage: { - ...encoded.definition?.capabilities.storage, - settings: { - ...encoded.definition?.capabilities.storage.settings, - analytics: { enabled: false }, - }, - }, - analytics: { - ...encoded.definition?.capabilities.analytics, - settings: { - ...encoded.definition?.capabilities.analytics.settings, - vector_port: 9001, - }, - }, - }, - }, - }; - yield* fs.writeFileString(statePath, yield* jsonText(oldSnapshot)); - const recovered = yield* store.read(stackId); - expect(recovered).toEqual({ ...complete, secrets: oldSnapshot.secrets }); - if (recovered === undefined) throw new Error("Expected recovered state"); - yield* store.replace(stackId, recovered); - const rewritten = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( - yield* fs.readFileString(statePath), - ); - expect(rewritten).toEqual({ ...encoded, secrets: oldSnapshot.secrets }); - - const missingDefault = { - ...encoded, - definition: { - ...encoded.definition, - capabilities: { - ...encoded.definition?.capabilities, - functions: { - ...encoded.definition?.capabilities.functions, - settings: { - ...encoded.definition?.capabilities.functions.settings, - functions: { - ...encoded.definition?.capabilities.functions.settings.functions, - hello: { - ...encoded.definition?.capabilities.functions.settings.functions?.hello, - verify_jwt: undefined, - }, - }, - }, - }, - }, - }, - }; - yield* fs.writeFileString(statePath, yield* jsonText(missingDefault)); - const missingDefaultExit = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(missingDefaultExit)).toBeInstanceOf(StackStateInvalidError); - - const invalidPreparation = { - ...encoded, - definition: { ...encoded.definition, preparation: "invalid" }, - }; - yield* fs.writeFileString(statePath, yield* jsonText(invalidPreparation)); - const invalidPreparationExit = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(invalidPreparationExit)).toBeInstanceOf(StackStateInvalidError); - - const invalidRecordKey = { - ...encoded, - definition: { - ...encoded.definition, - capabilities: { - ...encoded.definition?.capabilities, - functions: { - ...encoded.definition?.capabilities.functions, - settings: { - ...encoded.definition?.capabilities.functions.settings, - functions: { - "bad.slug": { - enabled: true, - verify_jwt: true, - import_map: null, - entrypoint: null, - static_files: null, - env: {}, - }, - }, - }, - }, - }, - }, - }; - yield* fs.writeFileString(statePath, yield* jsonText(invalidRecordKey)); - const recordExit = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(recordExit)).toBeInstanceOf(StackStateInvalidError); - - const invalidSecret = { - ...encoded, - secrets: { "": { policy: "managed", value: "x" } }, - }; - yield* fs.writeFileString(statePath, yield* jsonText(invalidSecret)); - const secretExit = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(secretExit)).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); - - it.live("rejects overlapping public and private ports at the persistence boundary", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-state-overlap-" }); - const value = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(value); - const store = yield* makeStackStateStore({ stateRoot: root }); - const before = { ...state(), identity: value }; - yield* store.initialize(stackId, before); - const candidate = { - ...before, - ports: [{ field: "api" as const, port: 23_100, intent: "exact" as const }], - privatePorts: [{ workloadId: "database:database", binding: "primary", port: 23_100 }], - }; - const result = yield* store.replace(stackId, candidate).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - const error = errorOf(result); - expect(error).toBeInstanceOf(StackStateInvalidError); - expect(error?.message).toContain("overlap"); - expect(yield* store.read(stackId)).toEqual(before); - }), - ), - ); - - it.live("fails closed when identity remnants exist without state and cleans exact identity", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-remnant-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const stackId = yield* deriveStackId(identity); - yield* fs.makeDirectory(path.join(root, stackId, "data"), { recursive: true }); - const sibling = yield* deriveStackId({ ...identity, stackName: "sibling" }); - yield* fs.makeDirectory(path.join(root, sibling, "data"), { recursive: true }); - const missing = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(missing)).toBeInstanceOf(StackStateInvalidError); - yield* store.cleanup(stackId); - expect(yield* fs.exists(path.join(root, stackId))).toBe(false); - expect(yield* fs.exists(path.join(root, sibling, "data"))).toBe(true); - }), - ), - ); - - it.live("treats a state document disappearing during read as an absent stack", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-read-race-" }); - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(stackId, state()); - const statePath = path.join(root, stackId, "state.json"); - const racingFs: FileSystem.FileSystem = { - ...fs, - readFileString: (candidate, encoding) => - candidate === statePath - ? fs.remove(candidate).pipe(Effect.andThen(fs.readFileString(candidate, encoding))) - : fs.readFileString(candidate, encoding), - }; - const racingStore = yield* makeStackStateStore({ stateRoot: root }).pipe( - Effect.provideService(FileSystem.FileSystem, racingFs), - ); - - expect(yield* racingStore.read(stackId)).toBeUndefined(); - }), - ), - ); - - it.live("does not recover a runtime remnant that contains an owner lock", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-live-remnant-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const stackId = yield* deriveStackId(identity); - const runtime = path.join(root, stackId, "runtime"); - const ownerLock = path.join(runtime, "owner.lock"); - yield* fs.makeDirectory(runtime, { recursive: true }); - yield* fs.writeFileString(ownerLock, "live"); - - const readExit = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(readExit)).toBeInstanceOf(StackStateInvalidError); - const recovered = yield* store.recoverRuntimeRemnant(stackId).pipe(Effect.exit); - - expect(errorOf(recovered)).toBeInstanceOf(StackStateInvalidError); - expect(yield* fs.exists(ownerLock)).toBe(true); - expect(yield* fs.exists(runtime)).toBe(true); - }), - ), - ); - - it.live("rejects unsupported state formats", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-format-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const stackId = yield* deriveStackId(identity); - yield* store.initialize(stackId, state()); - yield* fs.writeFileString( - path.join(root, stackId, "state.json"), - yield* jsonText({ ...state(), format: "supabase-stack-state-v2" }), - ); - const unsupported = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(unsupported)).toBeInstanceOf(StackStateFormatUnsupportedError); - yield* fs.writeFileString( - path.join(root, stackId, "state.json"), - yield* jsonText({ ...state(), format: 1 }), - ); - const malformed = yield* store.read(stackId).pipe(Effect.exit); - expect(errorOf(malformed)).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); - - it.live("rejects a forged identity tuple before writing", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-identity-" }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const stackId = yield* deriveStackId(identity); - const original = state(); - const forged = { - ...original, - identity: { ...original.identity, projectRoot: "/tmp/forged" }, - }; - const exit = yield* store.initialize(stackId, forged).pipe(Effect.exit); - expect(errorOf(exit)).toBeInstanceOf(StackStateInvalidError); - expect(yield* fs.exists(path.join(root, stackId))).toBe(false); - }), - ), - ); - - it.live("exposes only complete old or new values during repeated writes", () => - withPlatform( - Effect.gen(function* () { - const root = yield* (yield* FileSystem.FileSystem).makeTempDirectoryScoped({ - prefix: "supabase-stack-atomic-", - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const stackId = yield* deriveStackId(identity); - const oldValue = state(); - const newValue = { - ...oldValue, - desiredLifecycle: "running" as const, - ports: [{ field: "api" as const, port: 24_321, intent: "exact" as const }], - secrets: { "secret:test": { policy: "managed" as const, value: "new-value" } }, - }; - yield* store.initialize(stackId, oldValue); - const updates = Array.from({ length: 8 }, (_, index) => - store.replace(stackId, index % 2 === 0 ? oldValue : newValue), - ); - const writer = Effect.forEach(updates, (update) => update, { concurrency: 1 }); - const readers = Effect.forEach(Array.from({ length: 32 }), () => store.read(stackId), { - concurrency: 8, - }); - const [, observations] = yield* Effect.all([writer, readers], { concurrency: 2 }); - for (const observation of observations) { - expect([oldValue, newValue]).toContainEqual(observation); - } - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/BootstrapLog.ts b/packages/stack/src/supervisor/BootstrapLog.ts deleted file mode 100644 index 3ce81ba51f..0000000000 --- a/packages/stack/src/supervisor/BootstrapLog.ts +++ /dev/null @@ -1,42 +0,0 @@ -// oxlint-disable-next-line effecttsgo/node-builtin-import -- supervisor bootstrap log opens before Effect services exist. -import * as NodeFs from "node:fs"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- path join for the stack-owned bootstrap log. -import * as NodePath from "node:path"; - -export const SUPERVISOR_BOOTSTRAP_LOG = "bootstrap.log"; -export const SUPERVISOR_BOOTSTRAP_LOG_MAX_BYTES = 256 * 1024; - -export interface SupervisorBootstrapLog { - readonly fd: number; - readonly path: string; -} - -/** Opens a capped owner-only bootstrap log under `//`. */ -export const openSupervisorBootstrapLog = ( - stateRoot: string, - stackId: string, -): SupervisorBootstrapLog | undefined => { - try { - const stackRoot = NodePath.join(stateRoot, stackId); - NodeFs.mkdirSync(stackRoot, { recursive: true, mode: 0o700 }); - const path = NodePath.join(stackRoot, SUPERVISOR_BOOTSTRAP_LOG); - let fd = NodeFs.openSync(path, "a", 0o600); - try { - NodeFs.fchmodSync(fd, 0o600); - } catch { - // Open mode applies on create; fchmod can fail on some filesystems. - } - if (NodeFs.fstatSync(fd).size > SUPERVISOR_BOOTSTRAP_LOG_MAX_BYTES) { - NodeFs.closeSync(fd); - fd = NodeFs.openSync(path, "w", 0o600); - try { - NodeFs.fchmodSync(fd, 0o600); - } catch { - // Open mode applies on truncate; fchmod can fail on some filesystems. - } - } - return { fd, path }; - } catch { - return undefined; - } -}; diff --git a/packages/stack/src/supervisor/BootstrapLog.unit.test.ts b/packages/stack/src/supervisor/BootstrapLog.unit.test.ts deleted file mode 100644 index 9e1afc1bc0..0000000000 --- a/packages/stack/src/supervisor/BootstrapLog.unit.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- unit fixture inspects bootstrap.log mode and size. -import { mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- temp root for bootstrap.log. -import { tmpdir } from "node:os"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- join fixture paths. -import { join } from "node:path"; -import { - openSupervisorBootstrapLog, - SUPERVISOR_BOOTSTRAP_LOG, - SUPERVISOR_BOOTSTRAP_LOG_MAX_BYTES, -} from "./BootstrapLog.ts"; - -describe("openSupervisorBootstrapLog", () => { - it("creates an owner-only bootstrap log under the stack directory", () => { - const stateRoot = mkdtempSync(join(tmpdir(), "supervisor-bootstrap-")); - const stackId = "a".repeat(64); - const opened = openSupervisorBootstrapLog(stateRoot, stackId); - expect(opened).toBeDefined(); - if (opened === undefined) return; - expect(opened.path).toBe(join(stateRoot, stackId, SUPERVISOR_BOOTSTRAP_LOG)); - expect(statSync(opened.path).mode & 0o777).toBe(0o600); - }); - - it("truncates a bootstrap log that already exceeded the cap", () => { - const stateRoot = mkdtempSync(join(tmpdir(), "supervisor-bootstrap-")); - const stackId = "b".repeat(64); - const first = openSupervisorBootstrapLog(stateRoot, stackId); - expect(first).toBeDefined(); - if (first === undefined) return; - writeFileSync(first.path, "x".repeat(SUPERVISOR_BOOTSTRAP_LOG_MAX_BYTES + 1)); - const reopened = openSupervisorBootstrapLog(stateRoot, stackId); - expect(reopened).toBeDefined(); - if (reopened === undefined) return; - expect(readFileSync(reopened.path, "utf8")).toBe(""); - }); -}); diff --git a/packages/stack/src/supervisor/CapabilityState.ts b/packages/stack/src/supervisor/CapabilityState.ts deleted file mode 100644 index a5f3b33118..0000000000 --- a/packages/stack/src/supervisor/CapabilityState.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { Match, type Cause, type Deferred, type Exit, type Fiber } from "effect"; -import type { ActivationResult, BackendEndpoint } from "../gateway/Gateway.ts"; -import type { GatewayActivationError, StackError } from "../public/Errors.ts"; - -type EndpointState = - | { readonly _tag: "unresolved" } - | { - readonly _tag: "resolving"; - readonly deferred: Deferred.Deferred< - Exit.Exit, - never - >; - } - | { readonly _tag: "resolved"; readonly endpoint: BackendEndpoint }; - -type Retirement = - | { readonly _tag: "disarmed" } - | { - readonly _tag: "armed"; - readonly epoch: symbol; - readonly fiber: Fiber.Fiber; - }; - -type ActivationCompletion = Deferred.Deferred, never>; - -type WorkloadCompletion = Deferred.Deferred, never>; - -type StartingCompletion = - | { readonly _tag: "activation"; readonly deferred: ActivationCompletion } - | { readonly _tag: "workload"; readonly deferred: WorkloadCompletion }; - -export type CapabilityState = - | { readonly _tag: "disabled" } - | { readonly _tag: "stopped" } - | { - readonly _tag: "dormant"; - readonly sessionId: symbol; - readonly traffic: number; - readonly root: boolean; - readonly retirement: { readonly _tag: "disarmed" }; - } - | { - readonly _tag: "starting"; - readonly sessionId: symbol; - readonly traffic: number; - readonly operation: symbol; - readonly completion: StartingCompletion; - readonly prior: Extract; - readonly root: boolean; - readonly retirement: { readonly _tag: "disarmed" }; - } - | { - readonly _tag: "ready"; - readonly sessionId: symbol; - readonly traffic: number; - readonly endpoint: EndpointState; - readonly root: boolean; - readonly retirement: Retirement; - } - | { - readonly _tag: "stopping"; - readonly sessionId: symbol; - readonly traffic: number; - readonly operation: symbol; - readonly completion: Deferred.Deferred, never>; - readonly prior: Extract< - CapabilityState, - { readonly _tag: "dormant" | "ready" | "cleanup-failed" } - >; - readonly root: boolean; - readonly retirement: { readonly _tag: "disarmed" }; - } - | { - readonly _tag: "cleanup-failed"; - readonly sessionId: symbol; - readonly traffic: number; - readonly cause: Cause.Cause; - readonly root: boolean; - }; - -type DormantState = Extract; -type ReadyState = Extract; -type StartingState = Extract; -type StoppingState = Extract; -type CleanupFailedState = Extract; - -export const dormant = (sessionId: symbol, traffic = 0, root = false): DormantState => ({ - _tag: "dormant", - sessionId, - traffic, - root, - retirement: { _tag: "disarmed" }, -}); - -export const beginStarting = ( - prior: DormantState | ReadyState, - operation: symbol, - completion: StartingCompletion, - root = prior.root, -): StartingState => ({ - _tag: "starting", - sessionId: prior.sessionId, - traffic: prior.traffic, - operation, - completion, - prior, - root, - retirement: { _tag: "disarmed" }, -}); - -export const ready = ( - sessionId: symbol, - traffic: number, - root: boolean, - endpoint: EndpointState = { _tag: "unresolved" }, -): ReadyState => ({ - _tag: "ready", - sessionId, - traffic, - endpoint, - root, - retirement: { _tag: "disarmed" }, -}); - -export const completeStarting = ( - state: StartingState, - endpoint: EndpointState = { _tag: "unresolved" }, - root = state.root, -): ReadyState => ready(state.sessionId, state.traffic, root, endpoint); - -export const restoreStarting = (state: StartingState): DormantState | ReadyState => - Match.value(state.prior).pipe( - Match.tag("dormant", (prior) => dormant(prior.sessionId, state.traffic, prior.root)), - Match.tag("ready", (prior) => - ready(prior.sessionId, state.traffic, prior.root, prior.endpoint), - ), - Match.exhaustive, - ); - -export const promoteStartingPrior = (state: StartingState): StartingState => ({ - ...state, - prior: Match.value(state.prior).pipe( - Match.tag("dormant", (prior) => ready(prior.sessionId, state.traffic, prior.root)), - Match.tag("ready", (prior) => - ready(prior.sessionId, state.traffic, prior.root, prior.endpoint), - ), - Match.exhaustive, - ), -}); - -export const dormantFromReady = (state: ReadyState): DormantState => - dormant(state.sessionId, state.traffic, state.root); - -export const beginStopping = ( - prior: DormantState | ReadyState | CleanupFailedState, - operation: symbol, - completion: Deferred.Deferred, never>, - root = prior.root, -): StoppingState => ({ - _tag: "stopping", - sessionId: prior.sessionId, - traffic: prior.traffic, - operation, - completion, - prior, - root, - retirement: { _tag: "disarmed" }, -}); - -export const cleanupFailed = ( - state: StartingState | StoppingState | ReadyState, - cause: Cause.Cause, -): CapabilityState => ({ - _tag: "cleanup-failed", - sessionId: state.sessionId, - traffic: state.traffic, - cause, - root: state.root, -}); - -export const publicCapabilityState = ( - state: CapabilityState, -): "disabled" | "stopped" | "dormant" | "starting" | "ready" | "stopping" | "failed" => - Match.value(state).pipe( - Match.when({ _tag: "disabled" }, () => "disabled" as const), - Match.when({ _tag: "stopped" }, () => "stopped" as const), - Match.when({ _tag: "dormant" }, () => "dormant" as const), - Match.when({ _tag: "starting" }, () => "starting" as const), - Match.when({ _tag: "ready" }, () => "ready" as const), - Match.when({ _tag: "stopping" }, () => "stopping" as const), - Match.when({ _tag: "cleanup-failed" }, () => "failed" as const), - Match.exhaustive, - ); diff --git a/packages/stack/src/supervisor/HostListener.ts b/packages/stack/src/supervisor/HostListener.ts deleted file mode 100644 index 417ba94172..0000000000 --- a/packages/stack/src/supervisor/HostListener.ts +++ /dev/null @@ -1,365 +0,0 @@ -import { Effect, Queue, Scope } from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- pre-adoption HTTP and raw upgrade events must stay queued on the same bound server. -import { createServer as createHttpServer, type Server as HttpServer } from "node:http"; -import { createServer as createNetServer, isIP, type Server as NetServer } from "node:net"; -import type { Duplex } from "node:stream"; -import { PortUnavailableError } from "../public/Errors.ts"; -import { PORT_FIELD_PROTOCOL, type PortField } from "../public/Status.ts"; - -export interface HostListener { - readonly field: PortField; - readonly address: string; - readonly port: number; - readonly close: Effect.Effect; - /** The exact bound listener may be adopted by a gateway without rebind. */ - readonly binding: HostListenerBinding; - /** Sockets accepted since bind, shared with an adopting gateway for teardown. */ - readonly connections: HostListenerConnections; -} - -interface HostListenerConnections { - readonly sockets: Set; - /** Release the pre-adoption connection capture without resuming socket reads. */ - readonly release?: () => void; -} - -export type HostListenerHttpEvent = - | { - readonly _tag: "request"; - readonly request: import("node:http").IncomingMessage; - readonly response: import("node:http").ServerResponse; - } - | { - readonly _tag: "upgrade"; - readonly request: import("node:http").IncomingMessage; - readonly socket: Duplex; - readonly head: Buffer; - }; - -export interface HostListenerHttpEvents { - readonly queue: Queue.Queue; - /** Stop capturing events; queued events remain available for gateway adoption. */ - readonly detach: () => void; -} - -type HostListenerBinding = - | { - readonly kind: "http"; - readonly server: HttpServer; - readonly pendingEvents?: HostListenerHttpEvents; - } - | { readonly kind: "tcp"; readonly server: NetServer; readonly allowHalfOpen: true }; - -/** A scoped TCP reservation used while selecting a durable workload port. */ -export interface HeldPort { - readonly port: number; - readonly close: Effect.Effect; -} - -export interface HostListenerBindOptions { - readonly createHttpServer?: () => HttpServer; - readonly createTcpServer?: () => NetServer; - /** Test/embedding seam for a foreign listener API; production uses Server.listen directly. */ - readonly listen?: ( - server: HttpServer | NetServer, - address: string, - port: number, - onListening: () => void, - ) => void; -} - -interface ConnectionTracker extends HostListenerConnections { - readonly detach: () => void; -} - -const trackConnections = ( - server: HttpServer | NetServer, - holdConnections: boolean, -): ConnectionTracker => { - const sockets = new Set(); - const errorListeners = new Map void>(); - let holding = true; - const onConnection = (socket: Duplex) => { - sockets.add(socket); - const onError = (_cause: Error) => { - // A held socket has no gateway-owned error listener yet. Destroy only this exact socket so - // its error cannot become an uncaught event or affect other connections awaiting adoption. - socket.destroy(); - }; - const onClose = () => { - sockets.delete(socket); - errorListeners.delete(socket); - }; - errorListeners.set(socket, onError); - // TCP bytes must stay buffered until the gateway has connected its tunnel. HTTP request events - // are captured separately because Node/Bun parse them despite a paused socket. - if (holding && holdConnections) socket.pause(); - socket.once("close", onClose); - socket.once("error", onError); - }; - server.on("connection", onConnection); - const release = () => { - if (!holding) return; - holding = false; - server.off("connection", onConnection); - // Gateway adoption takes ownership of post-adoption socket errors. Keep close tracking until - // each socket closes so the exact resource set remains available to shutdown. - for (const [socket, onError] of errorListeners) socket.off("error", onError); - }; - return { - sockets, - release, - detach: () => { - release(); - }, - }; -}; - -const closeServer = ( - server: HttpServer | NetServer, - tracker: ConnectionTracker, -): Effect.Effect => - Effect.callback((resume) => { - const destroyConnections = () => { - for (const socket of tracker.sockets) socket.destroy(); - }; - const finish = () => { - tracker.detach(); - resume(Effect.void); - }; - destroyConnections(); - if (!server.listening) { - finish(); - return; - } - server.close(finish); - return Effect.sync(() => { - destroyConnections(); - tracker.detach(); - if (server.listening) server.close(() => undefined); - }); - }); - -interface BoundServer { - readonly server: T; - readonly connections: ConnectionTracker; - readonly pendingEvents?: HostListenerHttpEvents; -} - -const captureHttpEvents = ( - server: HttpServer, - queue: Queue.Queue, -): HostListenerHttpEvents => { - const requestHandler = ( - request: import("node:http").IncomingMessage, - response: import("node:http").ServerResponse, - ) => { - Queue.offerUnsafe(queue, { _tag: "request", request, response }); - }; - const upgradeHandler = ( - request: import("node:http").IncomingMessage, - socket: Duplex, - head: Buffer, - ) => { - socket.pause(); - Queue.offerUnsafe(queue, { _tag: "upgrade", request, socket, head }); - }; - server.on("request", requestHandler); - server.on("upgrade", upgradeHandler); - return { - queue, - detach: () => { - server.off("request", requestHandler); - server.off("upgrade", upgradeHandler); - }, - }; -}; - -const configureHttpTimeouts = (server: HttpServer): void => { - // A request can legitimately wait for a cold workload to become ready. Keep the native - // listener from terminating that request while the queue or lazy activation owns it. - server.requestTimeout = 0; - server.headersTimeout = 0; - server.timeout = 0; -}; - -const bind = ( - server: T, - address: string, - port: number, - field: string, - listen: HostListenerBindOptions["listen"] = (value, host, number, onListening) => - value.listen( - isIP(host) === 6 ? { host, port: number, ipv6Only: false } : { host, port: number }, - onListening, - ), - holdConnections = false, - prepare?: (server: T) => HostListenerHttpEvents, -): Effect.Effect, PortUnavailableError, Scope.Scope> => - Effect.callback, PortUnavailableError>((resume) => { - const tracker = trackConnections(server, holdConnections); - const pendingEvents = prepare?.(server); - let settled = false; - const cleanup = () => { - server.off("error", onError); - }; - const teardown = () => { - cleanup(); - pendingEvents?.detach(); - tracker.detach(); - const swallow = () => undefined; - server.once("error", swallow); - try { - for (const socket of tracker.sockets) socket.destroy(); - server.close(() => server.off("error", swallow)); - } catch { - server.off("error", swallow); - } - }; - const onError = (cause: Error) => { - if (settled) return; - settled = true; - teardown(); - resume( - Effect.fail( - new PortUnavailableError({ - field, - port, - message: "Host listener is unavailable", - cause, - }), - ), - ); - }; - server.once("error", onError); - try { - listen(server, address, port, () => { - if (settled) return; - settled = true; - cleanup(); - resume(Effect.succeed({ server, connections: tracker, pendingEvents })); - }); - } catch (cause) { - onError(cause instanceof Error ? cause : new Error(String(cause))); - } - return Effect.sync(() => { - if (settled) return; - settled = true; - // `listen` may still be completing when the fiber is interrupted. Close the server with a - // temporary error sink so a late bind failure can't become an uncaught process error; - // `close` throws synchronously if no handle exists, meaning there's nothing left to release. - teardown(); - }); - }); - -/** Verifies an address/port can be bound without retaining a listener. */ -export const checkHostPort = ( - address: string, - port: number, - field: string, -): Effect.Effect => - Effect.scoped(bindHeldPort(address, port, field).pipe(Effect.asVoid)); - -/** Bind and retain one TCP port until its enclosing scope is closed. */ -export const bindHeldPort = ( - address: string, - port: number, - field: string, -): Effect.Effect => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const server = createNetServer({ allowHalfOpen: true }); - const bound = yield* restore(bind(server, address, port, field, undefined, true)); - const close = yield* Effect.cached(closeServer(bound.server, bound.connections)); - yield* Effect.addFinalizer(() => close); - return { port: boundPort(bound.server, port), close } satisfies HeldPort; - }), - ); - -/** Bind and retain one public host listener for direct adoption by a gateway. */ -export const bindHostListener = ( - address: string, - port: number, - field: PortField, -): Effect.Effect => - bindHostListenerWithOptions(address, port, field); - -export const bindHostListenerWithOptions = ( - address: string, - port: number, - field: PortField, - options: HostListenerBindOptions = {}, -): Effect.Effect => { - if (PORT_FIELD_PROTOCOL[field] === "http") - return Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const queue = yield* Queue.unbounded(); - const server = options.createHttpServer?.() ?? createHttpServer(); - configureHttpTimeouts(server); - const bound = yield* restore( - bind(server, address, port, field, options.listen, false, (server) => - captureHttpEvents(server, queue), - ), - ); - const close = yield* Effect.cached( - Effect.sync(() => bound.pendingEvents?.detach()).pipe( - Effect.andThen( - bound.pendingEvents === undefined - ? Effect.void - : Queue.shutdown(bound.pendingEvents.queue), - ), - Effect.andThen(closeServer(bound.server, bound.connections)), - ), - ); - yield* Effect.addFinalizer(() => close); - return { - field, - address: boundAddress(bound.server, address), - port: boundPort(bound.server, port), - close, - connections: bound.connections, - binding: { kind: "http", server: bound.server, pendingEvents: bound.pendingEvents }, - } satisfies HostListener; - }), - ); - return Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const server = options.createTcpServer?.() ?? createNetServer({ allowHalfOpen: true }); - const bound = yield* restore(bind(server, address, port, field, options.listen, true)); - const close = yield* Effect.cached(closeServer(bound.server, bound.connections)); - yield* Effect.addFinalizer(() => close); - return { - field, - address: boundAddress(bound.server, address), - port: boundPort(bound.server, port), - close, - connections: bound.connections, - binding: { kind: "tcp", server: bound.server, allowHalfOpen: true }, - } satisfies HostListener; - }), - ); -}; - -export const isHttpPortField = (field: PortField): boolean => PORT_FIELD_PROTOCOL[field] === "http"; - -const boundAddress = (server: HttpServer | NetServer, fallback: string): string => { - const address = server.address(); - return typeof address === "object" && address !== null ? address.address : fallback; -}; - -const boundPort = (server: HttpServer | NetServer, fallback: number): number => { - const address = server.address(); - return typeof address === "object" && address !== null ? address.port : fallback; -}; - -const isIpv4 = (address: string): boolean => isIP(address) === 4; -const isIpv6 = (address: string): boolean => isIP(address) === 6; - -/** Whether a bound listener is known to cover an internal bind address. */ -export const hostListenerCoversAddress = (listener: HostListener, address: string): boolean => { - if (listener.address === address) return true; - if (listener.address === "0.0.0.0" && isIpv4(address)) return true; - // IPv6 wildcard listeners are dual-stack in bind(). - if (listener.address === "::" && (isIpv6(address) || isIpv4(address))) return true; - return false; -}; diff --git a/packages/stack/src/supervisor/Ingress.ts b/packages/stack/src/supervisor/Ingress.ts deleted file mode 100644 index 94305bc38a..0000000000 --- a/packages/stack/src/supervisor/Ingress.ts +++ /dev/null @@ -1,437 +0,0 @@ -import { Context, Crypto, Effect, Exit, FileSystem, Path, Ref, Scope, Semaphore } from "effect"; -import type { LifecycleInput } from "./Lifecycle.ts"; -import type { - ActivationResult, - GatewayRoute, - GatewayProxyRoute, - GatewayRouteRequest, - HttpGatewayListenerOptions, - StackGateway, -} from "../gateway/Gateway.ts"; -import type { GatewayActivity } from "../gateway/ActivityTracker.ts"; -import { - GatewayActivationError, - PortUnavailableError, - StackLifecycleConflictError, - StackPreparationError, - type StackError, -} from "../public/Errors.ts"; -import type { PortField } from "../public/Status.ts"; -import { routeCatalogFor, type GatewayApiMaterial } from "../gateway/RouteCatalog.ts"; -import { GatewayRouteNotFoundError, makeGateway } from "../gateway/Gateway.ts"; -import { - makePortCoordinator, - type ListenerIntents, - type PortReservation, -} from "../state/PortCoordinator.ts"; -import type { HostListener } from "./HostListener.ts"; -import type { StackStateStore } from "../state/StackStateStore.ts"; -import { privateBindingIntentsFor } from "../runtime/WorkloadRuntimeSpec.ts"; -import { - bindHeldPort, - bindHostListener, - hostListenerCoversAddress, - isHttpPortField, - type HeldPort, -} from "./HostListener.ts"; -import { - AUTH_ANON_KEY_SLOT, - AUTH_PUBLISHABLE_KEY_SLOT, - AUTH_SECRET_KEY_SLOT, - AUTH_SERVICE_ROLE_KEY_SLOT, -} from "../state/SecretStore.ts"; - -interface SupervisorIngressReservation extends PortReservation { - /** False when this accepted definition already owns the exact listeners and gateway. */ - readonly fresh: boolean; - /** Stable identity for the reservation across non-fresh reacquisition views. */ - readonly ownershipToken: symbol; -} - -export interface SupervisorIngress { - /** Reserve durable ports and bind public listeners before workload launch. */ - readonly acquire: ( - input: LifecycleInput, - ) => Effect.Effect; - /** Adopt acquired listeners into HTTP/TCP gateways after workloads are ready. */ - readonly open: ( - input: LifecycleInput, - reservation: SupervisorIngressReservation, - activate: ( - capability: import("../public/Capability.ts").CapabilityName, - ) => Effect.Effect, - activity?: GatewayActivity, - ) => Effect.Effect; - /** Close gateway, accepted sockets, and exact listeners; safe to call repeatedly. */ - readonly close: Effect.Effect; -} - -export interface SupervisorIngressOptions { - readonly stackId: string; - readonly stateRoot: string; - readonly store: StackStateStore; - readonly context: Context.Context; - readonly bindHost?: ( - address: string, - port: number, - field: PortField, - ) => Effect.Effect; - readonly bindPrivate?: ( - address: string, - port: number, - binding: string, - ) => Effect.Effect; - /** Resolves an internal host bind address for container callbacks when required. */ - readonly resolveInternalApiBindAddress?: () => Effect.Effect; - /** Resolver may be replaced by the production credential owner. */ - readonly apiMaterial?: ( - state: LifecycleInput["state"], - ) => Effect.Effect; - /** Resolves the accepted definition's Auth templates for live local serving. */ - readonly resolveAuthTemplates?: (state: LifecycleInput["state"]) => Effect.Effect< - ReadonlyArray<{ - readonly id: string; - readonly canonicalPath: string; - readonly extension: string; - }>, - StackPreparationError - >; -} - -const listenerIntents = (input: LifecycleInput): ListenerIntents => { - const usable = new Set(input.plan.routes.map(({ listener }) => listener)); - const select = (field: K): ListenerIntents[K] => - usable.has(field) - ? input.definition.listeners[field] - : { ...input.definition.listeners[field], enabled: false }; - return { - api: select("api"), - database: select("database"), - pooler: select("pooler"), - studio: select("studio"), - mailUi: select("mailUi"), - smtp: select("smtp"), - pop3: select("pop3"), - functionsInspector: select("functionsInspector"), - }; -}; - -const defaultApiMaterial = ( - state: LifecycleInput["state"], -): Effect.Effect => { - const get = (slot: string): string | undefined => state.secrets[slot]?.value; - const publishableKey = get(AUTH_PUBLISHABLE_KEY_SLOT); - const secretKey = get(AUTH_SECRET_KEY_SLOT); - const anonJwt = get(AUTH_ANON_KEY_SLOT); - const serviceRoleJwt = get(AUTH_SERVICE_ROLE_KEY_SLOT); - if ( - publishableKey === undefined || - secretKey === undefined || - anonJwt === undefined || - serviceRoleJwt === undefined - ) - return Effect.fail( - new StackPreparationError({ message: "Persisted API gateway material is incomplete" }), - ); - return Effect.succeed({ publishableKey, secretKey, anonJwt, serviceRoleJwt }); -}; - -const routeBackend = ( - input: LifecycleInput, - reservation: SupervisorIngressReservation, - route: Pick, - activation: ActivationResult, -) => { - if (route.binding === undefined) return Effect.succeed(activation.endpoint); - const workloadIds = new Set( - input.plan.workloads - .filter((entry) => entry.capability === route.capability) - .map((entry) => entry.id), - ); - const assignments = reservation.privateAssignments.filter( - (entry) => workloadIds.has(entry.workloadId) && entry.binding === route.binding, - ); - const [assignment, ...additionalAssignments] = assignments; - if (assignment === undefined || additionalAssignments.length > 0) - return Effect.fail( - new GatewayActivationError({ - message: - assignment === undefined - ? "Gateway private binding is unavailable" - : "Gateway private binding is ambiguous", - }), - ); - return Effect.succeed({ host: "127.0.0.1", port: assignment.port }); -}; - -const templateContentType = (extension: string): string => { - switch (extension.toLowerCase()) { - case ".html": - case ".htm": - return "text/html; charset=utf-8"; - case ".txt": - return "text/plain; charset=utf-8"; - case ".json": - return "application/json"; - default: - return "application/octet-stream"; - } -}; - -/** Compose PortCoordinator and StackGateway under one Supervisor owner scope. */ -export const makeSupervisorIngress = ( - options: SupervisorIngressOptions, -): Effect.Effect< - SupervisorIngress, - StackError, - Scope.Scope | Crypto.Crypto | FileSystem.FileSystem | Path.Path -> => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const ownerScope = yield* Scope.Scope; - const lock = yield* Semaphore.make(1); - const current = yield* Ref.make< - | { - readonly input: LifecycleInput; - readonly reservation: SupervisorIngressReservation; - readonly scope: Scope.Scope; - readonly gateway?: StackGateway; - } - | undefined - >(undefined); - const coordinator = makePortCoordinator({ - stateRoot: options.stateRoot, - store: options.store, - bindHost: options.bindHost ?? bindHostListener, - bindPrivate: options.bindPrivate ?? bindHeldPort, - }); - const acquire = ( - input: LifecycleInput, - ): Effect.Effect => - lock.withPermit( - Effect.gen(function* () { - const existing = yield* Ref.get(current); - // A Supervisor owns one ingress reservation for its running session. Definition - // changes are rejected while running and a stopped session closes this reservation, - // so a live reservation can always be reused without a configuration fingerprint. - if (existing !== undefined) return { ...existing.reservation, fresh: false }; - const reservationScope = Scope.forkUnsafe(ownerScope); - const reservation = yield* coordinator - .acquire( - options.stackId, - listenerIntents(input), - privateBindingIntentsFor(input.plan, input.state), - ) - .pipe( - Effect.provideContext(options.context), - Effect.provideService(Scope.Scope, reservationScope), - Effect.onExit((exit) => - Exit.isSuccess(exit) ? Effect.void : Scope.close(reservationScope, exit), - ), - ); - const owned: SupervisorIngressReservation = { - ...reservation, - fresh: true, - ownershipToken: Symbol(), - }; - yield* Ref.set(current, { input, reservation: owned, scope: reservationScope }); - return owned; - }), - ); - - const closeCurrent = (entry: { - readonly reservation: SupervisorIngressReservation; - readonly gateway?: StackGateway; - readonly scope: Scope.Scope; - }): Effect.Effect => - Effect.gen(function* () { - if (entry.gateway !== undefined) yield* entry.gateway.close; - yield* Scope.close(entry.scope, Exit.void); - }); - - const close: Effect.Effect = lock.withPermit( - Effect.gen(function* () { - const entry = yield* Ref.get(current); - if (entry === undefined) return; - yield* closeCurrent(entry); - yield* Ref.set(current, undefined); - }), - ); - - const open = ( - input: LifecycleInput, - reservation: SupervisorIngressReservation, - activate: ( - capability: import("../public/Capability.ts").CapabilityName, - ) => Effect.Effect, - activity?: GatewayActivity, - ): Effect.Effect => - lock.withPermit( - Effect.gen(function* () { - const entry = yield* Ref.get(current); - if ( - entry === undefined || - entry.reservation.ownershipToken !== reservation.ownershipToken - ) - return yield* new GatewayActivationError({ - message: "Gateway reservation is no longer current", - }); - if (entry.gateway !== undefined) return; - const intents = listenerIntents(input); - const material = intents.api.enabled - ? yield* (options.apiMaterial ?? defaultApiMaterial)(input.state) - : undefined; - const catalog = routeCatalogFor(input.plan, material); - const resolveTemplates = options.resolveAuthTemplates; - const templateRoute: GatewayRoute | undefined = - resolveTemplates === undefined - ? undefined - : { - match: (request) => { - const pathname = request.path.split("?", 1)[0] ?? request.path; - return pathname === "/email" || pathname.startsWith("/email/"); - }, - localResponse: (request) => { - const pathname = request.path.split("?", 1)[0] ?? request.path; - if (request.method !== "GET") - return Effect.fail( - new GatewayRouteNotFoundError({ message: "Auth template not found" }), - ); - return resolveTemplates(input.state).pipe( - Effect.mapError( - () => new GatewayRouteNotFoundError({ message: "Auth template not found" }), - ), - Effect.flatMap((templates) => { - const template = templates.find( - (entry) => `/email/${entry.id}${entry.extension}` === pathname, - ); - return template === undefined - ? Effect.fail( - new GatewayRouteNotFoundError({ - message: "Auth template not found", - }), - ) - : fs.readFile(template.canonicalPath).pipe( - Effect.mapError( - () => - new GatewayRouteNotFoundError({ - message: "Auth template not found", - }), - ), - Effect.map((body) => ({ - body, - contentType: templateContentType(template.extension), - })), - ); - }), - ); - }, - }; - const http: HttpGatewayListenerOptions[] = reservation.hostListeners - .filter((listener) => isHttpPortField(listener.field)) - .map((listener) => ({ - field: listener.field, - key: listener.field, - options: { - listener, - routes: - listener.field === "api" && templateRoute !== undefined - ? [templateRoute, ...(catalog.http.get(listener.field) ?? [])] - : (catalog.http.get(listener.field) ?? []), - resolveBackend: ( - route: GatewayProxyRoute, - _request: GatewayRouteRequest, - result: ActivationResult, - ) => routeBackend(input, reservation, route, result), - }, - })); - const internalApiAddress = - intents.api.enabled && - reservation.assignments.api !== undefined && - options.resolveInternalApiBindAddress !== undefined - ? yield* options.resolveInternalApiBindAddress() - : undefined; - let internalApi: HostListener | undefined; - if (internalApiAddress !== undefined && reservation.assignments.api !== undefined) { - const covered = reservation.hostListeners.some( - (listener) => - listener.field === "api" && - listener.port === reservation.assignments.api?.port && - hostListenerCoversAddress(listener, internalApiAddress), - ); - if (!covered) { - internalApi = yield* (options.bindHost ?? bindHostListener)( - internalApiAddress, - reservation.assignments.api.port, - "api", - ).pipe(Effect.provideService(Scope.Scope, entry.scope)); - http.push({ - field: "api", - key: "api:internal", - options: { - listener: internalApi, - routes: - templateRoute !== undefined - ? [templateRoute, ...(catalog.http.get("api") ?? [])] - : (catalog.http.get("api") ?? []), - resolveBackend: ( - route: GatewayProxyRoute, - _request: GatewayRouteRequest, - result: ActivationResult, - ) => routeBackend(input, reservation, route, result), - }, - }); - } - } - const tcp = reservation.hostListeners - .filter((listener) => !isHttpPortField(listener.field)) - .map((listener) => ({ - field: listener.field, - options: { - listener, - routes: catalog.tcp.get(listener.field) ?? [], - resolveBackend: ( - route: GatewayProxyRoute, - _request: GatewayRouteRequest, - result: ActivationResult, - ) => routeBackend(input, reservation, route, result), - }, - })); - const gatewayResult = yield* Effect.exit( - makeGateway({ - http, - tcp, - activate: (capability) => - activate(capability).pipe( - Effect.mapError((error) => - error instanceof GatewayActivationError - ? error - : new GatewayActivationError({ - message: error.message, - cause: error, - recovery: - error instanceof StackLifecycleConflictError - ? error.recovery - : undefined, - }), - ), - ), - activity, - }).pipe(Effect.provideService(Scope.Scope, entry.scope)), - ); - if (Exit.isFailure(gatewayResult)) { - if (internalApi !== undefined) yield* internalApi.close.pipe(Effect.ignore); - return yield* Effect.failCause(gatewayResult.cause); - } - const gateway = gatewayResult.value; - yield* Ref.set(current, { - input, - reservation, - scope: entry.scope, - gateway, - }); - }), - ); - - return { acquire, open, close } satisfies SupervisorIngress; - }); diff --git a/packages/stack/src/supervisor/LaunchProtocol.ts b/packages/stack/src/supervisor/LaunchProtocol.ts deleted file mode 100644 index 26540ea334..0000000000 --- a/packages/stack/src/supervisor/LaunchProtocol.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Schema } from "effect"; -import { OwnerSessionIdSchema } from "../control/MaintenanceProtocol.ts"; -import { StackIdSchema } from "../public/StackId.ts"; - -/** JSON payload shared by the detached launcher and Supervisor entrypoint. */ -export const SupervisorArgsSchema = Schema.Struct({ - stateRoot: Schema.String, - artifactCacheRoot: Schema.optional(Schema.String), - tempRoot: Schema.String, - platform: Schema.Literals(["posix", "windows"] as const), - stackId: StackIdSchema, - ownerSessionId: OwnerSessionIdSchema, -}); - -export type SupervisorArgs = Schema.Schema.Type; - -/** One-line readiness frame written by the Supervisor on fd3. */ -export const SupervisorReadySchema = Schema.Union([ - Schema.Struct({ - ok: Schema.Literal(true), - stackId: StackIdSchema, - ownerSessionId: Schema.String, - }), - Schema.Struct({ - ok: Schema.Literal(false), - code: Schema.Literals(["ownership-conflict", "failed"] as const), - message: Schema.String, - }), -]); - -export type SupervisorReady = Schema.Schema.Type; diff --git a/packages/stack/src/supervisor/Launcher.ts b/packages/stack/src/supervisor/Launcher.ts deleted file mode 100644 index 7dfeb80132..0000000000 --- a/packages/stack/src/supervisor/Launcher.ts +++ /dev/null @@ -1,447 +0,0 @@ -import { - Cause, - Config, - Crypto, - Effect, - Exit, - FileSystem, - Fiber, - Option, - Path, - Result, - Schedule, - Scope, - Schema, - Stream, -} from "effect"; -import { fileURLToPath } from "node:url"; -import { homedir } from "node:os"; -import { ChildProcess } from "effect/unstable/process"; -import { ChildProcessSpawner } from "effect/unstable/process"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { StackIdSchema, type StackId } from "../public/StackId.ts"; -import { - readOwnerMetadata, - StackRuntimeEnvironment, - type OwnerMetadata, - type StackRuntimeEnvironmentValue, -} from "../state/Ownership.ts"; -import { isMissingStateRemnantError, type StackStateStore } from "../state/StackStateStore.ts"; -import { makeControlClient } from "../control/ControlServer.ts"; -import { isMaintenanceTransportFailure } from "../control/MaintenanceProtocol.ts"; -import { STACK_RPC_RELEASE } from "../control/StackRpc.ts"; -import { - SupervisorReadySchema, - type SupervisorReady, - type SupervisorArgs, -} from "./LaunchProtocol.ts"; -import { - StackOwnershipConflictError, - StackStateFormatUnsupportedError, - StackStateInvalidError, - StackUpgradeRequiredError, -} from "../public/Errors.ts"; - -export { StackRuntimeEnvironment } from "../state/Ownership.ts"; -export type { StackRuntimeEnvironmentValue } from "../state/Ownership.ts"; - -export { SUPERVISOR_DISPATCH_SENTINEL } from "../internal/dispatch-markers.ts"; -import { SUPERVISOR_DISPATCH_SENTINEL } from "../internal/dispatch-markers.ts"; - -const isBunVirtualPath = (value: string): boolean => /(?:^|[\\/])\$bunfs(?:[\\/]|$)/.test(value); - -export const supervisorEntrypointFor = (moduleUrl: string): string => { - if (isBunVirtualPath(moduleUrl)) return SUPERVISOR_DISPATCH_SENTINEL; - const sourceEntrypoint = fileURLToPath(new URL("../entrypoints/supervisor-node.ts", moduleUrl)); - return isBunVirtualPath(sourceEntrypoint) ? SUPERVISOR_DISPATCH_SENTINEL : sourceEntrypoint; -}; - -/** Resolves default host values from the active Effect configuration provider. */ -export const defaultRuntimeEnvironment: Effect.Effect = Effect.gen( - function* () { - const optional = (name: string) => - Config.option(Config.string(name)).pipe(Effect.orElseSucceed(() => Option.none())); - const configuredHome = yield* optional("SUPABASE_HOME"); - const configuredUserHome = yield* optional("HOME"); - const home = Option.getOrElse( - configuredHome, - () => `${Option.getOrElse(configuredUserHome, homedir)}/.supabase`, - ); - const configuredTemp = yield* optional("TEMP"); - return { - stateRoot: `${home}/managed/stacks`, - tempRoot: - process.platform === "win32" - ? Option.getOrElse(configuredTemp, () => "C:\\Windows\\Temp") - : "/tmp", - platform: process.platform === "win32" ? "windows" : "posix", - supervisorCommand: process.execPath, - supervisorEntrypoint: supervisorEntrypointFor(import.meta.url), - }; - }, -); - -/** Injected `StackRuntimeEnvironment` when present, otherwise the process default. */ -export const resolveRuntimeEnvironment: Effect.Effect = - Effect.serviceOption(StackRuntimeEnvironment).pipe( - Effect.flatMap((configured) => - Option.isSome(configured) ? Effect.succeed(configured.value) : defaultRuntimeEnvironment, - ), - ); - -type ReadinessResult = - | { readonly kind: "ready"; readonly stackId: StackId; readonly ownerSessionId: string } - | { readonly kind: "ownership-conflict"; readonly message: string } - | { readonly kind: "failed"; readonly message: string }; - -type ChildResult = - | { readonly kind: "ready"; readonly metadata: OwnerMetadata } - | { readonly kind: "ownership-conflict"; readonly message: string } - | { readonly kind: "failed"; readonly message: string }; - -export interface EnsureSupervisorResult { - readonly owner: OwnerMetadata; - /** True only when this caller won the launch race and owns the new session. */ - readonly launched: boolean; -} - -const mapFailure = (message: string) => new StackStateInvalidError({ message }); -const SUPERVISOR_READINESS_TIMEOUT_MS = 30_000; -const SUPERVISOR_REREAD_INTERVAL_MS = 25; -const SUPERVISOR_READINESS_REREAD_TIMES = Math.floor( - SUPERVISOR_READINESS_TIMEOUT_MS / SUPERVISOR_REREAD_INTERVAL_MS, -); - -export interface LauncherOptions { - readonly stackId: StackId; - readonly stateStore: StackStateStore; - readonly environment: StackRuntimeEnvironmentValue; -} - -const encodeLaunchPayload = ( - payload: SupervisorArgs, -): Effect.Effect => - Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(payload).pipe( - Effect.mapError(() => mapFailure("Unable to encode supervisor launch arguments")), - ); - -const decodeReady = (bytes: Uint8Array): Effect.Effect => { - if (bytes.byteLength > 4096) - return Effect.fail(mapFailure("Supervisor readiness frame exceeds size limit")); - const text = new TextDecoder().decode(bytes); - if (text.trim().length === 0) - return Effect.succeed({ kind: "failed", message: "Supervisor exited before readiness" }); - const classify = (value: SupervisorReady): Effect.Effect => - value.ok - ? Effect.succeed({ - kind: "ready", - stackId: value.stackId, - ownerSessionId: value.ownerSessionId, - }) - : value.code === "ownership-conflict" - ? Effect.succeed({ kind: "ownership-conflict", message: value.message }) - : Effect.succeed({ kind: "failed", message: value.message }); - return Schema.decodeEffect(Schema.fromJsonString(SupervisorReadySchema))(text).pipe( - Effect.mapError(() => mapFailure("Invalid supervisor readiness frame")), - Effect.flatMap(classify), - ); -}; - -const validateReady = ( - ready: { readonly kind: "ready"; readonly stackId: StackId; readonly ownerSessionId: string }, - options: LauncherOptions, - expectedOwnerSessionId: string, -): Effect.Effect => - ready.stackId === options.stackId && ready.ownerSessionId === expectedOwnerSessionId - ? Effect.void - : Effect.fail(mapFailure("Supervisor readiness identity mismatch")); - -const validateCompatibleOwner = ( - options: LauncherOptions, - metadata: OwnerMetadata, -): Effect.Effect< - void, - StackOwnershipConflictError | StackStateInvalidError | StackUpgradeRequiredError, - Scope.Scope -> => - Effect.gen(function* () { - const probeExit = yield* makeControlClient(metadata.endpoint, { - stackId: options.stackId, - ownerSessionId: metadata.ownerSessionId, - }).probe.pipe(Effect.exit); - if (Exit.isFailure(probeExit)) { - const failure = Cause.findErrorOption(probeExit.cause); - if (Option.isSome(failure) && isMaintenanceTransportFailure(failure.value)) - return yield* mapFailure("Unable to probe existing Supervisor: transport unavailable"); - return yield* new StackOwnershipConflictError({ - message: "Existing Supervisor probe failed", - stackId: options.stackId, - }); - } - const probe = probeExit.value; - if ( - probe.ok && - probe.op === "probe" && - (metadata.rpcRelease !== STACK_RPC_RELEASE || probe.rpcRelease !== STACK_RPC_RELEASE) - ) - return yield* new StackUpgradeRequiredError({ - message: `Stack owner release ${probe.rpcRelease} requires stop before start`, - expectedRelease: STACK_RPC_RELEASE, - actualRelease: probe.rpcRelease, - }); - if ( - !probe.ok || - probe.op !== "probe" || - probe.stackId !== options.stackId || - probe.ownerSessionId !== metadata.ownerSessionId - ) - return yield* new StackOwnershipConflictError({ - message: "Existing Supervisor failed session probe", - stackId: options.stackId, - }); - }); - -const observeOwner = ( - options: LauncherOptions, - paths: { readonly stackRoot: string }, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const ownerFromEvents = Stream.filterMapEffect(fs.watch(paths.stackRoot), () => - readOwnerMetadata(options.environment.stateRoot, options.stackId, options.environment).pipe( - Effect.map((metadata) => - metadata === undefined ? Result.fail(undefined) : Result.succeed(metadata), - ), - ), - ); - // FileSystem.watch does not expose a subscription-ready signal. A bounded, - // Schedule-driven reread closes the publication gap between the initial absent - // read and watcher registration across the same budget as the launch guard. - const ownerFromReread = Stream.fromEffect( - readOwnerMetadata(options.environment.stateRoot, options.stackId, options.environment), - ).pipe( - Stream.repeat( - Schedule.spaced(`${SUPERVISOR_REREAD_INTERVAL_MS} millis`).pipe( - Schedule.upTo({ times: SUPERVISOR_READINESS_REREAD_TIMES }), - ), - ), - Stream.filterMap((metadata) => - metadata === undefined ? Result.fail(undefined) : Result.succeed(metadata), - ), - ); - const observed = yield* Stream.runHead(Stream.merge(ownerFromEvents, ownerFromReread)).pipe( - Effect.mapError((error) => mapFailure(String(error))), - ); - if (Option.isNone(observed)) - return yield* mapFailure("Supervisor owner metadata was not published"); - return observed.value; - }); - -const launchAndAwait = ( - options: LauncherOptions, - payload: SupervisorArgs, - paths: { readonly stackRoot: string }, -): Effect.Effect< - EnsureSupervisorResult, - StackOwnershipConflictError | StackStateInvalidError | StackUpgradeRequiredError, - FileSystem.FileSystem | Path.Path | Scope.Scope | ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const command = options.environment.supervisorCommand; - const entrypoint = options.environment.supervisorEntrypoint; - if (command === undefined || entrypoint === undefined) - return yield* mapFailure("Supervisor launcher is not configured"); - const encoded = yield* encodeLaunchPayload(payload); - // Subscribe before spawning: a loser can observe the winner's exact - // metadata publication even when its own child loses the atomic lock race. - const ownerFiber = yield* Effect.forkChild(observeOwner(options, paths), { - startImmediately: true, - }); - const child = yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const child = yield* restore( - ChildProcess.make(command, [entrypoint, encoded], { - detached: true, - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - additionalFds: { fd3: { type: "output" } }, - }), - ); - yield* Effect.asVoid(child.unref); - return child; - }), - ); - const readiness: Effect.Effect< - ChildResult, - StackStateInvalidError, - FileSystem.FileSystem | Path.Path - > = Effect.gen(function* () { - const chunks = yield* Stream.runCollect(child.getOutputFd(3)); - const values = chunks; - const length = values.reduce((sum, value) => sum + value.byteLength, 0); - const bytes = new Uint8Array(length); - let offset = 0; - for (const value of values) { - bytes.set(value, offset); - offset += value.byteLength; - } - const ready = yield* decodeReady(bytes); - if (ready.kind !== "ready") return ready; - yield* validateReady(ready, options, payload.ownerSessionId); - const metadata = yield* readOwnerMetadata( - options.environment.stateRoot, - options.stackId, - options.environment, - ); - if (metadata === undefined) - return yield* mapFailure("Supervisor readiness arrived before owner metadata"); - if ( - metadata.ownerSessionId !== ready.ownerSessionId || - metadata.rpcRelease !== STACK_RPC_RELEASE - ) - return yield* mapFailure("Supervisor readiness metadata identity mismatch"); - return { kind: "ready", metadata } satisfies ChildResult; - }).pipe(Effect.catchTag("PlatformError", (error) => Effect.fail(mapFailure(error.message)))); - const terminateLaunch: Effect.Effect = Effect.all([ - Effect.ignore(child.kill()), - Fiber.interrupt(ownerFiber), - ]).pipe(Effect.asVoid); - const childResult = yield* readiness.pipe( - Effect.timeoutOrElse({ - duration: SUPERVISOR_READINESS_TIMEOUT_MS, - orElse: () => Effect.fail(mapFailure("Supervisor did not publish readiness in time")), - }), - Effect.tapError(() => terminateLaunch), - Effect.onInterrupt(() => terminateLaunch), - ); - if (childResult.kind === "ownership-conflict") { - const current = yield* readOwnerMetadata( - options.environment.stateRoot, - options.stackId, - options.environment, - ); - if (current !== undefined) { - // The watcher is only a launch-time helper once the winner is visible. - yield* Fiber.interrupt(ownerFiber); - const childConflict = new StackOwnershipConflictError({ - message: childResult.message, - stackId: options.stackId, - }); - const preserveChildConflict = () => Effect.fail(childConflict); - return yield* validateCompatibleOwner(options, current).pipe( - Effect.map(() => ({ owner: current, launched: false }) satisfies EnsureSupervisorResult), - Effect.catchTags({ - StackStateInvalidError: preserveChildConflict, - StackOwnershipConflictError: preserveChildConflict, - }), - ); - } - return yield* Fiber.join(ownerFiber).pipe( - Effect.timeoutOrElse({ - duration: 5_000, - orElse: () => - Fiber.interrupt(ownerFiber).pipe( - Effect.andThen( - Effect.fail( - new StackOwnershipConflictError({ - message: "Supervisor ownership conflict did not publish a compatible owner", - stackId: options.stackId, - }), - ), - ), - ), - }), - Effect.map((owner) => ({ owner, launched: false }) satisfies EnsureSupervisorResult), - ); - } - yield* Fiber.interrupt(ownerFiber); - if (childResult.kind === "failed") { - yield* terminateLaunch; - return yield* mapFailure(childResult.message); - } - return { - owner: childResult.metadata, - launched: true, - } satisfies EnsureSupervisorResult; - }).pipe(Effect.catchTag("PlatformError", (error) => Effect.fail(mapFailure(error.message)))); - -/** - * Starts one detached owner process or joins the owner concurrently published - * by another caller. The lock itself is acquired in the supervisor process. - */ -export const ensureSupervisor = ( - options: LauncherOptions, -): Effect.Effect< - EnsureSupervisorResult, - | StackOwnershipConflictError - | StackStateFormatUnsupportedError - | StackStateInvalidError - | StackUpgradeRequiredError, - | FileSystem.FileSystem - | Path.Path - | Crypto.Crypto - | Scope.Scope - | ChildProcessSpawner.ChildProcessSpawner -> => - Effect.gen(function* () { - const state = yield* options.stateStore.read(options.stackId).pipe( - Effect.catchIf(isMissingStateRemnantError, () => - options.stateStore.recoverRuntimeRemnant(options.stackId).pipe(Effect.asVoid), - ), - Effect.mapError((error) => - error instanceof StackStateFormatUnsupportedError - ? error - : mapFailure(error instanceof Error ? error.message : String(error)), - ), - ); - if (state === undefined) return yield* mapFailure("Stack state is missing"); - const existing = yield* readOwnerMetadata( - options.environment.stateRoot, - options.stackId, - options.environment, - ); - if (existing !== undefined) { - const compatible = yield* validateCompatibleOwner(options, existing).pipe( - Effect.as(true), - // A published document with an unavailable control endpoint may be a - // crashed owner. Let the child arbitrate through the OS-held lease; - // malformed metadata remains fail-closed in readOwnerMetadata. - Effect.catchTag("StackStateInvalidError", () => Effect.succeed(false)), - ); - if (compatible) return { owner: existing, launched: false } satisfies EnsureSupervisorResult; - } - const stackId = yield* Schema.decodeEffect(StackIdSchema)(options.stackId).pipe( - Effect.mapError((error) => mapFailure(`Invalid StackId: ${String(error)}`)), - ); - const paths = yield* resolveStackPaths({ - stateRoot: options.environment.stateRoot, - stackId, - }).pipe(Effect.mapError((error) => mapFailure(String(error)))); - const crypto = yield* Crypto.Crypto; - const ownerSessionId = yield* crypto.randomUUIDv4.pipe( - Effect.mapError((error) => - mapFailure(`Unable to allocate owner session id: ${error.message}`), - ), - ); - const payload: SupervisorArgs = { - stateRoot: options.environment.stateRoot, - ...(options.environment.artifactCacheRoot === undefined - ? {} - : { artifactCacheRoot: options.environment.artifactCacheRoot }), - tempRoot: options.environment.tempRoot, - platform: options.environment.platform, - stackId: options.stackId, - ownerSessionId, - }; - yield* (yield* FileSystem.FileSystem) - .makeDirectory(paths.runtime, { recursive: true }) - .pipe( - Effect.mapError((error) => - mapFailure(`Unable to prepare owner runtime directory: ${error.message}`), - ), - ); - const owner = yield* launchAndAwait(options, payload, { stackRoot: paths.stackRoot }); - if (!owner.launched) yield* validateCompatibleOwner(options, owner.owner); - return owner; - }); diff --git a/packages/stack/src/supervisor/Lifecycle.ts b/packages/stack/src/supervisor/Lifecycle.ts deleted file mode 100644 index a47fab5257..0000000000 --- a/packages/stack/src/supervisor/Lifecycle.ts +++ /dev/null @@ -1,435 +0,0 @@ -import { Cause, Crypto, Effect, Exit, FileSystem, Path, Predicate, Redacted } from "effect"; -import type { StackDefinition, CompiledStack, SecretSlotInput } from "../model/Compiler.ts"; -import { compileStack, rebuildExecutionPlan, sameDefinition } from "../model/Compiler.ts"; -import type { ExecutionPlan } from "../model/ExecutionPlan.ts"; -import type { StackConfig } from "../public/Config.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import type { StackId } from "../public/StackId.ts"; -import { - StackLifecycleConflictError, - StackMustBeStoppedError, - StackStateInvalidError, - type StackError, -} from "../public/Errors.ts"; -import type { PersistedSecretValues, PersistedStackState } from "../state/StackState.ts"; -import type { StackStateStore } from "../state/StackStateStore.ts"; -import { - resolveSecrets, - type SecretCandidate, - type SecretDeclaration, -} from "../state/SecretStore.ts"; - -/** - * The runtime-facing contract has no Docker/native concepts. Concrete drivers own resources; - * this controller owns accepted durable intent and lifecycle transitions. - */ -export interface LifecycleInput { - readonly stackId: StackId; - readonly state: PersistedStackState; - readonly definition: StackDefinition; - readonly secrets: PersistedSecretValues; - readonly plan: ExecutionPlan; -} - -export interface LifecycleBackend { - /** Must complete all runtime/resource validation before the controller writes accepted intent. */ - readonly preflight: (input: LifecycleInput) => Effect.Effect; - /** Applies the desired lifecycle to runtime resources for one accepted definition. */ - readonly launch: ( - input: LifecycleInput, - session: "fresh" | "current", - ) => Effect.Effect; - /** Removes runtime resources while retaining durable state/data (stop path). */ - readonly cleanup: Effect.Effect; - /** Removes all exact runtime resources and persistent data (destroy path). */ - readonly destroyData: Effect.Effect; -} - -export type CleanupOutcome = - | { readonly _tag: "proven" } - | { readonly _tag: "unproven"; readonly cause: Cause.Cause }; - -export type LifecycleLaunchResult = - | { readonly _tag: "started"; readonly rollback: Effect.Effect } - | { - readonly _tag: "failed"; - readonly cause: Cause.Cause; - readonly cleanup: CleanupOutcome; - }; - -type LifecycleStartOutcome = - | { readonly _tag: "started"; readonly state: PersistedStackState } - | { - readonly _tag: "failed"; - readonly cause: Cause.Cause; - readonly cleanup: CleanupOutcome; - readonly durable: "stopped" | "unsafe"; - }; - -interface LifecycleStartOptions { - readonly config?: StackConfig; - /** A new Supervisor recovering running intent begins a fresh runtime session. */ - readonly freshSession?: boolean; -} - -export interface LifecycleController { - readonly start: ( - options?: LifecycleStartOptions, - ) => Effect.Effect; - readonly stop: Effect.Effect; - readonly destroy: Effect.Effect; -} - -type LifecycleRequirements = Crypto.Crypto | FileSystem.FileSystem | Path.Path; - -export interface LifecycleControllerOptions { - readonly stackId: StackId; - readonly runtime: StackRuntime; - readonly stateStore: StackStateStore; - readonly backend: LifecycleBackend; -} - -interface Candidate { - readonly definition: StackDefinition; - readonly secrets: PersistedSecretValues; - readonly plan: ExecutionPlan; -} - -const missingState = (stackId: StackId): StackStateInvalidError => - new StackStateInvalidError({ - stackId, - message: "Stack state is missing; refusing lifecycle mutation", - }); - -const lifecycleConflict = (message: string): StackLifecycleConflictError => - new StackLifecycleConflictError({ message }); - -const declarationsFromPersisted = (secrets: PersistedSecretValues): SecretCandidate => ({ - declarations: Object.entries(secrets).map(([slot, entry]) => ({ - slot, - policy: entry.policy, - ...(entry.policy === "passthrough" ? { value: Redacted.make(entry.value) } : {}), - })), -}); - -const declarationsFromCompiled = (compiled: CompiledStack): SecretCandidate => ({ - declarations: compiled.secrets.map((entry: SecretSlotInput): SecretDeclaration => ({ - slot: entry.slot, - policy: entry.policy, - ...(entry.value === undefined ? {} : { value: entry.value }), - ...(entry.generator === undefined ? {} : { generator: entry.generator }), - })), -}); - -const sameSecrets = (left: PersistedSecretValues, right: PersistedSecretValues): boolean => { - const leftEntries = Object.entries(left).sort(([a], [b]) => a.localeCompare(b)); - const rightEntries = Object.entries(right).sort(([a], [b]) => a.localeCompare(b)); - if (leftEntries.length !== rightEntries.length) return false; - return leftEntries.every(([slot, value], index) => { - const other = rightEntries[index]; - return ( - other !== undefined && - slot === other[0] && - value.policy === other[1].policy && - value.value === other[1].value - ); - }); -}; - -const materializeCandidate = ( - state: PersistedStackState, - runtime: StackRuntime, - config: StackConfig | undefined, -): Effect.Effect => - Effect.gen(function* () { - if (config === undefined && state.definition !== undefined) { - const plan = yield* rebuildExecutionPlan(runtime, state.definition); - const resolved = yield* resolveSecrets( - declarationsFromPersisted(state.secrets), - state.secrets, - state.desiredLifecycle, - ); - return { - definition: state.definition, - secrets: resolved.persisted, - plan, - }; - } - const compiled = yield* compileStack( - { - projectRoot: state.identity.projectRoot, - runtime, - config, - }, - state.definition === undefined ? undefined : { definition: state.definition }, - ); - const resolved = yield* resolveSecrets( - declarationsFromCompiled(compiled), - state.secrets, - state.desiredLifecycle, - ); - return { - definition: compiled.definition, - secrets: resolved.persisted, - plan: compiled.executionPlan, - }; - }); - -const lifecycleInput = ( - stackId: StackId, - state: PersistedStackState, - candidate: Candidate, -): LifecycleInput => ({ - stackId, - state, - definition: candidate.definition, - secrets: candidate.secrets, - plan: candidate.plan, -}); - -const stateWithCandidate = ( - state: PersistedStackState, - candidate: Candidate, - desiredLifecycle: PersistedStackState["desiredLifecycle"], -): PersistedStackState => ({ - ...state, - desiredLifecycle, - definition: candidate.definition, - secrets: candidate.secrets, -}); - -/** Creates one Supervisor-local lifecycle owner. Mutable coordination is allocated per Effect run. */ -export const makeLifecycleController = ( - options: LifecycleControllerOptions, -): Effect.Effect => - Effect.sync(() => { - const read = (): Effect.Effect => - options.stateStore - .read(options.stackId) - .pipe( - Effect.flatMap((state) => - state === undefined - ? Effect.fail(missingState(options.stackId)) - : Effect.succeed(state), - ), - ); - const persistNonRunningAfterFailure = ( - primary: Cause.Cause, - restore: { - readonly cleanup: boolean; - readonly restoreLifecycle: "stopped" | "unconfigured"; - }, - ): Effect.Effect => - Effect.gen(function* () { - const current = yield* options.stateStore.read(options.stackId).pipe(Effect.exit); - let cause = primary; - let durable: "stopped" | "unsafe" = "unsafe"; - if (Exit.isFailure(current)) { - cause = Cause.combine(cause, current.cause); - } else if (current.value === undefined) { - cause = Cause.combine(cause, Cause.fail(missingState(options.stackId))); - } else { - const persisted = yield* options.stateStore - .replace(options.stackId, { - ...current.value, - desiredLifecycle: restore.restoreLifecycle, - }) - .pipe(Effect.exit); - if (Exit.isFailure(persisted)) cause = Cause.combine(cause, persisted.cause); - else durable = "stopped"; - } - let cleanupOutcome: CleanupOutcome = { _tag: "proven" }; - if (restore.cleanup) { - const cleaned = yield* options.backend.cleanup.pipe(Effect.exit); - if (Exit.isFailure(cleaned)) { - cleanupOutcome = { _tag: "unproven", cause: cleaned.cause }; - cause = Cause.combine(cause, cleaned.cause); - } - } - return { _tag: "failed", cause, cleanup: cleanupOutcome, durable }; - }); - - const startOutcome = ( - startOptions?: LifecycleStartOptions, - ): Effect.Effect => { - const supplied = startOptions?.config; - const failed = ( - cause: Cause.Cause, - durable: "stopped" | "unsafe", - ): LifecycleStartOutcome => ({ - _tag: "failed", - cause, - cleanup: { _tag: "proven" }, - durable, - }); - return Effect.gen(function* () { - const initialRead = yield* options.stateStore.read(options.stackId).pipe(Effect.exit); - if (Exit.isFailure(initialRead)) return failed(initialRead.cause, "unsafe"); - if (initialRead.value === undefined) - return failed(Cause.fail(missingState(options.stackId)), "stopped"); - const initial = initialRead.value; - if (initial.desiredLifecycle === "destroying") - return yield* lifecycleConflict("Stack is being destroyed"); - - const freshSession = - initial.desiredLifecycle === "running" && startOptions?.freshSession === true; - const materialized = yield* materializeCandidate(initial, initial.runtime, supplied).pipe( - Effect.exit, - ); - if (Exit.isFailure(materialized)) { - if (freshSession) - return yield* persistNonRunningAfterFailure(materialized.cause, { - cleanup: false, - restoreLifecycle: "stopped", - }); - return failed( - materialized.cause, - initial.desiredLifecycle === "running" ? "unsafe" : "stopped", - ); - } - const candidate = materialized.value; - if (initial.desiredLifecycle === "running") { - if ( - supplied !== undefined && - (initial.definition === undefined || - !sameDefinition(candidate.definition, initial.definition)) - ) { - const error = new StackMustBeStoppedError({ - stackId: options.stackId, - message: "Running stack input changed; stop the stack before applying it", - guidance: "Use stop() followed by start() to apply stopped-time changes", - }); - if (freshSession) - return yield* persistNonRunningAfterFailure(Cause.fail(error), { - cleanup: false, - restoreLifecycle: "stopped", - }); - return failed(Cause.fail(error), "unsafe"); - } - if (supplied !== undefined && !sameSecrets(candidate.secrets, initial.secrets)) { - const error = new StackMustBeStoppedError({ - stackId: options.stackId, - message: "Running stack secrets changed; stop the stack before applying them", - guidance: "Use stop() followed by start() to apply stopped-time changes", - }); - if (freshSession) - return yield* persistNonRunningAfterFailure(Cause.fail(error), { - cleanup: false, - restoreLifecycle: "stopped", - }); - return failed(Cause.fail(error), "unsafe"); - } - if (freshSession) { - const preflighted = yield* options.backend - .preflight(lifecycleInput(options.stackId, initial, candidate)) - .pipe(Effect.exit); - if (Exit.isFailure(preflighted)) - return yield* persistNonRunningAfterFailure(preflighted.cause, { - cleanup: false, - restoreLifecycle: "stopped", - }); - } - const launched = yield* options.backend - .launch( - lifecycleInput(options.stackId, initial, candidate), - freshSession ? "fresh" : "current", - ) - .pipe(Effect.exit); - if (Exit.isFailure(launched) && freshSession) - return yield* persistNonRunningAfterFailure(launched.cause, { - cleanup: true, - restoreLifecycle: "stopped", - }); - if (Exit.isFailure(launched)) return failed(launched.cause, "unsafe"); - if (Predicate.isTagged(launched.value, "failed")) { - if (freshSession) - return yield* persistNonRunningAfterFailure(launched.value.cause, { - cleanup: true, - restoreLifecycle: "stopped", - }); - return { - _tag: "failed", - cause: launched.value.cause, - cleanup: launched.value.cleanup, - durable: "unsafe", - }; - } - return { _tag: "started", state: initial }; - } - - const preflighted = yield* options.backend - .preflight(lifecycleInput(options.stackId, initial, candidate)) - .pipe(Effect.exit); - if (Exit.isFailure(preflighted)) return failed(preflighted.cause, "stopped"); - const next = stateWithCandidate(initial, candidate, "running"); - const persisted = yield* options.stateStore - .replace(options.stackId, next) - .pipe(Effect.exit); - if (Exit.isFailure(persisted)) return failed(persisted.cause, "unsafe"); - const started = yield* options.backend - .launch(lifecycleInput(options.stackId, next, candidate), "fresh") - .pipe(Effect.exit); - if (Exit.isSuccess(started)) { - if (Predicate.isTagged(started.value, "failed")) - return yield* persistNonRunningAfterFailure(started.value.cause, { - cleanup: true, - restoreLifecycle: - initial.desiredLifecycle === "unconfigured" ? "unconfigured" : "stopped", - }); - return { _tag: "started", state: next }; - } - - return yield* persistNonRunningAfterFailure(started.cause, { - cleanup: true, - restoreLifecycle: - initial.desiredLifecycle === "unconfigured" ? "unconfigured" : "stopped", - }); - }); - }; - - const stop: Effect.Effect = - Effect.suspend(() => - Effect.gen(function* () { - const current = yield* read(); - if (current.desiredLifecycle === "unconfigured") { - // Even an unconfigured stack may have exact runtime remnants from an interrupted - // first start. Stop is the explicit retry boundary for that cleanup. - yield* options.backend.cleanup; - return current; - } - if (current.desiredLifecycle === "destroying") - return yield* lifecycleConflict("Stack is being destroyed"); - const stopped: PersistedStackState = - current.desiredLifecycle === "stopped" - ? current - : { ...current, desiredLifecycle: "stopped" }; - if (stopped !== current) yield* options.stateStore.replace(options.stackId, stopped); - yield* options.backend.cleanup; - return stopped; - }), - ); - - const destroy: Effect.Effect = Effect.suspend(() => - Effect.gen(function* () { - const current = yield* read(); - const destroying: PersistedStackState = - current.desiredLifecycle === "destroying" - ? current - : { ...current, desiredLifecycle: "destroying" }; - if (destroying !== current) yield* options.stateStore.replace(options.stackId, destroying); - yield* destroyRuntime; - }), - ); - - const destroyRuntime: Effect.Effect = Effect.gen( - function* () { - // Destructive cleanup is the single runtime teardown path. It is exact and idempotent, - // so it also handles an unconfigured state left behind by an interrupted first start. - // Persisted configuration need not compile in order to remove runtime remnants. - yield* options.backend.destroyData; - yield* options.stateStore.cleanup(options.stackId); - }, - ); - - return { start: startOutcome, stop, destroy } satisfies LifecycleController; - }); diff --git a/packages/stack/src/supervisor/LogStore.ts b/packages/stack/src/supervisor/LogStore.ts deleted file mode 100644 index 8b536bc62a..0000000000 --- a/packages/stack/src/supervisor/LogStore.ts +++ /dev/null @@ -1,463 +0,0 @@ -import { Data, DateTime, Effect, FileSystem, Path, Predicate, Schema, Semaphore } from "effect"; -import { - StackLogEntrySchema, - type LogCursor, - type LogQuery, - type StackLogEntry, - type StackLogSource, -} from "../public/Logs.ts"; -import { InvalidLogCursorError } from "../public/Errors.ts"; -import { redactKnownSecrets } from "../state/SecretStore.ts"; - -const CURSOR_PREFIX = "v1_"; -/** Keep at most twice the configured retention before compacting the file. */ -const COMPACTION_FACTOR = 2; -const DEFAULT_LOG_MAX_ENTRIES = 1_000; -const DEFAULT_LOG_MAX_BYTES = 1024 * 1024; - -/** File or decoding failure while opening or writing retained logs. */ -export class LogStoreError extends Data.TaggedError("LogStoreError")<{ - readonly path: string; - readonly message: string; - readonly cause?: unknown; -}> {} - -export interface LogStoreOptions { - /** Explicit owner-only retained log file. */ - readonly path: string; - /** Maximum number of retained records. Defaults to 1,000. */ - readonly maxEntries?: number; - /** Maximum encoded retained document size. Defaults to 1 MiB. */ - readonly maxBytes?: number; - /** Exact values redacted before persistence and publication. */ - readonly knownSecrets?: ReadonlyArray; -} - -export interface LogRecord { - readonly timestamp?: string; - readonly source: StackLogSource; - readonly stream: StackLogEntry["stream"]; - readonly message: string; -} - -/** Internal retained-log query. */ -interface LogStoreQuery { - readonly cursor?: LogCursor; -} - -export interface RetainedLogReadOptions extends LogStoreQuery { - readonly maxEntries?: number; - readonly maxBytes?: number; -} - -/** Cursor returned when a retained log file has no entries to advance to. */ -export const EMPTY_LOG_CURSOR: LogCursor = { opaque: "v1_0" }; - -/** Apply public log filtering and pagination to a retained read. */ -export const selectLogBatch = ( - scanned: ReadonlyArray, - query?: Pick, -): { readonly entries: ReadonlyArray; readonly cursor: LogCursor } => { - const capabilities = query?.capabilities === undefined ? undefined : new Set(query.capabilities); - const filtered = scanned.filter((entry) => - capabilities === undefined - ? true - : entry.source !== "gateway" && - entry.source !== "supervisor" && - capabilities.has(entry.source), - ); - const entries = - query?.tail === undefined - ? filtered - : query.tail <= 0 - ? [] - : filtered.slice(-Math.floor(query.tail)); - return { - entries, - cursor: scanned.at(-1)?.cursor ?? query?.cursor ?? EMPTY_LOG_CURSOR, - }; -}; - -export interface LogStore { - readonly path: string; - /** - * Appends one record. An individually oversized record is returned to the caller but is not - * retained, and does not advance the cursor or rewrite the retained log file. - */ - readonly append: (record: LogRecord) => Effect.Effect; - /** Returns retained records after the supplied cursor. */ - readonly read: ( - options?: LogStoreQuery, - ) => Effect.Effect, LogStoreError | InvalidLogCursorError>; -} - -interface LogDocument { - readonly nextCursor: number; - readonly entries: ReadonlyArray; -} - -interface LoadedLogDocument { - readonly document: LogDocument; - /** The source ended with a discarded, incomplete JSONL record. */ - readonly repairRequired: boolean; -} - -const emptyDocument = (): LogDocument => ({ - nextCursor: 1, - entries: [], -}); - -const fileError = (path: string, message: string, cause?: unknown) => - new LogStoreError({ path, message, ...(cause === undefined ? {} : { cause }) }); - -const opaqueCursor = (value: number): LogCursor => ({ - opaque: `${CURSOR_PREFIX}${value.toString(36)}`, -}); - -const parseOpaqueCursor = (path: string, opaque: string): Effect.Effect => { - if (!/^v1_[0-9a-z]+$/.test(opaque)) return Effect.fail(fileError(path, "Log cursor is invalid")); - const value = Number.parseInt(opaque.slice(CURSOR_PREFIX.length), 36); - return Number.isSafeInteger(value) && value > 0 - ? Effect.succeed(value) - : Effect.fail(fileError(path, "Log cursor is out of range")); -}; - -const decodeCursor = ( - path: string, - cursor: LogCursor | undefined, -): Effect.Effect => { - if (cursor === undefined) return Effect.map(Effect.void, () => undefined); - return parseOpaqueCursor(path, cursor.opaque).pipe( - Effect.mapError((error) => new InvalidLogCursorError({ message: error.message, cause: error })), - ); -}; - -const selected = ( - entries: ReadonlyArray, - options: LogStoreQuery | undefined, - path: string, -): Effect.Effect, InvalidLogCursorError> => - Effect.map(decodeCursor(path, options?.cursor), (after) => - after === undefined - ? entries - : entries.filter( - (entry) => Number.parseInt(entry.cursor.opaque.slice(CURSOR_PREFIX.length), 36) > after, - ), - ); - -const validateDocument = ( - path: string, - document: LogDocument, -): Effect.Effect => - Effect.gen(function* () { - if (!Number.isSafeInteger(document.nextCursor) || document.nextCursor < 1) - return yield* fileError(path, "Retained log next cursor is invalid"); - let previous = 0; - for (const entry of document.entries) { - const cursor = yield* parseOpaqueCursor(path, entry.cursor.opaque); - if (!Number.isSafeInteger(cursor) || cursor <= previous) - return yield* fileError(path, "Retained log cursors are not strictly increasing"); - previous = cursor; - } - if (document.nextCursor <= previous) - return yield* fileError(path, "Retained log next cursor is not ahead of entries"); - return document; - }); - -const readDocument = ( - fs: FileSystem.FileSystem, - path: string, -): Effect.Effect => { - const parse = (text: string, sourcePath: string) => { - const rawLines = text.split(/\r?\n/); - // A process crash can leave one unterminated tail after an append. Keep all complete JSONL - // records and drop only that final tail; a file containing no complete record remains an error. - const hasCompleteTail = text.endsWith("\n"); - const lines = ( - hasCompleteTail || rawLines.length === 1 ? rawLines : rawLines.slice(0, -1) - ).filter((line) => line.length > 0); - return Effect.forEach(lines, (line) => - Schema.decodeEffect(Schema.fromJsonString(StackLogEntrySchema))(line).pipe( - Effect.mapError((cause) => fileError(sourcePath, "Retained log file is malformed", cause)), - ), - ).pipe( - Effect.flatMap((entries) => { - const nextCursor = entries.reduce( - (next, entry) => - Math.max( - next, - Number.parseInt(entry.cursor.opaque.slice(CURSOR_PREFIX.length), 36) + 1, - ), - 1, - ); - return validateDocument(sourcePath, { nextCursor, entries }).pipe( - Effect.map((document) => ({ - document, - repairRequired: !hasCompleteTail && text.length > 0, - })), - ); - }), - ); - }; - return fs.readFileString(path).pipe( - Effect.flatMap((text) => { - return parse(text, path); - }), - Effect.catchTag("PlatformError", (error) => - Predicate.isTagged(error.reason, "NotFound") - ? fs.readFileString(`${path}.1`).pipe( - Effect.flatMap((text) => parse(text, `${path}.1`)), - Effect.catchTag("PlatformError", (backupError) => - Predicate.isTagged(backupError.reason, "NotFound") - ? Effect.succeed({ document: emptyDocument(), repairRequired: false }) - : Effect.fail(fileError(path, "Unable to read retained log file", backupError)), - ), - ) - : Effect.fail(fileError(path, "Unable to read retained log file", error)), - ), - ); -}; - -/** Read retained entries without opening or mutating a LogStore. */ -export const readRetainedLogs = ( - fs: FileSystem.FileSystem, - path: string, - options?: RetainedLogReadOptions, -): Effect.Effect, LogStoreError | InvalidLogCursorError> => - Effect.gen(function* () { - const loaded = yield* readDocument(fs, path); - const document = loaded.document; - const maxEntries = validLimit(options?.maxEntries, DEFAULT_LOG_MAX_ENTRIES); - const maxBytes = validLimit(options?.maxBytes, DEFAULT_LOG_MAX_BYTES); - if (maxEntries < 1 || maxBytes < 2) - return yield* fileError(path, "Log retention limits must be positive"); - const boundedDocument = yield* bounded(document, maxEntries, maxBytes, path); - return yield* selected(boundedDocument.document.entries, options, path); - }); - -const redactDocument = ( - document: LogDocument, - knownSecrets: ReadonlyArray, -): { readonly document: LogDocument; readonly changed: boolean } => { - let changed = false; - const entries = document.entries.map((entry) => { - const message = redactKnownSecrets(entry.message, knownSecrets); - if (message !== entry.message) changed = true; - return message === entry.message ? entry : { ...entry, message }; - }); - return { document: { ...document, entries }, changed }; -}; - -const encodedEntry = (entry: StackLogEntry, path: string): Effect.Effect => - Schema.encodeEffect(Schema.fromJsonString(StackLogEntrySchema))(entry).pipe( - Effect.mapError((cause) => fileError(path, "Unable to encode retained logs", cause)), - ); - -const encodedSize = ( - entries: ReadonlyArray, - path: string, -): Effect.Effect => - Effect.forEach(entries, (entry) => encodedEntry(entry, path), { concurrency: "unbounded" }).pipe( - Effect.map( - (lines) => - new TextEncoder().encode(lines.join("\n") + (lines.length > 0 ? "\n" : "")).byteLength, - ), - ); - -const bounded = ( - document: LogDocument, - maxEntries: number, - maxBytes: number, - path: string, - initialBytes?: number, -): Effect.Effect<{ readonly document: LogDocument; readonly bytes: number }, LogStoreError> => - Effect.gen(function* () { - const entries = [...document.entries]; - let bytes = initialBytes ?? (yield* encodedSize(entries, path)); - while (entries.length > maxEntries || bytes > maxBytes) { - if (entries.length === 0) break; - const removed = entries.shift(); - if (removed !== undefined) { - const encoded = yield* encodedEntry(removed, path); - bytes -= new TextEncoder().encode(`${encoded}\n`).byteLength; - } - } - return { document: { ...document, entries }, bytes }; - }); - -const persist = ( - fs: FileSystem.FileSystem, - path: string, - entries: ReadonlyArray, -): Effect.Effect => { - const temporary = `${path}.tmp`; - const write = Effect.gen(function* () { - const lines = yield* Effect.forEach(entries, (entry) => encodedEntry(entry, path)); - const text = lines.length === 0 ? "" : `${lines.join("\n")}\n`; - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(temporary, { flag: "w", mode: 0o600 }); - if (text.length > 0) yield* file.writeAll(new TextEncoder().encode(text)); - }), - ); - yield* fs.chmod(temporary, 0o600); - yield* fs.remove(`${path}.1`, { force: true }); - if (yield* fs.exists(path)) yield* fs.rename(path, `${path}.1`); - yield* fs.rename(temporary, path); - yield* fs.chmod(path, 0o600); - }); - return write.pipe( - Effect.ensuring( - fs - .remove(temporary, { force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ), - Effect.mapError((error) => - error instanceof LogStoreError - ? error - : fileError(path, "Unable to persist retained logs", error), - ), - ); -}; - -const validLimit = (value: number | undefined, fallback: number): number => - value === undefined || (Number.isFinite(value) && value >= 1) ? Math.floor(value ?? fallback) : 0; - -export const makeLogStore = ( - options: LogStoreOptions, -): Effect.Effect => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const pathService = yield* Path.Path; - const maxEntries = validLimit(options.maxEntries, DEFAULT_LOG_MAX_ENTRIES); - const maxBytes = validLimit(options.maxBytes, DEFAULT_LOG_MAX_BYTES); - if (maxEntries < 1 || maxBytes < 1) - return yield* fileError(options.path, "Log retention limits must be positive"); - - yield* fs - .makeDirectory(pathService.dirname(options.path), { recursive: true, mode: 0o700 }) - .pipe( - Effect.mapError((error) => - fileError(options.path, "Unable to create log directory", error), - ), - ); - const parentDirectory = pathService.dirname(options.path); - yield* fs - .chmod(parentDirectory, 0o700) - .pipe( - Effect.mapError((error) => - fileError(options.path, "Unable to secure retained log directory", error), - ), - ); - if (maxBytes < 2) - return yield* fileError(options.path, "Log byte retention limit is too small"); - const knownSecrets = options.knownSecrets ?? []; - const loaded = yield* readDocument(fs, options.path); - const redactedLoaded = redactDocument(loaded.document, knownSecrets); - const boundedLoaded = yield* bounded( - redactedLoaded.document, - maxEntries, - maxBytes, - options.path, - ); - let document = boundedLoaded.document; - let retainedBytes = boundedLoaded.bytes; - // The on-disk document may temporarily overshoot the configured limits between periodic - // compactions. Keep reads strictly bounded so the public retention contract remains stable. - let visibleEntries = boundedLoaded.document.entries; - const exists = yield* fs - .exists(options.path) - .pipe( - Effect.mapError((error) => - fileError(options.path, "Unable to inspect retained logs", error), - ), - ); - if (exists) - yield* fs - .chmod(options.path, 0o600) - .pipe( - Effect.mapError((error) => - fileError(options.path, "Unable to secure retained logs", error), - ), - ); - if ( - loaded.repairRequired || - redactedLoaded.changed || - boundedLoaded.document.entries.length !== loaded.document.entries.length - ) - yield* persist(fs, options.path, boundedLoaded.document.entries); - const semaphore = yield* Semaphore.make(1); - const append = (record: LogRecord) => - semaphore.withPermit( - // Persisting a record and advancing the in-memory cursor are one transaction. Without - // the mask, shutdown can interrupt after the file write but before `document = next`, - // causing the next append to reuse a cursor already present on disk. - Effect.uninterruptible( - Effect.gen(function* () { - const cursor = opaqueCursor(document.nextCursor); - const entry = { - cursor, - timestamp: record.timestamp ?? DateTime.formatIso(DateTime.nowUnsafe()), - source: record.source, - stream: record.stream, - message: redactKnownSecrets(record.message, knownSecrets), - } satisfies StackLogEntry; - const next = { - nextCursor: document.nextCursor + 1, - entries: [...document.entries, entry], - }; - const encoded = yield* encodedEntry(entry, options.path); - const entryBytes = new TextEncoder().encode(`${encoded}\n`).byteLength; - if (entryBytes > maxBytes) return entry; - const nextBytes = retainedBytes + entryBytes; - const withinCompactionWindow = - document.entries.length + 1 <= maxEntries * COMPACTION_FACTOR && - nextBytes <= maxBytes * COMPACTION_FACTOR; - if (withinCompactionWindow) { - const nextVisible = - next.entries.length <= maxEntries && nextBytes <= maxBytes - ? next.entries - : (yield* bounded(next, maxEntries, maxBytes, options.path, nextBytes)).document - .entries; - yield* Effect.scoped( - Effect.gen(function* () { - const file = yield* fs.open(options.path, { flag: "a", mode: 0o600 }); - yield* file.writeAll(new TextEncoder().encode(`${encoded}\n`)); - }), - ).pipe( - Effect.mapError((error) => - error instanceof LogStoreError - ? error - : fileError(options.path, "Unable to append retained logs", error), - ), - ); - document = next; - retainedBytes = nextBytes; - visibleEntries = nextVisible; - } else { - const boundedNext = yield* bounded( - next, - maxEntries, - maxBytes, - options.path, - nextBytes, - ); - yield* persist(fs, options.path, boundedNext.document.entries); - document = boundedNext.document; - retainedBytes = boundedNext.bytes; - visibleEntries = boundedNext.document.entries; - } - return entry; - }), - ), - ); - - const read = (readOptions?: LogStoreQuery) => - semaphore.withPermit(selected(visibleEntries, readOptions, options.path)); - - return { - path: options.path, - append, - read, - }; - }); diff --git a/packages/stack/src/supervisor/SessionLauncher.ts b/packages/stack/src/supervisor/SessionLauncher.ts deleted file mode 100644 index 39e7aaf749..0000000000 --- a/packages/stack/src/supervisor/SessionLauncher.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { Cause, Data, Deferred, Effect, Exit, Match, Ref, Semaphore } from "effect"; -import type { ExecutionPlan, PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { StackId } from "../public/StackId.ts"; -import { - RuntimeDriverError, - type RuntimeDriver, - type RuntimeWorkloadKey, -} from "../runtime/RuntimeDriver.ts"; - -interface SessionWorkload { - readonly key: RuntimeWorkloadKey; - readonly workload: PlannedWorkload; -} - -/** A cleanup failure retains ownership because at least one remove was unproven. */ -export class SessionCleanupError extends Data.TaggedError("SessionCleanupError")<{ - readonly cause: Cause.Cause; -}> {} - -export interface SessionLauncher { - /** Starts the supplied dependency closure as dependencies complete. */ - readonly launch: ( - plan: ExecutionPlan, - cancellation?: Deferred.Deferred, - ) => Effect.Effect; - /** Stops and removes every workload started in this session in reverse order. */ - readonly stop: Effect.Effect; - readonly stopCapabilities: ( - capabilities: ReadonlySet, - ) => Effect.Effect; - /** Clears the session after stack-wide runtime cleanup has completed. */ - readonly clear: Effect.Effect; -} - -/** Resources created by one launch attempt and a rollback scoped to that attempt. */ -interface SessionLaunch { - readonly rollback: Effect.Effect; -} - -export type SessionCleanupOutcome = - | { readonly _tag: "proven" } - | { - readonly _tag: "unproven"; - readonly cause: Cause.Cause; - }; - -export type SessionLaunchOutcome = - | { readonly _tag: "started"; readonly launch: SessionLaunch } - | { - readonly _tag: "failed"; - readonly cause: Cause.Cause; - readonly cleanup: SessionCleanupOutcome; - }; - -const START_CONCURRENCY = 4; - -const keyFor = (stackId: StackId, workload: PlannedWorkload): RuntimeWorkloadKey => ({ - stackId, - workloadId: workload.id, -}); - -type SessionError = RuntimeDriverError | SessionCleanupError; - -const combine = ( - primary: Cause.Cause, - cleanup: Cause.Cause, -): Cause.Cause => - cleanup.reasons.length === 0 ? primary : Cause.combine(primary, cleanup); - -const joinExit = (result: Exit.Exit): Effect.Effect => - Exit.isSuccess(result) ? Effect.succeed(result.value) : Effect.failCause(result.cause); - -/** - * Owns only the workloads started by the current Supervisor session. A launch starts - * dependency-ready workloads as soon as their prerequisites complete and a failure cleans - * that attempt's resources. - */ -export const makeSessionLauncher = (options: { - readonly stackId: StackId; - readonly driver: RuntimeDriver; -}): Effect.Effect => - Effect.gen(function* () { - const session = yield* Ref.make>([]); - const cleanup = ( - entries: ReadonlyArray, - ): Effect.Effect => - Effect.gen(function* () { - let cleanupCause: Cause.Cause = Cause.empty; - for (const entry of [...entries].reverse()) { - const stopped = yield* Effect.exit(options.driver.stop(entry.key)); - if (Exit.isFailure(stopped)) cleanupCause = combine(cleanupCause, stopped.cause); - const removed = yield* Effect.exit(options.driver.remove(entry.key)); - if (Exit.isFailure(removed)) cleanupCause = combine(cleanupCause, removed.cause); - else - yield* Ref.update(session, (current) => - current.filter((candidate) => candidate.key.workloadId !== entry.key.workloadId), - ); - } - return cleanupCause.reasons.length === 0 - ? ({ _tag: "proven" } satisfies SessionCleanupOutcome) - : ({ _tag: "unproven", cause: cleanupCause } satisfies SessionCleanupOutcome); - }); - - const launch = ( - plan: ExecutionPlan, - cancellation?: Deferred.Deferred, - ): Effect.Effect => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const cancel = cancellation ?? (yield* Deferred.make()); - const attempted: SessionWorkload[] = []; - const planEntries = plan.workloads.map((workload) => ({ - key: keyFor(options.stackId, workload), - workload, - })); - const ready = new Set((yield* Ref.get(session)).map(({ key }) => key.workloadId)); - const remaining = planEntries.filter((entry) => !ready.has(entry.workload.id)); - const unresolved = new Set(remaining.map((entry) => entry.workload.id)); - const graphReady = new Set(ready); - // Reject cycles before creating fibers that would otherwise wait forever. - while (unresolved.size > 0) { - const completed = [...unresolved].filter((id) => { - const entry = planEntries.find((candidate) => candidate.workload.id === id); - return ( - entry !== undefined && - entry.workload.dependencies.every((dependency) => graphReady.has(dependency)) - ); - }); - if (completed.length === 0) { - const failure = new RuntimeDriverError({ - message: "No workload is ready to start; dependencies are unsatisfied", - stackId: options.stackId, - }); - const cleaned = yield* cleanup(attempted); - return { - _tag: "failed", - cause: Match.value(cleaned).pipe( - Match.when({ _tag: "unproven" }, (value) => - combine(Cause.fail(failure), value.cause), - ), - Match.when({ _tag: "proven" }, () => Cause.fail(failure)), - Match.exhaustive, - ), - cleanup: cleaned, - } satisfies SessionLaunchOutcome; - } - completed.forEach((id) => unresolved.delete(id)); - completed.forEach((id) => graphReady.add(id)); - } - const startPermit = yield* Semaphore.make(START_CONCURRENCY); - const completions = new Map< - string, - Deferred.Deferred, never> - >(); - for (const entry of remaining) - completions.set( - entry.workload.id, - yield* Deferred.make>(), - ); - const startOne = (entry: SessionWorkload): Effect.Effect => { - const startBody = Effect.gen(function* () { - yield* Effect.forEach( - entry.workload.dependencies, - (dependency) => { - if (ready.has(dependency)) return Effect.void; - const completion = completions.get(dependency); - if (completion === undefined) - return Effect.fail( - new RuntimeDriverError({ - message: `Dependency ${dependency} is not part of the launch plan`, - stackId: options.stackId, - workloadId: entry.workload.id, - }), - ); - return Deferred.await(completion).pipe(Effect.flatMap(joinExit)); - }, - { discard: true }, - ); - yield* startPermit.withPermit( - Effect.gen(function* () { - // Record before entering the driver: a driver may acquire a resource and then - // fail or be interrupted before its start effect returns. - attempted.push(entry); - yield* Ref.update(session, (current) => - current.some((candidate) => candidate.key.workloadId === entry.key.workloadId) - ? current - : [...current, entry], - ); - yield* options.driver.start(entry.key, entry.workload); - }), - ); - }); - const cancelled = Deferred.await(cancel).pipe( - Effect.andThen( - Effect.fail( - new RuntimeDriverError({ - message: "Launch cancelled while another lifecycle prerequisite failed", - stackId: options.stackId, - workloadId: entry.workload.id, - }), - ), - ), - ); - const completion = completions.get(entry.workload.id); - return Effect.raceFirst(startBody, cancelled).pipe( - Effect.onExit((result) => - completion === undefined ? Effect.void : Deferred.succeed(completion, result), - ), - ); - }; - const outcome = yield* Effect.exit( - restore( - Effect.forEach(remaining, startOne, { concurrency: "unbounded", discard: true }), - ), - ); - if (Exit.isSuccess(outcome)) { - return { - _tag: "started", - launch: { rollback: cleanup(attempted) }, - } satisfies SessionLaunchOutcome; - } - const cleaned = yield* cleanup(attempted); - return { - _tag: "failed", - cause: Match.value(cleaned).pipe( - Match.when({ _tag: "unproven" }, (value) => combine(outcome.cause, value.cause)), - Match.when({ _tag: "proven" }, () => outcome.cause), - Match.exhaustive, - ), - cleanup: cleaned, - } satisfies SessionLaunchOutcome; - }), - ); - const cleanupOrFail = (entries: ReadonlyArray) => - cleanup(entries).pipe( - Effect.flatMap((outcome) => - Match.value(outcome).pipe( - Match.when({ _tag: "proven" }, () => Effect.void), - Match.when({ _tag: "unproven" }, (value) => - Effect.fail(new SessionCleanupError({ cause: value.cause })), - ), - Match.exhaustive, - ), - ), - ); - const stop = Effect.suspend(() => Ref.get(session).pipe(Effect.flatMap(cleanupOrFail))); - const stopCapabilities = ( - capabilities: ReadonlySet, - ) => - Ref.get(session).pipe( - Effect.flatMap((entries) => - cleanupOrFail(entries.filter(({ workload }) => capabilities.has(workload.capability))), - ), - ); - return { - launch, - stop, - stopCapabilities, - clear: Ref.set(session, []), - } satisfies SessionLauncher; - }); diff --git a/packages/stack/src/supervisor/StatusProjection.integration.test.ts b/packages/stack/src/supervisor/StatusProjection.integration.test.ts deleted file mode 100644 index eb15a1ee88..0000000000 --- a/packages/stack/src/supervisor/StatusProjection.integration.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Effect } from "effect"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { deriveStackId } from "../identity/Identity.ts"; -import type { ObservedWorkload } from "../runtime/RuntimeDriver.ts"; -import { ready } from "./CapabilityState.ts"; -import type { SupervisorSnapshot } from "./SupervisorState.ts"; -import { statusForSnapshot } from "./StatusProjection.ts"; - -const identity = { - projectRoot: "/tmp/status-projection", - branchContext: "ordinary-workspace", - stackName: "status-projection", -} as const; - -const snapshotFor = (): SupervisorSnapshot => { - const sessionId = Symbol("session"); - return { - stack: { _tag: "running" }, - sessionId, - plan: undefined, - capabilities: new Map([["studio", ready(sessionId, 0, false)]]), - }; -}; - -describe("status projection", () => { - it.effect("retains later workload errors when an earlier failure has no text", () => - Effect.gen(function* () { - const id = yield* deriveStackId(identity); - const compiled = yield* compileStack({ - projectRoot: identity.projectRoot, - runtime: { kind: "native" }, - config: {}, - }); - const state: PersistedStackState = { - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "running", - definition: compiled.definition, - ports: [], - privatePorts: [], - secrets: {}, - }; - const observed: ReadonlyArray = [ - { stackId: id, workloadId: "studio:studio", state: "failed" }, - { - stackId: id, - workloadId: "studio:pgmeta", - state: "failed", - error: "pg-meta failed", - }, - ]; - - const status = yield* statusForSnapshot( - id, - state, - { _tag: "available", workloads: observed }, - snapshotFor(), - ); - const studio = status.capabilities.find((capability) => capability.name === "studio"); - - expect(studio).toEqual({ - name: "studio", - activation: "lazy", - state: "failed", - error: "studio:pgmeta: pg-meta failed", - }); - - const allFailures = yield* statusForSnapshot( - id, - state, - { - _tag: "available", - workloads: [ - { - stackId: id, - workloadId: "studio:studio", - state: "failed", - error: "studio failed", - }, - { - stackId: id, - workloadId: "studio:pgmeta", - state: "failed", - error: "pg-meta failed", - }, - ], - }, - snapshotFor(), - ); - - expect(allFailures.capabilities.find((capability) => capability.name === "studio")).toEqual({ - name: "studio", - activation: "lazy", - state: "failed", - error: "studio:studio: studio failed; studio:pgmeta: pg-meta failed", - }); - }).pipe(Effect.provide(NodeServices.layer)), - ); -}); diff --git a/packages/stack/src/supervisor/StatusProjection.ts b/packages/stack/src/supervisor/StatusProjection.ts deleted file mode 100644 index 4b5be1d528..0000000000 --- a/packages/stack/src/supervisor/StatusProjection.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { Cause, Effect, Predicate } from "effect"; -import { CAPABILITY_NAMES, type CapabilityName } from "../public/Capability.ts"; -import type { StackId } from "../public/StackId.ts"; -import { - PORT_FIELD_PROTOCOL, - type ArtifactPreparationStatus, - type StackStatus, -} from "../public/Status.ts"; -import type { ObservedWorkload } from "../runtime/RuntimeDriver.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { recoveryForState, type SupervisorSnapshot } from "./SupervisorState.ts"; -import { publicCapabilityState } from "./CapabilityState.ts"; -import { publicPhase } from "./SupervisorTransitions.ts"; - -export type ActualPhase = "stopped" | "starting" | "running" | "stopping" | "destroying"; - -export type ObservedStatus = - | { readonly _tag: "available"; readonly workloads: ReadonlyArray } - | { readonly _tag: "unavailable" }; - -const observedForCapability = ( - name: CapabilityName, - observed: ReadonlyArray, -): ReadonlyArray => - observed.filter((entry) => entry.workloadId.startsWith(`${name}:`)); - -const observedFailureError = (failures: ReadonlyArray): string | undefined => { - if (failures.length === 1) return failures[0]?.error; - const details = failures.flatMap((entry) => - entry.error === undefined ? [] : [`${entry.workloadId}: ${entry.error}`], - ); - return details.length === 0 ? undefined : details.join("; "); -}; - -/** Projects one authoritative Supervisor snapshot while preserving observed runtime failures. */ -export const statusForSnapshot = ( - id: StackId, - state: PersistedStackState, - observedStatus: ObservedStatus, - snapshot: SupervisorSnapshot, - artifacts: ReadonlyArray = [], -): Effect.Effect => - Effect.sync(() => { - const definition = state.definition; - const observed: ReadonlyArray = Predicate.isTagged( - observedStatus, - "available", - ) - ? observedStatus.workloads - : []; - const observationAvailable = Predicate.isTagged(observedStatus, "available"); - const capabilities = CAPABILITY_NAMES.map((name) => { - const control = snapshot.capabilities.get(name); - const configured = definition?.capabilities[name]; - const observedEntries = observedForCapability(name, observed); - const observedFailures = observedEntries.filter((entry) => entry.state === "failed"); - const observedError = observedFailureError(observedFailures); - const projected = - control === undefined - ? configured === undefined || !configured.enabled - ? "disabled" - : "stopped" - : publicCapabilityState(control); - const observedStarting = observedEntries.some((entry) => entry.state === "starting"); - const observedUnready = observedEntries.some((entry) => entry.state !== "ready"); - const capability = - observedFailures.length > 0 && projected === "ready" - ? "failed" - : projected === "ready" && observedStarting - ? "starting" - : projected === "ready" && - observationAvailable && - observedEntries.length === 0 && - configured?.enabled === true - ? "stopped" - : projected === "ready" && observedEntries.length > 0 && observedUnready - ? "stopped" - : projected; - const cleanupError = Predicate.isTagged(control, "cleanup-failed") - ? Cause.pretty(control.cause) - : undefined; - return { - name, - activation: - configured?.activation ?? (name === "database" ? ("eager" as const) : ("lazy" as const)), - state: capability, - ...(cleanupError === undefined && observedError === undefined - ? {} - : { error: cleanupError ?? observedError }), - }; - }); - const versions: Partial> = {}; - if (definition !== undefined) - for (const name of CAPABILITY_NAMES) versions[name] = definition.capabilities[name].version; - const endpoints = state.ports.reduce((result, assignment) => { - const protocol = PORT_FIELD_PROTOCOL[assignment.field]; - const listener = definition?.listeners[assignment.field]; - return { - ...result, - [assignment.field]: { - protocol, - address: listener?.address ?? "127.0.0.1", - port: assignment.port, - url: `${protocol}://${listener?.address ?? "127.0.0.1"}:${assignment.port}`, - }, - }; - }, {}); - const recovery = recoveryForState(snapshot.stack); - const phase = publicPhase(snapshot.stack); - return { - id, - lifecycle: - phase === "stopped" && state.desiredLifecycle === "unconfigured" ? "unconfigured" : phase, - desiredLifecycle: state.desiredLifecycle, - runtime: state.runtime, - endpoints, - versions, - capabilities, - artifacts, - ...(recovery === undefined ? {} : { recovery }), - } satisfies StackStatus; - }); diff --git a/packages/stack/src/supervisor/Supervisor.ts b/packages/stack/src/supervisor/Supervisor.ts deleted file mode 100644 index d7f3b3bd39..0000000000 --- a/packages/stack/src/supervisor/Supervisor.ts +++ /dev/null @@ -1,1689 +0,0 @@ -import { - Cause, - Context, - Crypto, - Deferred, - Duration, - Effect, - Exit, - FileSystem, - Fiber, - FiberSet, - Match, - Option, - Path, - Predicate, - Redacted, - Ref, - Semaphore, - Scope, -} from "effect"; -import { rebuildExecutionPlan } from "../model/Compiler.ts"; -import { - activeExecutionPlan, - dependencyClosure, - eagerCapabilities, - type ExecutionPlan, -} from "../model/ExecutionPlan.ts"; -import { CAPABILITY_NAMES, type CapabilityName } from "../public/Capability.ts"; -import type { StackConfig } from "../public/Config.ts"; -import { - GatewayActivationError, - ContainerEngineError, - InvalidLogCursorError, - StackLifecycleConflictError, - StackNotRunningError, - StackRuntimeError, - StackCleanupError, - StackStateInvalidError, - isStackError, - isStackErrorTag, - type StackErrorTag, - type StackError, -} from "../public/Errors.ts"; -import type { ArtifactPreparationStatus, StackStatus } from "../public/Status.ts"; -import type { StackId } from "../public/StackId.ts"; -import type { LogQuery, StackLogBatch } from "../public/Logs.ts"; -import type { EffectStackCredentials } from "../public/Credentials.ts"; -import { - RuntimeDriverError, - type ObservedWorkload, - type RuntimeDriver, -} from "../runtime/RuntimeDriver.ts"; -import { withLeftoverPersistentDataGuidance } from "../runtime/Diagnostics.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { isMissingStateRemnantError, type StackStateStore } from "../state/StackStateStore.ts"; -import { - makeSessionLauncher, - SessionCleanupError, - type SessionCleanupOutcome, - type SessionLaunchOutcome, - type SessionLauncher, -} from "./SessionLauncher.ts"; -import { - makeLifecycleController, - type CleanupOutcome, - type LifecycleLaunchResult, - type LifecycleBackend, - type LifecycleInput, -} from "./Lifecycle.ts"; -import { EMPTY_LOG_CURSOR, selectLogBatch, type LogStore } from "./LogStore.ts"; -import type { SupervisorIngress } from "./Ingress.ts"; -import { - STACK_RPC_RELEASE, - StackRpcGroup, - type StackRpcError, - type StackRpcHandlers, -} from "../control/StackRpc.ts"; -import type { MaintenanceResponse } from "../control/MaintenanceProtocol.ts"; -import { statusForSnapshot, type ActualPhase, type ObservedStatus } from "./StatusProjection.ts"; -import { - recoveryForState, - type LifecycleKind, - type SupervisorSnapshot, -} from "./SupervisorState.ts"; -import { - AUTH_ANON_KEY_SLOT, - AUTH_PUBLISHABLE_KEY_SLOT, - AUTH_SECRET_KEY_SLOT, - AUTH_SERVICE_ROLE_KEY_SLOT, - DATABASE_INTERNAL_PASSWORD_SLOT, -} from "../state/SecretStore.ts"; - -import type { ActivationResult } from "../gateway/Gateway.ts"; -import { makeGatewayActivity } from "../gateway/ActivityTracker.ts"; -import { - admitLifecycle, - admitActivation, - activeLifecycle, - activationGate, - beginTraffic as transitionBeginTraffic, - beginRetirement as transitionBeginRetirement, - armRetirement as transitionArmRetirement, - claimWorkloads as transitionClaimWorkloads, - completeDormantCleanup as transitionCompleteDormantCleanup, - endTraffic as transitionEndTraffic, - enterCapabilityCleanup as transitionEnterCapabilityCleanup, - initializeSession, - promoteActivationSet as transitionPromoteActivationSet, - publicPhase, - planIdleTimer as transitionPlanIdleTimer, - settleLifecycleOwner, - settleRetirementOwner, - settleActivationTerminal as transitionSettleActivationTerminal, - readySet as transitionReadySet, - setRootSet as transitionSetRootSet, - settleCapabilityCleanup as transitionSettleCapabilityCleanup, - disarmAllRetirements as transitionDisarmAllRetirements, - type ActivationOwner, - type ActivationToken, - type ActivationClaims, - type ActivationTerminalOutcome, - type ActivationExit, - type ClaimedWorkload, - type CommandResult, - type CleanupHandle, - type EndpointExit, - isCleanupCandidate, - matchesActivationOwner, - type StartupHandle, - type SettlementOwner, - type SnapshotTransition, - type TransitionNotification, -} from "./SupervisorTransitions.ts"; - -/** Runtime construction is injected so catalog/artifact resolution can evolve independently. */ -export interface SupervisorRuntime { - readonly driver: RuntimeDriver; - readonly preflight: (input: LifecycleInput) => Effect.Effect; - /** Prepares artifacts before launching a newly selected workload closure. */ - readonly prepare: ( - input: LifecycleInput, - selected: ReadonlySet, - ) => Effect.Effect; - /** Best-effort preparation of lazy artifacts after a stack reaches running. */ - readonly prefetch: (state: PersistedStackState) => Effect.Effect; - /** Current in-memory preparation state; completed cache entries outlive the session. */ - readonly artifacts: Effect.Effect>; - readonly activate: ( - capability: CapabilityName, - input: LifecycleInput, - ) => Effect.Effect; - /** Supervisor-owned public ingress and lazy route activation lifecycle. */ - readonly ingress: SupervisorIngress; - readonly logStore: LogStore; -} - -export interface Supervisor { - readonly status: Effect.Effect; - readonly start: (options?: { - readonly config?: StackConfig; - }) => Effect.Effect; - readonly destroy: Effect.Effect; - /** Wipes Postgres data for the running stack and bootstraps a fresh cluster. */ - readonly resetDatabase: Effect.Effect; - /** Completes after a successful stop or destroy shutdown signal. */ - readonly shutdown: Effect.Effect; - /** Shuts down only when durable state is absent or cleanly non-running. */ - readonly shutdownIfIdle: Effect.Effect; - readonly logs: (query?: LogQuery) => Effect.Effect; - readonly activate: ( - capability: CapabilityName, - ) => Effect.Effect; - readonly maintenanceHandlers: { - readonly probe: Effect.Effect; - readonly stop: Effect.Effect; - }; - readonly rpcHandlers: StackRpcHandlers; -} - -export type SupervisorOptions = { - readonly stackId: StackId; - readonly ownerSessionId: string; - readonly stateStore: StackStateStore; - readonly context: Context.Context; - readonly runtime: SupervisorRuntime; -}; - -const RESET_DATABASE_BOUNCE_CAPABILITIES: ReadonlySet = new Set([ - "auth", - "storage", - "realtime", - "pooler", - "analytics", -]); - -const rpcError = (tag: StackRpcError["tag"], message: string): StackRpcError => ({ tag, message }); -const credentialsUnavailable = rpcError( - "StackNotRunningError", - "Stack credentials are unavailable", -); -const stateErrorMessage = (error: StackError | { readonly message?: string }): string => - typeof error.message === "string" ? error.message : "Stack operation failed"; - -const sessionCleanupMessage = (error: SessionCleanupError): string => { - const failure = Cause.findErrorOption(error.cause); - if (Option.isSome(failure)) { - if (failure.value instanceof RuntimeDriverError) { - const workload = - failure.value.workloadId === undefined ? "" : ` for ${failure.value.workloadId}`; - return `Session cleanup is unresolved${workload}: ${failure.value.message}`; - } - return sessionCleanupMessage(failure.value); - } - const details = Cause.pretty(error.cause); - return details.length === 0 - ? "Session cleanup is unresolved" - : `Session cleanup is unresolved: ${details}`; -}; - -const credentialHost = (address: string): string => - address.includes(":") && !address.startsWith("[") ? `[${address}]` : address; - -const mapRuntimeError = (error: unknown): StackError => { - if (error instanceof StackStateInvalidError) return error; - if (error instanceof ContainerEngineError) return error; - if (error instanceof RuntimeDriverError && isStackError(error.cause)) return error.cause; - if (error instanceof SessionCleanupError) - return new StackCleanupError({ message: sessionCleanupMessage(error), cause: error }); - return new StackRuntimeError({ - message: withLeftoverPersistentDataGuidance( - error instanceof Error ? error.message : String(error), - ), - cause: error, - }); -}; - -const mapCleanupError = (error: unknown): StackError => { - if (error instanceof StackStateInvalidError) return error; - if (error instanceof SessionCleanupError) - return new StackCleanupError({ message: sessionCleanupMessage(error), cause: error }); - return new StackCleanupError({ - message: error instanceof Error ? error.message : String(error), - cause: error, - }); -}; - -const combineCleanupOutcome = (left: CleanupOutcome, right: CleanupOutcome): CleanupOutcome => - Predicate.isTagged(left, "proven") && Predicate.isTagged(right, "proven") - ? { _tag: "proven" } - : { - _tag: "unproven", - cause: Cause.combine( - Predicate.isTagged(left, "unproven") ? left.cause : Cause.empty, - Predicate.isTagged(right, "unproven") ? right.cause : Cause.empty, - ), - }; - -const rpcTag = (error: StackError): StackRpcError["tag"] => error._tag; -const maintenanceStackErrorTag = (error: unknown): StackErrorTag | undefined => - Predicate.hasProperty(error, "_tag") && - typeof error._tag === "string" && - isStackErrorTag(error._tag) - ? error._tag - : undefined; - -/** Compose one owner process around the durable lifecycle controller and a runtime driver. */ -export const makeSupervisor = ( - options: SupervisorOptions, -): Effect.Effect => - Effect.gen(function* () { - const read = () => - options.stateStore.read(options.stackId).pipe(Effect.provideContext(options.context)); - const initial = yield* read().pipe( - Effect.catchIf(isMissingStateRemnantError, () => - options.stateStore - .recoverRuntimeRemnant(options.stackId) - .pipe(Effect.provideContext(options.context), Effect.asVoid), - ), - ); - if (initial === undefined) - return yield* new StackStateInvalidError({ message: "Stack state is missing" }); - const runtime = options.runtime; - const launcher: SessionLauncher = yield* makeSessionLauncher({ - stackId: options.stackId, - driver: runtime.driver, - }); - const machine = yield* Ref.make({ - stack: - initial.desiredLifecycle === "destroying" - ? { _tag: "destroy-required", evidence: { _tag: "persisted-intent" } } - : { _tag: "stopped", session: "uninitialized" }, - sessionId: Symbol("stack-session"), - plan: undefined, - capabilities: new Map(), - }); - const currentPhase = (): Effect.Effect => - Ref.get(machine).pipe(Effect.map((snapshot) => publicPhase(snapshot.stack))); - const activeCommand = (): Effect.Effect> => - Ref.get(machine).pipe(Effect.map(({ stack }) => activeLifecycle(stack))); - type ActivationHandler = ( - capability: CapabilityName, - ) => Effect.Effect; - // The ingress opens during an explicit lifecycle operation. A one-shot handoff keeps a request - // waiting for the handler instead of exposing a construction-time race. - const activationHandler = yield* Deferred.make(); - const initializeActivationInAdmission = (input: LifecycleInput) => - Effect.gen(function* () { - const eager = eagerCapabilities(input.plan); - const startup = new Map(); - for (const name of eager) - startup.set(name, { - completion: yield* Deferred.make, never>(), - operation: Symbol("startup"), - }); - const sessionId = Symbol("stack-session"); - const transition = initializeSession(yield* Ref.get(machine), input, sessionId, startup); - yield* applyTransitionInAdmission(transition); - yield* Ref.set( - idleTimeouts, - new Map( - CAPABILITY_NAMES.map((name) => [ - name, - input.definition.capabilities[name].idleTimeoutSeconds, - ]), - ), - ); - }); - const initializeActivation = (input: LifecycleInput) => - admission.withPermit(initializeActivationInAdmission(input)); - const resetForSession = (input: LifecycleInput) => initializeActivation(input); - const observe = () => - runtime.driver.observe(options.stackId).pipe(Effect.mapError(mapRuntimeError)); - const observedForStatus = () => - Ref.get(machine).pipe( - Effect.flatMap(({ stack }) => { - const available = (workloads: ReadonlyArray): ObservedStatus => ({ - _tag: "available", - workloads, - }); - const fallback: Effect.Effect = observe().pipe( - Effect.map(available), - Effect.orElseSucceed(() => ({ _tag: "unavailable" as const })), - ); - return Match.value(stack).pipe( - Match.when({ _tag: "stopped" }, () => Effect.succeed(available([]))), - Match.when({ _tag: "running" }, () => observe().pipe(Effect.map(available))), - Match.when({ _tag: "starting", prior: { _tag: "running" } }, () => - observe().pipe(Effect.map(available)), - ), - Match.when({ _tag: "starting" }, () => fallback), - Match.when({ _tag: "stopping" }, () => fallback), - Match.when({ _tag: "destroying" }, () => fallback), - Match.when({ _tag: "stop-required" }, () => fallback), - Match.when({ _tag: "start-recovery" }, () => fallback), - Match.when({ _tag: "destroy-required" }, () => fallback), - Match.exhaustive, - ); - }), - ); - - const snapshot = (): Effect.Effect => - Effect.gen(function* () { - const state = yield* read(); - if (state === undefined) - return yield* new StackStateInvalidError({ message: "Stack state is missing" }); - const status = yield* statusForSnapshot( - options.stackId, - state, - yield* observedForStatus(), - yield* Ref.get(machine), - yield* runtime.artifacts, - ); - return status; - }); - - // Admission rejects every overlapping lifecycle operation while execution serializes - // lifecycle and activation work against runtime access. - const admission = yield* Semaphore.make(1); - // Activation and lifecycle operations share one execution gate so they cannot race cleanup. - const execution = yield* Semaphore.make(1); - const supervisorScope = yield* Effect.scope; - const ownedFibers = yield* FiberSet.make().pipe( - Effect.provideService(Scope.Scope, supervisorScope), - ); - const backgroundPreparation = yield* Ref.make | undefined>(undefined); - const startBackgroundPreparation = (state: PersistedStackState): Effect.Effect => - Effect.gen(function* () { - const existing = yield* Ref.get(backgroundPreparation); - if (existing !== undefined) return; - const fiber = yield* Effect.forkIn(runtime.prefetch(state), supervisorScope, { - startImmediately: true, - }); - yield* Ref.set(backgroundPreparation, fiber); - }); - const joinExit = (result: Exit.Exit): Effect.Effect => - Exit.isSuccess(result) ? Effect.succeed(result.value) : Effect.failCause(result.cause); - const notify = (notification: TransitionNotification): Effect.Effect => - Match.value(notification).pipe( - Match.tag("endpoint", (value) => Deferred.succeed(value.completion, value.result)), - Match.tag("activation", (value) => Deferred.succeed(value.completion, value.result)), - Match.tag("stopping", (value) => Deferred.succeed(value.completion, value.result)), - Match.tag("lifecycle", (value) => Deferred.succeed(value.completion, value.result)), - Match.tag("workload", (value) => Deferred.succeed(value.completion, value.result)), - Match.exhaustive, - ); - const idleTimeouts = yield* Ref.make>(new Map()); - - const readySet = (): Effect.Effect> => - Ref.get(machine).pipe( - Effect.map( - (snapshot) => - new Set( - [...snapshot.capabilities].flatMap(([name, state]) => - Predicate.isTagged(state, "ready") ? [name] : [], - ), - ), - ), - ); - const selectedSet = (): Effect.Effect> => - Ref.get(machine).pipe( - Effect.map( - (snapshot) => - new Set( - [...snapshot.capabilities].flatMap(([name, state]) => - Predicate.isTagged(state, "ready") || Predicate.isTagged(state, "starting") - ? [name] - : [], - ), - ), - ), - ); - const setReadySetInAdmission = (names: ReadonlySet): Effect.Effect => - Effect.gen(function* () { - const transition = transitionReadySet(yield* Ref.get(machine), names); - yield* applyTransitionInAdmission(transition); - }); - const setReadySet = (names: ReadonlySet): Effect.Effect => - admission.withPermit(Effect.uninterruptible(setReadySetInAdmission(names))); - const promoteActivationSet = ( - names: ReadonlySet, - activationOwner: CapabilityName, - plan: ExecutionPlan, - ): Effect.Effect => - admission.withPermit( - Effect.uninterruptible( - Effect.gen(function* () { - const transition = transitionPromoteActivationSet( - yield* Ref.get(machine), - names, - activationOwner, - plan, - ); - yield* applyTransitionInAdmission(transition); - }), - ), - ); - const claimWorkloadsInAdmission = ( - names: ReadonlySet, - ): Effect.Effect> => - Effect.gen(function* () { - const handles = new Map(); - for (const name of names) { - handles.set(name, { - completion: yield* Deferred.make, never>(), - operation: Symbol("dependency-startup"), - }); - } - const transition = transitionClaimWorkloads(yield* Ref.get(machine), names, handles); - yield* applyTransitionInAdmission(transition); - return transition.claimed; - }); - const settleActivationTerminalInAdmission = ( - owner: ActivationOwner, - claims: ActivationClaims, - outcome: ActivationTerminalOutcome, - ): Effect.Effect => - Effect.gen(function* () { - const transition = transitionSettleActivationTerminal( - yield* Ref.get(machine), - owner, - claims, - outcome, - ); - yield* applyTransitionInAdmission(transition); - }); - const settleActivationTerminal = ( - owner: ActivationOwner, - claims: ActivationClaims, - outcome: ActivationTerminalOutcome, - ): Effect.Effect => - admission.withPermit(settleActivationTerminalInAdmission(owner, claims, outcome)); - const setRootSetInAdmission = (names: ReadonlySet): Effect.Effect => - Effect.gen(function* () { - yield* applyTransitionInAdmission(transitionSetRootSet(yield* Ref.get(machine), names)); - }); - const setRootSet = (names: ReadonlySet): Effect.Effect => - admission.withPermit(setRootSetInAdmission(names)); - const enterCapabilityCleanupInAdmission = (): Effect.Effect< - ReadonlyMap - > => - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const handles = new Map(); - for (const [name, state] of snapshot.capabilities) - if (isCleanupCandidate(state)) - handles.set(name, { - operation: Symbol("cleanup"), - completion: yield* Deferred.make, never>(), - }); - const transition = transitionEnterCapabilityCleanup(snapshot, handles); - yield* applyTransitionInAdmission(transition); - return handles; - }); - const settleCapabilityCleanupInAdmission = ( - handles: ReadonlyMap, - result: Exit.Exit, - ): Effect.Effect => - Effect.gen(function* () { - const transition = transitionSettleCapabilityCleanup( - yield* Ref.get(machine), - result, - handles, - ); - yield* applyTransitionInAdmission(transition); - }); - const completeDormantCleanupInAdmission = (): Effect.Effect => - Effect.gen(function* () { - yield* applyTransitionInAdmission( - transitionCompleteDormantCleanup(yield* Ref.get(machine)), - ); - }); - const completeDormantCleanup = (): Effect.Effect => - admission.withPermit(completeDormantCleanupInAdmission()); - const appendIdleLog = (message: string): Effect.Effect => - options.runtime.logStore - .append({ - source: "supervisor", - stream: "internal", - message, - }) - .pipe( - Effect.catchTag("LogStoreError", (error) => Effect.logWarning(message, error)), - Effect.asVoid, - ); - - function retireIdle( - capability: CapabilityName, - epoch: symbol, - ): Effect.Effect { - return execution.withPermit( - Effect.gen(function* () { - const completion = yield* Deferred.make, never>(); - const operation = Symbol("retirement"); - const retire = Effect.gen(function* () { - const fenced = yield* admission.withPermit( - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const transition = transitionBeginRetirement( - snapshot, - capability, - operation, - completion, - epoch, - ); - yield* applyTransitionInAdmission(transition); - return transition.admitted; - }), - ); - if (!fenced) return false; - yield* launcher - .stopCapabilities(new Set([capability])) - .pipe(Effect.mapError(mapCleanupError)); - return true; - }).pipe( - Effect.onExit((result) => - settleOwner({ - _tag: "retirement", - capability, - operation, - completion, - result, - }), - ), - ); - const stopped = yield* Effect.exit(retire); - if (Exit.isFailure(stopped)) { - const logged = yield* appendIdleLog( - `Failed to stop ${capability} after inactivity: ${Cause.pretty(stopped.cause)}`, - ).pipe(Effect.exit); - if (Exit.isFailure(logged)) - return yield* Effect.failCause(Cause.combine(stopped.cause, logged.cause)); - if (Cause.hasInterrupts(stopped.cause) || Cause.hasDies(stopped.cause)) - return yield* Effect.failCause(stopped.cause); - return false; - } - if (!stopped.value) return false; - - yield* appendIdleLog(`Stopped ${capability} after inactivity`); - return true; - }), - ); - } - - function armIdleTimerInAdmission(capability: CapabilityName): Effect.Effect { - return Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const timeouts = yield* Ref.get(idleTimeouts); - const plan = transitionPlanIdleTimer(snapshot, timeouts, capability); - if (plan === undefined) return; - const token = Symbol(); - yield* Effect.uninterruptibleMask(() => - Effect.gen(function* () { - const started = yield* Deferred.make(); - const fiber = yield* Effect.forkIn( - Deferred.await(started).pipe( - Effect.andThen( - Effect.sleep(Duration.seconds(plan.timeout)).pipe( - Effect.andThen( - FiberSet.run(ownedFibers, retireIdle(capability, token), { - startImmediately: true, - }).pipe(Effect.asVoid), - ), - ), - ), - ), - supervisorScope, - { startImmediately: true }, - ); - const transition = transitionArmRetirement(snapshot, plan, token, fiber); - yield* applyTransitionInAdmission(transition); - yield* Deferred.succeed(started, undefined); - }), - ); - }); - } - - function reevaluateIdleTimersInAdmission(): Effect.Effect { - return readySet().pipe( - Effect.flatMap((capabilities) => - Effect.forEach(capabilities, armIdleTimerInAdmission, { - discard: true, - }), - ), - ); - } - - function armIdleTimer(capability: CapabilityName): Effect.Effect { - return admission.withPermit(armIdleTimerInAdmission(capability)); - } - - const cancelIdleTimers: Effect.Effect = Effect.gen(function* () { - const timers = yield* admission.withPermit( - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const transition = transitionDisarmAllRetirements(snapshot); - yield* applyTransitionInAdmission(transition); - return transition.timers; - }), - ); - yield* Effect.forEach(timers, (fiber) => Fiber.interrupt(fiber), { - concurrency: "unbounded", - discard: true, - }); - }); - - type TrafficLease = Readonly<{ readonly sessionId: symbol }>; - const beginTraffic = (capability: CapabilityName): Effect.Effect => - Effect.gen(function* () { - const acquired = yield* admission.withPermit( - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const transition = transitionBeginTraffic(snapshot, capability); - yield* applyTransitionInAdmission(transition); - return { fiber: transition.timer, lease: { sessionId: snapshot.sessionId } }; - }), - ); - if (acquired.fiber !== undefined) yield* Fiber.interrupt(acquired.fiber); - return acquired.lease; - }); - const endTraffic = (capability: CapabilityName, lease: TrafficLease): Effect.Effect => - Effect.gen(function* () { - const shouldArm = yield* admission.withPermit( - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const transition = transitionEndTraffic(snapshot, capability, lease.sessionId); - yield* applyTransitionInAdmission(transition); - return transition.shouldArm; - }), - ); - if (shouldArm) yield* armIdleTimer(capability); - }); - - const activity = yield* makeGatewayActivity({ begin: beginTraffic, end: endTraffic }); - const ingressActivate = ( - capability: CapabilityName, - ): Effect.Effect => - Effect.gen(function* () { - // A request can reach the adopted gateway while the owning start operation is still - // installing its workloads. Wait for that shared lifecycle result before attempting lazy - // activation; otherwise the state check below would turn a valid cold request into 503. - const lifecycle = yield* activeCommand(); - if (lifecycle?.kind === "start") { - const started = yield* Deferred.await(lifecycle.result); - yield* joinExit(started); - } - const handler = yield* Deferred.await(activationHandler); - return yield* handler(capability); - }); - const ensureActivationStateAllowed = (): Effect.Effect< - PersistedStackState, - GatewayActivationError | StackError - > => - Effect.gen(function* () { - const state = yield* read(); - if (state === undefined) - return yield* new StackStateInvalidError({ message: "Stack state is missing" }); - if (state.desiredLifecycle !== "running") - return yield* new StackNotRunningError({ - message: "Stack must be running before activation", - }); - return state; - }); - const shutdownSignal = yield* Deferred.make(); - const signalShutdown = Deferred.succeed(shutdownSignal, undefined).pipe(Effect.asVoid); - const ensureAcceptingOperations = Deferred.poll(shutdownSignal).pipe( - Effect.flatMap((shutdown) => - Option.isNone(shutdown) - ? Effect.void - : Effect.fail( - new StackLifecycleConflictError({ - stackId: options.stackId, - message: "Stack owner is shutting down", - }), - ), - ), - ); - - const submitLifecycle = ( - kind: LifecycleKind, - effect: Effect.Effect, - ): Effect.Effect => - Effect.gen(function* () { - const owned = yield* admission.withPermit( - Effect.uninterruptible( - Effect.gen(function* () { - yield* ensureAcceptingOperations; - const deferred = yield* Deferred.make, never>(); - const snapshot = yield* Ref.get(machine); - const admitted = admitLifecycle(snapshot, kind, deferred, Symbol(kind)); - if (Predicate.isTagged(admitted, "rejected")) - return yield* new StackLifecycleConflictError({ - stackId: options.stackId, - message: - admitted.reason === "stop-required" - ? "Exact runtime cleanup is required; retry stop before starting" - : admitted.reason === "destroy-required" - ? "Destructive cleanup is required; retry destroy before proceeding" - : `Lifecycle operation ${admitted.activeKind ?? kind} is already active`, - recovery: - admitted.reason === "stop-required" || admitted.reason === "destroy-required" - ? recoveryForState(snapshot.stack) - : undefined, - }); - yield* applyTransitionInAdmission(admitted); - const finish = (result: Exit.Exit) => - Effect.gen(function* () { - const operation = Exit.isSuccess(result) - ? result.value - : { - _tag: "failed" as const, - cause: result.cause, - cleanup: { _tag: "unproven" as const, cause: result.cause }, - durable: "unsafe" as const, - }; - yield* settleOwner({ - _tag: "lifecycle", - completion: deferred, - result: operation, - }); - return operation; - }); - let entered = false; - const owner = Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const cancelled = yield* restore(cancelIdleTimers).pipe(Effect.exit); - if (Exit.isFailure(cancelled)) { - yield* finish(Exit.failCause(cancelled.cause)); - return yield* Effect.failCause(cancelled.cause); - } - const result = yield* restore( - execution.withPermit( - Effect.uninterruptibleMask((inner) => - Effect.gen(function* () { - entered = true; - const result = yield* inner(effect).pipe(Effect.exit); - const operation = yield* finish(result); - return yield* joinExit( - Predicate.isTagged(operation, "failed") - ? Exit.failCause(operation.cause) - : Exit.succeed(operation), - ); - }), - ), - ), - ).pipe(Effect.exit); - if (!entered && Exit.isFailure(result)) - yield* finish(Exit.failCause(result.cause)); - return yield* joinExit(result); - }), - ); - const ownerFiber = yield* FiberSet.run(ownedFibers, owner, { - startImmediately: true, - }); - // Effect rc112 has no safe Fiber.poll; FiberSet returns an already-interrupted fiber when closed. - const ownerExit = yield* Effect.sync(() => ownerFiber.pollUnsafe()); - if ( - ownerExit !== undefined && - Exit.isFailure(ownerExit) && - Cause.hasInterruptsOnly(ownerExit.cause) - ) { - const conflict = new StackLifecycleConflictError({ - stackId: options.stackId, - message: "Stack owner scope is closed", - }); - const cause = Cause.fail(conflict); - yield* settleOwnerInAdmission({ - _tag: "lifecycle", - completion: deferred, - result: { - _tag: "failed", - cause, - cleanup: { _tag: "unproven", cause }, - durable: "unsafe", - }, - }); - } - return deferred; - }), - ), - ); - return yield* joinExit(yield* Deferred.await(owned)); - }); - - const launchBackend = ( - input: LifecycleInput, - session: "fresh" | "current", - selectedOverride?: ReadonlySet, - ): Effect.Effect => - Effect.gen(function* () { - if (session === "fresh") yield* resetForSession(input); - const selected = selectedOverride ?? (yield* selectedSet()); - const plan = activeExecutionPlan(input.plan, selected); - const reservation = yield* runtime.ingress.acquire(input); - const launchCancellation = yield* Deferred.make(); - const preparedFiber = yield* Effect.forkChild( - runtime.prepare(input, selected).pipe(Effect.exit), - { - startImmediately: true, - }, - ); - const launchFiber = yield* Effect.forkChild(launcher.launch(plan, launchCancellation), { - startImmediately: true, - }); - const first = yield* Effect.raceFirst( - Fiber.await(preparedFiber).pipe( - Effect.map((value) => ({ _tag: "prepared", value }) as const), - ), - Fiber.await(launchFiber).pipe( - Effect.map((value) => ({ _tag: "launch", value }) as const), - ), - ); - let prepared: Exit.Exit; - let launch: SessionLaunchOutcome; - const preparedExit = (value: Exit.Exit, never>) => - Exit.isSuccess(value) ? value.value : Exit.failCause(value.cause); - const launchOutcome = ( - value: Exit.Exit, - ): SessionLaunchOutcome => - Exit.isSuccess(value) - ? value.value - : { - _tag: "failed", - cause: value.cause, - cleanup: { _tag: "unproven", cause: value.cause }, - }; - if (Predicate.isTagged(first, "prepared")) { - prepared = preparedExit(first.value); - if (Exit.isFailure(prepared)) yield* Deferred.succeed(launchCancellation, undefined); - launch = launchOutcome(yield* Fiber.await(launchFiber)); - } else { - launch = launchOutcome(first.value); - if (Predicate.isTagged(launch, "failed")) { - yield* Fiber.interrupt(preparedFiber); - prepared = preparedExit(yield* Fiber.await(preparedFiber)); - } else { - prepared = preparedExit(yield* Fiber.await(preparedFiber)); - } - } - const mapSessionCleanup = (cleanup: SessionCleanupOutcome): CleanupOutcome => - Match.value(cleanup).pipe( - Match.when({ _tag: "proven" }, () => ({ _tag: "proven" as const })), - Match.when({ _tag: "unproven" }, (value) => ({ - _tag: "unproven" as const, - cause: Cause.map(value.cause, mapRuntimeError), - })), - Match.exhaustive, - ); - if (Exit.isFailure(prepared)) { - const closed = reservation.fresh - ? yield* runtime.ingress.close.pipe(Effect.mapError(mapRuntimeError), Effect.exit) - : Exit.succeed(undefined); - let cause: Cause.Cause = Cause.map(prepared.cause, mapRuntimeError); - let workloadCleanup: CleanupOutcome = { _tag: "proven" }; - if (Predicate.isTagged(launch, "started")) { - workloadCleanup = mapSessionCleanup(yield* launch.launch.rollback); - if (Predicate.isTagged(workloadCleanup, "unproven")) - cause = Cause.combine(cause, workloadCleanup.cause); - } else { - const launchCause = Cause.map(launch.cause, mapRuntimeError); - cause = Cause.combine(cause, launchCause); - workloadCleanup = mapSessionCleanup(launch.cleanup); - } - const closedCleanup: CleanupOutcome = Exit.isFailure(closed) - ? { _tag: "unproven", cause: closed.cause } - : { _tag: "proven" }; - const cleanup = combineCleanupOutcome(workloadCleanup, closedCleanup); - if (Predicate.isTagged(closedCleanup, "unproven")) - cause = Cause.combine(cause, closedCleanup.cause); - return { - _tag: "failed", - cause, - cleanup, - } satisfies LifecycleLaunchResult; - } - if (Predicate.isTagged(launch, "failed")) { - const closed = reservation.fresh - ? yield* runtime.ingress.close.pipe(Effect.mapError(mapRuntimeError), Effect.exit) - : Exit.succeed(undefined); - const launchCause = Cause.map(launch.cause, mapRuntimeError); - let cause: Cause.Cause = launchCause; - const workloadCleanup = mapSessionCleanup(launch.cleanup); - const closedCleanup: CleanupOutcome = Exit.isFailure(closed) - ? { _tag: "unproven", cause: closed.cause } - : { _tag: "proven" }; - const cleanup = combineCleanupOutcome(workloadCleanup, closedCleanup); - if (Predicate.isTagged(closedCleanup, "unproven")) - cause = Cause.combine(cause, closedCleanup.cause); - return { - _tag: "failed", - cause, - cleanup, - } satisfies LifecycleLaunchResult; - } - const rollback: Effect.Effect = Effect.gen(function* () { - const workload = yield* launch.launch.rollback; - const closed = reservation.fresh - ? yield* runtime.ingress.close.pipe(Effect.mapError(mapRuntimeError), Effect.exit) - : Exit.succeed(undefined); - const workloadOutcome = mapSessionCleanup(workload); - const closedOutcome: CleanupOutcome = Exit.isSuccess(closed) - ? { _tag: "proven" } - : { _tag: "unproven", cause: closed.cause }; - return combineCleanupOutcome(workloadOutcome, closedOutcome); - }); - const opened = yield* runtime.ingress - .open(input, reservation, ingressActivate, activity) - .pipe(Effect.exit); - if (Exit.isFailure(opened)) { - const rolledBack = yield* rollback; - const cause = Predicate.isTagged(rolledBack, "unproven") - ? Cause.combine(opened.cause, rolledBack.cause) - : opened.cause; - return { - _tag: "failed", - cause, - cleanup: rolledBack, - } satisfies LifecycleLaunchResult; - } - return { _tag: "started", rollback } satisfies LifecycleLaunchResult; - }); - - const cleanupRuntime = (destroy: boolean): Effect.Effect => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - let handles: ReadonlyMap | undefined; - const result = yield* restore( - Effect.gen(function* () { - yield* admission.withPermit( - Effect.uninterruptible( - Effect.gen(function* () { - handles = yield* enterCapabilityCleanupInAdmission(); - }), - ), - ); - const background = yield* Ref.get(backgroundPreparation); - if (background !== undefined) yield* Fiber.interrupt(background); - yield* Ref.set(backgroundPreparation, undefined); - const ingress = yield* runtime.ingress.close.pipe( - Effect.mapError(mapCleanupError), - Effect.exit, - ); - const launched = destroy - ? Exit.succeed(undefined) - : yield* launcher.stop.pipe(Effect.mapError(mapCleanupError), Effect.exit); - const driver = yield* runtime.driver - .cleanup({ stackId: options.stackId, destroy }) - .pipe(Effect.mapError(mapCleanupError), Effect.exit); - let cause: Cause.Cause = Cause.empty; - for (const outcome of [ingress, launched, driver]) - if (Exit.isFailure(outcome)) cause = Cause.combine(cause, outcome.cause); - if (cause.reasons.length > 0) return yield* Effect.failCause(cause); - }), - ).pipe(Effect.exit); - if (handles !== undefined) - yield* admission.withPermit(settleCapabilityCleanupInAdmission(handles, result)); - if (Exit.isFailure(result)) return yield* Effect.failCause(result.cause); - yield* completeDormantCleanup(); - if (destroy) yield* launcher.clear; - yield* setRootSet(new Set()); - }), - ); - const backend: LifecycleBackend = { - preflight: runtime.preflight, - launch: launchBackend, - cleanup: cleanupRuntime(false), - destroyData: cleanupRuntime(true), - }; - const controller = yield* makeLifecycleController({ - stackId: options.stackId, - runtime: initial.runtime, - stateStore: options.stateStore, - backend, - }).pipe(Effect.provideContext(options.context)); - const status = snapshot(); - - type PreparedActivation = Readonly<{ - readonly input: LifecycleInput; - readonly selected: ReadonlySet; - }>; - const prepareActivation = ( - owner: ActivationOwner, - ): Effect.Effect => - Effect.gen(function* () { - const state = yield* ensureActivationStateAllowed(); - const definition = state.definition; - if (definition === undefined || !definition.capabilities[owner.capability].enabled) - return yield* new GatewayActivationError({ - message: `Capability ${owner.capability} is not enabled`, - }); - const plan = yield* rebuildExecutionPlan(state.runtime, definition).pipe( - Effect.provideContext(options.context), - Effect.mapError( - (error) => new StackStateInvalidError({ message: error.message, cause: error }), - ), - ); - return { - input: { - stackId: options.stackId, - state, - definition, - secrets: state.secrets, - plan, - }, - selected: new Set([ - ...(yield* readySet()), - ...dependencyClosure(plan, [owner.capability]), - ]), - }; - }); - - const runActivationOwner = (owner: ActivationOwner): Effect.Effect => - Effect.gen(function* () { - let entered = false; - let claims: ActivationClaims = { _tag: "none" }; - const activation = Effect.gen(function* () { - const fence = yield* admission.withPermit( - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - return { - ownerMatches: matchesActivationOwner(snapshot, owner), - gate: activationGate(snapshot, options.stackId), - }; - }), - ); - if (!fence.ownerMatches) - return yield* new StackLifecycleConflictError({ - stackId: options.stackId, - message: Predicate.isTagged(owner, "endpoint") - ? "Endpoint activation was superseded by a lifecycle transition" - : "Lazy activation was superseded by a lifecycle transition", - }); - if (Predicate.isTagged(fence.gate, "rejected")) return yield* fence.gate.error; - - const prepared = yield* prepareActivation(owner); - yield* admission.withPermit( - Effect.uninterruptible( - Effect.gen(function* () { - const claimed = yield* claimWorkloadsInAdmission(prepared.selected); - claims = { - _tag: "claimed", - claimed, - affected: prepared.selected, - }; - return claimed; - }), - ), - ); - const launched = yield* launchBackend(prepared.input, "current", prepared.selected).pipe( - Effect.exit, - ); - if (Exit.isFailure(launched)) - return { - _tag: "failed" as const, - cause: launched.cause, - cleanup: { _tag: "unproven" as const, cause: launched.cause }, - } satisfies ActivationTerminalOutcome; - if (Predicate.isTagged(launched.value, "failed")) - return { - _tag: "failed" as const, - cause: launched.value.cause, - cleanup: launched.value.cleanup, - } satisfies ActivationTerminalOutcome; - yield* setReadySet(prepared.selected); - const activated = yield* runtime - .activate(owner.capability, prepared.input) - .pipe(Effect.exit); - if (Exit.isFailure(activated)) { - const rolledBack = yield* launched.value.rollback; - const cause = Predicate.isTagged(rolledBack, "unproven") - ? Cause.combine(activated.cause, rolledBack.cause) - : activated.cause; - return { - _tag: "failed" as const, - cause, - cleanup: rolledBack, - } satisfies ActivationTerminalOutcome; - } - yield* promoteActivationSet(prepared.selected, owner.capability, prepared.input.plan); - return { - _tag: "succeeded" as const, - value: { capability: owner.capability, endpoint: activated.value }, - } satisfies ActivationTerminalOutcome; - }); - const fiber = Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const result = yield* restore( - execution.withPermit( - Effect.uninterruptibleMask((inner) => - Effect.gen(function* () { - entered = true; - const result = yield* inner(activation).pipe(Effect.exit); - const outcome: ActivationTerminalOutcome = Exit.isSuccess(result) - ? result.value - : { - _tag: "failed", - cause: result.cause, - cleanup: Predicate.isTagged(claims, "none") - ? { _tag: "proven" } - : { _tag: "unproven", cause: result.cause }, - }; - yield* settleActivationTerminal(owner, claims, outcome); - return yield* Predicate.isTagged(outcome, "failed") - ? Effect.failCause(outcome.cause) - : Effect.succeed(outcome.value); - }), - ), - ), - ).pipe(Effect.exit); - if (!entered && Exit.isFailure(result)) - yield* settleActivationTerminal( - owner, - { _tag: "none" }, - { _tag: "failed", cause: result.cause, cleanup: { _tag: "proven" } }, - ); - return yield* joinExit(result); - }), - ); - const ownerFiber = yield* FiberSet.run(ownedFibers, fiber, { startImmediately: true }); - const ownerExit = yield* Effect.sync(() => ownerFiber.pollUnsafe()); - if ( - ownerExit !== undefined && - Exit.isFailure(ownerExit) && - Cause.hasInterruptsOnly(ownerExit.cause) - ) - yield* settleActivationTerminalInAdmission( - owner, - { _tag: "none" }, - { - _tag: "failed", - cause: Cause.fail( - new StackLifecycleConflictError({ - stackId: options.stackId, - message: "Stack owner scope is closed", - }), - ), - cleanup: { _tag: "proven" }, - }, - ); - }); - const applyTransitionInAdmission = (transition: SnapshotTransition): Effect.Effect => - Effect.gen(function* () { - yield* Ref.set(machine, transition.snapshot); - if (transition.reconcile === "all-ready") yield* reevaluateIdleTimersInAdmission(); - yield* Effect.forEach(transition.notifications, notify, { discard: true }); - }); - const settleOwnerInAdmission = (owner: SettlementOwner): Effect.Effect => - Effect.gen(function* () { - const snapshot = yield* Ref.get(machine); - const settlement = Match.value(owner).pipe( - Match.when({ _tag: "lifecycle" }, (event) => settleLifecycleOwner(snapshot, event)), - Match.when({ _tag: "retirement" }, (event) => settleRetirementOwner(snapshot, event)), - Match.exhaustive, - ); - yield* applyTransitionInAdmission(settlement); - }); - const settleOwner = (owner: SettlementOwner): Effect.Effect => - admission.withPermit(settleOwnerInAdmission(owner)); - const activate: Supervisor["activate"] = (capability) => - Effect.gen(function* () { - const token = yield* admission.withPermit( - Effect.uninterruptible( - Effect.gen(function* () { - const decision = admitActivation( - yield* Ref.get(machine), - capability, - options.stackId, - ); - if (Predicate.isTagged(decision, "rejected")) return yield* decision.error; - if (Predicate.isTagged(decision, "respond")) return decision.token; - if (Predicate.isTagged(decision, "endpoint-owner")) { - const endpoint = yield* Deferred.make(); - const owner: ActivationOwner = { - _tag: "endpoint", - capability: decision.capability, - endpoint, - priorRoot: decision.priorRoot, - }; - yield* applyTransitionInAdmission(decision.transition(endpoint)); - yield* runActivationOwner(owner); - return { - _tag: "endpoint", - capability, - result: endpoint, - } satisfies ActivationToken; - } - const activation = yield* Deferred.make(); - const owner: ActivationOwner = { - _tag: "activation", - capability: decision.capability, - completion: activation, - }; - yield* applyTransitionInAdmission( - decision.transition(Symbol("activation"), activation), - ); - yield* runActivationOwner(owner); - return { _tag: "deferred", result: activation } satisfies ActivationToken; - }), - ), - ); - return yield* Match.value(token).pipe( - Match.when({ _tag: "deferred" }, (event) => - Deferred.await(event.result).pipe(Effect.flatMap(joinExit)), - ), - Match.when({ _tag: "await" }, (event) => - Effect.gen(function* () { - const completed = yield* Deferred.await(event.result); - if (Exit.isFailure(completed)) return yield* Effect.failCause(completed.cause); - return yield* activate(capability); - }), - ), - Match.when({ _tag: "endpoint" }, (event) => - Deferred.await(event.result).pipe( - Effect.flatMap(joinExit), - Effect.map((endpoint) => ({ capability: event.capability, endpoint })), - ), - ), - Match.when({ _tag: "exit" }, (event) => joinExit(event.result)), - Match.exhaustive, - ); - }); - yield* Deferred.succeed(activationHandler, activate); - - const startOperation = (startOptions?: { - readonly config?: StackConfig; - }): Effect.Effect => - Effect.gen(function* () { - const admitted = (yield* Ref.get(machine)).stack; - if (Predicate.isTagged(admitted, "start-recovery")) - return { - _tag: "failed", - cause: admitted.cause, - cleanup: { _tag: "unproven", cause: admitted.cause }, - durable: "unsafe", - } satisfies CommandResult; - const freshSession = - Predicate.isTagged(admitted, "starting") && - Predicate.isTagged(admitted.prior, "stopped") && - admitted.prior.session === "uninitialized"; - if (freshSession) { - const cleaned = yield* backend.cleanup.pipe(Effect.exit); - if (Exit.isFailure(cleaned)) { - return yield* Effect.failCause(cleaned.cause); - } - } - const started = yield* controller - .start({ - config: startOptions?.config, - freshSession, - }) - .pipe(Effect.provideContext(options.context), Effect.exit); - if (Exit.isFailure(started)) { - return { - _tag: "failed", - cause: started.cause, - cleanup: { _tag: "unproven", cause: started.cause }, - durable: "unsafe", - } satisfies CommandResult; - } - if (Predicate.isTagged(started.value, "failed")) { - return { - _tag: "failed", - cause: started.value.cause, - cleanup: started.value.cleanup, - durable: started.value.durable, - } satisfies CommandResult; - } - yield* setReadySet(yield* selectedSet()); - yield* startBackgroundPreparation(started.value.state); - return { _tag: "succeeded" } satisfies CommandResult; - }); - const start = (startOptions?: { readonly config?: StackConfig }) => - Effect.gen(function* () { - yield* submitLifecycle("start", startOperation(startOptions)); - return yield* snapshot(); - }); - const resetDatabaseOperation = (): Effect.Effect => - Effect.gen(function* () { - const notRunning = new StackNotRunningError({ - stackId: options.stackId, - message: "Stack is not running", - }); - const rejectNotRunning = { - _tag: "failed" as const, - cause: Cause.fail(notRunning), - cleanup: { _tag: "proven" as const }, - durable: "stopped" as const, - } satisfies CommandResult; - const failedWithoutMutation = (cause: Cause.Cause): CommandResult => ({ - _tag: "failed", - cause, - cleanup: { _tag: "proven" }, - durable: "unsafe", - }); - const failedAfterMutation = (cause: Cause.Cause): CommandResult => ({ - _tag: "failed", - cause, - cleanup: { _tag: "unproven", cause }, - durable: "unsafe", - }); - const control = (yield* Ref.get(machine)).stack; - if ( - !Predicate.isTagged(control, "starting") || - !Predicate.isTagged(control.prior, "running") - ) - return rejectNotRunning; - const state = yield* read(); - if (state === undefined || state.definition === undefined) - return failedWithoutMutation( - Cause.fail(new StackStateInvalidError({ message: "Stack state is missing" })), - ); - const status = yield* snapshot(); - const database = status.capabilities.find((capability) => capability.name === "database"); - if (database?.state !== "ready") - return { - ...rejectNotRunning, - cause: Cause.fail( - new StackNotRunningError({ - stackId: options.stackId, - message: "Database is not running", - }), - ), - }; - const plan = - (yield* Ref.get(machine)).plan ?? - (yield* rebuildExecutionPlan(state.runtime, state.definition).pipe( - Effect.mapError( - (error) => new StackStateInvalidError({ message: error.message, cause: error }), - ), - )); - const bounceNames = new Set( - status.capabilities.flatMap((capability) => - capability.state === "ready" && RESET_DATABASE_BOUNCE_CAPABILITIES.has(capability.name) - ? [capability.name] - : [], - ), - ); - const bounce = plan.workloads.filter((workload) => bounceNames.has(workload.capability)); - const databaseWorkload = plan.workloads.find( - (workload) => workload.id === "database:database", - ); - if (databaseWorkload === undefined) - return failedWithoutMutation( - Cause.fail(new StackStateInvalidError({ message: "Database workload is missing" })), - ); - const stopped = yield* launcher - .stopCapabilities(new Set(["database", ...bounceNames])) - .pipe(Effect.mapError(mapRuntimeError), Effect.exit); - if (Exit.isFailure(stopped)) return failedAfterMutation(stopped.cause); - const wiped = yield* runtime.driver - .wipePersistentData({ stackId: options.stackId, workloadId: databaseWorkload.id }) - .pipe(Effect.mapError(mapRuntimeError), Effect.exit); - if (Exit.isFailure(wiped)) return failedAfterMutation(wiped.cause); - const launched = yield* launcher.launch({ - ...plan, - workloads: [databaseWorkload, ...bounce], - }); - if (Predicate.isTagged(launched, "failed")) { - // Wipe emptied PGDATA; persist first-create so stop → start does not skip schema init. - const launchCause = Cause.map(launched.cause, mapRuntimeError); - const current = yield* read().pipe(Effect.exit); - let cause = launchCause; - if (Exit.isFailure(current)) { - cause = Cause.combine(cause, current.cause); - } else if (current.value === undefined) { - cause = Cause.combine( - cause, - Cause.fail(new StackStateInvalidError({ message: "Stack state is missing" })), - ); - } else { - const persisted = yield* options.stateStore - .replace(options.stackId, { - ...current.value, - desiredLifecycle: "unconfigured", - }) - .pipe(Effect.provideContext(options.context), Effect.exit); - if (Exit.isFailure(persisted)) cause = Cause.combine(cause, persisted.cause); - } - return failedAfterMutation(cause); - } - return { _tag: "succeeded" } satisfies CommandResult; - }); - const resetDatabase = Effect.gen(function* () { - const control = (yield* Ref.get(machine)).stack; - if (!Predicate.isTagged(control, "running")) - return yield* new StackNotRunningError({ - stackId: options.stackId, - message: "Stack is not running", - }); - yield* submitLifecycle("start", resetDatabaseOperation()); - return yield* snapshot(); - }); - const stopOperation = (): Effect.Effect => - Effect.gen(function* () { - const result = yield* controller.stop.pipe( - Effect.provideContext(options.context), - Effect.exit, - ); - if (Exit.isFailure(result)) - return { - _tag: "failed", - cause: result.cause, - cleanup: { _tag: "unproven", cause: result.cause }, - durable: "unsafe", - } satisfies CommandResult; - return { _tag: "succeeded" } satisfies CommandResult; - }); - const signalShutdownIfIdle = (): Effect.Effect => - Effect.gen(function* () { - const lifecycle = yield* activeCommand(); - if (lifecycle !== undefined) { - yield* Deferred.await(lifecycle.result); - return yield* signalShutdownIfIdle(); - } - yield* admission.withPermit( - Effect.gen(function* () { - // Recheck ownership after admission: a lifecycle may have started between the - // initial observation and this critical section. Keep the permit while making the - // final state/phase decision and signalling shutdown so no new start can slip in. - if ((yield* activeCommand()) !== undefined) return; - const state = yield* read().pipe(Effect.exit); - if (Exit.isFailure(state)) return; - const machineState = (yield* Ref.get(machine)).stack; - if ( - Predicate.isTagged(machineState, "stopped") && - (state.value === undefined || - state.value.desiredLifecycle === "stopped" || - state.value.desiredLifecycle === "unconfigured") - ) - yield* signalShutdown; - }), - ); - }); - const shutdownIfIdle = signalShutdownIfIdle(); - const stopWithShutdown = submitLifecycle("stop", stopOperation()); - const operation = (effect: Effect.Effect) => - effect.pipe(Effect.mapError((error) => rpcError(rpcTag(error), stateErrorMessage(error)))); - const destroyOperation: Effect.Effect = Effect.gen(function* () { - const result = yield* controller.destroy.pipe( - Effect.provideContext(options.context), - Effect.exit, - ); - if (Exit.isFailure(result)) - return { - _tag: "failed", - cause: result.cause, - cleanup: { _tag: "unproven", cause: result.cause }, - durable: "unsafe", - } satisfies CommandResult; - return { _tag: "succeeded" } satisfies CommandResult; - }); - const destroy = submitLifecycle("destroy", destroyOperation).pipe(Effect.asVoid); - const logs = (query?: LogQuery): Effect.Effect => - Effect.gen(function* () { - // Capture lifecycle phase before reading the log store. A stopping snapshot stays live - // and gives followers one more poll rather than racing a final batch. - const phaseAtRead = yield* currentPhase(); - const cursor = - query?.cursor?.opaque === EMPTY_LOG_CURSOR.opaque ? undefined : query?.cursor; - const scanned = yield* runtime.logStore - .read(cursor === undefined ? undefined : { cursor }) - .pipe( - Effect.mapError((error) => - error instanceof InvalidLogCursorError - ? error - : new StackStateInvalidError({ message: error.message, cause: error }), - ), - ); - const selected = selectLogBatch(scanned, query); - const running = phaseAtRead !== "stopped"; - return { - ...selected, - running, - } satisfies StackLogBatch; - }); - const maintenanceHandlers = { - probe: Effect.succeed({ - ok: true, - op: "probe", - ownerSessionId: options.ownerSessionId, - stackId: options.stackId, - rpcRelease: STACK_RPC_RELEASE, - } satisfies MaintenanceResponse), - stop: stopWithShutdown.pipe( - Effect.provideContext(options.context), - Effect.as({ ok: true, op: "stop" } satisfies MaintenanceResponse), - Effect.catch((error) => { - const stackErrorTag = maintenanceStackErrorTag(error); - return Effect.succeed({ - ok: false, - error: { - tag: "operation-failed", - message: stateErrorMessage(error), - ...(stackErrorTag === undefined ? {} : { stackErrorTag }), - }, - } satisfies MaintenanceResponse); - }), - ), - }; - const credentials: Effect.Effect = Effect.gen( - function* () { - const state = yield* read().pipe( - Effect.mapError((error) => rpcError(rpcTag(error), stateErrorMessage(error))), - ); - const actualPhase = yield* currentPhase(); - if ( - state === undefined || - actualPhase !== "running" || - state.desiredLifecycle !== "running" - ) - return yield* Effect.fail(credentialsUnavailable); - - const definition = state.definition; - const databaseListener = definition?.listeners.database; - const databaseAssignment = state.ports.find(({ field }) => field === "database"); - if (definition === undefined) - return yield* Effect.fail( - rpcError("InvalidStackConfigError", "Stack credentials require a stack definition"), - ); - if ( - databaseListener === undefined || - !databaseListener.enabled || - databaseAssignment === undefined - ) - return yield* Effect.fail( - rpcError( - "InvalidStackConfigError", - "Stack credentials require an enabled database listener and assigned database port", - ), - ); - - const requiredSecret = (slot: string): Effect.Effect => { - const value = state.secrets[slot]?.value; - return value === undefined || value.length === 0 - ? Effect.fail( - rpcError("StackSecretMismatchError", "Required stack credential is unavailable"), - ) - : Effect.succeed(value); - }; - - const databasePassword = yield* requiredSecret(DATABASE_INTERNAL_PASSWORD_SLOT); - const databaseHost = credentialHost(databaseListener.address); - const databaseUrl = `postgresql://${encodeURIComponent("postgres")}:${encodeURIComponent( - databasePassword, - )}@${databaseHost}:${databaseAssignment.port}/postgres`; - - const auth = definition.capabilities.auth; - const api = auth.enabled - ? yield* Effect.gen(function* () { - const publishableKey = yield* requiredSecret(AUTH_PUBLISHABLE_KEY_SLOT); - const secretKey = yield* requiredSecret(AUTH_SECRET_KEY_SLOT); - const anonJwt = yield* requiredSecret(AUTH_ANON_KEY_SLOT); - const serviceRoleJwt = yield* requiredSecret(AUTH_SERVICE_ROLE_KEY_SLOT); - return { - publishableKey, - secretKey: Redacted.make(secretKey), - anonJwt, - serviceRoleJwt: Redacted.make(serviceRoleJwt), - }; - }) - : undefined; - - const base: EffectStackCredentials = { - database: { - url: Redacted.make(databaseUrl), - password: Redacted.make(databasePassword), - }, - ...(api === undefined ? {} : { api }), - }; - const storage = definition.capabilities.storage; - const s3 = storage.settings.s3_protocol; - if (!storage.enabled || s3 === null || s3 === undefined || s3.enabled !== true) return base; - - const apiListener = definition.listeners.api; - const apiAssignment = state.ports.find(({ field }) => field === "api"); - if (apiListener === undefined || !apiListener.enabled || apiAssignment === undefined) - return yield* Effect.fail( - rpcError( - "InvalidStackConfigError", - "Stack credentials require an enabled API listener and assigned API port", - ), - ); - const accessKeyId = s3.access_key_id; - const region = s3.region; - if ( - accessKeyId === null || - accessKeyId === undefined || - accessKeyId.length === 0 || - region === null || - region === undefined || - region.length === 0 - ) - return yield* Effect.fail( - rpcError("StackStateInvalidError", "Storage credentials are unavailable"), - ); - const secretAccessKey = yield* requiredSecret( - "secret:storage.settings.s3_protocol.secret_access_key", - ); - return { - ...base, - storage: { - endpoint: `http://${credentialHost(apiListener.address)}:${apiAssignment.port}/storage/v1/s3`, - region, - accessKeyId, - secretAccessKey: Redacted.make(secretAccessKey), - }, - } satisfies EffectStackCredentials; - }, - ); - const rpcHandlers: StackRpcHandlers = StackRpcGroup.of({ - status: () => operation(status), - credentials: () => credentials, - start: ({ config }: { readonly config?: StackConfig }) => operation(start({ config })), - destroy: () => operation(destroy), - resetDatabase: () => operation(resetDatabase), - logs: (query: LogQuery) => operation(logs(query)), - }); - return { - status, - start, - resetDatabase, - destroy, - shutdown: Deferred.await(shutdownSignal), - shutdownIfIdle, - logs, - activate, - maintenanceHandlers, - rpcHandlers, - } satisfies Supervisor; - }); diff --git a/packages/stack/src/supervisor/SupervisorState.ts b/packages/stack/src/supervisor/SupervisorState.ts deleted file mode 100644 index 339b20abf0..0000000000 --- a/packages/stack/src/supervisor/SupervisorState.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Cause, Match, Option, Predicate, type Deferred, type Exit } from "effect"; -import type { ExecutionPlan } from "../model/ExecutionPlan.ts"; -import type { StackError } from "../public/Errors.ts"; -import type { StackRecovery } from "../public/Status.ts"; -import type { CapabilityName } from "../public/Capability.ts"; -import type { CapabilityState } from "./CapabilityState.ts"; - -type StableStackState = - | { readonly _tag: "stopped"; readonly session: "uninitialized" | "initialized" } - | { readonly _tag: "running" }; - -type RecoveryState = - | { - readonly _tag: "stop-required"; - readonly cause: Cause.Cause; - } - | { - readonly _tag: "destroy-required"; - readonly evidence: - | { readonly _tag: "persisted-intent" } - | { readonly _tag: "failed"; readonly cause: Cause.Cause }; - }; - -type StoppablePriorState = - | StableStackState - | Extract; - -type StartRecoveryState = { - readonly _tag: "start-recovery"; - readonly attempt: symbol; - readonly completion: LifecycleCompletion; - readonly cause: Cause.Cause; -}; - -export type LifecycleKind = "start" | "stop" | "destroy"; -type LifecycleCompletion = Deferred.Deferred, never>; - -const causeMessage = (cause: Cause.Cause, fallback: string): string => { - const error = Cause.findErrorOption(cause); - return Option.isSome(error) && error.value.message.length > 0 ? error.value.message : fallback; -}; - -export const recoveryForState = (state: StackControlState): StackRecovery | undefined => - Match.value(state).pipe( - Match.when({ _tag: "stop-required" }, (value) => ({ - operation: "stop" as const, - message: causeMessage( - value.cause, - "Runtime cleanup is required; retry stop before proceeding", - ), - })), - Match.when({ _tag: "destroy-required" }, (value) => ({ - operation: "destroy" as const, - message: Predicate.isTagged(value.evidence, "failed") - ? causeMessage( - value.evidence.cause, - "Destructive cleanup is required; retry destroy before proceeding", - ) - : "Destructive cleanup is required; retry destroy before proceeding", - })), - Match.when({ _tag: "stopped" }, () => undefined), - Match.when({ _tag: "running" }, () => undefined), - Match.when({ _tag: "starting" }, () => undefined), - Match.when({ _tag: "stopping" }, () => undefined), - Match.when({ _tag: "destroying" }, () => undefined), - Match.when({ _tag: "start-recovery" }, () => undefined), - Match.exhaustive, - ); - -export type StackControlState = - | StableStackState - | RecoveryState - | StartRecoveryState - | { - readonly _tag: "starting"; - readonly attempt: symbol; - readonly completion: LifecycleCompletion; - readonly prior: StableStackState; - } - | { - readonly _tag: "stopping"; - readonly attempt: symbol; - readonly completion: LifecycleCompletion; - readonly prior: StoppablePriorState; - } - | { - readonly _tag: "destroying"; - readonly attempt: symbol; - readonly completion: LifecycleCompletion; - readonly prior: StableStackState | RecoveryState; - }; - -export type SupervisorSnapshot = Readonly<{ - readonly stack: StackControlState; - readonly sessionId: symbol; - readonly plan: ExecutionPlan | undefined; - readonly capabilities: ReadonlyMap; -}>; - -export const isTransitioning = ( - state: StackControlState, -): state is Extract => - Predicate.isTagged("starting")(state) || - Predicate.isTagged("stopping")(state) || - Predicate.isTagged("destroying")(state) || - Predicate.isTagged("start-recovery")(state); diff --git a/packages/stack/src/supervisor/SupervisorTransitions.ts b/packages/stack/src/supervisor/SupervisorTransitions.ts deleted file mode 100644 index 76b5ba9ecc..0000000000 --- a/packages/stack/src/supervisor/SupervisorTransitions.ts +++ /dev/null @@ -1,1228 +0,0 @@ -import { Cause, Deferred, Exit, Fiber, Match, Predicate } from "effect"; -import type { ActivationResult } from "../gateway/Gateway.ts"; -import type { StackId } from "../public/StackId.ts"; -import { - GatewayActivationError, - StackLifecycleConflictError, - StackNotRunningError, - type StackError, -} from "../public/Errors.ts"; -import type { CleanupOutcome, LifecycleInput } from "./Lifecycle.ts"; -import { CAPABILITY_NAMES, type CapabilityName } from "../public/Capability.ts"; -import { - dependencyClosure, - eagerCapabilities, - type ExecutionPlan, -} from "../model/ExecutionPlan.ts"; -import { - beginStarting, - beginStopping, - cleanupFailed, - completeStarting, - dormant, - dormantFromReady, - promoteStartingPrior, - ready, - restoreStarting, - type CapabilityState, -} from "./CapabilityState.ts"; -import { - isTransitioning, - type LifecycleKind, - recoveryForState, - type StackControlState, - type SupervisorSnapshot, -} from "./SupervisorState.ts"; - -type ActivationFailure = GatewayActivationError | StackError; -export type ActivationExit = Exit.Exit; -export type EndpointExit = Exit.Exit; -export type CommandResult = - | { readonly _tag: "succeeded" } - | { - readonly _tag: "failed"; - readonly cause: Cause.Cause; - readonly cleanup: CleanupOutcome; - readonly durable: "stopped" | "unsafe"; - }; -type RetirementExit = Exit.Exit; - -export type TransitionNotification = - | { - readonly _tag: "endpoint"; - readonly completion: Deferred.Deferred; - readonly result: EndpointExit; - } - | { - readonly _tag: "activation"; - readonly completion: Deferred.Deferred; - readonly result: ActivationExit; - } - | { - readonly _tag: "stopping"; - readonly completion: Deferred.Deferred, never>; - readonly result: Exit.Exit; - } - | { - readonly _tag: "lifecycle"; - readonly completion: Deferred.Deferred, never>; - readonly result: Exit.Exit; - } - | { - readonly _tag: "workload"; - readonly completion: Deferred.Deferred, never>; - readonly result: Exit.Exit; - }; - -export type SnapshotTransition = Readonly<{ - readonly snapshot: SupervisorSnapshot; - readonly notifications: ReadonlyArray; - readonly reconcile: "all-ready" | "none"; -}>; - -type TransitionState = - | Extract - | Extract - | Extract; - -type LifecycleAdmission = - | (SnapshotTransition & { readonly _tag: "accepted" }) - | { - readonly _tag: "rejected"; - readonly reason: LifecycleAdmissionReason; - readonly activeKind?: LifecycleKind; - }; - -type LifecycleAdmissionReason = "lifecycle-transition" | "stop-required" | "destroy-required"; - -/** Projects control state into the phase visible to the supervisor and public projection. */ -export const publicPhase = ( - state: StackControlState, -): "stopped" | "starting" | "running" | "stopping" | "destroying" => { - return Match.value(state).pipe( - Match.when({ _tag: "stopped" }, () => "stopped" as const), - Match.when({ _tag: "running" }, () => "running" as const), - Match.when({ _tag: "starting", prior: { _tag: "running" } }, () => "running" as const), - Match.when({ _tag: "starting" }, () => "starting" as const), - Match.tag("stopping", "start-recovery", "stop-required", () => "stopping" as const), - Match.tag("destroying", "destroy-required", () => "destroying" as const), - Match.exhaustive, - ); -}; - -type ActiveLifecycle = Readonly<{ - readonly kind: LifecycleKind; - readonly result: Deferred.Deferred, never>; -}>; - -export const activeLifecycle = (state: StackControlState): ActiveLifecycle | undefined => - Match.value(state).pipe( - Match.tag("starting", "start-recovery", (value) => ({ - kind: "start" as const, - result: value.completion, - })), - Match.tag("stopping", (value) => ({ kind: "stop" as const, result: value.completion })), - Match.tag("destroying", (value) => ({ kind: "destroy" as const, result: value.completion })), - Match.tag("stopped", "running", "stop-required", "destroy-required", () => undefined), - Match.exhaustive, - ); - -/** Decides lifecycle admission once while the supervisor admission permit is held. */ -export const admitLifecycle = ( - snapshot: SupervisorSnapshot, - kind: LifecycleKind, - completion: Deferred.Deferred, never>, - attempt: symbol, -): LifecycleAdmission => { - const state = snapshot.stack; - const activeKind = activeLifecycle(state)?.kind; - const rejected = (reason: LifecycleAdmissionReason): LifecycleAdmission => ({ - _tag: "rejected", - reason, - ...(activeKind === undefined ? {} : { activeKind }), - }); - const accepted = (next: TransitionState): LifecycleAdmission => ({ - _tag: "accepted", - snapshot: { ...snapshot, stack: next }, - notifications: [], - reconcile: "none", - }); - return Match.value(kind).pipe( - Match.when("start", () => - Match.value(state).pipe( - Match.tag("stopped", "running", (prior) => - accepted({ _tag: "starting", attempt, completion, prior }), - ), - Match.tag("stop-required", () => rejected("stop-required")), - Match.tag("destroy-required", () => rejected("destroy-required")), - Match.tag("starting", "start-recovery", "stopping", "destroying", () => - rejected("lifecycle-transition"), - ), - Match.exhaustive, - ), - ), - Match.when("stop", () => - Match.value(state).pipe( - Match.tag("stopped", "running", "stop-required", (prior) => - accepted({ _tag: "stopping", attempt, completion, prior }), - ), - Match.tag("destroy-required", () => rejected("destroy-required")), - Match.tag("starting", "start-recovery", "stopping", "destroying", () => - rejected("lifecycle-transition"), - ), - Match.exhaustive, - ), - ), - Match.when("destroy", () => - Match.value(state).pipe( - Match.tag("stopped", "running", "stop-required", "destroy-required", (prior) => - accepted({ _tag: "destroying", attempt, completion, prior }), - ), - Match.tag("starting", "start-recovery", "stopping", "destroying", () => - rejected("lifecycle-transition"), - ), - Match.exhaustive, - ), - ), - Match.exhaustive, - ); -}; - -export type ActivationToken = - | { readonly _tag: "exit"; readonly result: ActivationExit } - | { readonly _tag: "deferred"; readonly result: Deferred.Deferred } - | { - readonly _tag: "endpoint"; - readonly capability: CapabilityName; - readonly result: Deferred.Deferred; - } - | { - readonly _tag: "await"; - readonly result: Deferred.Deferred, never>; - }; - -type ActivationDecision = - | { readonly _tag: "respond"; readonly token: ActivationToken } - | { - readonly _tag: "endpoint-owner"; - readonly capability: CapabilityName; - readonly priorRoot: boolean; - readonly transition: (endpoint: Deferred.Deferred) => SnapshotTransition; - } - | { - readonly _tag: "activation-owner"; - readonly capability: CapabilityName; - readonly transition: ( - operation: symbol, - completion: Deferred.Deferred, - ) => SnapshotTransition; - } - | { readonly _tag: "rejected"; readonly error: GatewayActivationError | StackError }; - -type ActivationGate = - | { readonly _tag: "accepted" } - | { readonly _tag: "rejected"; readonly error: GatewayActivationError | StackError }; - -export const activationGate = (snapshot: SupervisorSnapshot, stackId: StackId): ActivationGate => { - const inProgress = (kind: LifecycleKind): ActivationGate => ({ - _tag: "rejected", - error: new StackLifecycleConflictError({ - stackId, - message: `Cannot activate while ${kind} is in progress`, - }), - }); - return Match.value(snapshot.stack).pipe( - Match.when({ _tag: "running" }, () => ({ _tag: "accepted" as const })), - Match.when({ _tag: "stopped" }, () => ({ - _tag: "rejected" as const, - error: new StackNotRunningError({ message: "Stack must be running before activation" }), - })), - Match.when({ _tag: "starting" }, () => inProgress("start")), - Match.when({ _tag: "stopping" }, () => inProgress("stop")), - Match.when({ _tag: "destroying" }, () => inProgress("destroy")), - Match.when({ _tag: "stop-required" }, (state) => ({ - _tag: "rejected" as const, - error: new StackLifecycleConflictError({ - stackId, - message: "Exact runtime cleanup is required; retry stop before activating", - recovery: recoveryForState(state), - }), - })), - Match.when({ _tag: "start-recovery" }, () => inProgress("start")), - Match.when({ _tag: "destroy-required" }, (state) => ({ - _tag: "rejected" as const, - error: new StackLifecycleConflictError({ - stackId, - message: "Destructive cleanup is required; retry destroy before activating", - recovery: recoveryForState(state), - }), - })), - Match.exhaustive, - ); -}; - -export const admitActivation = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, - stackId: StackId, -): ActivationDecision => { - const gate = activationGate(snapshot, stackId); - if (Predicate.isTagged(gate, "rejected")) return gate; - const current = snapshot.capabilities.get(capability); - if (current === undefined) - return { - _tag: "rejected", - error: new GatewayActivationError({ - message: `Capability ${capability} is unavailable in this session`, - }), - }; - return Match.value(current).pipe( - Match.tag("disabled", () => ({ - _tag: "rejected" as const, - error: new GatewayActivationError({ message: `Capability ${capability} is not enabled` }), - })), - Match.tag("stopped", () => ({ - _tag: "rejected" as const, - error: new StackNotRunningError({ message: "Stack must be running before activation" }), - })), - Match.tag("cleanup-failed", () => ({ - _tag: "rejected" as const, - error: new StackLifecycleConflictError({ - stackId, - message: `Capability ${capability} cleanup failed; retry stop before activating`, - }), - })), - Match.tag("starting", (state) => - Match.value(state.completion).pipe( - Match.tag("activation", (completion) => ({ - _tag: "respond" as const, - token: { _tag: "deferred" as const, result: completion.deferred }, - })), - Match.tag("workload", (completion) => ({ - _tag: "respond" as const, - token: { _tag: "await" as const, result: completion.deferred }, - })), - Match.exhaustive, - ), - ), - Match.tag("stopping", (state) => ({ - _tag: "respond" as const, - token: { _tag: "await" as const, result: state.completion }, - })), - Match.tag("ready", (state) => - Match.value(state.endpoint).pipe( - Match.tag("resolved", (endpointState) => ({ - _tag: "respond" as const, - token: { - _tag: "exit" as const, - result: Exit.succeed({ capability, endpoint: endpointState.endpoint }), - }, - })), - Match.tag("resolving", (endpointState) => ({ - _tag: "respond" as const, - token: { _tag: "endpoint" as const, capability, result: endpointState.deferred }, - })), - Match.tag("unresolved", () => { - return { - _tag: "endpoint-owner" as const, - capability, - priorRoot: state.root, - transition: (endpoint: Deferred.Deferred) => ({ - snapshot: beginEndpointResolution(snapshot, capability, endpoint), - notifications: [], - reconcile: "none" as const, - }), - }; - }), - Match.exhaustive, - ), - ), - Match.tag("dormant", (state) => { - return { - _tag: "activation-owner" as const, - capability, - transition: (operation: symbol, completion: Deferred.Deferred) => ({ - snapshot: beginActivation(snapshot, capability, state, operation, completion), - notifications: [], - reconcile: "none" as const, - }), - }; - }), - Match.exhaustive, - ); -}; - -export type ActivationOwner = - | { - readonly _tag: "endpoint"; - readonly capability: CapabilityName; - readonly endpoint: Deferred.Deferred; - readonly priorRoot: boolean; - } - | { - readonly _tag: "activation"; - readonly capability: CapabilityName; - readonly completion: Deferred.Deferred; - }; -export type ActivationClaims = - | { readonly _tag: "none" } - | { - readonly _tag: "claimed"; - readonly claimed: ReadonlyArray; - readonly affected: ReadonlySet; - }; -export type ActivationTerminalOutcome = - | { readonly _tag: "succeeded"; readonly value: ActivationResult } - | { - readonly _tag: "failed"; - readonly cause: Cause.Cause; - readonly cleanup: CleanupOutcome; - }; -type RetirementOwner = { - readonly _tag: "retirement"; - readonly capability: CapabilityName; - readonly operation: symbol; - readonly completion: Deferred.Deferred, never>; - readonly result: RetirementExit; -}; -type LifecycleOwner = { - readonly _tag: "lifecycle"; - readonly completion: Deferred.Deferred, never>; - readonly result: CommandResult; -}; -export type SettlementOwner = RetirementOwner | LifecycleOwner; - -export const matchesActivationOwner = ( - snapshot: SupervisorSnapshot, - owner: ActivationOwner, -): boolean => { - const current = snapshot.capabilities.get(owner.capability); - return Match.value(owner).pipe( - Match.tag( - "endpoint", - (event) => - Predicate.isTagged(current, "ready") && - Predicate.isTagged(current.endpoint, "resolving") && - current.endpoint.deferred === event.endpoint, - ), - Match.tag( - "activation", - (event) => - Predicate.isTagged(current, "starting") && - Predicate.isTagged(current.completion, "activation") && - current.completion.deferred === event.completion, - ), - Match.exhaustive, - ); -}; - -export type StartupHandle = Readonly<{ - readonly completion: Deferred.Deferred, never>; - readonly operation: symbol; -}>; - -export const initializeSession = ( - snapshot: SupervisorSnapshot, - input: LifecycleInput, - sessionId: symbol, - startup: ReadonlyMap, -): SnapshotTransition => { - const roots = new Set( - CAPABILITY_NAMES.filter( - (name) => - input.definition.capabilities[name].enabled && input.plan.activation[name] === "eager", - ), - ); - const eager = eagerCapabilities(input.plan); - const capabilities = new Map(); - for (const name of CAPABILITY_NAMES) { - const configured = input.definition.capabilities[name]; - const handle = startup.get(name); - if (!configured.enabled) capabilities.set(name, { _tag: "disabled" }); - else if (eager.has(name) && handle !== undefined) - capabilities.set( - name, - beginStarting( - dormant(sessionId), - handle.operation, - { _tag: "workload", deferred: handle.completion }, - roots.has(name), - ), - ); - else capabilities.set(name, dormant(sessionId)); - } - return { - snapshot: { ...snapshot, sessionId, plan: input.plan, capabilities }, - notifications: [], - reconcile: "none", - }; -}; - -export const readySet = ( - snapshot: SupervisorSnapshot, - names: ReadonlySet, -): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - const notifications: Array = []; - for (const [name, state] of capabilities) { - if (Predicate.isTagged(state, "dormant") && names.has(name)) - capabilities.set(name, ready(state.sessionId, state.traffic, state.root)); - else if ( - Predicate.isTagged(state, "starting") && - Predicate.isTagged(state.completion, "workload") && - names.has(name) - ) { - notifications.push({ - _tag: "workload", - completion: state.completion.deferred, - result: Exit.void, - }); - capabilities.set(name, completeStarting(state)); - } - } - return { snapshot: { ...snapshot, capabilities }, notifications, reconcile: "none" }; -}; - -export const promoteActivationSet = ( - snapshot: SupervisorSnapshot, - names: ReadonlySet, - activationOwner: CapabilityName, - plan: ExecutionPlan, -): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - for (const name of names) { - const state = capabilities.get(name); - if ( - name !== activationOwner && - Predicate.isTagged(state, "starting") && - Predicate.isTagged(state.completion, "activation") - ) - capabilities.set(name, promoteStartingPrior(state)); - } - return { - snapshot: { ...snapshot, plan, capabilities }, - notifications: [], - reconcile: "none", - }; -}; - -export type ClaimedWorkload = Readonly<{ - readonly name: CapabilityName; - readonly completion: Deferred.Deferred, never>; - readonly prior: Extract; -}>; - -export const claimWorkloads = ( - snapshot: SupervisorSnapshot, - names: ReadonlySet, - handles: ReadonlyMap, -): SnapshotTransition & { readonly claimed: ReadonlyArray } => { - const capabilities = new Map(snapshot.capabilities); - const claimed: Array = []; - for (const name of names) { - const current = capabilities.get(name); - const handle = handles.get(name); - if (!Predicate.isTagged(current, "dormant") || handle === undefined) continue; - claimed.push({ name, completion: handle.completion, prior: current }); - capabilities.set( - name, - beginStarting(current, handle.operation, { - _tag: "workload", - deferred: handle.completion, - }), - ); - } - return { - snapshot: { ...snapshot, capabilities }, - claimed, - notifications: [], - reconcile: "none", - }; -}; - -const beginEndpointResolution = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, - endpoint: Deferred.Deferred, -): SupervisorSnapshot => { - const current = snapshot.capabilities.get(capability); - if (!Predicate.isTagged(current, "ready")) return snapshot; - return { - ...snapshot, - capabilities: new Map(snapshot.capabilities).set(capability, { - ...current, - root: true, - endpoint: { _tag: "resolving", deferred: endpoint }, - }), - }; -}; - -const beginActivation = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, - prior: Extract, - operation: symbol, - completion: Deferred.Deferred, -): SupervisorSnapshot => ({ - ...snapshot, - capabilities: new Map(snapshot.capabilities).set( - capability, - beginStarting(prior, operation, { _tag: "activation", deferred: completion }, true), - ), -}); - -export const setRootSet = ( - snapshot: SupervisorSnapshot, - names: ReadonlySet, -): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - for (const [name, state] of capabilities) - capabilities.set( - name, - Match.value(state).pipe( - Match.tag("disabled", "stopped", (value) => value), - Match.tag("dormant", "starting", "ready", "stopping", "cleanup-failed", (value) => ({ - ...value, - root: names.has(name), - })), - Match.exhaustive, - ), - ); - return { snapshot: { ...snapshot, capabilities }, notifications: [], reconcile: "none" }; -}; - -type TrafficTransition = SnapshotTransition & - Readonly<{ - readonly timer: Fiber.Fiber | undefined; - readonly shouldArm: boolean; - }>; - -export const beginTraffic = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, -): TrafficTransition => { - const current = snapshot.capabilities.get(capability); - const timer = - Predicate.isTagged(current, "ready") && Predicate.isTagged(current.retirement, "armed") - ? current.retirement.fiber - : undefined; - if (Predicate.isTagged(current, "ready")) { - const next = { - ...current, - traffic: current.traffic + 1, - retirement: { _tag: "disarmed" as const }, - }; - return { - snapshot: { ...snapshot, capabilities: new Map(snapshot.capabilities).set(capability, next) }, - notifications: [], - reconcile: "none", - timer, - shouldArm: false, - }; - } - if ( - Predicate.isTagged(current, "dormant") || - Predicate.isTagged(current, "starting") || - Predicate.isTagged(current, "stopping") || - Predicate.isTagged(current, "cleanup-failed") - ) { - const next = { ...current, traffic: current.traffic + 1 }; - return { - snapshot: { ...snapshot, capabilities: new Map(snapshot.capabilities).set(capability, next) }, - notifications: [], - reconcile: "none", - timer: undefined, - shouldArm: false, - }; - } - return { snapshot, notifications: [], reconcile: "none", timer: undefined, shouldArm: false }; -}; - -export const endTraffic = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, - sessionId: symbol, -): TrafficTransition => { - if (snapshot.sessionId !== sessionId) - return { snapshot, notifications: [], reconcile: "none", timer: undefined, shouldArm: false }; - const current = snapshot.capabilities.get(capability); - if ( - !Predicate.isTagged(current, "dormant") && - !Predicate.isTagged(current, "starting") && - !Predicate.isTagged(current, "ready") && - !Predicate.isTagged(current, "stopping") && - !Predicate.isTagged(current, "cleanup-failed") - ) - return { snapshot, notifications: [], reconcile: "none", timer: undefined, shouldArm: false }; - const next = { ...current, traffic: Math.max(0, current.traffic - 1) }; - return { - snapshot: { ...snapshot, capabilities: new Map(snapshot.capabilities).set(capability, next) }, - notifications: [], - reconcile: "none", - timer: undefined, - shouldArm: current.traffic <= 1, - }; -}; - -const canRetire = ( - plan: ExecutionPlan, - roots: ReadonlySet, - capability: CapabilityName, -): boolean => - ![...roots].some( - (root) => root !== capability && dependencyClosure(plan, [root]).has(capability), - ); - -const rootSet = (snapshot: SupervisorSnapshot): ReadonlySet => - new Set( - [...snapshot.capabilities].flatMap(([name, state]) => - "root" in state && state.root ? [name] : [], - ), - ); - -const idleTimeout = ( - timeouts: ReadonlyMap, - plan: ExecutionPlan, - capability: CapabilityName, -): number | false => - plan.activation[capability] === "lazy" ? (timeouts.get(capability) ?? false) : false; - -type IdleCandidate = Readonly<{ - readonly capability: CapabilityName; - readonly plan: ExecutionPlan; - readonly state: Extract; -}>; - -const idleCandidate = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, -): IdleCandidate | undefined => { - if (!Predicate.isTagged(snapshot.stack, "running")) return undefined; - const plan = snapshot.plan; - const state = snapshot.capabilities.get(capability); - if (plan === undefined || !Predicate.isTagged(state, "ready") || state.traffic !== 0) - return undefined; - return canRetire(plan, rootSet(snapshot), capability) ? { capability, plan, state } : undefined; -}; - -type IdleTimerPlan = IdleCandidate & Readonly<{ readonly timeout: number }>; - -export const planIdleTimer = ( - snapshot: SupervisorSnapshot, - timeouts: ReadonlyMap, - capability: CapabilityName, -): IdleTimerPlan | undefined => { - const candidate = idleCandidate(snapshot, capability); - if (candidate === undefined || !Predicate.isTagged(candidate.state.retirement, "disarmed")) - return undefined; - const timeout = idleTimeout(timeouts, candidate.plan, capability); - return timeout === false ? undefined : { ...candidate, timeout }; -}; - -export const beginRetirement = ( - snapshot: SupervisorSnapshot, - capability: CapabilityName, - operation: symbol, - completion: Deferred.Deferred, never>, - epoch: symbol, -): SnapshotTransition & { readonly admitted: boolean } => { - const candidate = idleCandidate(snapshot, capability); - const current = snapshot.capabilities.get(capability); - if ( - candidate === undefined || - !Predicate.isTagged(candidate.state.retirement, "armed") || - candidate.state.retirement.epoch !== epoch - ) { - if ( - Predicate.isTagged(current, "ready") && - Predicate.isTagged(current.retirement, "armed") && - current.retirement.epoch === epoch - ) { - return { - snapshot: { - ...snapshot, - capabilities: new Map(snapshot.capabilities).set(capability, { - ...current, - retirement: { _tag: "disarmed" }, - }), - }, - notifications: [], - reconcile: "none", - admitted: false, - }; - } - return { snapshot, notifications: [], reconcile: "none", admitted: false }; - } - return { - snapshot: { - ...snapshot, - capabilities: new Map(snapshot.capabilities).set( - capability, - beginStopping(candidate.state, operation, completion, false), - ), - }, - notifications: [], - reconcile: "none", - admitted: true, - }; -}; - -export const armRetirement = ( - snapshot: SupervisorSnapshot, - plan: IdleTimerPlan, - epoch: symbol, - fiber: Fiber.Fiber, -): SnapshotTransition => { - return { - snapshot: { - ...snapshot, - capabilities: new Map(snapshot.capabilities).set(plan.capability, { - ...plan.state, - retirement: { _tag: "armed", epoch, fiber }, - }), - }, - notifications: [], - reconcile: "none", - }; -}; - -export const disarmAllRetirements = ( - snapshot: SupervisorSnapshot, -): SnapshotTransition & { readonly timers: ReadonlyArray> } => { - const timers: Array> = []; - const capabilities = new Map(snapshot.capabilities); - for (const [name, state] of capabilities) - if (Predicate.isTagged(state, "ready") && Predicate.isTagged(state.retirement, "armed")) { - timers.push(state.retirement.fiber); - capabilities.set(name, { ...state, retirement: { _tag: "disarmed" } }); - } - return { - snapshot: { ...snapshot, capabilities }, - notifications: [], - reconcile: "none", - timers, - }; -}; - -export type CleanupHandle = Readonly<{ - readonly operation: symbol; - readonly completion: Deferred.Deferred, never>; -}>; - -export const isCleanupCandidate = ( - state: CapabilityState, -): state is Extract => - Predicate.isTagged(state, "ready") || Predicate.isTagged(state, "cleanup-failed"); - -export const enterCapabilityCleanup = ( - snapshot: SupervisorSnapshot, - handles: ReadonlyMap, -): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - for (const [name, state] of capabilities) { - const handle = handles.get(name); - if (handle !== undefined && isCleanupCandidate(state)) - capabilities.set(name, beginStopping(state, handle.operation, handle.completion)); - } - return { snapshot: { ...snapshot, capabilities }, notifications: [], reconcile: "none" }; -}; - -export const settleCapabilityCleanup = ( - snapshot: SupervisorSnapshot, - result: Exit.Exit, - handles: ReadonlyMap, -): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - const notifications: Array = []; - for (const [name, state] of capabilities) { - if (!Predicate.isTagged(state, "stopping")) continue; - const handle = handles.get(name); - if ( - handle === undefined || - handle.operation !== state.operation || - handle.completion !== state.completion - ) - continue; - capabilities.set( - name, - Exit.isSuccess(result) ? { _tag: "stopped" } : cleanupFailed(state, result.cause), - ); - notifications.push({ _tag: "stopping", completion: state.completion, result }); - } - return { snapshot: { ...snapshot, capabilities }, notifications, reconcile: "none" }; -}; - -export const completeDormantCleanup = (snapshot: SupervisorSnapshot): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - for (const [name, state] of capabilities) - if (Predicate.isTagged(state, "dormant")) capabilities.set(name, { _tag: "stopped" }); - return { snapshot: { ...snapshot, capabilities }, notifications: [], reconcile: "none" }; -}; - -const settleStartingCapabilities = ( - snapshot: SupervisorSnapshot, - cause: Cause.Cause, - cleanup: CleanupOutcome, - durable: "stopped" | "unsafe", -): SnapshotTransition => { - const capabilities = new Map(snapshot.capabilities); - const result = Exit.failCause(cause); - const notifications: Array = []; - for (const [name, state] of capabilities) { - if (!Predicate.isTagged(state, "starting") || !Predicate.isTagged(state.completion, "workload")) - continue; - capabilities.set( - name, - Predicate.isTagged(cleanup, "proven") - ? durable === "stopped" - ? { _tag: "stopped" } - : state.prior - : cleanupFailed(state, cause), - ); - notifications.push({ _tag: "workload", completion: state.completion.deferred, result }); - } - return { snapshot: { ...snapshot, capabilities }, notifications, reconcile: "none" }; -}; - -/** Applies one activation terminal event, including claims, roots and owner completion. */ -export const settleActivationTerminal = ( - snapshot: SupervisorSnapshot, - owner: ActivationOwner, - claims: ActivationClaims, - outcome: ActivationTerminalOutcome, -): SnapshotTransition => { - const failed = Predicate.isTagged(outcome, "failed"); - const result = failed ? Exit.failCause(outcome.cause) : Exit.void; - const ownerMatches = matchesActivationOwner(snapshot, owner); - const notification: TransitionNotification = Predicate.isTagged(owner, "endpoint") - ? { - _tag: "endpoint", - completion: owner.endpoint, - result: !failed ? Exit.succeed(outcome.value.endpoint) : Exit.failCause(outcome.cause), - } - : { - _tag: "activation", - completion: owner.completion, - result: !failed ? Exit.succeed(outcome.value) : Exit.failCause(outcome.cause), - }; - if (!ownerMatches) return { snapshot, notifications: [notification], reconcile: "all-ready" }; - - const cleanup = failed ? outcome.cleanup : { _tag: "proven" as const }; - const claimed = Predicate.isTagged(claims, "claimed") ? claims.claimed : []; - const affected = Predicate.isTagged(claims, "claimed") - ? claims.affected - : new Set(); - const capabilities = new Map(snapshot.capabilities); - const notifications: Array = []; - if (failed) { - for (const entry of claimed) { - const current = capabilities.get(entry.name); - if ( - Predicate.isTagged(current, "starting") && - Predicate.isTagged(current.completion, "workload") && - current.completion.deferred === entry.completion - ) { - capabilities.set( - entry.name, - Predicate.isTagged(cleanup, "unproven") - ? cleanupFailed(current, cleanup.cause) - : dormant(entry.prior.sessionId, current.traffic, entry.prior.root), - ); - notifications.push({ _tag: "workload", completion: entry.completion, result }); - } else if ( - Predicate.isTagged(current, "ready") && - current.sessionId === entry.prior.sessionId - ) { - capabilities.set( - entry.name, - Predicate.isTagged(cleanup, "unproven") - ? cleanupFailed(current, cleanup.cause) - : dormantFromReady(current), - ); - } - } - if (Predicate.isTagged(cleanup, "unproven")) - for (const name of affected) { - if (name === owner.capability) continue; - const current = capabilities.get(name); - if ( - !Predicate.isTagged(current, "starting") || - !Predicate.isTagged(current.completion, "activation") - ) - continue; - capabilities.set(name, cleanupFailed(current, cleanup.cause)); - notifications.push({ - _tag: "activation", - completion: current.completion.deferred, - result: Exit.failCause(cleanup.cause), - }); - } - } - const current = capabilities.get(owner.capability); - Match.value(owner).pipe( - Match.tag("endpoint", (event) => { - if ( - !Predicate.isTagged(current, "ready") || - !Predicate.isTagged(current.endpoint, "resolving") - ) - return; - capabilities.set( - event.capability, - failed - ? { ...current, root: event.priorRoot, endpoint: { _tag: "unresolved" } } - : { ...current, endpoint: { _tag: "resolved", endpoint: outcome.value.endpoint } }, - ); - }), - Match.tag("activation", (event) => { - if ( - !Predicate.isTagged(current, "starting") || - !Predicate.isTagged(current.completion, "activation") - ) - return; - capabilities.set( - event.capability, - failed - ? Predicate.isTagged(cleanup, "unproven") - ? cleanupFailed(current, cleanup.cause) - : Predicate.isTagged(snapshot.stack, "stopped") - ? { _tag: "stopped" } - : restoreStarting(current) - : completeStarting(current, { _tag: "resolved", endpoint: outcome.value.endpoint }, true), - ); - }), - Match.exhaustive, - ); - const next = - failed && Predicate.isTagged(cleanup, "unproven") - ? stopRecoverySnapshot({ ...snapshot, capabilities }, cleanup.cause) - : { ...snapshot, capabilities }; - notifications.push(notification); - return { snapshot: next, notifications, reconcile: "all-ready" }; -}; - -const stopRecoverySnapshot = ( - snapshot: SupervisorSnapshot, - cause: Cause.Cause, -): SupervisorSnapshot => - Match.value(snapshot.stack).pipe( - Match.tag("running", () => ({ - ...snapshot, - stack: { _tag: "stop-required" as const, cause }, - })), - Match.when({ _tag: "starting", prior: { _tag: "running" } }, (state) => ({ - ...snapshot, - stack: { - _tag: "start-recovery" as const, - cause, - attempt: state.attempt, - completion: state.completion, - }, - })), - Match.tag( - "stopped", - "stop-required", - "destroy-required", - "start-recovery", - "starting", - "stopping", - "destroying", - () => snapshot, - ), - Match.exhaustive, - ); - -export const settleRetirementOwner = ( - snapshot: SupervisorSnapshot, - owner: RetirementOwner, -): SnapshotTransition => { - const current = snapshot.capabilities.get(owner.capability); - if ( - Predicate.isTagged(current, "stopping") && - current.operation === owner.operation && - current.completion === owner.completion - ) { - if (Exit.isSuccess(owner.result) && owner.result.value) { - const next: CapabilityState = { - _tag: "dormant", - sessionId: current.sessionId, - traffic: current.traffic, - root: false, - retirement: { _tag: "disarmed" }, - }; - return { - snapshot: { - ...snapshot, - capabilities: new Map(snapshot.capabilities).set(owner.capability, next), - }, - notifications: [ - { - _tag: "stopping", - completion: owner.completion, - result: Exit.map(owner.result, () => undefined), - }, - ], - reconcile: "all-ready", - }; - } - if (Exit.isSuccess(owner.result)) - return { - snapshot, - notifications: [{ _tag: "stopping", completion: owner.completion, result: Exit.void }], - reconcile: "none", - }; - return { - snapshot: { - ...stopRecoverySnapshot(snapshot, owner.result.cause), - capabilities: new Map(snapshot.capabilities).set( - owner.capability, - cleanupFailed(current, owner.result.cause), - ), - }, - notifications: [ - { - _tag: "stopping", - completion: owner.completion, - result: Exit.failCause(owner.result.cause), - }, - ], - reconcile: "none", - }; - } - return { - snapshot, - notifications: [ - { - _tag: "stopping", - completion: owner.completion, - result: Exit.isSuccess(owner.result) ? Exit.void : Exit.failCause(owner.result.cause), - }, - ], - reconcile: "none", - }; -}; - -const commandResultExit = (operation: CommandResult): Exit.Exit => - Predicate.isTagged(operation, "failed") ? Exit.failCause(operation.cause) : Exit.void; - -const settleStartingFailure = ( - state: Extract, - operation: Extract, -): StackControlState => - Match.value(state.prior).pipe( - Match.tag("running", (prior) => - Match.value(operation.cleanup).pipe( - Match.tag("unproven", () => ({ _tag: "stop-required" as const, cause: operation.cause })), - Match.tag("proven", () => prior), - Match.exhaustive, - ), - ), - Match.tag("stopped", () => - Match.value(operation.cleanup).pipe( - Match.tag("proven", () => - operation.durable === "stopped" - ? { _tag: "stopped" as const, session: "initialized" as const } - : { _tag: "stop-required" as const, cause: operation.cause }, - ), - Match.tag("unproven", () => ({ _tag: "stop-required" as const, cause: operation.cause })), - Match.exhaustive, - ), - ), - Match.exhaustive, - ); - -export const settleLifecycleOwner = ( - snapshot: SupervisorSnapshot, - owner: LifecycleOwner, -): SnapshotTransition => { - const current = snapshot.stack; - const operation = owner.result; - const matches = isTransitioning(current) && current.completion === owner.completion; - if (!matches) - return { - snapshot, - notifications: [ - { - _tag: "lifecycle", - completion: owner.completion, - result: commandResultExit(operation), - }, - ], - reconcile: "all-ready", - }; - const completionCause = Match.value(current).pipe( - Match.tag("start-recovery", (state) => - Match.value(operation).pipe( - Match.tag("succeeded", () => state.cause), - Match.tag("failed", (failure) => Cause.combine(state.cause, failure.cause)), - Match.exhaustive, - ), - ), - Match.tag("starting", () => undefined), - Match.tag("stopping", () => undefined), - Match.tag("destroying", () => undefined), - Match.exhaustive, - ); - const completion: Exit.Exit = - completionCause === undefined ? commandResultExit(operation) : Exit.failCause(completionCause); - const next: StackControlState = Match.value(current).pipe( - Match.tag("start-recovery", (state) => ({ - _tag: "stop-required" as const, - cause: Match.value(operation).pipe( - Match.tag("succeeded", () => state.cause), - Match.tag("failed", (failure) => Cause.combine(state.cause, failure.cause)), - Match.exhaustive, - ), - })), - Match.tag("starting", (state) => - Match.value(operation).pipe( - Match.tag("succeeded", () => ({ _tag: "running" as const })), - Match.tag("failed", (failure) => settleStartingFailure(state, failure)), - Match.exhaustive, - ), - ), - Match.tag("stopping", () => - Match.value(operation).pipe( - Match.tag("succeeded", () => ({ - _tag: "stopped" as const, - session: "initialized" as const, - })), - Match.tag("failed", (failure) => ({ - _tag: "stop-required" as const, - cause: failure.cause, - })), - Match.exhaustive, - ), - ), - Match.tag("destroying", () => - Match.value(operation).pipe( - Match.tag("succeeded", () => ({ - _tag: "stopped" as const, - session: "initialized" as const, - })), - Match.tag("failed", (failure) => ({ - _tag: "destroy-required" as const, - evidence: { _tag: "failed" as const, cause: failure.cause }, - })), - Match.exhaustive, - ), - ), - Match.exhaustive, - ); - const startup = - Predicate.isTagged(operation, "failed") && - (Predicate.isTagged(current, "starting") || Predicate.isTagged(current, "start-recovery")) - ? settleStartingCapabilities(snapshot, operation.cause, operation.cleanup, operation.durable) - : undefined; - return { - snapshot: { - ...snapshot, - stack: next, - ...(startup === undefined ? {} : { capabilities: startup.snapshot.capabilities }), - }, - notifications: [ - ...(startup?.notifications ?? []), - { _tag: "lifecycle", completion: owner.completion, result: completion }, - ], - reconcile: "all-ready", - }; -}; diff --git a/packages/stack/src/supervisor/SupervisorTransitions.unit.test.ts b/packages/stack/src/supervisor/SupervisorTransitions.unit.test.ts deleted file mode 100644 index 68f4a49158..0000000000 --- a/packages/stack/src/supervisor/SupervisorTransitions.unit.test.ts +++ /dev/null @@ -1,638 +0,0 @@ -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, Predicate } from "effect"; -import type { ActivationResult, BackendEndpoint } from "../gateway/Gateway.ts"; -import { StackRuntimeError, type StackError } from "../public/Errors.ts"; -import { isStackId } from "../public/StackId.ts"; -import { - beginStarting, - beginStopping, - dormant, - ready, - type CapabilityState, -} from "./CapabilityState.ts"; -import { - activationGate, - admitActivation, - admitLifecycle, - endTraffic, - settleActivationTerminal, - settleLifecycleOwner, - settleRetirementOwner, - type ActivationExit, - type EndpointExit, -} from "./SupervisorTransitions.ts"; - -const snapshotFor = (capability: CapabilityState) => ({ - stack: { _tag: "running" as const }, - sessionId: Symbol("session"), - plan: undefined, - capabilities: new Map([["rest" as const, capability]]), -}); - -describe("supervisor transitions", () => { - it.effect("rejects activation during a stopped lifecycle", () => - Effect.gen(function* () { - const value = "a".repeat(64); - if (!isStackId(value)) return yield* Effect.die("invalid stack id fixture"); - const result = activationGate( - { - ...snapshotFor(ready(Symbol("session"), 0, false)), - stack: { _tag: "stopped", session: "initialized" }, - }, - value, - ); - expect(Predicate.isTagged(result, "rejected")).toBe(true); - if (Predicate.isTagged(result, "rejected")) - expect(result.error.message).toBe("Stack must be running before activation"); - }), - ); - - it.effect("uses the start-in-progress diagnostic during start recovery", () => - Effect.gen(function* () { - const value = "a".repeat(64); - if (!isStackId(value)) return yield* Effect.die("invalid stack id fixture"); - const cause = Cause.fail(new StackRuntimeError({ message: "cleanup pending" })); - const completion = yield* Deferred.make, never>(); - const result = activationGate( - { - ...snapshotFor(ready(Symbol("session"), 0, false)), - stack: { _tag: "start-recovery", attempt: Symbol("start"), completion, cause }, - }, - value, - ); - expect(Predicate.isTagged(result, "rejected")).toBe(true); - if (Predicate.isTagged(result, "rejected")) - expect(result.error.message).toBe("Cannot activate while start is in progress"); - }), - ); - - it.effect("rejects commands requiring recovery before admission", () => - Effect.gen(function* () { - const completion = yield* Deferred.make, never>(); - const cause = Cause.fail(new StackRuntimeError({ message: "cleanup pending" })); - const base = snapshotFor(ready(Symbol("session"), 0, false)); - const stop = admitLifecycle( - { ...base, stack: { _tag: "stop-required", cause } }, - "start", - completion, - Symbol("start"), - ); - const destroy = admitLifecycle( - { - ...base, - stack: { _tag: "destroy-required", evidence: { _tag: "failed", cause } }, - }, - "start", - completion, - Symbol("start"), - ); - const stopFromStopRequired = admitLifecycle( - { ...base, stack: { _tag: "stop-required", cause } }, - "stop", - completion, - Symbol("stop"), - ); - const stopFromDestroyRequired = admitLifecycle( - { - ...base, - stack: { _tag: "destroy-required", evidence: { _tag: "failed", cause } }, - }, - "stop", - completion, - Symbol("stop"), - ); - const destroyFromDestroyRequired = admitLifecycle( - { - ...base, - stack: { _tag: "destroy-required", evidence: { _tag: "failed", cause } }, - }, - "destroy", - completion, - Symbol("destroy"), - ); - expect(Predicate.isTagged(stop, "rejected") && stop.reason).toBe("stop-required"); - expect(Predicate.isTagged(destroy, "rejected") && destroy.reason).toBe("destroy-required"); - expect(Predicate.isTagged("accepted")(stopFromStopRequired)).toBe(true); - if (Predicate.isTagged("accepted")(stopFromStopRequired)) { - expect(stopFromStopRequired.snapshot.stack._tag).toBe("stopping"); - if (Predicate.isTagged("stopping")(stopFromStopRequired.snapshot.stack)) - expect(stopFromStopRequired.snapshot.stack.completion).toBe(completion); - } - expect(Predicate.isTagged("rejected")(stopFromDestroyRequired)).toBe(true); - if (Predicate.isTagged("rejected")(stopFromDestroyRequired)) - expect(stopFromDestroyRequired.reason).toBe("destroy-required"); - expect(Predicate.isTagged("accepted")(destroyFromDestroyRequired)).toBe(true); - if (Predicate.isTagged("accepted")(destroyFromDestroyRequired)) { - expect(destroyFromDestroyRequired.snapshot.stack._tag).toBe("destroying"); - if (Predicate.isTagged("destroying")(destroyFromDestroyRequired.snapshot.stack)) - expect(destroyFromDestroyRequired.snapshot.stack.completion).toBe(completion); - } - }), - ); - - it.effect("claims endpoint resolution from one pure activation admission", () => - Effect.gen(function* () { - const value = "a".repeat(64); - if (!isStackId(value)) return yield* Effect.die("invalid stack id fixture"); - const endpoint = yield* Deferred.make(); - const decision = admitActivation( - snapshotFor(ready(Symbol("session"), 0, false)), - "rest", - value, - ); - expect(Predicate.isTagged(decision, "endpoint-owner")).toBe(true); - if (Predicate.isTagged(decision, "endpoint-owner")) { - const current = decision.transition(endpoint).snapshot.capabilities.get("rest"); - expect(Predicate.isTagged(current, "ready")).toBe(true); - if (Predicate.isTagged(current, "ready")) - expect(current.endpoint).toEqual({ _tag: "resolving", deferred: endpoint }); - } - }), - ); - - it.effect("restores the saved root on a matching endpoint failure", () => - Effect.gen(function* () { - const endpoint = yield* Deferred.make, never>(); - const current = ready(Symbol("session"), 3, true, { _tag: "resolving", deferred: endpoint }); - const cause = Cause.fail(new StackRuntimeError({ message: "activation failed" })); - const settlement = settleActivationTerminal( - snapshotFor(current), - { - _tag: "endpoint", - capability: "rest", - endpoint, - priorRoot: false, - }, - { _tag: "none" }, - { _tag: "failed", cause, cleanup: { _tag: "proven" } }, - ); - const next = settlement.snapshot.capabilities.get("rest"); - expect(Predicate.isTagged("ready")(next)).toBe(true); - if (Predicate.isTagged("ready")(next)) { - expect(next.root).toBe(false); - expect(next.traffic).toBe(3); - expect(next.endpoint).toEqual({ _tag: "unresolved" }); - } - expect(settlement.reconcile).toBe("all-ready"); - }), - ); - - it.effect("rejects a deferred-mismatched endpoint completion", () => - Effect.gen(function* () { - const ownerEndpoint = yield* Deferred.make(); - const currentEndpoint = yield* Deferred.make(); - const snapshot = snapshotFor( - ready(Symbol("session"), 0, false, { _tag: "resolving", deferred: currentEndpoint }), - ); - const cause = Cause.fail(new StackRuntimeError({ message: "stale endpoint" })); - const settlement = settleActivationTerminal( - snapshot, - { _tag: "endpoint", capability: "rest", endpoint: ownerEndpoint, priorRoot: false }, - { _tag: "none" }, - { _tag: "failed", cause, cleanup: { _tag: "proven" } }, - ); - expect(settlement.snapshot).toBe(snapshot); - expect(settlement.notifications[0]).toMatchObject({ - _tag: "endpoint", - completion: ownerEndpoint, - result: Exit.failCause(cause), - }); - }), - ); - - it.effect("publishes a stale completion without replacing the newer state", () => - Effect.gen(function* () { - const endpoint = yield* Deferred.make, never>(); - const newer = ready(Symbol("newer"), 1, false); - const snapshot = snapshotFor(newer); - const endpointResult = { host: "127.0.0.1", port: 54321 }; - const settlement = settleActivationTerminal( - snapshot, - { - _tag: "endpoint", - capability: "rest", - endpoint, - priorRoot: true, - }, - { _tag: "none" }, - { - _tag: "succeeded", - value: { capability: "rest", endpoint: endpointResult }, - }, - ); - expect(settlement.snapshot).toBe(snapshot); - expect(settlement.snapshot.capabilities.get("rest")).toBe(newer); - const notification = settlement.notifications[0]; - if (notification !== undefined && Predicate.isTagged("endpoint")(notification)) { - expect(notification.completion).toBe(endpoint); - expect(notification.result).toEqual(Exit.succeed(endpointResult)); - } else expect.fail("expected endpoint notification"); - expect(settlement.reconcile).toBe("all-ready"); - }), - ); - - it.effect("ignores a stale unproven activation failure without claims", () => - Effect.gen(function* () { - const completion = yield* Deferred.make(); - const newer = ready(Symbol("newer"), 1, false); - const currentCompletion = yield* Deferred.make(); - const current = beginStarting(newer, Symbol("current"), { - _tag: "activation", - deferred: currentCompletion, - }); - const snapshot = { - ...snapshotFor(newer), - capabilities: new Map([["rest" as const, current]]), - }; - const cause = Cause.fail(new StackRuntimeError({ message: "stale activation failed" })); - // Execution serializes a live activation claim through settlement; an unmatched - // owner is stale/already-settled evidence and cannot impose recovery on its replacement. - const settlement = settleActivationTerminal( - snapshot, - { _tag: "activation", capability: "rest", completion }, - { _tag: "none" }, - { _tag: "failed", cause, cleanup: { _tag: "unproven", cause } }, - ); - - expect(settlement.snapshot).toBe(snapshot); - expect(settlement.snapshot.capabilities.get("rest")).toBe(current); - expect(settlement.snapshot.stack).toEqual({ _tag: "running" }); - const notification = settlement.notifications[0]; - if (notification !== undefined && Predicate.isTagged("activation")(notification)) { - expect(notification.completion).toBe(completion); - expect(notification.result).toEqual(Exit.failCause(cause)); - } else expect.fail("expected activation notification"); - }), - ); - - it.effect("does not restore claims into a newer session", () => - Effect.gen(function* () { - const endpoint = yield* Deferred.make(); - const completion = yield* Deferred.make, never>(); - const prior = dormant(Symbol("old-session")); - const current = ready(Symbol("new-session"), 0, false, { - _tag: "resolving", - deferred: endpoint, - }); - const claimedCurrent = ready(Symbol("new-session"), 0, false); - const snapshot = { - ...snapshotFor(current), - capabilities: new Map([ - ["rest" as const, current], - ["studio" as const, claimedCurrent], - ]), - }; - const cause = Cause.fail(new StackRuntimeError({ message: "activation failed" })); - const settlement = settleActivationTerminal( - snapshot, - { _tag: "endpoint", capability: "rest", endpoint, priorRoot: false }, - { - _tag: "claimed", - claimed: [{ name: "studio", completion, prior }], - affected: new Set(["studio"]), - }, - { _tag: "failed", cause, cleanup: { _tag: "proven" } }, - ); - expect(settlement.snapshot).not.toBe(snapshot); - expect(settlement.snapshot.capabilities.get("rest")).toMatchObject({ - _tag: "ready", - endpoint: { _tag: "unresolved" }, - }); - expect(settlement.snapshot.capabilities.get("studio")).toBe(claimedCurrent); - }), - ); - - it.effect("preserves lifecycle start recovery after retirement failure", () => - Effect.gen(function* () { - const lifecycleCompletion = yield* Deferred.make, never>(); - const retirementCompletion = yield* Deferred.make, never>(); - const operation = Symbol("retirement"); - const failure = new StackRuntimeError({ message: "retirement failed" }); - const cause = Cause.fail(failure); - const capability = beginStopping( - ready(Symbol("session"), 0, false), - operation, - retirementCompletion, - ); - const snapshot = { - ...snapshotFor(ready(Symbol("session"), 0, false)), - stack: { - _tag: "starting" as const, - attempt: Symbol("start"), - completion: lifecycleCompletion, - prior: { _tag: "running" as const }, - }, - capabilities: new Map([["rest" as const, capability]]), - }; - const settlement = settleRetirementOwner(snapshot, { - _tag: "retirement", - capability: "rest", - operation, - completion: retirementCompletion, - result: Exit.fail(failure), - }); - - expect(settlement.snapshot.stack).toEqual({ - _tag: "start-recovery", - attempt: snapshot.stack.attempt, - completion: lifecycleCompletion, - cause, - }); - }), - ); - - it.effect("preserves a stopping capability when settlement reports no retirement", () => - Effect.gen(function* () { - const completion = yield* Deferred.make, never>(); - const current = ready(Symbol("session"), 1, true); - const operation = Symbol("retirement"); - const capability = beginStopping(current, operation, completion); - const snapshot = { - ...snapshotFor(current), - capabilities: new Map([["rest" as const, capability]]), - }; - const settlement = settleRetirementOwner(snapshot, { - _tag: "retirement", - capability: "rest", - operation, - completion, - result: Exit.succeed(false), - }); - expect(settlement.snapshot.capabilities.get("rest")).toBe(capability); - expect(settlement.notifications).toEqual([ - { _tag: "stopping", completion, result: Exit.void }, - ]); - }), - ); - - it.effect("retires a capability only after retirement succeeds", () => - Effect.gen(function* () { - const completion = yield* Deferred.make, never>(); - const current = ready(Symbol("session"), 1, true); - const operation = Symbol("retirement"); - const capability = beginStopping(current, operation, completion); - const snapshot = { - ...snapshotFor(current), - capabilities: new Map([["rest" as const, capability]]), - }; - const settlement = settleRetirementOwner(snapshot, { - _tag: "retirement", - capability: "rest", - operation, - completion, - result: Exit.succeed(true), - }); - expect(settlement.snapshot.capabilities.get("rest")).toMatchObject({ - _tag: "dormant", - sessionId: current.sessionId, - root: false, - }); - expect(settlement.notifications).toEqual([ - { _tag: "stopping", completion, result: Exit.void }, - ]); - }), - ); - - it.effect("publishes a stale retirement result without changing state", () => - Effect.gen(function* () { - const currentCompletion = yield* Deferred.make, never>(); - const staleCompletion = yield* Deferred.make, never>(); - const current = beginStopping( - ready(Symbol("session"), 1, true), - Symbol("current"), - currentCompletion, - ); - const snapshot = { - ...snapshotFor(ready(Symbol("session"), 1, true)), - capabilities: new Map([["rest" as const, current]]), - }; - const settlement = settleRetirementOwner(snapshot, { - _tag: "retirement", - capability: "rest", - operation: Symbol("stale"), - completion: staleCompletion, - result: Exit.succeed(true), - }); - expect(settlement.snapshot).toBe(snapshot); - expect(settlement.notifications).toEqual([ - { _tag: "stopping", completion: staleCompletion, result: Exit.void }, - ]); - }), - ); - - it.effect("settles every starting failure disposition", () => - Effect.gen(function* () { - const cause = Cause.fail(new StackRuntimeError({ message: "start failed" })); - const scenarios = [ - { - name: "running-proven", - prior: ready(Symbol("running"), 0, false), - cleanup: { _tag: "proven" as const }, - durable: "stopped" as const, - stack: { _tag: "running" as const }, - capability: "stopped" as const, - }, - { - name: "running-unproven", - prior: ready(Symbol("running"), 0, false), - cleanup: { _tag: "unproven" as const, cause }, - durable: "unsafe" as const, - stack: { _tag: "stop-required" as const, cause }, - capability: "cleanup-failed" as const, - }, - { - name: "stopped-proven", - prior: dormant(Symbol("stopped")), - cleanup: { _tag: "proven" as const }, - durable: "stopped" as const, - stack: { _tag: "stopped" as const, session: "initialized" as const }, - capability: "stopped" as const, - }, - { - name: "stopped-unsafe", - prior: dormant(Symbol("stopped")), - cleanup: { _tag: "proven" as const }, - durable: "unsafe" as const, - stack: { _tag: "stop-required" as const, cause }, - capability: "dormant" as const, - }, - ]; - for (const scenario of scenarios) { - const lifecycleCompletion = yield* Deferred.make, never>(); - const workloadCompletion = yield* Deferred.make, never>(); - const current = beginStarting(scenario.prior, Symbol(scenario.name), { - _tag: "workload", - deferred: workloadCompletion, - }); - const snapshot = { - ...snapshotFor(current), - stack: { - _tag: "starting" as const, - attempt: Symbol(scenario.name), - completion: lifecycleCompletion, - prior: Predicate.isTagged(scenario.prior, "ready") - ? { _tag: "running" as const } - : { _tag: "stopped" as const, session: "initialized" as const }, - }, - }; - const settlement = settleLifecycleOwner(snapshot, { - _tag: "lifecycle", - completion: lifecycleCompletion, - result: { _tag: "failed", cause, cleanup: scenario.cleanup, durable: scenario.durable }, - }); - expect(settlement.snapshot.stack).toEqual(scenario.stack); - expect(settlement.snapshot.capabilities.get("rest")).toMatchObject({ - _tag: scenario.capability, - }); - expect(settlement.notifications).toHaveLength(2); - expect(settlement.notifications[0]).toMatchObject({ - _tag: "workload", - completion: workloadCompletion, - result: Exit.failCause(cause), - }); - expect(settlement.notifications[1]).toMatchObject({ - _tag: "lifecycle", - completion: lifecycleCompletion, - result: Exit.failCause(cause), - }); - } - }), - ); - - it.effect( - "restores the prior running capability when cleanup is proven but durability is unsafe", - () => - Effect.gen(function* () { - const cause = Cause.fail(new StackRuntimeError({ message: "state missing" })); - const lifecycleCompletion = yield* Deferred.make, never>(); - const workloadCompletion = yield* Deferred.make, never>(); - const prior = ready(Symbol("running"), 0, false); - const current = beginStarting(prior, Symbol("start"), { - _tag: "workload", - deferred: workloadCompletion, - }); - const snapshot = { - ...snapshotFor(current), - stack: { - _tag: "starting" as const, - attempt: Symbol("start"), - completion: lifecycleCompletion, - prior: { _tag: "running" as const }, - }, - }; - - const settlement = settleLifecycleOwner(snapshot, { - _tag: "lifecycle", - completion: lifecycleCompletion, - result: { _tag: "failed", cause, cleanup: { _tag: "proven" }, durable: "unsafe" }, - }); - - expect(settlement.snapshot.stack).toEqual({ _tag: "running" }); - expect(settlement.snapshot.capabilities.get("rest")).toBe(prior); - }), - ); - - it.effect("ignores traffic release from an older session", () => - Effect.sync(() => { - const snapshot = snapshotFor(ready(Symbol("current"), 1, false)); - const transition = endTraffic(snapshot, "rest", Symbol("old")); - - expect(transition.snapshot).toBe(snapshot); - expect(transition.shouldArm).toBe(false); - }), - ); - - it.effect("publishes a stale start recovery completion with its original result", () => - Effect.gen(function* () { - const oldCompletion = yield* Deferred.make, never>(); - const currentCompletion = yield* Deferred.make, never>(); - const cause = Cause.fail(new StackRuntimeError({ message: "new operation failed" })); - const snapshot = { - ...snapshotFor(ready(Symbol("session"), 0, false)), - stack: { - _tag: "start-recovery" as const, - attempt: Symbol("new-operation"), - completion: currentCompletion, - cause, - }, - }; - const settlement = settleLifecycleOwner(snapshot, { - _tag: "lifecycle", - completion: oldCompletion, - result: { _tag: "succeeded" }, - }); - expect(settlement.snapshot).toBe(snapshot); - const notification = settlement.notifications[0]; - if (notification !== undefined && Predicate.isTagged("lifecycle")(notification)) { - expect(notification.completion).toBe(oldCompletion); - expect(notification.result).toEqual(Exit.void); - expect(Exit.isSuccess(notification.result)).toBe(true); - } else { - expect.fail("expected lifecycle notification"); - } - }), - ); - - it.effect("keeps the original failure for a matching start recovery", () => - Effect.gen(function* () { - const completion = yield* Deferred.make, never>(); - const cause = Cause.fail(new StackRuntimeError({ message: "original failure" })); - const snapshot = { - ...snapshotFor(ready(Symbol("session"), 0, false)), - stack: { - _tag: "start-recovery" as const, - attempt: Symbol("operation"), - completion, - cause, - }, - }; - const settlement = settleLifecycleOwner(snapshot, { - _tag: "lifecycle", - completion, - result: { _tag: "succeeded" }, - }); - expect(settlement.snapshot.stack).toEqual({ _tag: "stop-required", cause }); - const notification = settlement.notifications[0]; - if (notification !== undefined && Predicate.isTagged("lifecycle")(notification)) - expect(notification.result).toEqual(Exit.failCause(cause)); - else expect.fail("expected lifecycle notification"); - }), - ); - - it.effect("stops a queued activation that settles after the stack stopped", () => - Effect.gen(function* () { - const completion = yield* Deferred.make, never>(); - const sessionId = Symbol("session"); - const current = beginStarting(dormant(sessionId), Symbol("operation"), { - _tag: "activation", - deferred: completion, - }); - const snapshot = { - stack: { _tag: "stopped" as const, session: "initialized" as const }, - sessionId, - plan: undefined, - capabilities: new Map([["rest" as const, current]]), - }; - const cause = Cause.fail(new StackRuntimeError({ message: "activation rejected" })); - const settlement = settleActivationTerminal( - snapshot, - { - _tag: "activation", - capability: "rest", - completion, - }, - { _tag: "none" }, - { _tag: "failed", cause, cleanup: { _tag: "proven" } }, - ); - - expect(settlement.snapshot.capabilities.get("rest")).toEqual({ _tag: "stopped" }); - const notification = settlement.notifications[0]; - if (notification !== undefined && Predicate.isTagged("activation")(notification)) { - expect(notification.completion).toBe(completion); - expect(notification.result).toEqual(Exit.failCause(cause)); - } else { - expect.fail("expected activation notification"); - } - }), - ); -}); diff --git a/packages/stack/src/supervisor/handles.integration.test.ts b/packages/stack/src/supervisor/handles.integration.test.ts deleted file mode 100644 index fe573a421f..0000000000 --- a/packages/stack/src/supervisor/handles.integration.test.ts +++ /dev/null @@ -1,836 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Crypto, - Deferred, - Effect, - Exit, - FileSystem, - Fiber, - Option, - Path, - Predicate, - Redacted, - Result, - Schema, - Sink, - Scope, - Stream, -} from "effect"; -import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import { - createStack, - findStack, - inspectStack, - openStack, - listStacks, -} from "../public/EffectStack.ts"; -import type { StackStatus } from "../public/Status.ts"; -import type { CreateStackError } from "../public/Errors.ts"; -import { deriveStackId, resolveStackIdentity } from "../identity/Identity.ts"; -import { - defaultRuntimeEnvironment, - ensureSupervisor, - SUPERVISOR_DISPATCH_SENTINEL, - supervisorEntrypointFor, - type StackRuntimeEnvironmentValue, -} from "./Launcher.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import type { StackId } from "../public/StackId.ts"; -import { - acquireOwnership, - controlEndpointFor, - publishOwnership, - readOwnerMetadata, - StackRuntimeEnvironment, -} from "../state/Ownership.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { makeControlClient, startControlServer } from "../control/ControlServer.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { STACK_RPC_RELEASE, type StackRpcHandlers } from "../control/StackRpc.ts"; -import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; -import { - StackOwnershipConflictError, - StackPreparationError, - StackRuntimeMismatchError, -} from "../public/Errors.ts"; -import type { ContainerEngine } from "../runtime/ContainerEngine.ts"; -import { - ContainerEngineResolver, - defaultContainerEngineResolver, -} from "../runtime/ContainerEngineResolver.ts"; - -const databaseRelease = catalogReleaseFor("database:database"); -if (databaseRelease === undefined) throw new Error("Missing default database release"); - -const withRuntimeRoot = (effect: (project: string) => Effect.Effect) => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-handles-" }); - const path = yield* Path.Path; - const project = path.join(root, "project"); - yield* fs.makeDirectory(project); - const defaults = yield* defaultRuntimeEnvironment; - const runtime: StackRuntimeEnvironmentValue = { - ...defaults, - stateRoot: path.join(root, "managed", "stacks"), - tempRoot: "/tmp", - platform: "posix", - }; - const cleanupOwners = Effect.gen(function* () { - const exists = yield* fs.exists(runtime.stateRoot); - if (!exists) return; - const entries = yield* fs.readDirectory(runtime.stateRoot); - yield* Effect.forEach( - entries, - (entry) => - Schema.is(StackIdSchema)(entry) ? stopOwner(StackIdSchema.make(entry)) : Effect.void, - { discard: true }, - ); - }); - return yield* effect(project).pipe( - Effect.onExit(() => cleanupOwners), - Effect.provideService(StackRuntimeEnvironment, runtime), - Effect.provideService(ContainerEngineResolver, { - isInstalled: () => Effect.succeed(false), - resolve: (kind) => defaultContainerEngineResolver.resolve(kind), - }), - ); - }), - ).pipe(Effect.provide(NodeServices.layer)); - -const stopOwner = (id: StackId) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const fs = yield* FileSystem.FileSystem; - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: id }); - const owner = yield* readOwnerMetadata(env.stateRoot, id, env); - if (owner === undefined) return; - const removed = Stream.runHead( - Stream.filterMapEffect(fs.watch(paths.stackRoot), () => - readOwnerMetadata(env.stateRoot, id, env).pipe( - Effect.map((metadata) => - metadata === undefined ? Result.succeed(true) : Result.fail(undefined), - ), - ), - ), - ); - const watcher = yield* Effect.forkChild(removed); - yield* Effect.scoped( - makeControlClient(owner.endpoint, { - stackId: id, - ownerSessionId: owner.ownerSessionId, - rpcRelease: owner.rpcRelease, - }).stop, - ); - const remaining = yield* readOwnerMetadata(env.stateRoot, id, env); - if (remaining === undefined) { - yield* Fiber.interrupt(watcher); - return; - } - yield* Fiber.join(watcher); - }); - -const quoteModuleSpecifier = (value: string): string => - `'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`; - -const fakeContainerEngine = (kind: "docker" | "podman", calls: string[]): ContainerEngine => ({ - kind, - preflight: Effect.succeed({ host: "host.containers.internal" }), - probe: Effect.sync(() => { - calls.push(`${kind}:probe`); - }), - inspectImage: (image) => - Effect.sync(() => { - calls.push(`${kind}:inspect:${image}`); - return { present: true }; - }), - pullImage: (image) => - Effect.sync(() => { - calls.push(`${kind}:pull:${image}`); - }), - listResources: () => Effect.succeed([]), - createNetwork: () => Effect.die("unused"), - removeNetwork: () => Effect.void, - createVolume: () => Effect.die("unused"), - removeVolume: () => Effect.void, - createContainer: () => Effect.die("unused"), - copyToContainer: () => Effect.void, - startContainer: () => Effect.void, - waitContainer: () => Effect.succeed(0), - stopContainer: () => Effect.void, - removeContainer: () => Effect.void, - streamLogs: () => Stream.empty, -}); - -describe("managed stack handles", { timeout: 30_000 }, () => { - it("selects the private dispatch marker only for compiled Bun paths", () => { - expect( - supervisorEntrypointFor("file:///$bunfs/root/packages/stack/src/supervisor/Launcher.ts"), - ).toBe(SUPERVISOR_DISPATCH_SENTINEL); - expect(supervisorEntrypointFor(import.meta.url)).not.toBe(SUPERVISOR_DISPATCH_SENTINEL); - }); - - it("selects the private dispatch marker for Windows compiled Bun paths", () => { - expect( - supervisorEntrypointFor("C:\\$bunfs\\root\\packages\\stack\\src\\supervisor\\Launcher.ts"), - ).toBe(SUPERVISOR_DISPATCH_SENTINEL); - }); - - it.live("preserves the child ownership detail for metadata without a lease lock", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const runtime: StackRuntimeEnvironmentValue = { - ...env, - platform: "windows", - tempRoot: project, - }; - const stack = yield* createStack({ projectRoot: project }); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId: stack.id }); - const owner = { - format: "supabase-stack-owner-v1" as const, - stackId: stack.id, - ownerSessionId: "stale-owner", - leasePort: 45_001, - endpoint: controlEndpointFor(stack.id, runtime, 45_001), - rpcRelease: STACK_RPC_RELEASE, - }; - const ownerJson = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(owner); - yield* fs.writeFileString(paths.controlMetadata, ownerJson); - const frameJson = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ - ok: false, - code: "ownership-conflict", - message: "Owner metadata exists without a lease lock; refusing recovery", - }); - const frame = new TextEncoder().encode(`${frameJson}\n`); - const spawner = ChildProcessSpawner.make(() => - Effect.succeed( - ChildProcessSpawner.makeHandle({ - pid: ChildProcessSpawner.ProcessId(1), - exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), - isRunning: Effect.succeed(false), - kill: () => Effect.void, - stdin: Sink.drain, - stdout: Stream.empty, - stderr: Stream.empty, - all: Stream.empty, - getInputFd: () => Sink.drain, - getOutputFd: (fd) => (fd === 3 ? Stream.succeed(frame) : Stream.empty), - unref: Effect.succeed(Effect.void), - }), - ), - ); - const result = yield* ensureSupervisor({ - stackId: stack.id, - stateStore: store, - environment: runtime, - }).pipe( - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), - Effect.exit, - ); - yield* fs.remove(paths.controlMetadata, { force: true }); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const failure = Cause.findErrorOption(result.cause); - expect(Option.isSome(failure)).toBe(true); - if (Option.isSome(failure)) { - expect(failure.value).toBeInstanceOf(StackOwnershipConflictError); - expect(failure.value.message).toContain("without a lease lock"); - } - } - }), - ), - ); - - it.live("persists Docker for an omitted container engine without probing", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const resolver = { - isInstalled: () => Effect.die("resolver must not be called"), - resolve: () => Effect.die("resolver must not be called"), - }; - yield* createStack({ projectRoot: project, runtime: { kind: "container" } }).pipe( - Effect.provideService(ContainerEngineResolver, resolver), - ); - expect( - (yield* findStack({ projectRoot: project })).pipe(Option.getOrUndefined)?.runtime, - ).toEqual({ kind: "container", engine: "docker" }); - }), - ), - ); - - it.live("selects Docker for a new stack when the client is installed", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const calls: string[] = []; - const resolver = { - isInstalled: (kind: "docker" | "podman") => - Effect.sync(() => { - calls.push(kind); - return true; - }), - resolve: () => Effect.die("engine construction must not run during create"), - }; - yield* createStack({ projectRoot: project }).pipe( - Effect.provideService(ContainerEngineResolver, resolver), - ); - expect(calls).toEqual(["docker"]); - expect( - (yield* findStack({ projectRoot: project })).pipe(Option.getOrUndefined)?.runtime, - ).toEqual({ kind: "container", engine: "docker" }); - }), - ), - ); - - it.live("selects native for a new stack when the Docker client is absent", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const calls: string[] = []; - const resolver = { - isInstalled: (kind: "docker" | "podman") => - Effect.sync(() => { - calls.push(kind); - return false; - }), - resolve: () => Effect.die("engine construction must not run during create"), - }; - yield* createStack({ projectRoot: project }).pipe( - Effect.provideService(ContainerEngineResolver, resolver), - ); - expect(calls).toEqual(["docker"]); - expect( - (yield* findStack({ projectRoot: project })).pipe(Option.getOrUndefined)?.runtime, - ).toEqual({ kind: "native" }); - }), - ), - ); - - it.live("persists explicit Podman without probing", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const resolver = { - isInstalled: () => Effect.die("resolver must not be called"), - resolve: () => Effect.die("resolver must not be called"), - }; - yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "podman" }, - }).pipe(Effect.provideService(ContainerEngineResolver, resolver)); - expect( - (yield* findStack({ projectRoot: project })).pipe(Option.getOrUndefined)?.runtime, - ).toEqual({ kind: "container", engine: "podman" }); - }), - ), - ); - - it.live("reuses an existing runtime without probing the container resolver", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const created = yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "podman" }, - }); - const resolver = { - isInstalled: () => Effect.die("resolver must not be called"), - resolve: () => Effect.die("resolver must not be called"), - }; - const reopened = yield* createStack({ projectRoot: project }).pipe( - Effect.provideService(ContainerEngineResolver, resolver), - ); - expect(reopened.id).toBe(created.id); - expect((yield* reopened.status).runtime).toEqual({ kind: "container", engine: "podman" }); - }), - ), - ); - - it.live("prepares through only the explicitly selected Podman engine", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const calls: string[] = []; - const dockerCalls: string[] = []; - const resolver = { - isInstalled: () => Effect.succeed(true), - resolve: (kind: "docker" | "podman") => - Effect.succeed( - kind === "podman" - ? fakeContainerEngine(kind, calls) - : fakeContainerEngine(kind, dockerCalls), - ), - }; - const stack = yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "podman" }, - }).pipe(Effect.provideService(ContainerEngineResolver, resolver)); - const prepared = yield* stack.prepare({ capabilities: ["database"] }); - expect(prepared.capabilities).toHaveLength(1); - expect(calls).toEqual(["podman:probe", `podman:inspect:${databaseRelease.containerImage}`]); - expect(dockerCalls).toEqual([]); - }), - ), - ); - - it.live("does not probe a persisted container engine when reopening", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const created = yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "podman" }, - }); - const resolver = { - isInstalled: () => Effect.die("resolver must not be called"), - resolve: () => Effect.die("resolver must not be called"), - }; - const stack = yield* openStack(created.id).pipe( - Effect.provideService(ContainerEngineResolver, resolver), - ); - expect((yield* stack.status).runtime).toEqual({ kind: "container", engine: "podman" }); - }), - ), - ); - - it.live("rejects a conflicting explicit engine before probing", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "docker" }, - }); - const result = yield* createStack({ - projectRoot: project, - runtime: { kind: "container", engine: "podman" }, - }).pipe( - Effect.provideService(ContainerEngineResolver, { - isInstalled: () => Effect.die("resolver must not be called"), - resolve: () => Effect.die("resolver must not be called"), - }), - Effect.exit, - ); - expect(Exit.isFailure(result)).toBe(true); - if (Exit.isFailure(result)) { - const error = Option.getOrUndefined(Cause.findErrorOption(result.cause)); - expect(error).toBeInstanceOf(StackRuntimeMismatchError); - } - }), - ), - ); - - it.live("does not probe an explicitly native identity", () => - withRuntimeRoot((project) => - createStack({ projectRoot: project, runtime: { kind: "native" } }).pipe( - Effect.provideService(ContainerEngineResolver, { - isInstalled: () => Effect.die("native stack must not resolve a container engine"), - resolve: () => Effect.die("native stack must not resolve a container engine"), - }), - ), - ), - ); - - it.live("creates an unconfigured stack without reading config or starting workloads", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot: project }); - const status = yield* stack.status; - expect(status.lifecycle).toBe("unconfigured"); - expect(status.desiredLifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("preserves preparation failures returned by the owner start RPC", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const crypto = yield* Crypto.Crypto; - const identity = yield* resolveStackIdentity({ projectRoot: project }); - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - yield* store.initialize(stackId, { - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "stopped", - ports: [], - privatePorts: [], - secrets: {}, - }); - const ownerSessionId = yield* crypto.randomUUIDv4; - const lease = yield* acquireOwnership({ - stateRoot: env.stateRoot, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - environment: env, - }); - yield* publishOwnership(lease); - const status: StackStatus = { - id: stackId, - lifecycle: "stopped", - desiredLifecycle: "stopped", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - artifacts: [], - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: "eager", - state: "stopped", - })), - }; - const handlers: StackRpcHandlers = { - status: () => Effect.succeed(status), - credentials: () => - Effect.succeed({ - database: { - url: Redacted.make("postgres://localhost"), - password: Redacted.make("secret"), - }, - api: { - publishableKey: "publishable", - secretKey: Redacted.make("secret"), - anonJwt: "anon", - serviceRoleJwt: Redacted.make("service"), - }, - }), - start: () => - Effect.fail({ tag: "StackPreparationError", message: "artifact is incomplete" }), - destroy: () => Effect.void, - resetDatabase: () => Effect.succeed(status), - logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), - }; - yield* startControlServer({ - endpoint: lease.metadata.endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - rpcHandlers: handlers, - }); - const stack = yield* openStack(stackId); - const failed = yield* stack.start({ config: {} }).pipe(Effect.exit); - const error = Exit.isFailure(failed) - ? Option.getOrUndefined(Cause.findErrorOption(failed.cause)) - : undefined; - yield* lease.release; - expect(Exit.isFailure(failed)).toBe(true); - expect(error).toBeInstanceOf(StackPreparationError); - expect(error).toMatchObject({ message: "artifact is incomplete" }); - }), - ), - ); - - it.live("waits for the owner control socket to close before destroy resolves", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const env = yield* StackRuntimeEnvironment; - const crypto = yield* Crypto.Crypto; - const identity = yield* resolveStackIdentity({ projectRoot: project }); - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: env.stateRoot }); - yield* store.initialize(stackId, { - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "stopped", - ports: [], - privatePorts: [], - secrets: {}, - }); - const ownerSessionId = yield* crypto.randomUUIDv4; - const lease = yield* acquireOwnership({ - stateRoot: env.stateRoot, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - environment: env, - }); - yield* publishOwnership(lease); - const ownerScope = yield* Scope.make(); - const status: StackStatus = { - id: stackId, - lifecycle: "stopped", - desiredLifecycle: "stopped", - runtime: { kind: "native" }, - endpoints: {}, - versions: {}, - artifacts: [], - capabilities: CAPABILITY_NAMES.map((name) => ({ - name, - activation: "eager", - state: "stopped", - })), - }; - const destroyStarted = yield* Deferred.make(); - const responseRelease = yield* Deferred.make(); - const callbackStarted = yield* Deferred.make(); - const callbackRelease = yield* Deferred.make(); - const callbackCompleted = yield* Deferred.make(); - const destroyDone = yield* Deferred.make(); - yield* startControlServer({ - endpoint: lease.metadata.endpoint, - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - maintenanceHandlers: { - probe: Effect.succeed({ - ok: true, - op: "probe", - stackId, - ownerSessionId, - rpcRelease: STACK_RPC_RELEASE, - }), - stop: Effect.succeed({ ok: true, op: "stop" }), - }, - rpcHandlers: { - status: () => Effect.succeed(status), - credentials: () => - Effect.fail({ tag: "StackNotRunningError" as const, message: "not running" }), - start: () => Effect.succeed(status), - destroy: () => - Deferred.succeed(destroyStarted, undefined).pipe( - Effect.andThen(Deferred.await(responseRelease)), - Effect.asVoid, - ), - resetDatabase: () => Effect.succeed(status), - logs: () => Effect.succeed({ entries: [], cursor: { opaque: "v1_0" }, running: false }), - }, - onShutdownReady: Deferred.succeed(callbackStarted, undefined).pipe( - Effect.andThen(Deferred.await(callbackRelease)), - Effect.andThen(Deferred.succeed(callbackCompleted, undefined)), - Effect.asVoid, - ), - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - const stack = yield* openStack(stackId); - const destroyFiber = yield* Effect.forkChild( - stack.destroy.pipe(Effect.andThen(Deferred.succeed(destroyDone, undefined))), - { startImmediately: true }, - ); - yield* Deferred.await(destroyStarted); - yield* Deferred.succeed(responseRelease, undefined); - yield* Deferred.await(callbackStarted); - expect(yield* Deferred.isDone(destroyDone)).toBe(false); - yield* Deferred.succeed(callbackRelease, undefined); - yield* Deferred.await(callbackCompleted); - yield* Scope.close(ownerScope, Exit.void); - yield* Fiber.join(destroyFiber); - yield* lease.release; - }), - ), - ); - - it.live("concurrent equivalent creates join one owner", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const [first, second] = yield* Effect.all( - [createStack({ projectRoot: project }), createStack({ projectRoot: project })], - { concurrency: 2 }, - ); - expect(second.id).toBe(first.id); - expect((yield* second.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("discovery never creates an identity", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const found = yield* findStack({ projectRoot: project }); - expect(Option.isNone(found)).toBe(true); - const absentId = StackIdSchema.make("f".repeat(64)); - const absent = yield* inspectStack(absentId).pipe(Effect.exit); - expect(Exit.isFailure(absent)).toBe(true); - }), - ), - ); - - it.live("openStack is observational and rejects unknown ids", () => - withRuntimeRoot((_project) => - Effect.gen(function* () { - const result = yield* openStack(StackIdSchema.make("0".repeat(64))).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - }), - ), - ); - - it.live("filters read-only discovery by project root", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const other = path.join(path.dirname(project), "other-project"); - yield* fs.makeDirectory(other); - const first = yield* createStack({ projectRoot: project }); - yield* createStack({ projectRoot: other }); - const filtered = yield* listStacks({ projectRoot: project }); - expect(filtered.map((entry) => entry.id)).toEqual([first.id]); - }), - ), - ); - - it.live("concurrent creates preserve the published state across an advisory read race", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const env = yield* StackRuntimeEnvironment; - const identity = yield* resolveStackIdentity({ projectRoot: project }); - const stackId = yield* deriveStackId(identity); - const paths = yield* resolveStackPaths({ stateRoot: env.stateRoot, stackId }); - const path = yield* Path.Path; - const registryLock = path.join(path.resolve(env.stateRoot), ".stack-registry.lock"); - const writerReady = yield* Deferred.make(); - const releaseWriter = yield* Deferred.make(); - const writerPublished = yield* Deferred.make(); - const firstFs: FileSystem.FileSystem = { - ...fs, - rename: (from, to) => - to === paths.stateDocument - ? Deferred.succeed(writerReady, undefined).pipe( - Effect.andThen(Deferred.await(releaseWriter)), - Effect.andThen(fs.rename(from, to)), - Effect.tap(() => Deferred.succeed(writerPublished, undefined)), - ) - : fs.rename(from, to), - }; - const secondFs: FileSystem.FileSystem = { - ...fs, - readFileString: (candidate, encoding) => { - if (candidate === registryLock) - return Deferred.succeed(releaseWriter, undefined).pipe( - Effect.andThen(fs.readFileString(candidate, encoding)), - ); - if (candidate !== paths.stateDocument) return fs.readFileString(candidate, encoding); - return fs - .readFileString(candidate, encoding) - .pipe( - Effect.catchTag("PlatformError", (error) => - Predicate.isTagged(error.reason, "NotFound") - ? Deferred.succeed(releaseWriter, undefined).pipe( - Effect.andThen(Deferred.await(writerPublished)), - Effect.andThen(Effect.fail(error)), - ) - : Effect.fail(error), - ), - ); - }, - }; - const create = (fileSystem: FileSystem.FileSystem) => - createStack({ projectRoot: project }).pipe( - Effect.provideService(FileSystem.FileSystem, fileSystem), - ); - const first = yield* Effect.forkChild( - create(firstFs).pipe( - Effect.catchCause((cause) => - Deferred.failCause(writerReady, cause).pipe(Effect.andThen(Effect.failCause(cause))), - ), - ), - { startImmediately: true }, - ); - yield* Deferred.await(writerReady); - const second = yield* Effect.forkChild(create(secondFs), { startImmediately: true }); - const [firstHandle, secondHandle] = yield* Effect.all( - [Fiber.join(first), Fiber.join(second)], - { - concurrency: 2, - }, - ); - expect(secondHandle.id).toBe(firstHandle.id); - expect((yield* secondHandle.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("concurrent caller processes share one stack identity after exit", () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const path = yield* Path.Path; - const supabaseHome = path.dirname(project); - const stackModule = new URL("../public/EffectStack.ts", import.meta.url).href; - const encodedStackModule = quoteModuleSpecifier(stackModule); - // bare imports in a `node -e` script resolve from cwd, so run from the package root - const cwd = path.resolve(import.meta.dirname, "../.."); - const script = ` - const { Effect } = await import("effect"); - const { NodeServices } = await import("@effect/platform-node"); - const { createStack } = await import(${encodedStackModule}); - const stack = await Effect.runPromise( - Effect.scoped( - createStack({ projectRoot: process.argv[1], runtime: { kind: "native" } }).pipe( - Effect.provide(NodeServices.layer), - ), - ), - ); - process.stdout.write(stack.id); - `; - const spawnCaller = () => - Effect.gen(function* () { - const child = yield* ChildProcess.make( - process.execPath, - ["--input-type=module", "-e", script, project], - { - cwd, - env: { SUPABASE_HOME: supabaseHome }, - extendEnv: true, - stdout: "pipe", - stderr: "pipe", - }, - ); - const [chunks, stderrChunks, code] = yield* Effect.all( - [Stream.runCollect(child.stdout), Stream.runCollect(child.stderr), child.exitCode], - { concurrency: 3 }, - ); - const bytes = new Uint8Array(chunks.reduce((sum, value) => sum + value.byteLength, 0)); - let offset = 0; - for (const value of chunks) { - bytes.set(value, offset); - offset += value.byteLength; - } - const stderrBytes = new Uint8Array( - stderrChunks.reduce((sum, value) => sum + value.byteLength, 0), - ); - offset = 0; - for (const value of stderrChunks) { - stderrBytes.set(value, offset); - offset += value.byteLength; - } - const stderr = new TextDecoder().decode(stderrBytes); - return { id: new TextDecoder().decode(bytes), code, stderr }; - }); - const [first, second] = yield* Effect.all([spawnCaller(), spawnCaller()], { - concurrency: 2, - }); - expect(first.code, first.stderr).toBe(0); - expect(second.code, second.stderr).toBe(0); - expect(first.id).toBe(second.id); - const attached = yield* openStack(StackIdSchema.make(first.id)); - expect((yield* attached.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live( - "maintenance stop keeps an unconfigured owner usable without fabricating lifecycle state", - () => - withRuntimeRoot((project) => - Effect.gen(function* () { - const stack = yield* createStack({ projectRoot: project }); - yield* stack.stop; - const status = yield* stack.status; - expect(status.lifecycle).toBe("unconfigured"); - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/host-listener.integration.test.ts b/packages/stack/src/supervisor/host-listener.integration.test.ts deleted file mode 100644 index aac3ec66f5..0000000000 --- a/packages/stack/src/supervisor/host-listener.integration.test.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Fiber, Option, Predicate, Queue } from "effect"; -// oxlint-disable-next-line effecttsgo/node-builtin-import -- fixture passes the native server instance into HostListener adoption APIs. -import { createServer, request as httpRequest, type Server as HttpServer } from "node:http"; -import { connect as connectNet, type Server as NetServer, type Socket } from "node:net"; -import { PortUnavailableError } from "../public/Errors.ts"; -import { bindHostListener, bindHostListenerWithOptions, checkHostPort } from "./HostListener.ts"; - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -describe("host listener binding", () => { - it.live("allocates independent TCP resources for each evaluation", () => - run( - Effect.gen(function* () { - const effect = bindHostListener("127.0.0.1", 0, "database"); - const [first, second] = yield* Effect.all([effect, effect], { concurrency: 2 }); - expect(first.port).toBeGreaterThan(0); - expect(second.port).toBeGreaterThan(0); - expect(first.port).not.toBe(second.port); - yield* first.close; - expect(first.binding.server.listening).toBe(false); - const socket = yield* Effect.acquireRelease( - Effect.callback((resume) => { - const connection = connectNet(second.port, "127.0.0.1"); - connection.once("connect", () => resume(Effect.succeed(connection))); - connection.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => connection.destroy()); - }), - (connection) => Effect.sync(() => connection.destroy()), - ); - expect(socket.destroyed).toBe(false); - }), - ), - ); - - it.live("evaluates concurrent port checks with independent temporary servers", () => - run( - Effect.gen(function* () { - const effect = checkHostPort("127.0.0.1", 0, "database"); - const ports = yield* Effect.all([effect, effect], { concurrency: 2 }); - expect(ports).toEqual([undefined, undefined]); - }), - ), - ); - - it.live("binds an IPv6 wildcard listener with dual-stack IPv4 access", () => - run( - Effect.gen(function* () { - const listener = yield* bindHostListener("::", 0, "api"); - expect(listener.port).toBeGreaterThan(0); - if (listener.binding.kind !== "http" || listener.binding.pendingEvents === undefined) - return yield* Effect.die("IPv6 API listener did not expose HTTP events"); - const request = yield* Effect.forkChild( - Effect.callback((resume) => { - const request = httpRequest( - { host: "127.0.0.1", port: listener.port, path: "/" }, - (response) => resume(Effect.succeed(response)), - ); - request.once("error", (error) => resume(Effect.fail(error))); - request.end(); - return Effect.sync(() => request.destroy()); - }), - { startImmediately: true }, - ); - const event = yield* Queue.take(listener.binding.pendingEvents.queue); - if (!Predicate.isTagged(event, "request")) - return yield* Effect.die("IPv4 request was not captured"); - event.response.end("ok"); - const response = yield* Fiber.join(request); - response.resume(); - }), - ), - ); - - it.live("binds HTTP and TCP listeners for direct gateway adoption", () => - run( - Effect.gen(function* () { - const http = yield* bindHostListener("127.0.0.1", 0, "api"); - const tcp = yield* bindHostListener("127.0.0.1", 0, "database"); - expect(http.binding.kind).toBe("http"); - expect(tcp.binding.kind).toBe("tcp"); - expect(http.binding.server.listening).toBe(true); - expect(tcp.binding.server.listening).toBe(true); - yield* http.close; - yield* http.close; - yield* tcp.close; - expect(http.binding.server.listening).toBe(false); - expect(tcp.binding.server.listening).toBe(false); - }), - ), - ); - - it.live("reports an occupied exact listener as a typed unavailable error", () => - run( - Effect.gen(function* () { - const held = yield* bindHostListener("127.0.0.1", 0, "api"); - const address = held.binding.server.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("bound listener did not expose an address"); - let failedServer: HttpServer | undefined; - const failed = yield* bindHostListenerWithOptions("127.0.0.1", address.port, "api", { - createHttpServer: () => { - const created = createServer(); - failedServer = created; - return created; - }, - }).pipe(Effect.exit); - const error = errorOf(failed); - expect(error).toBeInstanceOf(PortUnavailableError); - expect(error).toMatchObject({ cause: { code: "EADDRINUSE" } }); - expect(failedServer?.listening).toBe(false); - }), - ), - ); - - it.live("cancels a listener whose bind callback has not fired", () => - run( - Effect.gen(function* () { - let server: HttpServer | undefined; - // The real server factory is intentionally used through a deferred listen seam below; - // this keeps the cancellation test deterministic without racing a kernel bind. - const effect = bindHostListenerWithOptions("127.0.0.1", 0, "api", { - createHttpServer: () => { - const created = createServer(); - server = created; - return created; - }, - listen: () => undefined, - }); - const fiber = yield* Effect.forkChild(effect); - yield* Effect.yieldNow; - yield* Fiber.interrupt(fiber); - expect(server?.listening).toBe(false); - }), - ), - ); - - it.live("closes connections accepted before gateway adoption", () => - run( - Effect.gen(function* () { - const http = yield* bindHostListener("127.0.0.1", 0, "api"); - const tcp = yield* bindHostListener("127.0.0.1", 0, "database"); - const connect = (server: HttpServer | NetServer) => - Effect.gen(function* () { - const address = server.address(); - if (typeof address !== "object" || address === null) { - return yield* Effect.die("listener did not expose an address"); - } - const accepted = yield* Effect.forkChild( - Effect.callback((resume) => { - const onConnection = (socket: Socket) => resume(Effect.succeed(socket)); - server.once("connection", onConnection); - return Effect.sync(() => server.off("connection", onConnection)); - }), - { startImmediately: true }, - ); - yield* Effect.acquireRelease( - Effect.callback((resume) => { - const socket = connectNet(address.port, "127.0.0.1"); - socket.once("connect", () => resume(Effect.succeed(socket))); - socket.once("error", (error) => resume(Effect.fail(error))); - return Effect.sync(() => { - socket.destroy(); - }); - }), - (socket) => Effect.sync(() => socket.destroy()), - ); - const acceptedSocket = yield* Fiber.join(accepted).pipe(Effect.timeout("5 seconds")); - return acceptedSocket; - }); - const httpSocket = yield* connect(http.binding.server); - const tcpSocket = yield* connect(tcp.binding.server); - yield* http.close; - yield* tcp.close; - expect(httpSocket.destroyed).toBe(true); - expect(tcpSocket.destroyed).toBe(true); - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/ingress.integration.test.ts b/packages/stack/src/supervisor/ingress.integration.test.ts deleted file mode 100644 index 0c83dfd1c0..0000000000 --- a/packages/stack/src/supervisor/ingress.integration.test.ts +++ /dev/null @@ -1,822 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Context, Crypto, Effect, Exit, FileSystem, Option, Path, Ref, Scope } from "effect"; -import { - createServer as createHttpServer, - request as requestHttp, - type IncomingMessage, - type ServerResponse, - // oxlint-disable-next-line effecttsgo/node-builtin-import -- fixture needs Node request clients and server instances to exercise ingress forwarding and adoption. -} from "node:http"; -import type { Duplex } from "node:stream"; -import { deriveStackId, type StackIdentity } from "../identity/Identity.ts"; -import { compileStack } from "../model/Compiler.ts"; -import { excludeStackCapabilities } from "../model/Exclusions.ts"; -import type { ExecutionPlan } from "../model/ExecutionPlan.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import { - GatewayActivationError, - PortUnavailableError, - StackLifecycleConflictError, - StackPreparationError, -} from "../public/Errors.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { makeSupervisorIngress } from "./Ingress.ts"; -import { bindHostListener } from "./HostListener.ts"; -import type { HostListener } from "./HostListener.ts"; -import { privateBindingIntentsFor } from "../runtime/WorkloadRuntimeSpec.ts"; - -const identity: StackIdentity = { - projectRoot: "/tmp/supabase-ingress", - branchContext: "ordinary-workspace", - stackName: "ingress", -}; - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const bindPrivate = (_address: string, port: number, _binding: string) => - Effect.succeed({ port, close: Effect.void }); - -const closeServer = (server: ReturnType): Effect.Effect => - Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }); - -const listenBackend = (server: ReturnType) => - Effect.acquireRelease( - Effect.callback((resume) => { - server.once("error", (error) => resume(Effect.fail(error))); - server.listen(0, "127.0.0.1", () => resume(Effect.void)); - }), - () => closeServer(server), - ).pipe(Effect.as(server)); - -const makeIngressContext = (prefix: string) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const context = Context.make(FileSystem.FileSystem, fs).pipe( - Context.add(Path.Path, path), - Context.add(Crypto.Crypto, crypto), - ); - const root = yield* fs.makeTempDirectoryScoped({ prefix }); - return { context, fs, path, root }; - }); - -const persistedIngressState = ( - identity: StackIdentity, - compiled: { - readonly definition: PersistedStackState["definition"]; - readonly executionPlan: ExecutionPlan; - }, - privatePortBase: number, -): PersistedStackState => ({ - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "running", - definition: compiled.definition, - ports: [], - privatePorts: privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }).map((binding, index) => ({ - ...binding, - port: privatePortBase + index, - })), - secrets: {}, -}); - -const request = (port: number, path = "/rest/v1/items", method = "GET", host = "127.0.0.1") => - Effect.callback<{ readonly status: number; readonly body: string }, Error>((resume) => { - const client = requestHttp({ host, port, path, method }, (response: IncomingMessage) => { - const chunks: Buffer[] = []; - response.on("data", (chunk: Buffer) => chunks.push(chunk)); - response.once("end", () => - resume( - Effect.succeed({ - status: response.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - }), - ), - ); - }); - client.once("error", (error) => resume(Effect.fail(error))); - client.end(); - return Effect.sync(() => client.destroy()); - }); - -describe("Supervisor ingress", () => { - it.live("does not allocate a public listener for a disabled pooler workload", () => - run( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const context = Context.make(FileSystem.FileSystem, fs).pipe( - Context.add(Path.Path, path), - Context.add(Crypto.Crypto, crypto), - ); - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-ingress-pooler-" }); - const stackIdentity = { ...identity, projectRoot: root }; - const stackId = yield* deriveStackId(stackIdentity); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { pooler: { enabled: false } }, - listeners: { pooler: { enabled: true, port: 55_329 } }, - }, - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(stackId, { - format: "supabase-stack-state-v1", - identity: stackIdentity, - runtime: { kind: "native" }, - desiredLifecycle: "running", - definition: compiled.definition, - ports: [], - privatePorts: privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }).map((binding, index) => ({ ...binding, port: 30_000 + index })), - secrets: {}, - }); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindPrivate, - }); - const state = yield* store.read(stackId).pipe(Effect.map((value) => value!)); - yield* ingress.acquire({ - stackId, - state, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }); - const acquired = yield* store.read(stackId).pipe(Effect.map((value) => value!)); - expect(acquired.ports.some(({ field }) => field === "pooler")).toBe(false); - expect(acquired.ports.some(({ field }) => field === "api")).toBe(true); - }), - ), - ); - - it.live("closes reservation scopes after repeated failed acquire attempts", () => - run( - Effect.gen(function* () { - const { context, fs, path, root } = yield* makeIngressContext("supabase-ingress-scope-"); - const projectRoot = path.join(root, "project"); - yield* fs.makeDirectory(projectRoot); - const stackIdentity = { - ...identity, - projectRoot, - }; - const stackId = yield* deriveStackId(stackIdentity); - const databasePort = 50_000 + (Number.parseInt(stackId.slice(0, 4), 16) % 10_000); - const compiled = yield* compileStack({ - projectRoot, - runtime: { kind: "native" }, - config: { listeners: { database: { port: databasePort } } }, - }); - const store = yield* makeStackStateStore({ - stateRoot: path.join(root, "managed", "stacks"), - }); - yield* store.initialize(stackId, { - format: "supabase-stack-state-v1", - identity: stackIdentity, - runtime: { kind: "native" }, - desiredLifecycle: "running", - definition: compiled.definition, - ports: [], - privatePorts: privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }).map((binding, index) => ({ - ...binding, - port: 30_000 + index, - })), - secrets: {}, - }); - const apiCloseCount = yield* Ref.make(0); - const bindHost = ( - address: string, - port: number, - field: HostListener["field"], - ): Effect.Effect => { - if (field === "database") - return Effect.fail( - new PortUnavailableError({ - field, - port, - message: "Injected database listener failure", - }), - ); - if (field !== "api") - return Effect.fail( - new PortUnavailableError({ - field, - port, - message: "Unexpected listener bind", - }), - ); - const close = Ref.update(apiCloseCount, (count) => count + 1); - return Effect.gen(function* () { - yield* Effect.addFinalizer(() => close); - return { - field, - address, - port, - close, - connections: { sockets: new Set() }, - binding: { kind: "http", server: createHttpServer() }, - } satisfies HostListener; - }); - }; - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: path.join(root, "managed", "stacks"), - store, - context, - bindHost, - bindPrivate, - }); - const state = yield* store.read(stackId).pipe(Effect.map((value) => value!)); - const input = { - stackId, - state, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }; - for (let attempt = 1; attempt <= 3; attempt += 1) { - expect(Exit.isFailure(yield* ingress.acquire(input).pipe(Effect.exit))).toBe(true); - expect(yield* Ref.get(apiCloseCount)).toBe(attempt); - } - }), - ), - ); - - it.live("adopts a coordinated listener and forwards a public request", () => - run( - Effect.gen(function* () { - const { context, root } = yield* makeIngressContext("supabase-ingress-"); - const stackIdentity = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(stackIdentity); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - listeners: { - database: { enabled: false }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(stackId, persistedIngressState(stackIdentity, compiled, 30000)); - const listenerCloseCount = yield* Ref.make(0); - const bindHost = (address: string, port: number, field: HostListener["field"]) => - bindHostListener(address, port, field).pipe( - Effect.map((listener) => ({ - ...listener, - close: listener.close.pipe( - Effect.andThen(Ref.update(listenerCloseCount, (count) => count + 1)), - ), - })), - ); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindPrivate, - apiMaterial: () => - Effect.succeed({ - publishableKey: "sb_publishable_test", - secretKey: "sb_secret_test", - anonJwt: "anon-jwt", - serviceRoleJwt: "service-jwt", - }), - bindHost, - resolveInternalApiBindAddress: () => Effect.succeed("::1"), - }); - const input = { - stackId, - desiredLifecycle: "running" as const, - state: yield* store.read(stackId).pipe(Effect.map((value) => value!)), - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }; - const reservation = yield* ingress.acquire(input); - expect(reservation.hostListeners).toHaveLength(1); - const backend = yield* listenBackend( - createHttpServer((_request: IncomingMessage, response: ServerResponse) => { - response.statusCode = 200; - response.end("forwarded"); - }), - ); - const backendAddress = backend.address(); - if (typeof backendAddress !== "object" || backendAddress === null) - return yield* Effect.die("backend did not expose an address"); - let recoveryRequired = false; - yield* ingress.open(input, reservation, (capability) => - recoveryRequired - ? Effect.fail( - new StackLifecycleConflictError({ - message: "cleanup failed", - recovery: { operation: "stop", message: "backend cleanup failed" }, - }), - ) - : Effect.succeed({ - capability, - endpoint: { host: "127.0.0.1", port: backendAddress.port }, - }), - ); - const api = reservation.assignments.api; - if (api === undefined) return yield* Effect.die("API listener was not assigned"); - const response = yield* request(api.port); - expect(response.status).toBe(200); - expect(response.body).toBe("forwarded"); - const internalResponse = yield* request(api.port, "/rest/v1/items", "GET", "::1"); - expect(internalResponse.status).toBe(200); - expect(internalResponse.body).toBe("forwarded"); - recoveryRequired = true; - const recoveryResponse = yield* request(api.port); - expect(recoveryResponse.status).toBe(503); - expect(recoveryResponse.body).toContain('"error":"STACK_RECOVERY_REQUIRED"'); - expect(recoveryResponse.body).toContain('"operation":"stop"'); - expect(recoveryResponse.body).toContain( - '"message":"Retry stack stop before activating workloads"', - ); - const reused = yield* ingress.acquire(input); - expect(reused.fresh).toBe(false); - recoveryRequired = false; - yield* ingress.open(input, reused, (capability) => - Effect.succeed({ - capability, - endpoint: { host: "127.0.0.1", port: backendAddress.port }, - }), - ); - const reusedResponse = yield* request(api.port); - expect(reusedResponse.status).toBe(200); - expect(reusedResponse.body).toBe("forwarded"); - yield* ingress.close; - expect(yield* Ref.get(listenerCloseCount)).toBe(2); - const reacquired = yield* ingress.acquire(input); - expect(reacquired.fresh).toBe(true); - const stale = yield* ingress - .open(input, reservation, (capability) => - Effect.succeed({ - capability, - endpoint: { host: "127.0.0.1", port: backendAddress.port }, - }), - ) - .pipe(Effect.exit); - expect(Exit.isFailure(stale)).toBe(true); - yield* ingress.close; - }), - ), - ); - - it.live("reuses a wildcard API listener for the internal loopback bind", () => - run( - Effect.gen(function* () { - const { context, root } = yield* makeIngressContext("supabase-ingress-wildcard-"); - const stackIdentity = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(stackIdentity); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - listeners: { - api: { address: "0.0.0.0" }, - database: { enabled: false }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const persisted = persistedIngressState(stackIdentity, compiled, 30_000); - yield* store.initialize(stackId, persisted); - const binds: Array<{ readonly address: string; readonly field: HostListener["field"] }> = - []; - const bindHost = (address: string, port: number, field: HostListener["field"]) => - bindHostListener(address, port, field).pipe( - Effect.tap((listener) => - Effect.sync(() => { - binds.push({ address: listener.address, field }); - }), - ), - ); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindHost, - bindPrivate, - apiMaterial: () => - Effect.succeed({ - publishableKey: "sb_publishable_test", - secretKey: "sb_secret_test", - anonJwt: "anon-jwt", - serviceRoleJwt: "service-jwt", - }), - resolveInternalApiBindAddress: () => Effect.succeed("127.0.0.1"), - }); - const input = { - stackId, - desiredLifecycle: "running" as const, - state: persisted, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }; - const reservation = yield* ingress.acquire(input); - const backend = yield* listenBackend( - createHttpServer((_request: IncomingMessage, response: ServerResponse) => { - response.statusCode = 200; - response.end("wildcard-forwarded"); - }), - ); - const backendAddress = backend.address(); - if (typeof backendAddress !== "object" || backendAddress === null) - return yield* Effect.die("backend did not expose an address"); - yield* ingress.open(input, reservation, (capability) => - Effect.succeed({ - capability, - endpoint: { host: "127.0.0.1", port: backendAddress.port }, - }), - ); - const api = reservation.assignments.api; - if (api === undefined) return yield* Effect.die("API listener was not assigned"); - const response = yield* request(api.port); - expect(response.status).toBe(200); - expect(response.body).toBe("wildcard-forwarded"); - expect(binds.filter((entry) => entry.field === "api")).toHaveLength(1); - expect(binds.find((entry) => entry.field === "api")?.address).toBe("0.0.0.0"); - yield* ingress.close; - }), - ), - ); - - it.live("rejects incomplete persisted gateway material before opening", () => - run( - Effect.gen(function* () { - const { context, root } = yield* makeIngressContext("supabase-ingress-material-"); - const stackId = yield* deriveStackId({ - ...identity, - projectRoot: root, - }); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { capabilities: { rest: {} } }, - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(stackId, { - format: "supabase-stack-state-v1", - identity: { - ...identity, - projectRoot: root, - }, - runtime: { kind: "native" }, - desiredLifecycle: "running", - definition: compiled.definition, - ports: [], - privatePorts: privateBindingIntentsFor(compiled.executionPlan, { - definition: compiled.definition, - runtime: { kind: "native" }, - }).map((binding, index) => ({ - ...binding, - port: 30100 + index, - })), - secrets: {}, - }); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindPrivate, - }); - const state = yield* store.read(stackId).pipe(Effect.map((value) => value!)); - const reservation = yield* ingress.acquire({ - stackId, - state, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }); - const failed = yield* ingress - .open( - { - stackId, - state, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }, - reservation, - () => Effect.fail(new GatewayActivationError({ message: "not reached" })), - ) - .pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - if (Exit.isFailure(failed)) { - const error = Cause.findErrorOption(failed.cause); - expect(Option.isSome(error)).toBe(true); - if (Option.isSome(error)) { - expect(error.value).toBeInstanceOf(StackPreparationError); - expect(error.value.message).toBe("Persisted API gateway material is incomplete"); - } - } - }), - ), - ); - - it.live("opens non-API listeners without resolving API gateway material", () => - run( - Effect.gen(function* () { - const { context, root } = yield* makeIngressContext("supabase-ingress-no-api-"); - const stackIdentity = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(stackIdentity); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - listeners: { - api: { enabled: false }, - database: { enabled: true }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const persisted = persistedIngressState(stackIdentity, compiled, 30200); - yield* store.initialize(stackId, persisted); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindPrivate, - apiMaterial: () => - Effect.fail(new StackPreparationError({ message: "API material must not resolve" })), - }); - const input = { - stackId, - desiredLifecycle: "running" as const, - state: persisted, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }; - const reservation = yield* ingress.acquire(input); - expect(reservation.assignments.api).toBeUndefined(); - expect(reservation.assignments.database?.port).toEqual(expect.any(Number)); - expect(reservation.privateAssignments).toEqual( - expect.arrayContaining([ - { workloadId: "database:database", binding: "primary", port: expect.any(Number) }, - ]), - ); - yield* ingress.open(input, reservation, () => - Effect.fail(new GatewayActivationError({ message: "not reached" })), - ); - yield* ingress.close; - }), - ), - ); - - it.live("opens postgres-only stacks without resolving API gateway material", () => - run( - Effect.gen(function* () { - const { context, root } = yield* makeIngressContext("supabase-ingress-pg-only-"); - const stackIdentity = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(stackIdentity); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: excludeStackCapabilities( - {}, - CAPABILITY_NAMES.filter((name) => name !== "database"), - ), - }); - expect(compiled.definition.listeners.api.enabled).toBe(true); - const store = yield* makeStackStateStore({ stateRoot: root }); - const persisted = persistedIngressState(stackIdentity, compiled, 30300); - yield* store.initialize(stackId, persisted); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindPrivate, - }); - const input = { - stackId, - desiredLifecycle: "running" as const, - state: persisted, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }; - const reservation = yield* ingress.acquire(input); - expect(reservation.assignments.api).toBeUndefined(); - expect(reservation.assignments.database?.port).toEqual(expect.any(Number)); - yield* ingress.open(input, reservation, () => - Effect.fail(new GatewayActivationError({ message: "not reached" })), - ); - yield* ingress.close; - }), - ), - ); - - it.live("serves accepted Auth templates locally with live content and no activation", () => - run( - Effect.gen(function* () { - const { context, fs, path, root } = yield* makeIngressContext("supabase-ingress-template-"); - const stackIdentity = { - ...identity, - projectRoot: root, - }; - const stackId = yield* deriveStackId(stackIdentity); - const templatePath = path.join(root, "templates", "confirmation.html"); - const outsideRoot = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-ingress-outside-", - }); - const outsidePath = path.join(outsideRoot, "outside.html"); - yield* fs.makeDirectory(path.join(root, "templates"), { recursive: true }); - yield* fs.writeFileString(templatePath, "first"); - const compiled = yield* compileStack({ - projectRoot: root, - runtime: { kind: "native" }, - config: { - capabilities: { - rest: { enabled: false }, - auth: { - settings: { - email: { - template: { confirmation: { content_path: "templates/confirmation.html" } }, - }, - }, - }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - listeners: { - database: { enabled: false }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }); - const store = yield* makeStackStateStore({ stateRoot: root }); - const persisted = persistedIngressState(stackIdentity, compiled, 30300); - yield* store.initialize(stackId, persisted); - const activated: string[] = []; - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindPrivate, - apiMaterial: () => - Effect.succeed({ - publishableKey: "sb_publishable_test", - secretKey: "sb_secret_test", - anonJwt: "anon-jwt", - serviceRoleJwt: "service-jwt", - }), - resolveAuthTemplates: () => - Effect.all({ root: fs.realPath(root), template: fs.realPath(templatePath) }).pipe( - Effect.mapError( - (cause) => new StackPreparationError({ message: "Template is unavailable", cause }), - ), - Effect.flatMap(({ root: canonicalRoot, template: canonicalPath }) => - !path.relative(canonicalRoot, canonicalPath).startsWith("..") && - !path.isAbsolute(path.relative(canonicalRoot, canonicalPath)) - ? Effect.succeed([ - { - id: "confirmation", - extension: ".html", - canonicalPath, - }, - ]) - : Effect.fail(new StackPreparationError({ message: "Template escaped root" })), - ), - ), - }); - const input = { - stackId, - desiredLifecycle: "running" as const, - state: persisted, - definition: compiled.definition, - secrets: {}, - plan: compiled.executionPlan, - }; - const reservation = yield* ingress.acquire(input); - yield* ingress.open(input, reservation, (capability) => { - activated.push(capability); - return Effect.fail(new GatewayActivationError({ message: "not reached" })); - }); - const api = reservation.assignments.api; - if (api === undefined) return yield* Effect.die("API listener was not assigned"); - const first = yield* request(api.port, "/email/confirmation.html"); - expect(first.status).toBe(200); - expect(first.body).toBe("first"); - expect(activated).toEqual([]); - yield* fs.writeFileString(templatePath, "second"); - const second = yield* request(api.port, "/email/confirmation.html?live=1"); - expect(second.status).toBe(200); - expect(second.body).toBe("second"); - expect((yield* request(api.port, "/email/unknown.html")).status).toBe(404); - expect((yield* request(api.port, "/email/confirmation.html", "POST")).status).toBe(404); - expect((yield* request(api.port, "/email/confirmation.html", "OPTIONS")).status).toBe(404); - expect((yield* request(api.port, "/email/../outside.html")).status).toBe(404); - expect((yield* request(api.port, "/email/%2e%2e/outside.html")).status).toBe(404); - yield* fs.remove(templatePath); - expect((yield* request(api.port, "/email/confirmation.html")).status).toBe(404); - yield* fs.writeFileString(outsidePath, "outside"); - yield* fs.symlink(outsidePath, templatePath); - expect((yield* request(api.port, "/email/confirmation.html")).status).toBe(404); - expect(activated).toEqual([]); - yield* ingress.close; - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/lifecycle.integration.test.ts b/packages/stack/src/supervisor/lifecycle.integration.test.ts deleted file mode 100644 index bb06ef9162..0000000000 --- a/packages/stack/src/supervisor/lifecycle.integration.test.ts +++ /dev/null @@ -1,498 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, FileSystem, Option, Redacted, Ref } from "effect"; -import { deriveStackId } from "../identity/Identity.ts"; -import { - StackRuntimeError, - StackCleanupError, - StackStateInvalidError, - StackMustBeStoppedError, - type StackError, -} from "../public/Errors.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import type { PersistedStackState } from "../state/StackState.ts"; -import { makeStackStateStore, type StackStateStore } from "../state/StackStateStore.ts"; -import { - makeLifecycleController, - type LifecycleBackend, - type LifecycleInput, - type LifecycleLaunchResult, -} from "./Lifecycle.ts"; - -const layer = NodeServices.layer; - -const identity = { - projectRoot: "/tmp/supabase-lifecycle", - branchContext: "ordinary-workspace", - stackName: "lifecycle", -} as const; - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -interface BackendState { - readonly calls: Array; - readonly preflight: Array; - failPreflight?: boolean; - failLaunch?: boolean; - failDestroyData?: boolean; - failDestroyDataOnce?: boolean; - failCleanupOnce?: boolean; - gate?: Deferred.Deferred; - preflightStarted?: Deferred.Deferred; - waitBeforeLaunch?: Deferred.Deferred; - launchMutation?: () => Effect.Effect; - failCurrentRead?: Ref.Ref; - stopLaunchStarted?: Deferred.Deferred; - stopLaunchGate?: Deferred.Deferred; - startLaunchStarted?: Deferred.Deferred; - lastLifecycle?: PersistedStackState["desiredLifecycle"]; -} - -const backend = (state: BackendState): LifecycleBackend => ({ - preflight: (input) => - Effect.gen(function* () { - state.calls.push("preflight"); - state.preflight.push(input); - if (state.preflightStarted !== undefined) - yield* Deferred.succeed(state.preflightStarted, undefined); - if (state.failPreflight) return yield* new StackRuntimeError({ message: "preflight failed" }); - if (state.gate !== undefined) yield* Deferred.await(state.gate); - }), - launch: (input) => - Effect.gen(function* () { - state.lastLifecycle = input.state.desiredLifecycle; - state.calls.push(`launch:${input.state.desiredLifecycle}`); - if (input.state.desiredLifecycle === "running" && state.startLaunchStarted !== undefined) - yield* Deferred.succeed(state.startLaunchStarted, undefined); - if (state.waitBeforeLaunch !== undefined) yield* Deferred.await(state.waitBeforeLaunch); - if (state.launchMutation !== undefined) yield* state.launchMutation(); - if (state.failLaunch) return yield* new StackRuntimeError({ message: "launch failed" }); - return { - _tag: "started", - rollback: Effect.succeed({ _tag: "proven" }), - } satisfies LifecycleLaunchResult; - }), - cleanup: Effect.gen(function* () { - state.calls.push(`cleanup:${state.lastLifecycle ?? "invalid"}`); - if (state.failCleanupOnce) { - state.failCleanupOnce = false; - return yield* new StackCleanupError({ message: "cleanup failed" }); - } - }), - destroyData: Effect.gen(function* () { - state.calls.push(`destroy-data:${state.lastLifecycle ?? "invalid"}`); - if (state.failDestroyDataOnce) { - state.failDestroyDataOnce = false; - return yield* new StackCleanupError({ message: "destroy-data failed" }); - } - if (state.failDestroyData) - return yield* new StackCleanupError({ message: "destroy-data failed" }); - }), -}); - -const makeFixture = (runtime: StackRuntime = { kind: "native" }) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-lifecycle-" }); - const id = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(id, { - format: "supabase-stack-state-v1", - identity, - runtime, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, - }); - const state: BackendState = { calls: [], preflight: [] }; - const persistedStore: StackStateStore = { - ...store, - read: (stackId) => - Effect.gen(function* () { - if (state.failCurrentRead !== undefined && (yield* Ref.get(state.failCurrentRead))) { - yield* Ref.set(state.failCurrentRead, false); - return yield* new StackStateInvalidError({ - message: "injected current-state read failure", - }); - } - return yield* store.read(stackId); - }), - }; - const controller = yield* makeLifecycleController({ - stackId: id, - runtime, - stateStore: persistedStore, - backend: backend(state), - }); - const testController = { - ...controller, - start: (options?: Parameters[0]) => - controller - .start(options) - .pipe( - Effect.flatMap((outcome) => - outcome._tag === "started" - ? Effect.succeed(outcome.state) - : Effect.failCause(outcome.cause), - ), - ), - }; - return { id, root, store: persistedStore, state, controller: testController }; - }); - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(layer)); - -describe("durable lifecycle controller", () => { - it.live("commits a complete running definition before reconciling", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const result = yield* fixture.controller.start(); - expect(result).toMatchObject({ - desiredLifecycle: "running", - definition: { preparation: "background" }, - }); - expect(fixture.state.calls).toEqual(["preflight", "launch:running"]); - expect(yield* fixture.store.read(fixture.id)).toEqual(result); - }), - ), - ); - - it.live("reuses persisted definition for an omitted start", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const first = yield* fixture.controller.start({ config: { capabilities: { rest: {} } } }); - fixture.state.calls.length = 0; - const second = yield* fixture.controller.stop; - expect(second.desiredLifecycle).toBe("stopped"); - fixture.state.calls.length = 0; - const third = yield* fixture.controller.start(); - expect(third.definition).toEqual(first.definition); - expect(fixture.state.preflight.at(-1)?.definition).toEqual(first.definition); - }), - ), - ); - - it.live("is idempotent for an identical running input", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const first = yield* fixture.controller.start({ config: { capabilities: { rest: {} } } }); - fixture.state.calls.length = 0; - const second = yield* fixture.controller.start({ config: { capabilities: { rest: {} } } }); - expect(second).toEqual(first); - expect(fixture.state.calls).toEqual(["launch:running"]); - }), - ), - ); - - it.live("persists stopped when a fresh running session cannot launch", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.controller.start(); - fixture.state.failLaunch = true; - - const failed = yield* fixture.controller.start({ freshSession: true }).pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackRuntimeError); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("stopped"); - expect(fixture.state.calls.at(-1)).toBe("cleanup:running"); - }), - ), - ); - - it.live("preserves sticky ports allocated during a failed cold launch", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - fixture.state.failLaunch = true; - fixture.state.launchMutation = () => - Effect.provide( - fixture.store.read(fixture.id).pipe( - Effect.flatMap((current) => - current === undefined - ? Effect.fail(new StackStateInvalidError({ message: "state disappeared" })) - : fixture.store.replace(fixture.id, { - ...current, - ports: [{ field: "api", port: 54_321, intent: "automatic" }], - privatePorts: [{ workloadId: "rest:rest", binding: "http", port: 54_322 }], - }), - ), - Effect.asVoid, - ), - layer, - ); - - const failed = yield* fixture.controller.start().pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackRuntimeError); - const stopped = yield* fixture.store.read(fixture.id); - expect(stopped?.desiredLifecycle).toBe("unconfigured"); - expect(stopped?.ports).toEqual([{ field: "api", port: 54_321, intent: "automatic" }]); - expect(stopped?.privatePorts).toEqual([ - { workloadId: "rest:rest", binding: "http", port: 54_322 }, - ]); - }), - ), - ); - - it.live("does not write a stale stopped state when current-state read fails", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - fixture.state.failLaunch = true; - const failCurrentRead = yield* Ref.make(false); - fixture.state.failCurrentRead = failCurrentRead; - fixture.state.launchMutation = () => Ref.set(failCurrentRead, true); - const failed = yield* fixture.controller.start().pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackRuntimeError); - expect(fixture.state.calls).toContain("cleanup:running"); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("running"); - }), - ), - ); - - it.live("accepts explicit materialized defaults for a running stack", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const first = yield* fixture.controller.start(); - const second = yield* fixture.controller.start({ - config: { capabilities: { rest: { enabled: true } } }, - }); - expect(second.definition).toEqual(first.definition); - }), - ), - ); - - it.live("allows pass-through secret changes after a failed cold launch", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - fixture.state.failLaunch = true; - const original = { - capabilities: { - functions: { - settings: { - functions: { hello: { env: { TOKEN: Redacted.make("one") } } }, - }, - }, - }, - }; - const failed = yield* fixture.controller.start({ config: original }).pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackRuntimeError); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("unconfigured"); - fixture.state.failLaunch = false; - const changed = { - capabilities: { - functions: { - settings: { - functions: { hello: { env: { TOKEN: Redacted.make("two") } } }, - }, - }, - }, - }; - const restarted = yield* fixture.controller.start({ config: changed }); - expect(restarted.desiredLifecycle).toBe("running"); - expect(restarted.secrets).toMatchObject({ - "secret:functions.settings.functions.hello.env.TOKEN": { value: "two" }, - }); - }), - ), - ); - - it.live("requires stop before applying pass-through secret changes", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const original = { - capabilities: { - functions: { - settings: { - functions: { hello: { env: { TOKEN: Redacted.make("one") } } }, - }, - }, - }, - }; - const changed = { - capabilities: { - functions: { - settings: { - functions: { hello: { env: { TOKEN: Redacted.make("two") } } }, - }, - }, - }, - }; - yield* fixture.controller.start({ config: original }); - const running = yield* fixture.controller.start({ config: changed }).pipe(Effect.exit); - expect(errorOf(running)).toBeInstanceOf(StackMustBeStoppedError); - yield* fixture.controller.stop; - const restarted = yield* fixture.controller.start({ config: changed }); - expect(restarted.desiredLifecycle).toBe("running"); - expect(restarted.secrets).toMatchObject({ - "secret:functions.settings.functions.hello.env.TOKEN": { value: "two" }, - }); - }), - ), - ); - - it.live("rejects a changed running input before backend mutation", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.controller.start({ config: { capabilities: { rest: {} } } }); - const before = [...fixture.state.calls]; - const exit = yield* fixture.controller - .start({ config: { capabilities: { rest: { settings: { schemas: ["private"] } } } } }) - .pipe(Effect.exit); - expect(errorOf(exit)).toBeInstanceOf(StackMustBeStoppedError); - expect(fixture.state.calls).toEqual(before); - }), - ), - ); - - it.live("retains stopped state and makes stop idempotent", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.controller.start(); - fixture.state.calls.length = 0; - const first = yield* fixture.controller.stop; - expect(first).toMatchObject({ - desiredLifecycle: "stopped", - definition: { preparation: "background" }, - }); - expect(fixture.state.calls).toEqual(["cleanup:running"]); - fixture.state.calls.length = 0; - const second = yield* fixture.controller.stop; - expect(second).toEqual(first); - expect(fixture.state.calls).toEqual(["cleanup:running"]); - }), - ), - ); - - it.live("retries stopped cleanup after a prior cleanup failure", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.controller.start(); - fixture.state.calls.length = 0; - fixture.state.failCleanupOnce = true; - const first = yield* fixture.controller.stop.pipe(Effect.exit); - expect(errorOf(first)).toBeInstanceOf(StackCleanupError); - const stopped = yield* fixture.store.read(fixture.id); - expect(stopped?.desiredLifecycle).toBe("stopped"); - fixture.state.calls.length = 0; - const second = yield* fixture.controller.stop; - expect(second.desiredLifecycle).toBe("stopped"); - expect(fixture.state.calls).toEqual(["cleanup:running"]); - }), - ), - ); - - it.live("leaves old state untouched when preflight fails", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const first = yield* fixture.controller.start(); - yield* fixture.controller.stop; - fixture.state.calls.length = 0; - fixture.state.failPreflight = true; - const exit = yield* fixture.controller - .start({ config: { capabilities: { rest: {} } } }) - .pipe(Effect.exit); - expect(errorOf(exit)).toBeInstanceOf(StackRuntimeError); - expect(yield* fixture.store.read(fixture.id)).toMatchObject({ - desiredLifecycle: "stopped", - definition: first.definition, - }); - expect(fixture.state.calls).toEqual(["preflight"]); - }), - ), - ); - - it.live("destroys runtime data before deleting the exact identity root", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(`${fixture.root}/${fixture.id}/runtime`, { recursive: true }); - yield* fixture.controller.start(); - fixture.state.calls.length = 0; - yield* fixture.controller.destroy; - expect(fixture.state.calls).toEqual(["destroy-data:running"]); - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - expect(yield* fs.exists(`${fixture.root}/${fixture.id}`)).toBe(false); - const recreated = yield* fixture.store.initialize(fixture.id, { - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, - }); - expect(recreated.desiredLifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("retains the destroying fence when data cleanup fails", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.controller.start(); - fixture.state.calls.length = 0; - fixture.state.failDestroyData = true; - const exit = yield* fixture.controller.destroy.pipe(Effect.exit); - expect(errorOf(exit)).toBeInstanceOf(StackCleanupError); - expect(yield* fixture.store.read(fixture.id)).toMatchObject({ - desiredLifecycle: "destroying", - }); - expect(fixture.state.calls).toEqual(["destroy-data:running"]); - fixture.state.failDestroyData = false; - yield* fixture.controller.destroy; - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("destroys exact runtime remnants from an unconfigured state", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.controller.destroy; - expect(fixture.state.calls).toEqual(["destroy-data:invalid"]); - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("retries destructive cleanup after a transient failure", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - fixture.state.failDestroyDataOnce = true; - const first = yield* fixture.controller.destroy.pipe(Effect.exit); - expect(errorOf(first)).toBeInstanceOf(StackCleanupError); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("destroying"); - yield* fixture.controller.destroy; - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("fails closed when state is missing", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.store.cleanup(fixture.id); - const exit = yield* fixture.controller.start().pipe(Effect.exit); - expect(errorOf(exit)).toBeInstanceOf(StackStateInvalidError); - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/observability.integration.test.ts b/packages/stack/src/supervisor/observability.integration.test.ts deleted file mode 100644 index a6c1cfb798..0000000000 --- a/packages/stack/src/supervisor/observability.integration.test.ts +++ /dev/null @@ -1,408 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Exit, FileSystem, Fiber, Option, Path } from "effect"; -import { InvalidLogCursorError } from "../public/Errors.ts"; -import { LogStoreError, makeLogStore, readRetainedLogs, selectLogBatch } from "./LogStore.ts"; - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -const boundedFixture = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-logs-" }); - const store = yield* makeLogStore({ path: path.join(root, "logs.json"), maxEntries: 2 }); - return { fs, path, root, store }; - }); - -describe("observability", () => { - it.live("retains only the newest records at the configured entry bound", () => - withPlatform( - Effect.gen(function* () { - const { store } = yield* boundedFixture(); - yield* store.append({ - source: "database", - stream: "stdout", - message: "one", - }); - yield* store.append({ source: "auth", stream: "stderr", message: "two" }); - yield* store.append({ - source: "gateway", - stream: "internal", - message: "three", - }); - expect((yield* store.read()).map((entry) => entry.message)).toEqual(["two", "three"]); - }), - ), - ); - - it.live("resumes a bounded read after an opaque cursor", () => - withPlatform( - Effect.gen(function* () { - const { store } = yield* boundedFixture(); - const first = yield* store.append({ - source: "database", - stream: "stdout", - message: "one", - }); - const second = yield* store.append({ source: "auth", stream: "stderr", message: "two" }); - const third = yield* store.append({ - source: "gateway", - stream: "internal", - message: "three", - }); - expect((yield* store.read({ cursor: first.cursor })).map((entry) => entry.cursor)).toEqual([ - second.cursor, - third.cursor, - ]); - }), - ), - ); - - it.live("filters retained records by capability before publishing a batch", () => - withPlatform( - Effect.gen(function* () { - const { store } = yield* boundedFixture(); - yield* store.append({ source: "database", stream: "stdout", message: "database" }); - yield* store.append({ source: "auth", stream: "stderr", message: "auth" }); - const scanned = yield* store.read(); - expect( - selectLogBatch(scanned, { capabilities: ["auth"] }).entries.map((entry) => entry.message), - ).toEqual(["auth"]); - }), - ), - ); - - it.live("rejects invalid cursors while keeping the retained log file private", () => - withPlatform( - Effect.gen(function* () { - const { fs, path, root, store } = yield* boundedFixture(); - yield* store.append({ source: "auth", stream: "stderr", message: "auth" }); - const invalidCursor = yield* store - .read({ cursor: { opaque: "not-a-cursor" } }) - .pipe(Effect.exit); - expect(errorOf(invalidCursor)).toBeInstanceOf(InvalidLogCursorError); - expect((yield* fs.stat(path.join(root, "logs.json"))).mode & 0o077).toBe(0); - }), - ), - ); - - it.live("redacts known secrets in retained entries before writing them to disk", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-log-redaction-" }); - const logPath = path.join(root, "logs.json"); - const store = yield* makeLogStore({ path: logPath, knownSecrets: ["top-secret"] }); - - const entry = yield* store.append({ - source: "database", - stream: "stdout", - message: "password=top-secret", - }); - - expect(entry.message).toBe("password=[REDACTED]"); - expect((yield* store.read()).map((retained) => retained.message)).toEqual([ - "password=[REDACTED]", - ]); - expect(yield* fs.readFileString(logPath)).not.toContain("top-secret"); - }), - ), - ); - - it.live("bounds retained records by encoded size", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-log-size-" }); - const store = yield* makeLogStore({ path: path.join(root, "logs.json"), maxBytes: 300 }); - yield* store.append({ source: "database", stream: "stdout", message: "x".repeat(200) }); - expect(yield* store.read()).toEqual([]); - expect(yield* fs.exists(path.join(root, "logs.json"))).toBe(false); - }), - ), - ); - - it.live("keeps reads bounded while compacting retained logs periodically", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-log-compaction-", - }); - const logPath = path.join(root, "logs.json"); - const store = yield* makeLogStore({ path: logPath, maxEntries: 2 }); - yield* store.append({ source: "database", stream: "stdout", message: "one" }); - yield* store.append({ source: "database", stream: "stdout", message: "two" }); - yield* store.append({ source: "database", stream: "stdout", message: "three" }); - - expect((yield* store.read()).map((entry) => entry.message)).toEqual(["two", "three"]); - expect( - (yield* readRetainedLogs(fs, logPath, { maxEntries: 2 })).map((entry) => entry.message), - ).toEqual(["two", "three"]); - // The third append is retained in the append-only window; compaction is deferred until - // the bounded overshoot is exhausted. - expect((yield* fs.readFileString(logPath)).trim().split("\n")).toHaveLength(3); - - yield* store.append({ source: "database", stream: "stdout", message: "four" }); - yield* store.append({ source: "database", stream: "stdout", message: "five" }); - expect((yield* store.read()).map((entry) => entry.message)).toEqual(["four", "five"]); - expect((yield* fs.readFileString(logPath)).trim().split("\n")).toHaveLength(2); - }), - ), - ); - - it.live("drops an oversized record without evicting retained logs or reusing cursors", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-log-cursor-" }); - const logPath = path.join(root, "logs.json"); - const firstStore = yield* makeLogStore({ path: logPath, maxBytes: 300 }); - const first = yield* firstStore.append({ - source: "database", - stream: "stdout", - message: "retained", - }); - const beforeOversized = yield* fs.readFileString(logPath); - - yield* firstStore.append({ - source: "database", - stream: "stdout", - message: "x".repeat(300), - }); - - expect(yield* fs.readFileString(logPath)).toBe(beforeOversized); - expect((yield* firstStore.read()).map((entry) => entry.cursor)).toEqual([first.cursor]); - - const secondStore = yield* makeLogStore({ path: logPath, maxBytes: 300 }); - const second = yield* secondStore.append({ - source: "database", - stream: "stdout", - message: "after restart", - }); - const followed = yield* secondStore.read({ cursor: first.cursor }); - - expect(second.cursor.opaque).not.toBe(first.cursor.opaque); - expect(followed.map((entry) => entry.cursor)).toEqual([second.cursor]); - }), - ), - ); - - it.live("does not reuse a cursor when an append is interrupted after its write", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-log-interrupt-" }); - const logPath = path.join(root, "logs.json"); - const writeCompleted = yield* Deferred.make(); - const interruptionRequested = yield* Deferred.make(); - const releaseWrite = yield* Deferred.make(); - const wrappedFs = { - ...fs, - open: (target: string, options?: Parameters[1]) => - fs.open(target, options).pipe( - Effect.map((file) => - target === logPath && options?.flag === "a" - ? new Proxy(file, { - get: (current, property, receiver) => - property === "writeAll" - ? (bytes: Uint8Array) => - current - .writeAll(bytes) - .pipe( - Effect.andThen(Deferred.succeed(writeCompleted, undefined)), - Effect.andThen(Deferred.await(releaseWrite)), - ) - : Reflect.get(current, property, receiver), - }) - : file, - ), - ), - } satisfies FileSystem.FileSystem; - const store = yield* makeLogStore({ path: logPath }).pipe( - Effect.provideService(FileSystem.FileSystem, wrappedFs), - ); - const interrupted = yield* Effect.forkChild( - store.append({ source: "auth", stream: "stdout", message: "interrupted" }), - ); - yield* Deferred.await(writeCompleted); - const interrupt = yield* Effect.forkChild( - Deferred.succeed(interruptionRequested, undefined).pipe( - Effect.andThen(Fiber.interrupt(interrupted)), - ), - ); - yield* Deferred.await(interruptionRequested); - yield* Deferred.succeed(releaseWrite, undefined); - yield* Fiber.join(interrupt); - const second = yield* store.append({ - source: "rest", - stream: "stderr", - message: "after interruption", - }); - expect(second.cursor.opaque).not.toBe("v1_1"); - expect((yield* store.read()).map((entry) => entry.cursor.opaque)).toEqual(["v1_1", "v1_2"]); - }), - ), - ); - - it.live("keeps in-memory retention unchanged when compaction persistence fails", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-log-persist-failure-", - }); - const logPath = path.join(root, "logs.json"); - const store = yield* makeLogStore({ path: logPath, maxEntries: 1 }); - yield* store.append({ source: "database", stream: "stdout", message: "one" }); - yield* store.append({ source: "database", stream: "stdout", message: "two" }); - yield* fs.chmod(root, 0o500); - const failed = yield* store - .append({ source: "database", stream: "stdout", message: "three" }) - .pipe(Effect.exit); - yield* fs.chmod(root, 0o700); - - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* store.read()).map((entry) => entry.message)).toEqual(["two"]); - expect((yield* fs.readFileString(logPath)).trim().split("\n")).toHaveLength(2); - }), - ), - ); - - it.live("secures a pre-existing log directory and rejects unsafe limits", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-log-permissions-", - }); - const directory = path.join(root, "logs"); - yield* fs.makeDirectory(directory, { recursive: true, mode: 0o755 }); - yield* fs.chmod(directory, 0o755); - const logPath = path.join(directory, "logs.json"); - yield* makeLogStore({ path: logPath }); - expect((yield* fs.stat(directory)).mode & 0o077).toBe(0); - - for (const value of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { - const invalid = yield* makeLogStore({ - path: path.join(root, `invalid-${String(value)}.json`), - maxEntries: value, - }).pipe(Effect.exit); - expect(errorOf(invalid)).toBeInstanceOf(LogStoreError); - } - const tooSmall = yield* makeLogStore({ - path: path.join(root, "too-small.json"), - maxBytes: 1, - }).pipe(Effect.exit); - expect(errorOf(tooSmall)).toBeInstanceOf(LogStoreError); - }), - ), - ); - - it.live("reloads retained logs and rejects malformed files", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-log-reload-" }); - const logPath = path.join(root, "logs.json"); - const first = yield* makeLogStore({ path: logPath }); - yield* first.append({ source: "supervisor", stream: "internal", message: "old-secret" }); - const second = yield* makeLogStore({ path: logPath, knownSecrets: ["old-secret"] }); - expect((yield* second.read()).map((entry) => entry.message)).toEqual(["[REDACTED]"]); - expect(yield* fs.readFileString(logPath)).not.toContain("old-secret"); - const complete = yield* fs.readFileString(logPath); - yield* fs.writeFileString(logPath, `${complete}{"cursor"`); - const recovered = yield* makeLogStore({ path: logPath }); - expect((yield* recovered.read()).map((entry) => entry.message)).toEqual(["[REDACTED]"]); - yield* fs.writeFileString(logPath, "not json"); - const malformed = yield* makeLogStore({ path: logPath }).pipe(Effect.exit); - expect(errorOf(malformed)).toBeInstanceOf(LogStoreError); - }), - ), - ); - - it.live("repairs a valid unterminated final record before the next append", () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-stack-log-tail-" }); - const logPath = path.join(root, "logs.json"); - const firstStore = yield* makeLogStore({ path: logPath }); - yield* firstStore.append({ source: "auth", stream: "stdout", message: "first" }); - const complete = yield* fs.readFileString(logPath); - yield* fs.writeFileString(logPath, complete.slice(0, -1)); - - expect((yield* readRetainedLogs(fs, logPath)).map((entry) => entry.message)).toEqual([ - "first", - ]); - expect(yield* fs.readFileString(logPath)).toBe(complete.slice(0, -1)); - - const repairedStore = yield* makeLogStore({ path: logPath }); - yield* repairedStore.append({ source: "rest", stream: "stdout", message: "second" }); - const reopened = yield* makeLogStore({ path: logPath }); - expect((yield* reopened.read()).map((entry) => entry.message)).toEqual(["first", "second"]); - }), - ), - ); - - it.live( - "repairs a torn tail before the next append and preserves recovery for the next owner", - () => - withPlatform( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: "supabase-stack-log-recovery-", - }); - const logPath = path.join(root, "logs.json"); - const first = yield* makeLogStore({ path: logPath }); - const before = yield* first.append({ - source: "supervisor", - stream: "internal", - message: "before crash", - }); - yield* fs.writeFileString(logPath, '{"cursor":{"opaque":"v1_2"},"timestamp":', { - flag: "a", - }); - - const readonlyBefore = yield* fs.readFileString(logPath); - expect(yield* readRetainedLogs(fs, logPath)).toHaveLength(1); - expect(yield* fs.readFileString(logPath)).toBe(readonlyBefore); - - const recovered = yield* makeLogStore({ path: logPath }); - expect((yield* recovered.read()).map((entry) => entry.message)).toEqual(["before crash"]); - const after = yield* recovered.append({ - source: "supervisor", - stream: "internal", - message: "after restart", - }); - expect(after.cursor.opaque).toBe("v1_2"); - expect((yield* readRetainedLogs(fs, logPath)).map((entry) => entry.message)).toEqual([ - "before crash", - "after restart", - ]); - - const nextOwner = yield* makeLogStore({ path: logPath }); - expect((yield* nextOwner.read()).map((entry) => entry.cursor.opaque)).toEqual([ - before.cursor.opaque, - after.cursor.opaque, - ]); - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/session-launcher.integration.test.ts b/packages/stack/src/supervisor/session-launcher.integration.test.ts deleted file mode 100644 index 2101fdd913..0000000000 --- a/packages/stack/src/supervisor/session-launcher.integration.test.ts +++ /dev/null @@ -1,288 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { Cause, Deferred, Effect, Fiber, Option } from "effect"; -import type { CapabilityName } from "../public/Capability.ts"; -import { CAPABILITY_NAMES } from "../public/Capability.ts"; -import { StackIdSchema } from "../public/StackId.ts"; -import type { ExecutionPlan, PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { RuntimeDriverError, type RuntimeDriver } from "../runtime/RuntimeDriver.ts"; -import { makeNativeRuntime } from "../runtime/NativeRuntime.ts"; -import { makeSessionLauncher } from "./SessionLauncher.ts"; - -const stackId = StackIdSchema.make("b".repeat(64)); - -const workload = (id: string, dependencies: ReadonlyArray = []): PlannedWorkload => ({ - id, - capability: "database", - dependencies, - readiness: {}, - artifacts: { - native: { kind: "native", release: "test" }, - container: { kind: "container", image: "test/image" }, - }, - selected: { kind: "native", release: "test" }, -}); - -const plan = (workloads: ReadonlyArray): ExecutionPlan => { - const activation = { - database: "eager", - rest: "lazy", - auth: "lazy", - realtime: "lazy", - storage: "lazy", - functions: "lazy", - studio: "lazy", - mail: "lazy", - analytics: "lazy", - pooler: "lazy", - } satisfies { [Name in CapabilityName]: "eager" | "lazy" }; - const dependencies = { - database: [], - rest: [], - auth: [], - realtime: [], - storage: [], - functions: [], - studio: [], - mail: [], - analytics: [], - pooler: [], - } satisfies { [Name in CapabilityName]: ReadonlyArray }; - return { - runtime: { kind: "native" }, - activation, - startOrder: CAPABILITY_NAMES, - dependencies, - routes: [], - workloads, - }; -}; - -const ready = (workload: PlannedWorkload) => ({ - stackId, - workloadId: workload.id, - state: "ready" as const, -}); - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -describe("session launcher", () => { - it.live("starts independent workloads in parallel before their dependants", () => - Effect.gen(function* () { - const databaseEntered = yield* Deferred.make(); - const releaseDatabase = yield* Deferred.make(); - const mailEntered = yield* Deferred.make(); - const calls: string[] = []; - const database = workload("database:database"); - const mail = workload("mail:mail"); - const rest = workload("rest:rest", [database.id]); - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: (_key, current) => - Effect.gen(function* () { - calls.push(`start:${current.id}`); - if (current.id === database.id) { - yield* Deferred.succeed(databaseEntered, undefined); - yield* Deferred.await(releaseDatabase); - } - if (current.id === mail.id) yield* Deferred.succeed(mailEntered, undefined); - return ready(current); - }), - stop: (key) => Effect.sync(() => calls.push(`stop:${key.workloadId}`)), - remove: (key) => Effect.sync(() => calls.push(`remove:${key.workloadId}`)), - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const launcher = yield* makeSessionLauncher({ stackId, driver }); - const launching = yield* Effect.forkChild(launcher.launch(plan([database, mail, rest])), { - startImmediately: true, - }); - - yield* Deferred.await(databaseEntered); - yield* Deferred.await(mailEntered); - expect(calls).toContain("start:database:database"); - expect(calls).toContain("start:mail:mail"); - expect(calls).not.toContain("start:rest:rest"); - - yield* Deferred.succeed(releaseDatabase, undefined); - yield* Fiber.join(launching); - expect(calls).toContain("start:rest:rest"); - }), - ); - - it.live("starts a dependant when its own prerequisite completes", () => - Effect.gen(function* () { - const databaseEntered = yield* Deferred.make(); - const mailEntered = yield* Deferred.make(); - const releaseMail = yield* Deferred.make(); - const restEntered = yield* Deferred.make(); - const database = workload("database:database"); - const mail = workload("mail:mail"); - const rest = workload("rest:rest", [database.id]); - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: (_key, current) => - Effect.gen(function* () { - if (current.id === database.id) yield* Deferred.succeed(databaseEntered, undefined); - if (current.id === mail.id) { - yield* Deferred.succeed(mailEntered, undefined); - yield* Deferred.await(releaseMail); - } - if (current.id === rest.id) yield* Deferred.succeed(restEntered, undefined); - return ready(current); - }), - stop: () => Effect.void, - remove: () => Effect.void, - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const launcher = yield* makeSessionLauncher({ stackId, driver }); - const launching = yield* Effect.forkChild(launcher.launch(plan([database, mail, rest])), { - startImmediately: true, - }); - yield* Deferred.await(databaseEntered); - yield* Deferred.await(mailEntered); - yield* Deferred.await(restEntered); - yield* Deferred.succeed(releaseMail, undefined); - yield* Fiber.join(launching); - }), - ); - - it.live("interrupts and removes an in-flight native workload when a sibling fails", () => - withPlatform( - Effect.gen(function* () { - const databaseEntered = yield* Deferred.make(); - const databaseInterrupted = yield* Deferred.make(); - const database = workload("database:database"); - const mail = workload("mail:mail"); - const driver = yield* makeNativeRuntime({ - resolveProcess: (key) => { - if (key.workloadId === database.id) - return Effect.gen(function* () { - yield* Deferred.succeed(databaseEntered, undefined); - return yield* Effect.never; - }).pipe(Effect.ensuring(Deferred.succeed(databaseInterrupted, undefined))); - return Deferred.await(databaseEntered).pipe( - Effect.andThen( - Effect.fail( - new RuntimeDriverError({ - message: "mail failed", - stackId, - workloadId: key.workloadId, - }), - ), - ), - ); - }, - waitForReadiness: () => Effect.never, - }); - const launcher = yield* makeSessionLauncher({ stackId, driver }); - const launching = yield* Effect.forkChild(launcher.launch(plan([database, mail])), { - startImmediately: true, - }); - yield* Deferred.await(databaseEntered); - const result = yield* Fiber.join(launching); - - expect(result._tag).toBe("failed"); - yield* Deferred.await(databaseInterrupted); - expect(yield* driver.observe(stackId)).toEqual([]); - }), - ), - ); - - it.live("fails with a typed error when no pending workload can become ready", () => - Effect.gen(function* () { - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: () => Effect.die("unreachable"), - stop: () => Effect.die("unreachable"), - remove: () => Effect.die("unreachable"), - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const launcher = yield* makeSessionLauncher({ stackId, driver }); - const result = yield* launcher.launch( - plan([workload("cycle:a", ["cycle:b"]), workload("cycle:b", ["cycle:a"])]), - ); - expect(result._tag).toBe("failed"); - const error = - result._tag === "failed" - ? Option.getOrUndefined(Cause.findErrorOption(result.cause)) - : undefined; - expect(error).toBeInstanceOf(RuntimeDriverError); - }), - ); - - it.live("retains a partially started workload when rollback removal fails", () => - Effect.gen(function* () { - const owned = new Set(); - let failRemove = true; - const database = workload("database:database"); - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: (_key, current) => - Effect.gen(function* () { - owned.add(current.id); - return yield* new RuntimeDriverError({ - message: "database start failed after acquiring resource", - stackId, - workloadId: current.id, - }); - }), - stop: () => Effect.void, - remove: (key) => - Effect.gen(function* () { - if (key.workloadId === database.id && failRemove) { - failRemove = false; - return yield* new RuntimeDriverError({ - message: "database remove failed", - stackId, - workloadId: key.workloadId, - }); - } - owned.delete(key.workloadId); - }), - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const launcher = yield* makeSessionLauncher({ stackId, driver }); - const result = yield* launcher.launch(plan([database])); - expect(result._tag).toBe("failed"); - if (result._tag === "failed") expect(result.cleanup._tag).toBe("unproven"); - expect(owned).toEqual(new Set([database.id])); - yield* launcher.stop; - expect(owned).toEqual(new Set()); - }), - ); - - it.live("interrupts an owner without stranding dependent completion or cleanup ownership", () => - Effect.gen(function* () { - const databaseEntered = yield* Deferred.make(); - const owned = new Set(); - const database = workload("database:database"); - const rest = workload("rest:rest", [database.id]); - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: (_key, current) => - current.id === database.id - ? Deferred.succeed(databaseEntered, undefined).pipe( - Effect.andThen(Effect.sync(() => owned.add(current.id))), - Effect.andThen(Effect.never), - ) - : Effect.die("dependent must await database"), - stop: () => Effect.void, - remove: (key) => Effect.sync(() => void owned.delete(key.workloadId)), - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const launcher = yield* makeSessionLauncher({ stackId, driver }); - const launching = yield* Effect.forkChild(launcher.launch(plan([database, rest])), { - startImmediately: true, - }); - yield* Deferred.await(databaseEntered); - yield* Fiber.interrupt(launching).pipe(Effect.timeout("5 seconds")); - expect(owned).toEqual(new Set()); - }), - ); -}); diff --git a/packages/stack/src/supervisor/startup-ingress.integration.test.ts b/packages/stack/src/supervisor/startup-ingress.integration.test.ts deleted file mode 100644 index e492273861..0000000000 --- a/packages/stack/src/supervisor/startup-ingress.integration.test.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Context, - Crypto, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Path, - Ref, -} from "effect"; -import { - request as requestHttp, - createServer, - type ClientRequest, - type IncomingMessage, - type ServerResponse, - // oxlint-disable-next-line effecttsgo/node-builtin-import -- fixture needs ClientRequest.end's sent callback and a mutable ClientRequest for cancellation; Effect HttpClient exposes neither. -} from "node:http"; -import { deriveStackId } from "../identity/Identity.ts"; -import type { StackError } from "../public/Errors.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { bindHostListener, type HostListener } from "./HostListener.ts"; -import { makeSupervisorIngress } from "./Ingress.ts"; -import { makeSupervisor, type SupervisorRuntime } from "./Supervisor.ts"; -import type { RuntimeDriver } from "../runtime/RuntimeDriver.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import type { StackLogEntry } from "../public/Logs.ts"; - -const withPlatform = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const response = ( - port: number, - sent: Deferred.Deferred, - finished: Deferred.Deferred, - requestRef: { value?: ClientRequest }, -) => - Effect.callback<{ readonly status: number; readonly body: string }, Error>((resume) => { - const client = requestHttp({ host: "127.0.0.1", port, path: "/rest/v1/items" }, (incoming) => { - const chunks: Buffer[] = []; - incoming.on("data", (chunk: Buffer) => chunks.push(chunk)); - incoming.once("end", () => - resume( - Effect.succeed({ - status: incoming.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - }).pipe(Effect.tap(() => Deferred.succeed(finished, undefined))), - ), - ); - }); - requestRef.value = client; - client.once("error", (error) => resume(Effect.fail(error))); - client.end(() => Deferred.doneUnsafe(sent, Effect.void)); - return Effect.sync(() => client.destroy()); - }); - -const backend = Effect.acquireRelease( - Effect.callback, Error>((resume) => { - const server = createServer((_request: IncomingMessage, result: ServerResponse) => { - result.statusCode = 200; - result.end("backend-ready"); - }); - server.once("error", (error) => resume(Effect.fail(error))); - server.listen(0, "127.0.0.1", () => resume(Effect.succeed(server))); - return Effect.sync(() => { - if (server.listening) server.close(); - }); - }), - (server) => - Effect.callback((resume) => { - if (!server.listening) return resume(Effect.void); - server.close(() => resume(Effect.void)); - }), -); - -const makeStartupFixture = () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-startup-ingress-" }); - const identity = { - projectRoot: root, - branchContext: "ordinary-workspace", - stackName: "startup-ingress", - } as const; - const stackId = yield* deriveStackId(identity); - const store = yield* makeStackStateStore({ stateRoot: root }); - yield* store.initialize(stackId, { - format: "supabase-stack-state-v1", - identity, - runtime: { kind: "native" }, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, - }); - const context = Context.make(FileSystem.FileSystem, fs).pipe( - Context.add(Path.Path, path), - Context.add(Crypto.Crypto, crypto), - ); - const listenerBound = yield* Deferred.make(); - const startEntered = yield* Deferred.make(); - const releaseStart = yield* Deferred.make(); - const activationCalls = yield* Ref.make(0); - const service = yield* backend; - const address = service.address(); - if (typeof address !== "object" || address === null) - return yield* Effect.die("backend did not expose an address"); - - const bindHost = (host: string, port: number, field: import("../public/Status.ts").PortField) => - bindHostListener(host, port, field).pipe( - Effect.tap((listener) => - field === "api" ? Deferred.succeed(listenerBound, listener) : Effect.void, - ), - ); - const ingress = yield* makeSupervisorIngress({ - stackId, - stateRoot: root, - store, - context, - bindHost, - apiMaterial: () => - Effect.succeed({ - publishableKey: "publishable", - secretKey: "secret", - anonJwt: "anon", - serviceRoleJwt: "service", - }), - }); - const driver: RuntimeDriver = { - observe: () => Effect.succeed([]), - start: (key, _workload: PlannedWorkload) => - Effect.gen(function* () { - if (key.workloadId === "database:database") { - yield* Deferred.succeed(startEntered, undefined); - yield* Deferred.await(releaseStart); - } - return { ...key, state: "ready" as const }; - }), - stop: () => Effect.void, - remove: () => Effect.void, - cleanup: () => Effect.void, - wipePersistentData: () => Effect.void, - }; - const entry: StackLogEntry = { - cursor: { opaque: "v1_1" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "supervisor", - stream: "internal", - message: "startup", - }; - const runtime: SupervisorRuntime = { - driver, - preflight: () => Effect.void, - prepare: () => Effect.void, - prefetch: () => Effect.void, - artifacts: Effect.succeed([]), - activate: () => - Ref.update(activationCalls, (count) => count + 1).pipe( - Effect.andThen(Effect.succeed({ host: "127.0.0.1", port: address.port })), - ), - ingress, - logStore: { - path: "memory://startup-ingress", - append: () => Effect.succeed(entry), - read: () => Effect.succeed([entry]), - }, - }; - const supervisor = yield* makeSupervisor({ - stackId, - ownerSessionId: "startup-ingress-test", - stateStore: store, - context, - runtime, - }); - const start = supervisor - .start({ - config: { - listeners: { - api: { enabled: true }, - database: { enabled: false }, - pooler: { enabled: false }, - studio: { enabled: false }, - mailUi: { enabled: false }, - smtp: { enabled: false }, - pop3: { enabled: false }, - functionsInspector: { enabled: false }, - }, - }, - }) - .pipe(Effect.tapCause((cause) => Deferred.failCause(listenerBound, cause))); - return { listenerBound, startEntered, releaseStart, activationCalls, supervisor, start }; - }); - -describe("startup ingress", () => { - it.live("holds requests during startup and forwards them after the service is ready", () => - withPlatform( - Effect.gen(function* () { - const fixture = yield* makeStartupFixture(); - const starting = yield* Effect.forkChild(fixture.start); - const listener = yield* Deferred.await(fixture.listenerBound); - if (listener.binding.kind !== "http") return yield* Effect.die("API listener is not HTTP"); - - const requestsAccepted = yield* Deferred.make(); - let acceptedCount = 0; - const onRequest = () => { - acceptedCount += 1; - if (acceptedCount === 2) Deferred.doneUnsafe(requestsAccepted, Effect.void); - }; - listener.binding.server.on("request", onRequest); - const firstSent = yield* Deferred.make(); - const secondSent = yield* Deferred.make(); - const secondFinished = yield* Deferred.make(); - const firstRequest: { value?: ClientRequest } = {}; - const first = yield* Effect.forkChild( - response(listener.port, firstSent, yield* Deferred.make(), firstRequest), - ); - yield* Deferred.await(firstSent); - const second = yield* Effect.forkChild( - response(listener.port, secondSent, secondFinished, {}), - ); - yield* Deferred.await(secondSent); - yield* Deferred.await(requestsAccepted); - yield* Deferred.await(fixture.startEntered); - expect(Option.isNone(yield* Deferred.poll(secondFinished))).toBe(true); - firstRequest.value?.destroy(); - yield* Deferred.succeed(fixture.releaseStart, undefined); - - expect(yield* Fiber.join(second)).toEqual({ status: 200, body: "backend-ready" }); - expect((yield* Fiber.join(starting)).lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.activationCalls)).toBe(1); - expect(Exit.isFailure(yield* Fiber.join(first).pipe(Effect.exit))).toBe(true); - listener.binding.server.off("request", onRequest); - yield* fixture.supervisor.destroy; - }), - ), - ); -}); diff --git a/packages/stack/src/supervisor/supervisor.integration.test.ts b/packages/stack/src/supervisor/supervisor.integration.test.ts deleted file mode 100644 index 60cbc1bb27..0000000000 --- a/packages/stack/src/supervisor/supervisor.integration.test.ts +++ /dev/null @@ -1,4680 +0,0 @@ -import { NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { - Cause, - Deferred, - Effect, - Exit, - Fiber, - FileSystem, - Option, - Path, - Queue, - Redacted, - Ref, - Scheduler, - Scope, -} from "effect"; -import * as TestClock from "effect/testing/TestClock"; -import { Headers } from "effect/unstable/http"; -import { Rpc } from "effect/unstable/rpc"; -import { RequestId } from "effect/unstable/rpc/RpcMessage"; -import type { LogQuery, StackLogEntry } from "../public/Logs.ts"; -import type { StackRuntime } from "../public/Runtime.ts"; -import type { ArtifactPreparationStatus, StackStatus } from "../public/Status.ts"; -import { - GatewayActivationError, - InvalidLogCursorError, - PortUnavailableError, - StackLifecycleConflictError, - StackNotRunningError, - StackRuntimeError, - StackCleanupError, - StackStateInvalidError, - StackMustBeStoppedError, - StackVersionUnsupportedError, - ArtifactIntegrityError, - ContainerEngineError, -} from "../public/Errors.ts"; -import { - RuntimeDriverError, - type RuntimeDriver, - type ObservedWorkload, -} from "../runtime/RuntimeDriver.ts"; -import type { EffectStackCredentials } from "../public/Credentials.ts"; -import type { PlannedWorkload } from "../model/ExecutionPlan.ts"; -import { deriveStackId } from "../identity/Identity.ts"; -import { makeStackStateStore } from "../state/StackStateStore.ts"; -import { resolveStackPaths } from "../state/Paths.ts"; -import { StackRpcGroup, type StackRpcError } from "../control/StackRpc.ts"; -import { makeSupervisor, type Supervisor, type SupervisorRuntime } from "./Supervisor.ts"; -import type { SupervisorIngress } from "./Ingress.ts"; -import type { GatewayActivity } from "../gateway/ActivityTracker.ts"; - -const identity = { - projectRoot: "/tmp/supabase-supervisor", - branchContext: "ordinary-workspace", - stackName: "supervisor", -} as const; - -const errorOf = (exit: Exit.Exit): E | undefined => - Exit.isFailure(exit) ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) : undefined; - -const invokeCredentials = ( - supervisor: Supervisor, -): Effect.Effect => - Effect.gen(function* () { - const handler = yield* StackRpcGroup.accessHandler("credentials").pipe( - Effect.provide( - StackRpcGroup.toLayerHandler("credentials", supervisor.rpcHandlers.credentials), - ), - ); - const value = yield* handler(undefined, { - client: new Rpc.ServerClient(1), - requestId: RequestId(1), - headers: Headers.empty, - }); - if (Deferred.isDeferred(value)) - return yield* Deferred.await(value); - return value; - }); - -type ReadGate = { - readonly started: Deferred.Deferred; - readonly gate: Deferred.Deferred; - readonly beforeRead?: boolean; -}; - -const makeFixture = ( - fixtureOptions: { - readonly ingress?: SupervisorIngress; - readonly timeline?: Ref.Ref>; - readonly logRecords?: Ref.Ref>; - readonly logWritten?: Deferred.Deferred; - readonly logWrittenFor?: string; - readonly logWrittenAdditional?: Deferred.Deferred; - readonly logWrittenAdditionalFor?: string; - readonly logQueue?: Queue.Queue; - readonly runtime?: StackRuntime; - readonly supervisorScope?: Scope.Scope; - readonly startGate?: Deferred.Deferred; - readonly startStarted?: Deferred.Deferred; - readonly startWorkload?: string; - readonly startFinished?: Deferred.Deferred; - readonly activationGate?: Deferred.Deferred; - readonly activationGateAfterFirst?: Deferred.Deferred; - readonly activationStarted?: Deferred.Deferred; - readonly activationStartedAfterFirst?: Deferred.Deferred; - readonly activationCalls?: Ref.Ref; - readonly activationFailFirst?: Ref.Ref; - readonly startFailures?: Ref.Ref; - readonly startFailureWorkload?: string; - readonly preflightFailFirst?: Ref.Ref; - readonly preflightCalls?: Ref.Ref; - readonly preflightGate?: Deferred.Deferred; - readonly preflightStarted?: Deferred.Deferred; - readonly stopGate?: Deferred.Deferred; - readonly stopStarted?: Deferred.Deferred; - readonly workloadStopFailFirst?: Ref.Ref; - readonly workloadStopGate?: Deferred.Deferred; - readonly workloadStopStarted?: Deferred.Deferred; - readonly workloadRemoveFailFirst?: Ref.Ref; - readonly workloadRemoveFailWorkload?: string; - readonly workloadRemoveDieFirst?: Ref.Ref; - readonly stopFailFirst?: Ref.Ref; - readonly destroyGate?: Deferred.Deferred; - readonly destroyStarted?: Deferred.Deferred; - readonly destroyPreFenceFail?: Ref.Ref; - readonly observeFailure?: Ref.Ref; - readonly stoppedReplaceFail?: Ref.Ref; - readonly startQueue?: Queue.Queue; - readonly readGateQueue?: Ref.Ref>; - readonly readCalls?: Ref.Ref; - readonly prefetchStarted?: Deferred.Deferred; - readonly prefetchFinished?: Deferred.Deferred; - readonly prefetchGate?: Deferred.Deferred; - readonly prefetchInterrupted?: Deferred.Deferred; - readonly prefetchCalls?: Ref.Ref; - readonly prepareStarted?: Deferred.Deferred; - readonly prepareActivationStarted?: Deferred.Deferred; - readonly prepareGate?: Deferred.Deferred; - readonly prepareGateEnabledRef?: Ref.Ref; - readonly prepareFailure?: boolean; - readonly prepareFailureRef?: Ref.Ref; - readonly artifactStatuses?: Ref.Ref>; - } = {}, -) => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "supabase-supervisor-" }); - const id = yield* deriveStackId(identity); - const baseStore = yield* makeStackStateStore({ stateRoot: root }); - const destroyPreFenceFail = fixtureOptions.destroyPreFenceFail; - const stoppedReplaceFail = fixtureOptions.stoppedReplaceFail; - const persistedStore = - destroyPreFenceFail === undefined && stoppedReplaceFail === undefined - ? baseStore - : { - ...baseStore, - replace: (stackId: string, state: Parameters[1]) => - Effect.gen(function* () { - if (state.desiredLifecycle === "destroying" && destroyPreFenceFail !== undefined) { - const fail = yield* Ref.get(destroyPreFenceFail); - if (fail) { - yield* Ref.set(destroyPreFenceFail, false); - return yield* new StackStateInvalidError({ - message: "injected destroy fence failure", - }); - } - } - if (state.desiredLifecycle === "stopped" && stoppedReplaceFail !== undefined) { - const fail = yield* Ref.get(stoppedReplaceFail); - if (fail) { - yield* Ref.set(stoppedReplaceFail, false); - return yield* new StackStateInvalidError({ - message: "injected stopped-state persistence failure", - }); - } - } - return yield* baseStore.replace(stackId, state); - }), - }; - const runtimeStore = - fixtureOptions.readGateQueue !== undefined - ? { - ...persistedStore, - read: (stackId: string) => - Effect.gen(function* () { - const gate = yield* Ref.modify(fixtureOptions.readGateQueue!, (gates) => [ - gates[0], - gates.slice(1), - ]); - if (gate !== undefined) { - if (gate.beforeRead === true) { - yield* Deferred.succeed(gate.started, undefined); - yield* Deferred.await(gate.gate); - } - } - const state = yield* persistedStore.read(stackId); - if (gate !== undefined && gate.beforeRead !== true) { - yield* Deferred.succeed(gate.started, undefined); - yield* Deferred.await(gate.gate); - } - return state; - }), - } - : persistedStore; - const store = - fixtureOptions.readCalls === undefined - ? runtimeStore - : { - ...runtimeStore, - read: (stackId: string) => - Ref.update(fixtureOptions.readCalls!, (count) => count + 1).pipe( - Effect.andThen(runtimeStore.read(stackId)), - ), - }; - yield* store.initialize(id, { - format: "supabase-stack-state-v1", - identity, - runtime: fixtureOptions.runtime ?? { kind: "native" }, - desiredLifecycle: "unconfigured", - ports: [], - privatePorts: [], - secrets: {}, - }); - const resources = yield* Ref.make>([]); - const calls = yield* Ref.make>([]); - const logOptions = yield* Ref.make>([]); - const entry: StackLogEntry = { - cursor: { opaque: "v1_1" }, - timestamp: "2026-01-01T00:00:00.000Z", - source: "auth", - stream: "internal", - message: "hello", - }; - const finalEntry: StackLogEntry = { - ...entry, - cursor: { opaque: "v1_2" }, - timestamp: "2026-01-01T00:00:01.000Z", - message: "stopped", - }; - const logEntries = yield* Ref.make>([entry]); - const failDestroy = yield* Ref.make(false); - let gateStopCleanup = false; - const driver: RuntimeDriver = { - observe: () => - Effect.gen(function* () { - if ( - fixtureOptions.observeFailure !== undefined && - (yield* Ref.get(fixtureOptions.observeFailure)) - ) - return yield* new RuntimeDriverError({ - cause: new ContainerEngineError({ - engine: "docker", - message: "injected observe failure", - }), - message: "injected observe failure", - }); - return yield* Ref.get(resources); - }), - start: (key, workload: PlannedWorkload) => - Effect.gen(function* () { - gateStopCleanup = true; - if (fixtureOptions.timeline !== undefined) - yield* Ref.update(fixtureOptions.timeline, (current) => [ - ...current, - `start:${workload.id}`, - ]); - yield* Ref.update(calls, (current) => [...current, `start:${workload.id}`]); - if ( - fixtureOptions.startStarted !== undefined && - (fixtureOptions.startWorkload === undefined || - fixtureOptions.startWorkload === workload.id) - ) - yield* Deferred.succeed(fixtureOptions.startStarted, undefined); - if ( - fixtureOptions.startGate !== undefined && - (fixtureOptions.startWorkload === undefined || - fixtureOptions.startWorkload === workload.id) - ) - yield* Deferred.await(fixtureOptions.startGate); - if ( - fixtureOptions.startFailures !== undefined && - key.workloadId === (fixtureOptions.startFailureWorkload ?? "functions:edge-runtime") - ) { - const remaining = yield* Ref.get(fixtureOptions.startFailures); - if (remaining > 0) { - yield* Ref.set(fixtureOptions.startFailures, remaining - 1); - if (fixtureOptions.stopFailFirst !== undefined) - yield* Ref.set(fixtureOptions.stopFailFirst, true); - const failed = { ...key, state: "failed" as const, error: "injected start failure" }; - yield* Ref.update(resources, (current) => [ - ...current.filter((entry) => entry.workloadId !== key.workloadId), - failed, - ]); - return yield* new RuntimeDriverError({ - message: "injected start failure", - stackId: key.stackId, - workloadId: key.workloadId, - }); - } - } - const ready = { ...key, state: "ready" as const }; - yield* Ref.update(resources, (current) => [ - ...current.filter((entry) => entry.workloadId !== key.workloadId), - ready, - ]); - if ( - fixtureOptions.startFinished !== undefined && - workload.id === (fixtureOptions.startWorkload ?? "rest:rest") - ) - yield* Deferred.succeed(fixtureOptions.startFinished, undefined); - if (fixtureOptions.startQueue !== undefined) - yield* Queue.offer(fixtureOptions.startQueue, workload.id); - return ready; - }), - stop: (key) => - Effect.gen(function* () { - if (fixtureOptions.workloadStopStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.workloadStopStarted, undefined); - if (fixtureOptions.workloadStopGate !== undefined) - yield* Deferred.await(fixtureOptions.workloadStopGate); - if (fixtureOptions.workloadStopFailFirst !== undefined) { - const fail = yield* Ref.get(fixtureOptions.workloadStopFailFirst); - if (fail) { - yield* Ref.set(fixtureOptions.workloadStopFailFirst, false); - return yield* new RuntimeDriverError({ - message: "injected workload stop failure", - stackId: key.stackId, - workloadId: key.workloadId, - }); - } - } - if (fixtureOptions.timeline !== undefined) - yield* Ref.update(fixtureOptions.timeline, (current) => [ - ...current, - `stop:${key.workloadId}`, - ]); - yield* Ref.update(resources, (current) => - current.map((entry) => - entry.workloadId === key.workloadId ? { ...entry, state: "stopped" as const } : entry, - ), - ); - }), - remove: (key) => - Effect.gen(function* () { - if ( - fixtureOptions.workloadRemoveFailFirst !== undefined && - (fixtureOptions.workloadRemoveFailWorkload === undefined || - fixtureOptions.workloadRemoveFailWorkload === key.workloadId) - ) { - const fail = yield* Ref.get(fixtureOptions.workloadRemoveFailFirst); - if (fail) { - yield* Ref.set(fixtureOptions.workloadRemoveFailFirst, false); - return yield* new RuntimeDriverError({ - message: "injected workload remove failure", - stackId: key.stackId, - workloadId: key.workloadId, - }); - } - } - if (fixtureOptions.workloadRemoveDieFirst !== undefined) { - const fail = yield* Ref.get(fixtureOptions.workloadRemoveDieFirst); - if (fail) { - yield* Ref.set(fixtureOptions.workloadRemoveDieFirst, false); - return yield* Effect.die("injected workload remove defect"); - } - } - yield* Ref.update(resources, (current) => - current.filter((entry) => entry.workloadId !== key.workloadId), - ); - }), - cleanup: ({ destroy }) => - Effect.gen(function* () { - if (destroy && (yield* Ref.get(failDestroy))) - return yield* new RuntimeDriverError({ message: "destroy failed" }); - if (!destroy && fixtureOptions.stopFailFirst !== undefined) { - const fail = yield* Ref.get(fixtureOptions.stopFailFirst); - if (fail) { - yield* Ref.set(fixtureOptions.stopFailFirst, false); - return yield* new RuntimeDriverError({ message: "injected stop cleanup failure" }); - } - } - if (fixtureOptions.timeline !== undefined) - yield* Ref.update(fixtureOptions.timeline, (current) => [ - ...current, - `cleanup:${destroy ? "destroy" : "stop"}`, - ]); - yield* Ref.update(calls, (current) => [ - ...current, - `cleanup:${destroy ? "destroy" : "stop"}`, - ]); - if (destroy && fixtureOptions.destroyStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.destroyStarted, undefined); - if (destroy && fixtureOptions.destroyGate !== undefined) - yield* Deferred.await(fixtureOptions.destroyGate); - if (!destroy && gateStopCleanup && fixtureOptions.stopStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.stopStarted, undefined); - if (!destroy && gateStopCleanup && fixtureOptions.stopGate !== undefined) - yield* Deferred.await(fixtureOptions.stopGate); - yield* Ref.set(resources, []); - if (!destroy && gateStopCleanup) - yield* Ref.update(logEntries, (current) => [...current, finalEntry]); - }), - wipePersistentData: () => Effect.void, - }; - const runtime: SupervisorRuntime = { - driver, - preflight: (_input) => - Effect.gen(function* () { - if (fixtureOptions.preflightCalls !== undefined) - yield* Ref.update(fixtureOptions.preflightCalls, (count) => count + 1); - if (fixtureOptions.preflightStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.preflightStarted, undefined); - if (fixtureOptions.preflightGate !== undefined) - yield* Deferred.await(fixtureOptions.preflightGate); - if (fixtureOptions.preflightFailFirst !== undefined) { - const fail = yield* Ref.get(fixtureOptions.preflightFailFirst); - if (fail) { - yield* Ref.set(fixtureOptions.preflightFailFirst, false); - return yield* new StackRuntimeError({ message: "injected preflight failure" }); - } - } - }), - prepare: () => - fixtureOptions.prepareStarted === undefined && fixtureOptions.prepareGate === undefined - ? Effect.void - : Effect.gen(function* () { - if (fixtureOptions.prepareStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.prepareStarted, undefined); - if ( - fixtureOptions.prepareActivationStarted !== undefined && - fixtureOptions.prepareGateEnabledRef !== undefined && - (yield* Ref.get(fixtureOptions.prepareGateEnabledRef)) - ) - yield* Deferred.succeed(fixtureOptions.prepareActivationStarted, undefined); - if ( - fixtureOptions.prepareGate !== undefined && - (fixtureOptions.prepareGateEnabledRef === undefined || - (yield* Ref.get(fixtureOptions.prepareGateEnabledRef))) - ) - yield* Deferred.await(fixtureOptions.prepareGate); - if ( - fixtureOptions.prepareFailure === true || - (fixtureOptions.prepareFailureRef !== undefined && - (yield* Ref.get(fixtureOptions.prepareFailureRef))) - ) - return yield* new StackRuntimeError({ message: "injected preparation failure" }); - }), - prefetch: (state) => { - if (state.definition?.preparation === "on-demand") return Effect.void; - return Effect.gen(function* () { - if (fixtureOptions.prefetchCalls !== undefined) - yield* Ref.update(fixtureOptions.prefetchCalls, (count) => count + 1); - if (fixtureOptions.artifactStatuses !== undefined) - yield* Ref.set(fixtureOptions.artifactStatuses, [ - { workloadId: "rest:rest", capability: "rest", state: "downloading" }, - ]); - if (fixtureOptions.prefetchStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.prefetchStarted, undefined); - if (fixtureOptions.prefetchGate !== undefined) - yield* Deferred.await(fixtureOptions.prefetchGate); - if (fixtureOptions.artifactStatuses !== undefined) - yield* Ref.set(fixtureOptions.artifactStatuses, [ - { workloadId: "rest:rest", capability: "rest", state: "ready" }, - ]); - if (fixtureOptions.prefetchFinished !== undefined) - yield* Deferred.succeed(fixtureOptions.prefetchFinished, undefined); - }).pipe( - Effect.onInterrupt(() => - Effect.gen(function* () { - if (fixtureOptions.artifactStatuses !== undefined) - yield* Ref.set(fixtureOptions.artifactStatuses, []); - if (fixtureOptions.prefetchInterrupted !== undefined) - yield* Deferred.succeed(fixtureOptions.prefetchInterrupted, undefined); - }), - ), - ); - }, - artifacts: - fixtureOptions.artifactStatuses === undefined - ? Effect.succeed([]) - : Ref.get(fixtureOptions.artifactStatuses), - activate: () => - Effect.gen(function* () { - const callNumber = - fixtureOptions.activationCalls === undefined - ? undefined - : yield* Ref.updateAndGet(fixtureOptions.activationCalls, (count) => count + 1); - if (fixtureOptions.activationStarted !== undefined) - yield* Deferred.succeed(fixtureOptions.activationStarted, undefined); - if (fixtureOptions.activationGateAfterFirst !== undefined && callNumber !== 1) { - if (fixtureOptions.activationStartedAfterFirst !== undefined) - yield* Deferred.succeed(fixtureOptions.activationStartedAfterFirst, undefined); - yield* Deferred.await(fixtureOptions.activationGateAfterFirst); - } else if (fixtureOptions.activationGate !== undefined) - yield* Deferred.await(fixtureOptions.activationGate); - if (fixtureOptions.activationFailFirst !== undefined) { - const fail = yield* Ref.get(fixtureOptions.activationFailFirst); - if (fail) { - yield* Ref.set(fixtureOptions.activationFailFirst, false); - return yield* new GatewayActivationError({ message: "injected activation failure" }); - } - } - return { host: "127.0.0.1", port: 9999 }; - }), - ingress: fixtureOptions.ingress ?? { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: () => Effect.void, - close: Effect.void, - }, - logStore: { - path: "memory://logs", - append: (record) => - (fixtureOptions.logRecords === undefined - ? Effect.void - : Ref.update(fixtureOptions.logRecords, (current) => [...current, record.message]) - ).pipe( - Effect.andThen( - fixtureOptions.logQueue === undefined - ? Effect.void - : Queue.offer(fixtureOptions.logQueue, record.message), - ), - Effect.andThen( - fixtureOptions.logWritten === undefined || - (fixtureOptions.logWrittenFor !== undefined && - !record.message.includes(fixtureOptions.logWrittenFor)) - ? Effect.void - : Deferred.succeed(fixtureOptions.logWritten, undefined), - ), - Effect.andThen( - fixtureOptions.logWrittenAdditional === undefined || - (fixtureOptions.logWrittenAdditionalFor !== undefined && - !record.message.includes(fixtureOptions.logWrittenAdditionalFor)) - ? Effect.void - : Deferred.succeed(fixtureOptions.logWrittenAdditional, undefined), - ), - Effect.andThen( - Effect.succeed({ - ...entry, - source: record.source, - stream: record.stream, - message: record.message, - }), - ), - ), - read: (options) => - options?.cursor?.opaque === "not-a-cursor" - ? Effect.fail(new InvalidLogCursorError({ message: "Log cursor is invalid" })) - : Ref.update(logOptions, (current) => [...current, options]).pipe( - Effect.andThen(Ref.get(logEntries)), - ), - }, - }; - const context = yield* Effect.context< - FileSystem.FileSystem | Path.Path | import("effect").Crypto.Crypto - >(); - const supervisorEffect = makeSupervisor({ - stackId: id, - ownerSessionId: "owner-session", - stateStore: store, - context, - runtime, - }); - const supervisor = - fixtureOptions.supervisorScope === undefined - ? yield* supervisorEffect - : yield* supervisorEffect.pipe( - Effect.provideService(Scope.Scope, fixtureOptions.supervisorScope), - ); - yield* Ref.set(calls, []); - return { - supervisor, - calls, - logOptions, - failDestroy, - context, - store, - id, - runtime, - resources, - root, - }; - }); - -const run = (effect: Effect.Effect) => - Effect.scoped(effect).pipe(Effect.provide(NodeServices.layer)); - -const makeCredentialsFixture = ({ authEnabled = true }: { readonly authEnabled?: boolean } = {}) => - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: {}, auth: { enabled: authEnabled } } }, - }); - const running = yield* fixture.store - .read(fixture.id) - .pipe(Effect.provideContext(fixture.context)); - if (running === undefined) - return yield* new StackStateInvalidError({ message: "running fixture state is missing" }); - if (running.definition === undefined) - return yield* new StackStateInvalidError({ message: "running definition is missing" }); - const state = { - ...running, - ports: [ - { field: "api", port: 55433, intent: "exact" as const }, - { field: "database", port: 55432, intent: "exact" as const }, - ] as const, - }; - yield* fixture.store.replace(fixture.id, state).pipe(Effect.provideContext(fixture.context)); - const baseSecrets = { - ...state.secrets, - "secret:auth.settings.publishable_key": { - policy: "managed" as const, - value: "publishable", - }, - "secret:auth.settings.secret_key": { policy: "managed" as const, value: "secret" }, - "secret:auth.settings.anon_key": { policy: "managed" as const, value: "anon" }, - "secret:auth.settings.service_role_key": { policy: "managed" as const, value: "service" }, - }; - return { fixture, state, definition: running.definition, baseSecrets }; - }); - -describe("Supervisor composition", () => { - it.live("rejects new work after owner shutdown begins", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.shutdownIfIdle; - - const start = yield* fixture.supervisor.start().pipe(Effect.exit); - expect(errorOf(start)).toBeInstanceOf(StackLifecycleConflictError); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("does not admit a lifecycle while idle shutdown makes its final decision", () => - run( - Effect.gen(function* () { - const readGateQueue = yield* Ref.make>([]); - const readStarted = yield* Deferred.make(); - const readGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - readGateQueue, - }); - yield* Ref.set(readGateQueue, [{ started: readStarted, gate: readGate, beforeRead: true }]); - const shutdown = yield* Effect.forkChild(fixture.supervisor.shutdownIfIdle); - yield* Deferred.await(readStarted); - const start = yield* Effect.forkChild(fixture.supervisor.start({ config: {} })); - yield* Deferred.succeed(readGate, undefined); - yield* Fiber.join(shutdown); - expect(errorOf(yield* Fiber.join(start).pipe(Effect.exit))).toBeInstanceOf( - StackLifecycleConflictError, - ); - }), - ), - ); - - it.live("starts through the composed lifecycle and reports observed readiness", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - const status = yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - expect(status.lifecycle).toBe("running"); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - expect(yield* Ref.get(fixture.calls)).toContain("start:database:database"); - }), - ), - ); - - it.live("publishes starting until an eager workload reaches readiness", () => - run( - Effect.gen(function* () { - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ startStarted, startGate }); - const starting = yield* Effect.forkChild( - fixture.supervisor.start({ config: { capabilities: { rest: { activation: "eager" } } } }), - { startImmediately: true }, - ); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - expect((yield* fixture.supervisor.status).lifecycle).toBe("starting"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("starting"); - yield* Deferred.succeed(startGate, undefined); - const ready = yield* Fiber.join(starting); - expect(ready.lifecycle).toBe("running"); - expect(ready.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - }), - ), - ); - - it.live("retires lazy traffic after its lease ends and reactivates on demand", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const timeline = yield* Ref.make>([]); - const activationStarted = yield* Deferred.make(); - const logWritten = yield* Deferred.make(); - const fixture = yield* makeFixture({ - timeline, - activationStarted, - logWritten, - logWrittenFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy", idleTimeoutSeconds: 1 }, - auth: { activation: "lazy" }, - }, - }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - const release = yield* Deferred.make(); - const request = yield* Effect.forkChild( - tracker.track( - "rest", - fixture.supervisor.activate("rest").pipe(Effect.andThen(Deferred.await(release))), - ), - ); - yield* Deferred.await(activationStarted); - yield* TestClock.adjust("1 second"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(request); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(logWritten); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - expect(yield* Ref.get(timeline)).toContain("stop:rest:rest"); - yield* fixture.supervisor.activate("rest"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("keeps a lazy dependency pinned until its dependent retires", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const timeline = yield* Ref.make>([]); - const activationStarted = yield* Deferred.make(); - const logWritten = yield* Deferred.make(); - const restLogWritten = yield* Deferred.make(); - const fixture = yield* makeFixture({ - timeline, - activationStarted, - logWritten, - logWrittenFor: "Stopped studio after inactivity", - logWrittenAdditional: restLogWritten, - logWrittenAdditionalFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy", idleTimeoutSeconds: 1 }, - studio: { activation: "lazy", idleTimeoutSeconds: 1 }, - }, - }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - const release = yield* Deferred.make(); - const request = yield* Effect.forkChild( - tracker.track( - "studio", - fixture.supervisor.activate("studio").pipe(Effect.andThen(Deferred.await(release))), - ), - ); - yield* Deferred.await(activationStarted); - yield* Deferred.succeed(release, undefined); - yield* Fiber.join(request); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(logWritten); - const afterStudio = yield* fixture.supervisor.status; - expect(afterStudio.capabilities.find(({ name }) => name === "studio")?.state).toBe( - "dormant", - ); - expect(afterStudio.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(restLogWritten); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - expect(yield* Ref.get(timeline)).toEqual( - expect.arrayContaining(["stop:studio:pgmeta", "stop:rest:rest"]), - ); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("re-arms a dependency idle timer after a pinned dependent retires", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const timeline = yield* Ref.make>([]); - const restReady = yield* Deferred.make(); - const studioReady = yield* Deferred.make(); - const restStopped = yield* Deferred.make(); - const studioStopped = yield* Deferred.make(); - const fixture = yield* makeFixture({ - timeline, - logWritten: studioStopped, - logWrittenFor: "Stopped studio after inactivity", - logWrittenAdditional: restStopped, - logWrittenAdditionalFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy", idleTimeoutSeconds: 1 }, - studio: { activation: "lazy", idleTimeoutSeconds: 1 }, - }, - }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - - const releaseRest = yield* Deferred.make(); - const restRequest = yield* Effect.forkChild( - tracker.track( - "rest", - fixture.supervisor.activate("rest").pipe( - Effect.tap(() => Deferred.succeed(restReady, undefined)), - Effect.andThen(Deferred.await(releaseRest)), - ), - ), - ); - yield* TestClock.withLive(Deferred.await(restReady).pipe(Effect.timeout("5 seconds"))); - yield* Deferred.succeed(releaseRest, undefined); - yield* TestClock.withLive(Fiber.join(restRequest).pipe(Effect.timeout("5 seconds"))); - - const releaseStudio = yield* Deferred.make(); - const studioRequest = yield* Effect.forkChild( - tracker.track( - "studio", - fixture.supervisor.activate("studio").pipe( - Effect.tap(() => Deferred.succeed(studioReady, undefined)), - Effect.andThen(Deferred.await(releaseStudio)), - ), - ), - ); - yield* TestClock.withLive(Deferred.await(studioReady).pipe(Effect.timeout("5 seconds"))); - yield* TestClock.adjust("1 second"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - yield* Deferred.succeed(releaseStudio, undefined); - yield* TestClock.withLive(Fiber.join(studioRequest).pipe(Effect.timeout("5 seconds"))); - yield* TestClock.adjust("1 second"); - yield* TestClock.withLive(Deferred.await(studioStopped).pipe(Effect.timeout("5 seconds"))); - yield* TestClock.adjust("1 second"); - yield* TestClock.withLive(Deferred.await(restStopped).pipe(Effect.timeout("5 seconds"))); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - expect(yield* Ref.get(timeline)).toEqual( - expect.arrayContaining(["stop:studio:pgmeta", "stop:rest:rest"]), - ); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("retires an unresolved dependency after its cancelled lookup waiter finishes", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const lookupStarted = yield* Deferred.make(); - const lookupStartedAfterFirst = yield* Deferred.make(); - const lookupGate = yield* Deferred.make(); - const activationCalls = yield* Ref.make(0); - const studioStopped = yield* Deferred.make(); - const restStopped = yield* Deferred.make(); - const fixture = yield* makeFixture({ - activationGateAfterFirst: lookupGate, - activationStarted: lookupStarted, - activationStartedAfterFirst: lookupStartedAfterFirst, - activationCalls, - logWritten: studioStopped, - logWrittenFor: "Stopped studio after inactivity", - logWrittenAdditional: restStopped, - logWrittenAdditionalFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy", idleTimeoutSeconds: 1 }, - studio: { activation: "lazy", idleTimeoutSeconds: 1 }, - }, - }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - - yield* tracker.track("studio", fixture.supervisor.activate("studio")); - yield* TestClock.adjust("1 second"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "studio") - ?.state, - ).toBe("dormant"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - - const cancelled = yield* Effect.forkChild( - tracker.track("rest", fixture.supervisor.activate("rest")), - ); - yield* TestClock.withLive( - Deferred.await(lookupStartedAfterFirst).pipe(Effect.timeout("5 seconds")), - ); - yield* Fiber.interrupt(cancelled); - const owner = yield* Effect.forkChild(fixture.supervisor.activate("rest")); - yield* Deferred.succeed(lookupGate, undefined); - yield* TestClock.withLive(Fiber.join(owner).pipe(Effect.timeout("5 seconds"))); - yield* TestClock.adjust("1 second"); - yield* TestClock.withLive(Deferred.await(restStopped).pipe(Effect.timeout("5 seconds"))); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("joins concurrent activation while a lazy dependency is starting", () => - run( - Effect.gen(function* () { - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - startStarted, - startGate, - startWorkload: "rest:rest", - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy" }, - studio: { activation: "lazy" }, - }, - }, - }); - const studio = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds")); - const rest = yield* Effect.forkChild(fixture.supervisor.activate("rest"), { - startImmediately: true, - }); - yield* Deferred.succeed(startGate, undefined); - yield* Fiber.join(rest).pipe(Effect.timeout("5 seconds")); - yield* Fiber.join(studio).pipe(Effect.timeout("5 seconds")); - const status = yield* fixture.supervisor.status; - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - expect(status.capabilities.find(({ name }) => name === "studio")?.state).toBe("ready"); - }), - ), - ); - - it.live( - "propagates a recovered owner read failure instead of fabricating an empty running stack", - () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - const failRead = yield* Ref.make(false); - const stateStore = { - ...fixture.store, - read: (stackId: string) => - Effect.gen(function* () { - if (yield* Ref.getAndSet(failRead, false)) - return yield* new StackStateInvalidError({ - message: "transient state read failure", - }); - return yield* fixture.store.read(stackId); - }), - }; - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore, - context: fixture.context, - runtime: fixture.runtime, - }); - yield* Ref.set(failRead, true); - const failed = yield* successor.start().pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* successor.status).lifecycle).toBe("stopping"); - expect(Exit.isFailure(yield* successor.start().pipe(Effect.exit))).toBe(true); - expect((yield* successor.maintenanceHandlers.stop).ok).toBe(true); - const restarted = yield* successor.start().pipe(Effect.exit); - expect(Exit.isSuccess(restarted)).toBe(true); - if (Exit.isSuccess(restarted)) expect(restarted.value.lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "database:database", state: "ready" }), - ); - }), - ), - ); - - it.live("retries lazy activation after a preclaim state read failure", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const failRead = yield* Ref.make(false); - const stateStore = { - ...fixture.store, - read: (stackId: string) => - Effect.gen(function* () { - if (yield* Ref.getAndSet(failRead, false)) - return yield* new StackStateInvalidError({ - message: "transient activation state read failure", - }); - return yield* fixture.store.read(stackId); - }), - }; - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore, - context: fixture.context, - runtime: fixture.runtime, - }); - yield* successor.start(); - const before = yield* Ref.get(fixture.resources); - yield* Ref.set(failRead, true); - const failed = yield* successor - .activate("functions") - .pipe(Effect.timeout("5 seconds"), Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(errorOf(failed)).toBeInstanceOf(StackStateInvalidError); - expect(errorOf(failed)?.message).toBe("transient activation state read failure"); - expect(yield* Ref.get(fixture.resources)).toEqual(before); - const retry = yield* successor.activate("functions").pipe(Effect.timeout("5 seconds")); - expect(retry.endpoint).toEqual({ host: "127.0.0.1", port: 9999 }); - expect( - (yield* successor.status).capabilities.find(({ name }) => name === "functions")?.state, - ).toBe("ready"); - }), - ), - ); - - it.live("shares the original workload start failure with concurrent dependency callers", () => - run( - Effect.gen(function* () { - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const startFailures = yield* Ref.make(1); - const fixture = yield* makeFixture({ - startStarted, - startGate, - startWorkload: "rest:rest", - startFailures, - startFailureWorkload: "rest:rest", - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy" }, - studio: { activation: "lazy" }, - }, - }, - }); - const studio = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - const rest = yield* Effect.forkChild(fixture.supervisor.activate("rest"), { - startImmediately: true, - }); - yield* Deferred.succeed(startGate, undefined); - const studioExit = yield* Fiber.join(studio).pipe(Effect.exit); - const restExit = yield* Fiber.join(rest).pipe(Effect.exit); - expect(Exit.isFailure(studioExit)).toBe(true); - expect(Exit.isFailure(restExit)).toBe(true); - expect(errorOf(studioExit)?._tag).toBe(errorOf(restExit)?._tag); - expect(errorOf(studioExit)?.message).toBe(errorOf(restExit)?.message); - }), - ), - ); - - it.live("fences a lazy launch when root rollback removal is unproven", () => - run( - Effect.gen(function* () { - const startFailures = yield* Ref.make(1); - const workloadRemoveFailFirst = yield* Ref.make(true); - const fixture = yield* makeFixture({ - startFailures, - startFailureWorkload: "studio:pgmeta", - workloadRemoveFailFirst, - workloadRemoveFailWorkload: "studio:pgmeta", - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy" }, - studio: { activation: "lazy" }, - }, - }, - }); - const activation = yield* fixture.supervisor.activate("studio").pipe(Effect.exit); - expect(Exit.isFailure(activation)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("failed"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "studio") - ?.state, - ).toBe("failed"); - expect(Exit.isFailure(yield* fixture.supervisor.activate("rest").pipe(Effect.exit))).toBe( - true, - ); - expect(Exit.isFailure(yield* fixture.supervisor.start().pipe(Effect.exit))).toBe(true); - }), - ), - ); - - it.live("rejects activation of a disabled capability without changing its state", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ - config: { - capabilities: { rest: { enabled: false }, studio: { enabled: false } }, - }, - }); - const activation = yield* fixture.supervisor.activate("rest").pipe(Effect.exit); - expect(Exit.isFailure(activation)).toBe(true); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("disabled"); - }), - ), - ); - - it.live("re-arms an idle timer after a failed root activation releases execution", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const prepareGateEnabledRef = yield* Ref.make(false); - const prepareActivationStarted = yield* Deferred.make(); - const prepareGate = yield* Deferred.make(); - const prepareFailureRef = yield* Ref.make(false); - const logWritten = yield* Deferred.make(); - const fixture = yield* makeFixture({ - prepareGateEnabledRef, - prepareActivationStarted, - prepareGate, - prepareFailureRef, - logWritten, - logWrittenFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy", idleTimeoutSeconds: 1 }, - auth: { activation: "lazy" }, - studio: { activation: "lazy", idleTimeoutSeconds: 1 }, - }, - }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* Ref.set(prepareGateEnabledRef, true); - const auth = yield* Effect.forkChild( - tracker.track("auth", fixture.supervisor.activate("auth")), - { startImmediately: true }, - ); - yield* Deferred.await(prepareActivationStarted); - yield* TestClock.adjust("1 second"); - const studio = yield* Effect.forkChild( - tracker.track("studio", fixture.supervisor.activate("studio")), - { startImmediately: true }, - ); - const beforeRelease = yield* fixture.supervisor.status; - expect(beforeRelease.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - expect(beforeRelease.capabilities.find(({ name }) => name === "studio")?.state).toBe( - "starting", - ); - yield* Ref.set(prepareFailureRef, true); - yield* Deferred.succeed(prepareGate, undefined); - const authResult = yield* Fiber.join(auth).pipe(Effect.exit); - expect(Exit.isFailure(authResult)).toBe(true); - expect(errorOf(authResult)).toBeInstanceOf(StackRuntimeError); - const studioResult = yield* Fiber.join(studio).pipe(Effect.exit); - expect(Exit.isFailure(studioResult)).toBe(true); - expect(errorOf(studioResult)).toBeInstanceOf(StackRuntimeError); - const afterFailure = yield* fixture.supervisor.status; - expect(afterFailure.lifecycle).toBe("running"); - expect(afterFailure.capabilities.find(({ name }) => name === "studio")?.state).toBe( - "dormant", - ); - yield* TestClock.adjust("1 second"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - yield* Deferred.await(logWritten); - expect(yield* Ref.get(fixture.resources)).toEqual( - expect.not.arrayContaining([expect.objectContaining({ workloadId: "rest:rest" })]), - ); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("accepts very small idle timeout values", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const logWritten = yield* Deferred.make(); - const fixture = yield* makeFixture({ - logWritten, - logWrittenFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1e-7 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 millis"); - yield* Deferred.await(logWritten); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("rearms a zero-rounded idle timeout after its first retirement", () => - run( - Effect.gen(function* () { - const logs = yield* Queue.unbounded(); - const fixture = yield* makeFixture({ - logQueue: logs, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1e-12 } } }, - }); - yield* fixture.supervisor.activate("rest"); - yield* TestClock.adjust("1 millis"); - expect(yield* Queue.take(logs)).toContain("Stopped rest after inactivity"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - - yield* fixture.supervisor.activate("rest"); - yield* TestClock.adjust("1 millis"); - expect(yield* Queue.take(logs)).toContain("Stopped rest after inactivity"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("resets the idle deadline when a second request arrives", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const activationStarted = yield* Deferred.make(); - const logWritten = yield* Deferred.make(); - const fixture = yield* makeFixture({ - activationStarted, - logWritten, - logWrittenFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - const first = yield* Effect.forkChild( - tracker.track("rest", fixture.supervisor.activate("rest")), - ); - yield* Deferred.await(activationStarted); - yield* Fiber.join(first); - yield* TestClock.adjust("500 millis"); - yield* tracker.track("rest", Effect.void); - yield* TestClock.adjust("500 millis"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(logWritten); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("queues traffic arriving during idle cleanup for a fresh activation", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const activationCalls = yield* Ref.make(0); - const logQueue = yield* Queue.unbounded(); - const stopStarted = yield* Deferred.make(); - const stopGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - activationCalls, - logQueue, - workloadStopStarted: stopStarted, - workloadStopGate: stopGate, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopStarted); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "auth") - ?.state, - ).toBe("dormant"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("stopping"); - - const requestStarted = yield* Deferred.make(); - const cancelled = yield* Effect.forkChild( - Deferred.succeed(requestStarted, undefined).pipe( - Effect.andThen(tracker.track("rest", fixture.supervisor.activate("rest"))), - ), - ); - yield* Deferred.await(requestStarted); - expect(yield* Ref.get(activationCalls)).toBe(1); - yield* Fiber.interrupt(cancelled); - - const survivorReady = yield* Deferred.make(); - const survivorRelease = yield* Deferred.make(); - const request = yield* Effect.forkChild( - tracker.track( - "rest", - fixture.supervisor - .activate("rest") - .pipe( - Effect.andThen(Deferred.succeed(survivorReady, undefined)), - Effect.andThen(Deferred.await(survivorRelease)), - ), - ), - ); - - yield* Deferred.succeed(stopGate, undefined); - expect(yield* Queue.take(logQueue)).toContain("Stopped rest after inactivity"); - yield* Deferred.await(survivorReady); - expect(yield* Ref.get(activationCalls)).toBe(2); - yield* TestClock.adjust("1 second"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - yield* Deferred.succeed(survivorRelease, undefined); - yield* Fiber.join(request); - yield* TestClock.adjust("1 second"); - expect(yield* Queue.take(logQueue)).toContain("Stopped rest after inactivity"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("rejects an idempotent start admitted during idle retirement", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const stopStarted = yield* Deferred.make(); - const stopGate = yield* Deferred.make(); - const workloadRemoveFailFirst = yield* Ref.make(true); - const config = { - capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } }, - } as const; - const fixture = yield* makeFixture({ - workloadStopStarted: stopStarted, - workloadStopGate: stopGate, - workloadRemoveFailFirst, - workloadRemoveFailWorkload: "rest:rest", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ config }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopStarted); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("stopping"); - - const admitted = yield* Effect.forkChild(fixture.supervisor.start({ config }), { - startImmediately: true, - }); - yield* Effect.yieldNow; - const rejected = yield* fixture.supervisor.start({ config }).pipe(Effect.exit); - expect(errorOf(rejected)).toBeInstanceOf(StackLifecycleConflictError); - - yield* Deferred.succeed(stopGate, undefined); - const failed = yield* Fiber.join(admitted).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("failed"); - expect((yield* Ref.get(fixture.calls)).filter((call) => call.startsWith("start:"))).toEqual( - ["start:database:database", "start:rest:rest"], - ); - expect( - errorOf(yield* fixture.supervisor.start({ config }).pipe(Effect.exit)), - ).toBeInstanceOf(StackLifecycleConflictError); - - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("keeps a ready capability alive while an overlapping request lease remains", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const firstAcquired = yield* Deferred.make(); - const firstRelease = yield* Deferred.make(); - const secondAcquired = yield* Deferred.make(); - const secondRelease = yield* Deferred.make(); - const logQueue = yield* Queue.unbounded(); - const fixture = yield* makeFixture({ - logQueue, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - const first = yield* Effect.forkChild( - tracker.track( - "rest", - fixture.supervisor - .activate("rest") - .pipe( - Effect.andThen(Deferred.succeed(firstAcquired, undefined)), - Effect.andThen(Deferred.await(firstRelease)), - ), - ), - ); - yield* Deferred.await(firstAcquired); - const second = yield* Effect.forkChild( - tracker.track( - "rest", - Deferred.succeed(secondAcquired, undefined).pipe( - Effect.andThen(Deferred.await(secondRelease)), - ), - ), - ); - yield* Deferred.await(secondAcquired); - yield* Deferred.succeed(firstRelease, undefined); - yield* Fiber.join(first); - yield* TestClock.adjust("2 seconds"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - yield* Deferred.succeed(secondRelease, undefined); - yield* Fiber.join(second); - yield* TestClock.adjust("1 second"); - expect(yield* Queue.take(logQueue)).toContain("Stopped rest after inactivity"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("settles activation waiters when the owner scope is interrupted during retirement", () => - run( - Effect.gen(function* () { - const ownerScope = yield* Scope.make(); - const stopStarted = yield* Deferred.make(); - const stopGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - workloadStopStarted: stopStarted, - workloadStopGate: stopGate, - }).pipe(Effect.provideService(Scope.Scope, ownerScope)); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - yield* fixture.supervisor.activate("rest"); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopStarted); - - const waiter = yield* Effect.forkChild(fixture.supervisor.activate("rest"), { - startImmediately: true, - }); - yield* TestClock.withLive( - Scope.close(ownerScope, Exit.void).pipe(Effect.timeout("5 seconds")), - ); - const settled = yield* TestClock.withLive( - Fiber.await(waiter).pipe(Effect.timeout("5 seconds")), - ); - expect(Exit.isFailure(settled)).toBe(true); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("settles overlapping dependency activation when launch owner scope closes", () => - run( - Effect.gen(function* () { - const ownerScope = yield* Scope.make(); - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - startStarted, - startGate, - startWorkload: "rest:rest", - supervisorScope: ownerScope, - }); - const config = { - capabilities: { - rest: { activation: "lazy" as const }, - studio: { activation: "lazy" as const }, - functions: { activation: "lazy" as const }, - }, - }; - yield* fixture.supervisor.start({ config }); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "database:database", state: "ready" }), - ); - const studio = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(startStarted); - const rest = yield* Effect.forkChild(fixture.supervisor.activate("rest"), { - startImmediately: true, - }); - const functions = yield* Effect.forkChild(fixture.supervisor.activate("functions"), { - startImmediately: true, - }); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("starting"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "functions") - ?.state, - ).toBe("starting"); - - yield* Scope.close(ownerScope, Exit.void).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("owner scope did not close"), - }), - ); - const studioExit = yield* Fiber.await(studio).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("Studio activation remained pending after owner scope close"), - }), - ); - const restExit = yield* Fiber.await(rest).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("REST activation remained pending after owner scope close"), - }), - ); - expect(Exit.isFailure(studioExit)).toBe(true); - if (Exit.isFailure(studioExit)) expect(Cause.hasInterrupts(studioExit.cause)).toBe(true); - expect(Exit.isFailure(restExit)).toBe(true); - if (Exit.isFailure(restExit)) expect(Cause.hasInterrupts(restExit.cause)).toBe(true); - const functionsExit = yield* Fiber.await(functions).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => - Effect.die("Functions activation remained pending after owner scope close"), - }), - ); - expect(Exit.isFailure(functionsExit)).toBe(true); - if (Exit.isFailure(functionsExit)) - expect(Cause.hasInterrupts(functionsExit.cause)).toBe(true); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopping"); - expect(status.recovery?.operation).toBe("stop"); - expect(status.capabilities.find(({ name }) => name === "studio")?.state).toBe("failed"); - expect(status.capabilities.find(({ name }) => name === "functions")?.state).toBe("dormant"); - }), - ), - ); - - it.live("rejects lifecycle admission after the owner scope closes", () => - run( - Effect.gen(function* () { - const ownerScope = yield* Scope.make(); - const fixture = yield* makeFixture({ supervisorScope: ownerScope }); - const config = { capabilities: { rest: { activation: "lazy" as const } } }; - yield* fixture.supervisor.start({ config }); - yield* Scope.close(ownerScope, Exit.void); - const activation = yield* Effect.exit(fixture.supervisor.activate("rest")); - expect(errorOf(activation)).toBeInstanceOf(StackLifecycleConflictError); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - const retry = yield* Effect.exit(fixture.supervisor.start({ config })); - expect(errorOf(retry)).toBeInstanceOf(StackLifecycleConflictError); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopping"); - expect(status.recovery).toEqual({ - operation: "stop", - message: "Stack owner scope is closed", - }); - }), - ), - ); - - it.live("settles endpoint activation when launch owner scope closes", () => - run( - Effect.gen(function* () { - const ownerScope = yield* Scope.make(); - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const activationGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - startStarted, - startGate, - startWorkload: "rest:rest", - activationStarted, - activationGate, - supervisorScope: ownerScope, - }); - const config = { - capabilities: { - rest: { activation: "lazy" as const }, - studio: { activation: "lazy" as const }, - }, - }; - yield* fixture.supervisor.start({ config }); - const studio = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(startStarted); - const rest = yield* Effect.forkChild(fixture.supervisor.activate("rest"), { - startImmediately: true, - }); - yield* Deferred.succeed(startGate, undefined); - yield* Deferred.await(activationStarted); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "rest:rest", state: "ready" }), - ); - - yield* Scope.close(ownerScope, Exit.void).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("owner scope did not close"), - }), - ); - const studioExit = yield* Fiber.await(studio).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("Studio activation remained pending after owner scope close"), - }), - ); - expect(Exit.isFailure(studioExit)).toBe(true); - if (Exit.isFailure(studioExit)) expect(Cause.hasInterrupts(studioExit.cause)).toBe(true); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "rest:rest", state: "ready" }), - ); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopping"); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("failed"); - const restExit = yield* Fiber.await(rest).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => - Effect.die("REST activation remained pending after endpoint owner scope close"), - }), - ); - expect(Exit.isFailure(restExit)).toBe(true); - }), - ), - ); - - it.live("settles lazy activation interrupted while waiting for admission", () => - run( - Effect.gen(function* () { - const ownerScope = yield* Scope.make(); - const readGateQueue = yield* Ref.make>([]); - const activationReadStarted = yield* Deferred.make(); - const activationReadGate = yield* Deferred.make(); - const shutdownReadStarted = yield* Deferred.make(); - const shutdownReadGate = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - readGateQueue, - supervisorScope: ownerScope, - activationStarted, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy" } } }, - }); - yield* Ref.set(readGateQueue, [ - { started: activationReadStarted, gate: activationReadGate }, - ]); - - const activation = yield* Effect.forkChild( - fixture.supervisor - .activate("rest") - .pipe(Effect.provideService(Scheduler.PreventSchedulerYield, true)), - { startImmediately: true }, - ); - yield* Deferred.await(activationReadStarted).pipe( - Effect.timeout("5 seconds"), - Effect.orDie, - ); - yield* Ref.update(readGateQueue, (gates) => [ - ...gates, - { started: shutdownReadStarted, gate: shutdownReadGate }, - ]); - const shutdown = yield* Effect.forkChild(fixture.supervisor.shutdownIfIdle, { - startImmediately: true, - }); - yield* Deferred.await(shutdownReadStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - - // Deferred resumes synchronously to the masked wait; wake-up is queued. - // Only FiberSet has a live finalizer, so scope close installs interruption first. - yield* Deferred.succeed(activationReadGate, undefined); - const closing = yield* Effect.forkChild( - Scope.close(ownerScope, Exit.void).pipe( - Effect.provideService(Scheduler.PreventSchedulerYield, true), - ), - { startImmediately: true }, - ); - yield* Deferred.succeed(shutdownReadGate, undefined); - - yield* Fiber.join(closing).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("owner scope did not close"), - }), - ); - const activationExit = yield* Fiber.await(activation).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("activation remained pending after owner scope close"), - }), - ); - yield* Fiber.join(shutdown); - expect(Exit.isFailure(activationExit)).toBe(true); - if (Exit.isFailure(activationExit)) - expect(Cause.hasInterrupts(activationExit.cause)).toBe(true); - expect(yield* Deferred.isDone(activationStarted)).toBe(false); - - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("running"); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("dormant"); - - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - yield* successor.start(); - const retry = yield* successor.activate("rest"); - expect(retry.endpoint).toEqual({ host: "127.0.0.1", port: 9999 }); - }), - ), - ); - - it.live("keeps unrelated ready traffic flowing during idle cleanup", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const stopStarted = yield* Deferred.make(); - const stopGate = yield* Deferred.make(); - const authCompleted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - workloadStopStarted: stopStarted, - workloadStopGate: stopGate, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - const body = Effect.gen(function* () { - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy", idleTimeoutSeconds: 1 }, - auth: { activation: "eager" }, - }, - }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("auth", fixture.supervisor.activate("auth")); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopStarted); - - const authRequest = yield* Effect.forkChild( - tracker.track( - "auth", - fixture.supervisor - .activate("auth") - .pipe(Effect.andThen(Deferred.succeed(authCompleted, undefined))), - ), - { startImmediately: true }, - ); - yield* Deferred.await(authCompleted); - yield* Deferred.succeed(stopGate, undefined); - yield* Fiber.join(authRequest); - }); - yield* body.pipe(Effect.ensuring(Deferred.succeed(stopGate, undefined))); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("rejects queued demand when explicit stop wins idle retirement", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const stopStarted = yield* Deferred.make(); - const stopGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - workloadStopStarted: stopStarted, - workloadStopGate: stopGate, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(stopStarted); - const request = yield* Effect.forkChild( - tracker.track("rest", fixture.supervisor.activate("rest")), - { - startImmediately: true, - }, - ); - const stopping = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop, { - startImmediately: true, - }); - yield* Deferred.succeed(stopGate, undefined); - expect((yield* Fiber.join(stopping)).ok).toBe(true); - const requestResult = yield* Fiber.join(request).pipe(Effect.exit); - expect(Exit.isFailure(requestResult)).toBe(true); - expect(yield* Ref.get(fixture.resources)).toEqual([]); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("records the original cause when idle cleanup fails", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const logRecords = yield* Ref.make>([]); - const logWritten = yield* Deferred.make(); - const workloadStopFailFirst = yield* Ref.make(true); - const workloadStopStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - logRecords, - logWritten, - workloadStopFailFirst, - workloadStopStarted, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(workloadStopStarted).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("workload stop did not start"), - }), - ); - yield* Deferred.await(logWritten); - const messages = yield* Ref.get(logRecords); - expect(messages).toEqual( - expect.arrayContaining([ - expect.stringContaining("Failed to stop rest after inactivity"), - expect.stringContaining("injected workload stop failure"), - ]), - ); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopping"); - expect(status.recovery).toEqual({ - operation: "stop", - message: expect.stringContaining("injected workload stop failure"), - }); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("fences the session after an idle cleanup defect", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const logRecords = yield* Ref.make>([]); - const logWritten = yield* Deferred.make(); - const workloadRemoveDieFirst = yield* Ref.make(true); - const fixture = yield* makeFixture({ - logRecords, - logWritten, - workloadRemoveDieFirst, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(logWritten); - - const messages = yield* Ref.get(logRecords); - expect(messages).toEqual( - expect.arrayContaining([ - expect.stringContaining("Failed to stop rest after inactivity"), - expect.stringContaining("injected workload remove defect"), - ]), - ); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - const activation = yield* fixture.supervisor.activate("rest").pipe(Effect.exit); - expect(errorOf(activation)).toBeInstanceOf(StackLifecycleConflictError); - - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const restartedTracker = yield* Ref.get(activity); - if (restartedTracker === undefined) - return yield* Effect.die("restarted gateway activity was not installed"); - yield* restartedTracker.track("rest", fixture.supervisor.activate("rest")); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("cancels idle timers when a stack session stops", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const firstTracker = yield* Ref.get(activity); - if (firstTracker === undefined) - return yield* Effect.die("first gateway activity was not installed"); - yield* firstTracker.track("rest", fixture.supervisor.activate("rest")); - yield* fixture.supervisor.maintenanceHandlers.stop; - - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 10 } } }, - }); - const secondTracker = yield* Ref.get(activity); - if (secondTracker === undefined) - return yield* Effect.die("second gateway activity was not installed"); - yield* secondTracker.track("rest", fixture.supervisor.activate("rest")); - yield* TestClock.adjust("1 second"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("ready"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("restores idle deadlines after a rejected running start", () => - run( - Effect.gen(function* () { - const activity = yield* Ref.make(undefined); - const logWritten = yield* Deferred.make(); - const fixture = yield* makeFixture({ - logWritten, - logWrittenFor: "Stopped rest after inactivity", - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: false, - ownershipToken: Symbol(), - }), - open: (_input, _reservation, _activate, tracker) => - tracker === undefined ? Effect.void : Ref.set(activity, tracker), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy", idleTimeoutSeconds: 1 } } }, - }); - const tracker = yield* Ref.get(activity); - if (tracker === undefined) return yield* Effect.die("gateway activity was not installed"); - yield* tracker.track("rest", fixture.supervisor.activate("rest")); - const rejected = yield* fixture.supervisor - .start({ config: { capabilities: { rest: { settings: { schemas: ["private"] } } } } }) - .pipe(Effect.exit); - expect(errorOf(rejected)).toBeInstanceOf(StackMustBeStoppedError); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(logWritten); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("dormant"); - }).pipe(Effect.provide(TestClock.layer())), - ), - ); - - it.live("persists unconfigured after a cold startup ingress failure", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Effect.fail( - new PortUnavailableError({ - field: "api", - port: 54_321, - message: "injected ingress failure", - }), - ), - open: () => Effect.void, - close: Effect.void, - }, - }); - expect( - Exit.isFailure(yield* fixture.supervisor.start({ config: {} }).pipe(Effect.exit)), - ).toBe(true); - const status = yield* fixture.supervisor.status; - expect(status.desiredLifecycle).toBe("unconfigured"); - expect(status.lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("persists stopped after a restart ingress failure", () => - run( - Effect.gen(function* () { - const acquireCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Ref.updateAndGet(acquireCalls, (count) => count + 1).pipe( - Effect.flatMap((count) => - count !== 2 - ? Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }) - : Effect.fail( - new PortUnavailableError({ - field: "api", - port: 54_321, - message: "injected start ingress failure", - }), - ), - ), - ), - open: () => Effect.void, - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ config: {} }); - yield* fixture.supervisor.maintenanceHandlers.stop; - - expect(Exit.isFailure(yield* fixture.supervisor.start().pipe(Effect.exit))).toBe(true); - const status = yield* fixture.supervisor.status; - expect(status.desiredLifecycle).toBe("stopped"); - expect(status.lifecycle).toBe("stopped"); - expect(errorOf(yield* invokeCredentials(fixture.supervisor).pipe(Effect.exit))?.tag).toBe( - "StackNotRunningError", - ); - yield* fixture.supervisor.shutdownIfIdle; - const retry = yield* fixture.supervisor.start().pipe(Effect.exit); - expect(errorOf(retry)).toBeInstanceOf(StackLifecycleConflictError); - expect(yield* Ref.get(fixture.calls)).toContain("start:database:database"); - }), - ), - ); - - it.live("exits after a stopped-owner restart ingress failure", () => - run( - Effect.gen(function* () { - const acquireCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Ref.updateAndGet(acquireCalls, (count) => count + 1).pipe( - Effect.flatMap((count) => - count !== 2 - ? Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }) - : Effect.fail( - new PortUnavailableError({ - field: "api", - port: 54_321, - message: "injected stopped-owner ingress failure", - }), - ), - ), - ), - open: () => Effect.void, - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ config: {} }); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - - expect(Exit.isFailure(yield* fixture.supervisor.start().pipe(Effect.exit))).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - yield* fixture.supervisor.shutdownIfIdle; - expect(errorOf(yield* fixture.supervisor.start().pipe(Effect.exit))).toBeInstanceOf( - StackLifecycleConflictError, - ); - }), - ), - ); - - it.live("starts only database by default and keeps other capabilities dormant", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const status = yield* fixture.supervisor.start({ config: {} }); - expect(status.lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.calls)).toEqual(["cleanup:stop", "start:database:database"]); - expect(status.capabilities.find(({ name }) => name === "database")?.state).toBe("ready"); - for (const name of [ - "rest", - "auth", - "realtime", - "storage", - "functions", - "studio", - "mail", - "analytics", - "pooler", - ] as const) - expect(status.capabilities.find((capability) => capability.name === name)?.state).toBe( - "dormant", - ); - }), - ), - ); - - it.live("returns database readiness while background preparation remains observable", () => - run( - Effect.gen(function* () { - const prefetchStarted = yield* Deferred.make(); - const prefetchGate = yield* Deferred.make(); - const prefetchInterrupted = yield* Deferred.make(); - const prefetchCalls = yield* Ref.make(0); - const artifactStatuses = yield* Ref.make>([]); - const fixture = yield* makeFixture({ - prefetchStarted, - prefetchGate, - prefetchInterrupted, - prefetchCalls, - artifactStatuses, - }); - const startReturned = yield* Deferred.make(); - yield* Effect.forkChild( - fixture.supervisor - .start({ config: {} }) - .pipe( - Effect.tap((status) => Deferred.succeed(startReturned, status).pipe(Effect.asVoid)), - ), - { startImmediately: true }, - ); - const started = yield* Deferred.await(startReturned).pipe( - Effect.timeout("5 seconds"), - Effect.orDie, - ); - expect(started.capabilities.find(({ name }) => name === "database")?.state).toBe("ready"); - expect(started.capabilities.find(({ name }) => name === "rest")?.state).toBe("dormant"); - yield* Deferred.await(prefetchStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - const during = yield* fixture.supervisor.status; - expect(during.capabilities.find(({ name }) => name === "rest")?.state).toBe("dormant"); - expect(during.artifacts).toEqual([ - { workloadId: "rest:rest", capability: "rest", state: "downloading" }, - ]); - - const repeated = yield* fixture.supervisor.start(); - expect(repeated.artifacts).toEqual(during.artifacts); - expect(yield* Ref.get(prefetchCalls)).toBe(1); - - const stopped = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(stopped.ok).toBe(true); - yield* Deferred.await(prefetchInterrupted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - }), - ), - ); - - it.live("launches workloads while selected preparation is still in flight", () => - run( - Effect.gen(function* () { - const prepareStarted = yield* Deferred.make(); - const prepareGate = yield* Deferred.make(); - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - prepareStarted, - prepareGate, - startStarted, - startGate, - }); - const starting = yield* Effect.forkChild(fixture.supervisor.start({ config: {} }), { - startImmediately: true, - }); - yield* Deferred.await(prepareStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.succeed(startGate, undefined); - yield* Deferred.succeed(prepareGate, undefined); - const status = yield* Fiber.join(starting); - expect(status.lifecycle).toBe("running"); - }), - ), - ); - - it.live("cleans launched workloads when preparation fails after launch completes", () => - run( - Effect.gen(function* () { - const prepareStarted = yield* Deferred.make(); - const prepareGate = yield* Deferred.make(); - const startFinished = yield* Deferred.make(); - const fixture = yield* makeFixture({ - prepareStarted, - prepareGate, - prepareFailure: true, - startFinished, - }); - const starting = yield* Effect.forkChild( - fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }), - { startImmediately: true }, - ); - yield* Deferred.await(prepareStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.await(startFinished).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.succeed(prepareGate, undefined); - const result = yield* Fiber.join(starting).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(yield* Ref.get(fixture.calls)).toContain("cleanup:stop"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("cancels a blocked launcher when preparation fails first", () => - run( - Effect.gen(function* () { - const prepareFailureRef = yield* Ref.make(false); - const prepareGateEnabledRef = yield* Ref.make(false); - const prepareStarted = yield* Deferred.make(); - const prepareActivationStarted = yield* Deferred.make(); - const prepareGate = yield* Deferred.make(); - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - prepareStarted, - prepareActivationStarted, - prepareGate, - prepareGateEnabledRef, - prepareFailureRef, - startStarted, - startGate, - startWorkload: "studio:studio", - }); - const starting = yield* Effect.forkChild( - fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "eager" }, - studio: { activation: "lazy" }, - }, - }, - }), - { startImmediately: true }, - ); - yield* Fiber.join(starting).pipe(Effect.timeout("5 seconds")); - yield* Ref.set(prepareFailureRef, true); - yield* Ref.set(prepareGateEnabledRef, true); - const activation = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(prepareActivationStarted).pipe( - Effect.timeout("5 seconds"), - Effect.orDie, - ); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.succeed(prepareGate, undefined); - const result = yield* Fiber.join(activation).pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - const statusAfterFailure = yield* fixture.supervisor.status; - expect(statusAfterFailure.lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.resources)).toEqual( - expect.arrayContaining([ - expect.objectContaining({ workloadId: "database:database", state: "ready" }), - expect.objectContaining({ workloadId: "rest:rest", state: "ready" }), - ]), - ); - yield* Deferred.succeed(startGate, undefined); - yield* Ref.set(prepareFailureRef, false); - expect((yield* fixture.supervisor.activate("studio")).endpoint).toEqual({ - host: "127.0.0.1", - port: 9999, - }); - }), - ), - ); - - it.live("retains preparation and cleanup causes after a launched workload", () => - run( - Effect.gen(function* () { - const prepareFailureRef = yield* Ref.make(false); - const prepareGateEnabledRef = yield* Ref.make(false); - const prepareActivationStarted = yield* Deferred.make(); - const prepareGate = yield* Deferred.make(); - const startStarted = yield* Deferred.make(); - const startFinished = yield* Deferred.make(); - const workloadRemoveFailFirst = yield* Ref.make(true); - const ingressCloseFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ - prepareActivationStarted, - prepareGate, - prepareGateEnabledRef, - prepareFailureRef, - startStarted, - startWorkload: "studio:studio", - startFinished, - workloadRemoveFailFirst, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }), - open: () => Effect.void, - close: Effect.gen(function* () { - if (yield* Ref.get(ingressCloseFailFirst)) { - yield* Ref.set(ingressCloseFailFirst, false); - return yield* new StackCleanupError({ message: "injected ingress close failure" }); - } - }), - }, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "eager" }, - studio: { activation: "lazy" }, - }, - }, - }); - yield* Ref.set(ingressCloseFailFirst, true); - yield* Ref.set(prepareFailureRef, true); - yield* Ref.set(prepareGateEnabledRef, true); - const activation = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(prepareActivationStarted).pipe( - Effect.timeout("5 seconds"), - Effect.orDie, - ); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.await(startFinished).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Deferred.succeed(prepareGate, undefined); - - const failed = yield* Fiber.join(activation).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - if (Exit.isFailure(failed)) { - const message = Cause.pretty(failed.cause); - expect(message).toContain("injected preparation failure"); - expect(message).toContain("injected workload remove failure"); - expect(message).toContain("injected ingress close failure"); - } - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - }), - ), - ); - - it.live("keeps accepted start work alive when its caller is interrupted", () => - run( - Effect.gen(function* () { - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const prefetchStarted = yield* Deferred.make(); - const prefetchGate = yield* Deferred.make(); - const prefetchFinished = yield* Deferred.make(); - const artifactStatuses = yield* Ref.make>([]); - const fixture = yield* makeFixture({ - startStarted, - startGate, - prefetchStarted, - prefetchGate, - prefetchFinished, - artifactStatuses, - }); - const waiter = yield* Effect.forkChild(fixture.supervisor.start({ config: {} }), { - startImmediately: true, - }); - yield* Deferred.await(startStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Fiber.interrupt(waiter); - const interrupted = yield* Fiber.join(waiter).pipe(Effect.exit); - expect(Exit.isFailure(interrupted)).toBe(true); - yield* Deferred.succeed(startGate, undefined); - yield* Deferred.await(prefetchStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - const during = yield* fixture.supervisor.status; - expect(during.lifecycle).toBe("running"); - expect(during.artifacts).toEqual([ - { workloadId: "rest:rest", capability: "rest", state: "downloading" }, - ]); - yield* Deferred.succeed(prefetchGate, undefined); - yield* Deferred.await(prefetchFinished).pipe(Effect.timeout("5 seconds"), Effect.orDie); - expect((yield* fixture.supervisor.status).lifecycle).toBe("running"); - yield* fixture.supervisor.maintenanceHandlers.stop; - }), - ), - ); - - it.live("interrupts background preparation before destroying the stack", () => - run( - Effect.gen(function* () { - const prefetchStarted = yield* Deferred.make(); - const prefetchGate = yield* Deferred.make(); - const prefetchInterrupted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - prefetchStarted, - prefetchGate, - prefetchInterrupted, - }); - yield* fixture.supervisor.start({ config: {} }); - yield* Deferred.await(prefetchStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - const destroying = yield* Effect.forkChild(fixture.supervisor.destroy, { - startImmediately: true, - }); - yield* Deferred.await(prefetchInterrupted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - yield* Fiber.join(destroying); - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("skips background preparation for on-demand and failed starts", () => - run( - Effect.gen(function* () { - const onDemandStarted = yield* Deferred.make(); - const onDemandCalls = yield* Ref.make(0); - const onDemand = yield* makeFixture({ - prefetchStarted: onDemandStarted, - prefetchCalls: onDemandCalls, - }); - yield* onDemand.supervisor.start({ config: { preparation: "on-demand" } }); - expect(Option.isNone(yield* Deferred.poll(onDemandStarted))).toBe(true); - expect(yield* Ref.get(onDemandCalls)).toBe(0); - - const failedStarted = yield* Deferred.make(); - const failedCalls = yield* Ref.make(0); - const startFailures = yield* Ref.make(1); - const failed = yield* makeFixture({ - prefetchStarted: failedStarted, - prefetchCalls: failedCalls, - startFailures, - }); - const result = yield* failed.supervisor - .start({ config: { capabilities: { functions: { activation: "eager" } } } }) - .pipe(Effect.exit); - expect(Exit.isFailure(result)).toBe(true); - expect(Option.isNone(yield* Deferred.poll(failedStarted))).toBe(true); - expect(yield* Ref.get(failedCalls)).toBe(0); - }), - ), - ); - - it.live("reports starting while relaunching a stopped owner", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - - const blocked = yield* Ref.make(false); - const launchStarted = yield* Deferred.make(); - const launchGate = yield* Deferred.make(); - const baseDriver = fixture.runtime.driver; - const driver: RuntimeDriver = { - ...baseDriver, - start: (key, workload) => - Effect.gen(function* () { - if (yield* Ref.get(blocked)) { - yield* Deferred.succeed(launchStarted, undefined); - yield* Deferred.await(launchGate); - } - return yield* baseDriver.start(key, workload); - }), - }; - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "stopped-relaunch-successor", - stateStore: fixture.store, - context: fixture.context, - runtime: { ...fixture.runtime, driver }, - }); - yield* successor.start({ config: {} }); - expect((yield* successor.maintenanceHandlers.stop).ok).toBe(true); - yield* Ref.set(blocked, true); - - const starting = yield* Effect.forkChild(successor.start(), { startImmediately: true }); - yield* Deferred.await(launchStarted); - expect((yield* successor.status).lifecycle).toBe("starting"); - expect((yield* successor.logs()).running).toBe(true); - yield* Deferred.succeed(launchGate, undefined); - expect((yield* Fiber.join(starting)).lifecycle).toBe("running"); - yield* successor.maintenanceHandlers.stop; - yield* successor.shutdownIfIdle; - yield* fixture.supervisor.shutdownIfIdle; - }), - ), - ); - - it.live("cleans stale runtime resources before the first start of a new supervisor", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(fixture.calls, []); - yield* Ref.set(fixture.resources, [ - { - stackId: fixture.id, - workloadId: "database:database", - state: "ready", - }, - ]); - - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - yield* successor.start(); - expect(yield* Ref.get(fixture.calls)).toEqual(["cleanup:stop", "start:database:database"]); - - yield* Ref.set(fixture.calls, []); - yield* successor.start(); - expect(yield* Ref.get(fixture.calls)).toEqual([]); - }), - ), - ); - - it.live("skips preflight for a live session and preflights a new owner", () => - run( - Effect.gen(function* () { - const preflightCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ preflightCalls }); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(preflightCalls, 0); - - yield* fixture.supervisor.start(); - expect(yield* Ref.get(preflightCalls)).toBe(0); - yield* fixture.supervisor.maintenanceHandlers.stop; - yield* fixture.supervisor.start(); - expect(yield* Ref.get(preflightCalls)).toBe(1); - - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - yield* Ref.set(preflightCalls, 0); - yield* successor.start(); - expect(yield* Ref.get(preflightCalls)).toBe(1); - }), - ), - ); - - it.live("keeps the owner for retryable first-start cleanup", () => - run( - Effect.gen(function* () { - const stopFailFirst = yield* Ref.make(true); - const fixture = yield* makeFixture({ stopFailFirst }); - const failed = yield* fixture.supervisor.start({ config: {} }).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("unconfigured"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect(yield* Ref.get(fixture.calls)).toEqual([]); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("keeps the owner when eager launch cleanup is unproven", () => - run( - Effect.gen(function* () { - const startFailures = yield* Ref.make(1); - const stopFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ startFailures, stopFailFirst }); - const failed = yield* fixture.supervisor - .start({ config: { capabilities: { functions: { activation: "eager" } } } }) - .pipe(Effect.exit); - - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("unconfigured"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect(yield* Ref.get(fixture.calls)).toContain("cleanup:stop"); - - const retry = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(retry.ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("shuts down after a proven eager launch failure", () => - run( - Effect.gen(function* () { - const startFailures = yield* Ref.make(1); - const fixture = yield* makeFixture({ startFailures }); - const failed = yield* fixture.supervisor - .start({ config: { capabilities: { functions: { activation: "eager" } } } }) - .pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("unconfigured"); - const stopped = yield* fixture.supervisor.status; - expect(stopped.lifecycle).toBe("unconfigured"); - expect(stopped.capabilities.find(({ name }) => name === "database")?.state).toBe("stopped"); - yield* fixture.supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdown; - }), - ), - ); - - it.live("persists unconfigured after a wipe-then-relaunch reset failure", () => - run( - Effect.gen(function* () { - const startFailures = yield* Ref.make(0); - const fixture = yield* makeFixture({ - startFailures, - startFailureWorkload: "database:database", - }); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(startFailures, 1); - const failed = yield* fixture.supervisor.resetDatabase.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("unconfigured"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("unconfigured"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("publishes stopped after launch cleanup retry succeeds", () => - run( - Effect.gen(function* () { - const startFailures = yield* Ref.make(1); - const stopFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ startFailures, stopFailFirst }); - const failed = yield* fixture.supervisor - .start({ config: { capabilities: { functions: { activation: "eager" } } } }) - .pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - const stopped = yield* fixture.supervisor.status; - expect(stopped.lifecycle).toBe("unconfigured"); - expect(stopped.capabilities.some(({ state }) => state === "failed")).toBe(false); - expect((yield* fixture.supervisor.start()).lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.resources)).not.toEqual([]); - }), - ), - ); - - it.live("persists stopped after a proven fresh-session preflight failure", () => - run( - Effect.gen(function* () { - const preflightFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ preflightFailFirst }); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(preflightFailFirst, true); - - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - const failed = yield* successor.start().pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("stopped"); - expect((yield* successor.status).lifecycle).toBe("stopped"); - yield* successor.shutdownIfIdle; - yield* successor.shutdown; - - const retryOwner = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "retry-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - expect((yield* retryOwner.start()).lifecycle).toBe("running"); - }), - ), - ); - - it.live("persists stopped after a proven fresh-session materialization failure", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(fixture.calls, []); - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "materialization-successor", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - const failed = yield* successor - .start({ config: { capabilities: { database: { version: "99" } } } }) - .pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(errorOf(failed)).toBeInstanceOf(StackVersionUnsupportedError); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("stopped"); - expect((yield* successor.status).lifecycle).toBe("stopped"); - expect(yield* Ref.get(fixture.calls)).toEqual(["cleanup:stop"]); - yield* successor.shutdownIfIdle; - yield* successor.shutdown; - const retryOwner = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "materialization-retry", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - expect((yield* retryOwner.start()).lifecycle).toBe("running"); - }), - ), - ); - - it.live("persists stopped after a proven fresh-session changed-input rejection", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "changed-input-successor", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - const failed = yield* successor - .start({ config: { capabilities: { rest: { settings: { schemas: ["private"] } } } } }) - .pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(errorOf(failed)).toBeInstanceOf(StackMustBeStoppedError); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("stopped"); - expect((yield* successor.status).lifecycle).toBe("stopped"); - yield* successor.shutdownIfIdle; - yield* successor.shutdown; - }), - ), - ); - - it.live("keeps stopping when fresh-ingress cleanup fails, then recovers on explicit stop", () => - run( - Effect.gen(function* () { - const closeFailures = yield* Ref.make(2); - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }), - open: () => Effect.void, - close: Effect.gen(function* () { - const remaining = yield* Ref.get(closeFailures); - if (remaining > 0) { - yield* Ref.set(closeFailures, remaining - 1); - return yield* new StackCleanupError({ message: "injected close failure" }); - } - }), - }, - }); - const failed = yield* fixture.supervisor.start({ config: {} }).pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(false); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - }), - ), - ); - - it.live("keeps stopping when persisting the stopped fence fails", () => - run( - Effect.gen(function* () { - const stoppedReplaceFail = yield* Ref.make(true); - const fixture = yield* makeFixture({ stoppedReplaceFail }); - yield* fixture.supervisor.start({ config: {} }); - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "stopped-fence-successor", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - const failed = yield* successor.maintenanceHandlers.stop; - expect(failed.ok).toBe(false); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("running"); - expect((yield* successor.status).lifecycle).toBe("stopping"); - expect(errorOf(yield* successor.activate("functions").pipe(Effect.exit))).toBeInstanceOf( - StackLifecycleConflictError, - ); - expect((yield* successor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* successor.status).lifecycle).toBe("stopped"); - yield* successor.shutdownIfIdle; - yield* fixture.supervisor.shutdownIfIdle; - }), - ), - ); - - it.live("retries first-create after a proven cold-start cleanup", () => - run( - Effect.gen(function* () { - const stoppedReplaceFail = yield* Ref.make(true); - const acquireCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - stoppedReplaceFail, - ingress: { - acquire: () => - Ref.updateAndGet(acquireCalls, (count) => count + 1).pipe( - Effect.flatMap((count) => - count === 1 - ? Effect.fail( - new PortUnavailableError({ - field: "api", - port: 54_321, - message: "injected cold start failure", - }), - ) - : Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }), - ), - ), - open: () => Effect.void, - close: Effect.void, - }, - }); - expect(Exit.isFailure(yield* fixture.supervisor.start().pipe(Effect.exit))).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("unconfigured"); - expect((yield* fixture.supervisor.start()).lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.resources)).not.toEqual([]); - }), - ), - ); - - it.live("clears failed capability status after successful cleanup retry", () => - run( - Effect.gen(function* () { - const removeFailure = yield* Ref.make(true); - const fixture = yield* makeFixture({ workloadRemoveFailFirst: removeFailure }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - const failed = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(failed.ok).toBe(false); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopped"); - expect(status.capabilities.some(({ state }) => state === "failed")).toBe(false); - expect((yield* fixture.supervisor.start()).lifecycle).toBe("running"); - }), - ), - ); - - it.live("returns persisted database, API, and storage credentials while running", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - const running = yield* fixture.store - .read(fixture.id) - .pipe(Effect.provideContext(fixture.context)); - if (running === undefined || running.definition === undefined) - return yield* new StackStateInvalidError({ message: "running fixture state is missing" }); - yield* fixture.store - .replace(fixture.id, { - ...running, - ports: [ - { field: "api", port: 55433, intent: "exact" }, - { field: "database", port: 55432, intent: "exact" }, - ] as const, - }) - .pipe(Effect.provideContext(fixture.context)); - const credentials = yield* invokeCredentials(fixture.supervisor); - expect(credentials.database.url).toEqual(expect.anything()); - expect(Redacted.value(credentials.database.url)).toMatch( - /^postgresql:\/\/postgres:.+@127\.0\.0\.1:\d+\/postgres$/, - ); - expect(Redacted.value(credentials.database.password)).toEqual(expect.any(String)); - if (credentials.api === undefined) - return yield* new StackStateInvalidError({ message: "API credentials are missing" }); - expect(credentials.api.publishableKey).toEqual(expect.any(String)); - expect(Redacted.value(credentials.api.secretKey)).toEqual(expect.any(String)); - expect(credentials.api.anonJwt).toEqual(expect.any(String)); - expect(Redacted.value(credentials.api.serviceRoleJwt)).toEqual(expect.any(String)); - expect(credentials.storage).toEqual( - expect.objectContaining({ - region: "local", - accessKeyId: "625729a08b95bf1b7ff351a663f3a23c", - }), - ); - if (credentials.storage === undefined) - return yield* new StackStateInvalidError({ message: "storage credentials are missing" }); - const persistedStorageSecret = - running.secrets["secret:storage.settings.s3_protocol.secret_access_key"]?.value; - expect(persistedStorageSecret).toEqual(expect.any(String)); - expect(Redacted.value(credentials.storage.secretAccessKey)).toBe(persistedStorageSecret); - }), - ), - ); - - it.live("URL-encodes persisted database credentials and brackets IPv6 listeners", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - const running = yield* fixture.store - .read(fixture.id) - .pipe(Effect.provideContext(fixture.context)); - if (running === undefined || running.definition === undefined) - return yield* new StackStateInvalidError({ message: "running fixture state is missing" }); - const definition = { - ...running.definition, - listeners: { - ...running.definition.listeners, - database: { ...running.definition.listeners.database, address: "2001:db8::1" }, - }, - }; - const state = { - ...running, - definition, - ports: [ - { field: "api", port: 55433, intent: "exact" as const }, - { field: "database", port: 55432, intent: "exact" as const }, - ] as const, - secrets: { - ...running.secrets, - "secret:database.internal.password": { - policy: "managed" as const, - value: "p@ss:word", - }, - }, - }; - yield* fixture.store - .replace(fixture.id, state) - .pipe(Effect.provideContext(fixture.context)); - const credentials = yield* invokeCredentials(fixture.supervisor); - expect(Redacted.value(credentials.database.url)).toBe( - "postgresql://postgres:p%40ss%3Aword@[2001:db8::1]:55432/postgres", - ); - }), - ), - ); - - it.live("omits storage credentials when Storage is disabled", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: {}, storage: { enabled: false } } }, - }); - const running = yield* fixture.store - .read(fixture.id) - .pipe(Effect.provideContext(fixture.context)); - if (running === undefined) - return yield* new StackStateInvalidError({ message: "running fixture state is missing" }); - const state = { - ...running, - ports: [ - { field: "api", port: 55433, intent: "exact" as const }, - { field: "database", port: 55432, intent: "exact" as const }, - ] as const, - }; - yield* fixture.store - .replace(fixture.id, state) - .pipe(Effect.provideContext(fixture.context)); - const credentials = yield* invokeCredentials(fixture.supervisor); - expect(credentials.storage).toBeUndefined(); - }), - ), - ); - - it.live( - "returns database credentials when Auth is disabled and fails closed for missing secrets", - () => - run( - Effect.gen(function* () { - const { fixture } = yield* makeCredentialsFixture({ authEnabled: false }); - const authDisabled = yield* invokeCredentials(fixture.supervisor); - expect(authDisabled.database.url).toEqual(expect.anything()); - expect(authDisabled.api).toBeUndefined(); - }), - ), - ); - - it.live("fails closed when an enabled Auth secret slot is absent", () => - run( - Effect.gen(function* () { - const { fixture, state, baseSecrets } = yield* makeCredentialsFixture(); - const missingSecret = { - ...state, - secrets: Object.fromEntries( - Object.entries(baseSecrets).filter( - ([slot]) => slot !== "secret:auth.settings.publishable_key", - ), - ), - }; - yield* fixture.store - .replace(fixture.id, missingSecret) - .pipe(Effect.provideContext(fixture.context)); - const failed = yield* invokeCredentials(fixture.supervisor).pipe(Effect.exit); - expect(errorOf(failed)).toMatchObject({ tag: "StackSecretMismatchError" }); - }), - ), - ); - - it.live("fails closed when a required Storage secret slot is absent", () => - run( - Effect.gen(function* () { - const { fixture, state, baseSecrets } = yield* makeCredentialsFixture(); - const missingSecret = { - ...state, - secrets: Object.fromEntries( - Object.entries(baseSecrets).filter( - ([slot]) => slot !== "secret:storage.settings.s3_protocol.secret_access_key", - ), - ), - }; - yield* fixture.store - .replace(fixture.id, missingSecret) - .pipe(Effect.provideContext(fixture.context)); - const failed = yield* invokeCredentials(fixture.supervisor).pipe(Effect.exit); - expect(errorOf(failed)).toMatchObject({ tag: "StackSecretMismatchError" }); - }), - ), - ); - - it.live("fails closed when the API listener is absent", () => - run( - Effect.gen(function* () { - const { fixture, state, baseSecrets } = yield* makeCredentialsFixture(); - const missingApiListener = { - ...state, - secrets: baseSecrets, - ports: [{ field: "database", port: 55432, intent: "exact" as const }] as const, - }; - yield* fixture.store - .replace(fixture.id, missingApiListener) - .pipe(Effect.provideContext(fixture.context)); - const failed = yield* invokeCredentials(fixture.supervisor).pipe(Effect.exit); - expect(errorOf(failed)).toMatchObject({ tag: "InvalidStackConfigError" }); - }), - ), - ); - - it.live("fails closed when the database listener is disabled", () => - run( - Effect.gen(function* () { - const { fixture, state, definition, baseSecrets } = yield* makeCredentialsFixture(); - const disabledDatabase = { - ...state, - secrets: baseSecrets, - definition: { - ...definition, - listeners: { - ...definition.listeners, - database: { ...definition.listeners.database, enabled: false }, - }, - }, - }; - yield* fixture.store - .replace(fixture.id, disabledDatabase) - .pipe(Effect.provideContext(fixture.context)); - const failed = yield* invokeCredentials(fixture.supervisor).pipe(Effect.exit); - expect(errorOf(failed)).toMatchObject({ tag: "InvalidStackConfigError" }); - }), - ), - ); - - it.live("acknowledges stop only after runtime cleanup", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - const response = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(response.ok).toBe(true); - expect(yield* Ref.get(fixture.calls)).toContain("cleanup:stop"); - }), - ), - ); - - it.live("reports stopping while stop cleanup is still in progress", () => - run( - Effect.gen(function* () { - const stopGate = yield* Deferred.make(); - const stopStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ stopGate, stopStarted }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: {}, auth: { activation: "lazy" } } }, - }); - - const stop = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop); - yield* Deferred.await(stopStarted); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "auth") - ?.state, - ).toBe("dormant"); - const duringStop = yield* fixture.supervisor.logs(); - expect(duringStop.running).toBe(true); - expect(duringStop.entries).toHaveLength(1); - - yield* Deferred.succeed(stopGate, undefined); - expect((yield* Fiber.join(stop)).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "auth") - ?.state, - ).toBe("stopped"); - const afterStop = yield* fixture.supervisor.logs(); - expect(afterStop.running).toBe(false); - expect(afterStop.entries).toHaveLength(2); - expect(afterStop.entries.filter(({ message }) => message === "stopped")).toHaveLength(1); - }), - ), - ); - - it.live("settles stop when the owner scope closes during workload cleanup", () => - run( - Effect.gen(function* () { - const ownerScope = yield* Scope.make(); - const workloadStopStarted = yield* Deferred.make(); - const workloadStopGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - supervisorScope: ownerScope, - workloadStopStarted, - workloadStopGate, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - const stopping = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop, { - startImmediately: true, - }); - yield* Deferred.await(workloadStopStarted); - yield* Scope.close(ownerScope, Exit.void).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("owner scope did not close"), - }), - ); - const result = yield* Fiber.await(stopping).pipe( - Effect.timeoutOrElse({ - duration: "5 seconds", - orElse: () => Effect.die("stop remained pending after owner scope close"), - }), - ); - expect(Exit.isFailure(result)).toBe(true); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopping"); - expect(status.recovery).toMatchObject({ - operation: "stop", - message: expect.any(String), - }); - expect(status.recovery?.message.length).toBeGreaterThan(0); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("failed"); - }), - ), - ); - - it.live("preserves stop cleanup diagnostics through maintenance responses", () => - run( - Effect.gen(function* () { - const stopFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ stopFailFirst }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - yield* Ref.set(stopFailFirst, true); - - const response = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(response).toEqual({ - ok: false, - error: { - tag: "operation-failed", - message: "injected stop cleanup failure", - stackErrorTag: "StackCleanupError", - }, - }); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - }), - ), - ); - - it.live("attempts runtime cleanup when session workload stop fails", () => - run( - Effect.gen(function* () { - const workloadStopFailFirst = yield* Ref.make(true); - const fixture = yield* makeFixture({ workloadStopFailFirst }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - yield* Ref.set(fixture.calls, []); - - const response = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(response).toEqual({ - ok: false, - error: { - tag: "operation-failed", - message: "Session cleanup is unresolved for rest:rest: injected workload stop failure", - stackErrorTag: "StackCleanupError", - }, - }); - expect(yield* Ref.get(fixture.calls)).toContain("cleanup:stop"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - }), - ), - ); - - it.live("stops the launched session in reverse dependency order", () => - run( - Effect.gen(function* () { - const timeline = yield* Ref.make>([]); - const fixture = yield* makeFixture({ timeline }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - yield* Ref.set(timeline, []); - - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect(yield* Ref.get(timeline)).toEqual([ - "stop:rest:rest", - "stop:database:database", - "cleanup:stop", - ]); - }), - ), - ); - - it.live("keeps stopping state while an explicit stop is active", () => - run( - Effect.gen(function* () { - const stopGate = yield* Deferred.make(); - const stopStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ stopGate, stopStarted }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - - const stopping = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop); - yield* Deferred.await(stopStarted); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - yield* Deferred.succeed(stopGate, undefined); - yield* Fiber.join(stopping); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - yield* fixture.supervisor.shutdownIfIdle; - const restart = yield* fixture.supervisor.start().pipe(Effect.exit); - expect(errorOf(restart)).toBeInstanceOf(StackLifecycleConflictError); - }), - ), - ); - - it.live("reports destroying while persistent data cleanup is in progress", () => - run( - Effect.gen(function* () { - const destroyGate = yield* Deferred.make(); - const destroyStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ destroyGate, destroyStarted }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - - const destroy = yield* Effect.forkChild(fixture.supervisor.destroy); - yield* Deferred.await(destroyStarted); - expect((yield* fixture.supervisor.status).lifecycle).toBe("destroying"); - - yield* Deferred.succeed(destroyGate, undefined); - yield* Fiber.join(destroy); - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("fences activation when the destroy durable pre-fence fails", () => - run( - Effect.gen(function* () { - const destroyPreFenceFail = yield* Ref.make(true); - const fixture = yield* makeFixture({ destroyPreFenceFail }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "lazy" } } }, - }); - expect((yield* fixture.supervisor.status).lifecycle).toBe("running"); - const destroyed = yield* fixture.supervisor.destroy.pipe(Effect.exit); - expect(Exit.isFailure(destroyed)).toBe(true); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("running"); - expect((yield* fixture.supervisor.status).lifecycle).toBe("destroying"); - const resourcesBefore = yield* Ref.get(fixture.resources); - const activation = yield* fixture.supervisor.activate("rest").pipe(Effect.exit); - expect(errorOf(activation)).toBeInstanceOf(StackLifecycleConflictError); - const readyActivation = yield* fixture.supervisor.activate("database").pipe(Effect.exit); - expect(errorOf(readyActivation)).toBeInstanceOf(StackLifecycleConflictError); - expect(yield* Ref.get(fixture.resources)).toEqual(resourcesBefore); - expect(yield* Ref.get(fixture.calls)).not.toContain("start:rest:rest"); - yield* fixture.supervisor.destroy; - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("requires destroy retry after persistent cleanup fails", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - yield* Ref.set(fixture.failDestroy, true); - - const failed = yield* fixture.supervisor.destroy.pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("destroying"); - expect(errorOf(yield* fixture.supervisor.start().pipe(Effect.exit))).toBeInstanceOf( - StackLifecycleConflictError, - ); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(false); - - yield* Ref.set(fixture.failDestroy, false); - yield* fixture.supervisor.destroy; - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("propagates typed workload observation failures while running", () => - run( - Effect.gen(function* () { - const observeFailure = yield* Ref.make(false); - const fixture = yield* makeFixture({ observeFailure }); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(observeFailure, true); - const status = yield* fixture.supervisor.status.pipe(Effect.exit); - expect(Exit.isFailure(status)).toBe(true); - expect(errorOf(status)).toBeInstanceOf(ContainerEngineError); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("running"); - }), - ), - ); - - it.live("preserves stopping recovery status when observation also fails", () => - run( - Effect.gen(function* () { - const observeFailure = yield* Ref.make(false); - const stopFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ observeFailure, stopFailFirst }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - yield* Ref.set(stopFailFirst, true); - const failedStop = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(failedStop.ok).toBe(false); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - - yield* Ref.set(observeFailure, true); - const status = yield* fixture.supervisor.status.pipe(Effect.exit); - expect(Exit.isSuccess(status)).toBe(true); - if (Exit.isSuccess(status)) { - expect(status.value.lifecycle).toBe("stopping"); - const rest = status.value.capabilities.find(({ name }) => name === "rest"); - expect(rest?.state).toBe("failed"); - expect(rest?.error).toContain("injected stop cleanup failure"); - } - - yield* Ref.set(observeFailure, false); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - }), - ), - ); - - it.live("projects an observed empty workload as stopped", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - yield* Ref.set(fixture.resources, []); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("running"); - expect(status.capabilities.find(({ name }) => name === "database")?.state).toBe("stopped"); - }), - ), - ); - - it.live("destroys observed remnants from an unconfigured supervisor", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* Ref.set(fixture.resources, [ - { stackId: fixture.id, workloadId: "database:database", state: "ready" }, - ]); - yield* fixture.supervisor.destroy; - expect(yield* Ref.get(fixture.resources)).toEqual([]); - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); - - it.live("shuts down after a start rejects missing state following destroy", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: {} }); - yield* fixture.supervisor.destroy; - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - - const failed = yield* fixture.supervisor.start().pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackStateInvalidError); - yield* fixture.supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdown.pipe( - Effect.timeoutOrElse({ - duration: "1 second", - orElse: () => Effect.die("destroyed Supervisor did not shut down"), - }), - ); - }), - ), - ); - - it.live("removes a runtime remnant when state disappears before supervisor initialization", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const fs = yield* FileSystem.FileSystem; - const paths = yield* resolveStackPaths({ stateRoot: fixture.root, stackId: fixture.id }); - yield* fixture.store.cleanup(fixture.id); - yield* fs.makeDirectory(paths.runtime, { recursive: true }); - - const failed = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "missing-state-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }).pipe(Effect.exit); - - expect(errorOf(failed)).toBeInstanceOf(StackStateInvalidError); - expect(yield* fs.exists(paths.runtime)).toBe(false); - }), - ), - ); - - it.live("passes log query through the Supervisor log batch", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const options: LogQuery = { capabilities: ["auth"] }; - expect((yield* fixture.supervisor.logs(options)).entries).toHaveLength(1); - expect(yield* Ref.get(fixture.logOptions)).toEqual([undefined]); - }), - ), - ); - - it.live("returns filtered log batches with a running marker", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const batch = yield* fixture.supervisor.logs({ capabilities: ["auth"], tail: 20 }); - expect(batch.entries.every((entry) => entry.source === "auth")).toBe(true); - expect(batch.running).toBe(false); - }), - ), - ); - - it.live("keeps running state and explicit stop guidance for changed start input", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - const failed = yield* fixture.supervisor - .start({ config: { capabilities: { rest: { settings: { schemas: ["private"] } } } } }) - .pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(StackMustBeStoppedError); - expect((yield* fixture.supervisor.status).lifecycle).toBe("running"); - }), - ), - ); - - it.live("keeps the owner when stop cannot prove its cleanup", () => - run( - Effect.gen(function* () { - const stopFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ stopFailFirst }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - yield* Ref.set(stopFailFirst, true); - - const stopped = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(stopped.ok).toBe(false); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("stopped"); - - const shutdown = yield* Effect.forkChild(fixture.supervisor.shutdown, { - startImmediately: true, - }); - yield* fixture.supervisor.shutdownIfIdle; - expect(shutdown.pollUnsafe()).toBeUndefined(); - - const stoppedAgain = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(stoppedAgain.ok).toBe(true); - yield* fixture.supervisor.shutdownIfIdle; - yield* Fiber.join(shutdown); - expect(yield* Ref.get(fixture.resources)).toEqual([]); - }), - ), - ); - - it.live("requires explicit stop cleanup before starting again", () => - run( - Effect.gen(function* () { - const stopFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ stopFailFirst }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - yield* Ref.set(stopFailFirst, true); - const failedStop = yield* fixture.supervisor.maintenanceHandlers.stop; - expect(failedStop.ok).toBe(false); - - expect(errorOf(yield* fixture.supervisor.start().pipe(Effect.exit))).toBeInstanceOf( - StackLifecycleConflictError, - ); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - yield* Ref.set(fixture.calls, []); - const started = yield* fixture.supervisor.start(); - - expect(started.lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.calls)).toContain("start:database:database"); - }), - ), - ); - - it.live("keeps lazy capabilities dormant until explicit activation", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const status = yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { activation: "lazy" }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - expect(status.capabilities.find(({ name }) => name === "functions")?.state).toBe("dormant"); - expect(yield* Ref.get(fixture.calls)).toEqual(["cleanup:stop", "start:database:database"]); - const activation = yield* fixture.supervisor.activate("functions"); - expect(activation.endpoint).toEqual({ host: "127.0.0.1", port: 9999 }); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "functions") - ?.state, - ).toBe("ready"); - }), - ), - ); - - it.live("preserves lazy artifact preparation failures during activation", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const artifactFailure = new ArtifactIntegrityError({ - message: "functions artifact checksum mismatch", - }); - const baseDriver = fixture.runtime.driver; - const driver: RuntimeDriver = { - ...baseDriver, - start: (key, workload) => - key.workloadId === "functions:edge-runtime" - ? Effect.fail( - new RuntimeDriverError({ - message: artifactFailure.message, - stackId: key.stackId, - workloadId: key.workloadId, - cause: artifactFailure, - }), - ) - : baseDriver.start(key, workload), - }; - const supervisor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "artifact-failure-supervisor", - stateStore: fixture.store, - context: fixture.context, - runtime: { ...fixture.runtime, driver }, - }); - yield* supervisor.start({ - config: { - capabilities: { - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { activation: "lazy" }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const result = yield* supervisor.activate("functions").pipe(Effect.exit); - expect(errorOf(result)).toBeInstanceOf(ArtifactIntegrityError); - yield* supervisor.maintenanceHandlers.stop; - yield* supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdownIfIdle; - }), - ), - ); - - it.live("keeps accepted start work alive when its waiter is interrupted", () => - run( - Effect.gen(function* () { - const startGate = yield* Deferred.make(); - const startStarted = yield* Deferred.make(); - const startFinished = yield* Deferred.make(); - const fixture = yield* makeFixture({ startGate, startStarted, startFinished }); - const config = { - capabilities: { rest: { activation: "eager" } }, - } satisfies import("../public/Config.ts").StackConfig; - const waiter = yield* Effect.forkChild(fixture.supervisor.start({ config })); - yield* Deferred.await(startStarted); - yield* Fiber.interrupt(waiter); - yield* Deferred.succeed(startGate, undefined); - yield* Deferred.await(startFinished); - // shutdownIfIdle acquires the lifecycle admission permit, so it waits until the accepted - // owner operation has released admission after publishing its running phase. - yield* fixture.supervisor.shutdownIfIdle; - - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("running"); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - const starts = (yield* Ref.get(fixture.calls)).filter((call) => call.startsWith("start:")); - expect(starts).toContain("start:database:database"); - expect(starts).toContain("start:rest:rest"); - }), - ), - ); - - it.live("shuts down after an interrupted pre-commit start later fails", () => - run( - Effect.gen(function* () { - const preflightFailFirst = yield* Ref.make(true); - const preflightGate = yield* Deferred.make(); - const preflightStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - preflightFailFirst, - preflightGate, - preflightStarted, - }); - const waiter = yield* Effect.forkChild(fixture.supervisor.start({ config: {} })); - yield* Deferred.await(preflightStarted); - yield* Fiber.interrupt(waiter); - yield* Deferred.succeed(preflightGate, undefined); - yield* Effect.yieldNow; - yield* fixture.supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdown.pipe( - Effect.timeoutOrElse({ - duration: "1 second", - orElse: () => Effect.die("idle Supervisor did not shut down"), - }), - ); - }), - ), - ); - - it.live("rejects concurrent starts even with identical input", () => - run( - Effect.gen(function* () { - const preflightFailFirst = yield* Ref.make(true); - const preflightGate = yield* Deferred.make(); - const preflightStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - preflightFailFirst, - preflightGate, - preflightStarted, - }); - const firstConfig = { capabilities: {} }; - const first = yield* Effect.forkChild(fixture.supervisor.start({ config: firstConfig })); - yield* Deferred.await(preflightStarted); - const second = yield* fixture.supervisor.start({ config: firstConfig }).pipe(Effect.exit); - expect(Exit.isFailure(second)).toBe(true); - expect(errorOf(second)).toBeInstanceOf(StackLifecycleConflictError); - yield* Deferred.succeed(preflightGate, undefined); - const firstExit = yield* Fiber.join(first).pipe(Effect.exit); - expect(Exit.isFailure(firstExit)).toBe(true); - expect(yield* Ref.get(preflightFailFirst)).toBe(false); - }), - ), - ); - - it.live("does not join concurrent starts with distinct secret values", () => - run( - Effect.gen(function* () { - const preflightFailFirst = yield* Ref.make(true); - const preflightGate = yield* Deferred.make(); - const preflightStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ - preflightFailFirst, - preflightGate, - preflightStarted, - }); - const firstConfig = { - security: { - jwt: { signing: { kind: "symmetric" as const, secret: Redacted.make("a") } }, - }, - }; - const distinctConfig = { - security: { - jwt: { signing: { kind: "symmetric" as const, secret: Redacted.make("b") } }, - }, - }; - const first = yield* Effect.forkChild(fixture.supervisor.start({ config: firstConfig })); - yield* Deferred.await(preflightStarted); - const secondExit = yield* fixture.supervisor - .start({ config: distinctConfig }) - .pipe(Effect.exit); - expect(Exit.isFailure(secondExit)).toBe(true); - expect(errorOf(secondExit)).toBeInstanceOf(StackLifecycleConflictError); - yield* Deferred.succeed(preflightGate, undefined); - const firstExit = yield* Fiber.join(first).pipe(Effect.exit); - expect(Exit.isFailure(firstExit)).toBe(true); - }), - ), - ); - - it.live("single-flights lazy activation and retains its endpoint", () => - run( - Effect.gen(function* () { - const gate = yield* Deferred.make(); - const started = yield* Deferred.make(); - const activationCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - activationGate: gate, - activationStarted: started, - activationCalls, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { activation: "lazy" }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - const first = yield* Effect.forkChild(fixture.supervisor.activate("functions")); - yield* Deferred.await(started); - const second = yield* Effect.forkChild(fixture.supervisor.activate("functions")); - yield* Deferred.succeed(gate, undefined); - const [left, right] = yield* Effect.all([Fiber.join(first), Fiber.join(second)]); - expect(left).toEqual(right); - expect(yield* Ref.get(activationCalls)).toBe(1); - expect(yield* fixture.supervisor.activate("functions")).toEqual(left); - expect(yield* Ref.get(activationCalls)).toBe(1); - }), - ), - ); - - it.live("keeps a ready capability while concurrent endpoint lookup is shared", () => - run( - Effect.gen(function* () { - const gate = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const activationCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - activationGate: gate, - activationStarted, - activationCalls, - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { activation: "eager" }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "functions") - ?.state, - ).toBe("ready"); - const first = yield* Effect.forkChild(fixture.supervisor.activate("functions")); - yield* Deferred.await(activationStarted); - const second = yield* Effect.forkChild(fixture.supervisor.activate("functions")); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "functions") - ?.state, - ).toBe("ready"); - yield* Deferred.succeed(gate, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); - expect(yield* Ref.get(activationCalls)).toBe(1); - }), - ), - ); - - it.live("reuses a ready lazy activation without rereading durable state", () => - run( - Effect.gen(function* () { - const readCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ readCalls }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - functions: { activation: "lazy" }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }, - }); - yield* Ref.set(readCalls, 0); - - yield* fixture.supervisor.activate("functions"); - const afterFirst = yield* Ref.get(readCalls); - yield* fixture.supervisor.activate("functions"); - - expect(yield* Ref.get(readCalls)).toBe(afterFirst); - }), - ), - ); - - it.live("keeps activated lazy workloads ready across an idempotent start", () => - run( - Effect.gen(function* () { - const fixture = yield* makeFixture(); - const config = { - capabilities: { - rest: { enabled: false }, - auth: { enabled: false }, - realtime: { enabled: false }, - storage: { enabled: false }, - functions: { activation: "lazy" as const }, - studio: { enabled: false }, - mail: { enabled: false }, - analytics: { enabled: false }, - pooler: { enabled: false }, - }, - }; - yield* fixture.supervisor.start({ config }); - yield* fixture.supervisor.activate("functions"); - yield* Ref.set(fixture.calls, []); - - const status = yield* fixture.supervisor.start({ config }); - - expect(yield* Ref.get(fixture.calls)).toEqual([]); - expect(status.lifecycle).toBe("running"); - expect(status.capabilities.find(({ name }) => name === "functions")?.state).toBe("ready"); - }), - ), - ); - - it.live("keeps a dependency ready when its overlapping endpoint activation fails", () => - run( - Effect.gen(function* () { - const startStarted = yield* Deferred.make(); - const startGate = yield* Deferred.make(); - const activationCalls = yield* Ref.make(0); - const activationStartedAfterFirst = yield* Deferred.make(); - const activationGateAfterFirst = yield* Deferred.make(); - const activationFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ - startStarted, - startGate, - startWorkload: "rest:rest", - activationCalls, - activationStartedAfterFirst, - activationGateAfterFirst, - activationFailFirst, - }); - const config = { - capabilities: { - rest: { activation: "lazy" as const }, - studio: { activation: "lazy" as const }, - }, - }; - yield* fixture.supervisor.start({ config }); - const studio = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(startStarted); - const rest = yield* Effect.forkChild(fixture.supervisor.activate("rest"), { - startImmediately: true, - }); - yield* Deferred.succeed(startGate, undefined); - yield* Deferred.await(activationStartedAfterFirst); - expect(yield* Ref.get(activationCalls)).toBe(2); - yield* Ref.set(activationFailFirst, true); - yield* Deferred.succeed(activationGateAfterFirst, undefined); - - const studioResult = yield* Fiber.join(studio).pipe(Effect.exit); - const restResult = yield* Fiber.join(rest).pipe(Effect.exit); - expect(Exit.isSuccess(studioResult)).toBe(true); - expect(errorOf(restResult)).toBeInstanceOf(GatewayActivationError); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "rest:rest", state: "ready" }), - ); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "studio:pgmeta", state: "ready" }), - ); - const status = yield* fixture.supervisor.status; - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - expect(status.capabilities.find(({ name }) => name === "studio")?.state).toBe("ready"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - }), - ), - ); - - it.live( - "reports a post-ready workload failure without restarting it or stopping unrelated services", - () => - run( - Effect.gen(function* () { - const starts = yield* Queue.unbounded(); - const fixture = yield* makeFixture({ startQueue: starts }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: { activation: "eager" } } }, - }); - yield* Queue.take(starts); - yield* Queue.take(starts); - yield* Ref.set(fixture.calls, []); - yield* Ref.update(fixture.resources, (current) => - current.map((entry) => - entry.workloadId === "database:database" - ? { ...entry, state: "failed" as const, error: "crashed" } - : entry, - ), - ); - const status = yield* fixture.supervisor.status; - expect(status.capabilities.find(({ name }) => name === "database")?.state).toBe("failed"); - expect(status.capabilities.find(({ name }) => name === "rest")?.state).toBe("ready"); - expect(yield* Ref.get(fixture.calls)).toEqual([]); - }), - ), - ); - - it.live("allows a failed activation to retry", () => - run( - Effect.gen(function* () { - const activationFailFirst = yield* Ref.make(true); - const activationCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ activationFailFirst, activationCalls }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const failed = yield* fixture.supervisor.activate("functions").pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - const retry = yield* fixture.supervisor.activate("functions"); - expect(retry.endpoint).toEqual({ host: "127.0.0.1", port: 9999 }); - expect(yield* Ref.get(activationCalls)).toBe(2); - expect( - (yield* Ref.get(fixture.calls)).filter((call) => call === "start:functions:edge-runtime"), - ).toHaveLength(2); - }), - ), - ); - - it.live("reports a failed lazy activation as dormant until it is retried", () => - run( - Effect.gen(function* () { - const startFailures = yield* Ref.make(1); - const fixture = yield* makeFixture({ startFailures }); - yield* fixture.supervisor.start({ config: { capabilities: { functions: {} } } }); - - const failed = yield* fixture.supervisor.activate("functions").pipe(Effect.exit); - - expect(Exit.isFailure(failed)).toBe(true); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "functions") - ?.state, - ).toBe("dormant"); - const retry = yield* fixture.supervisor.activate("functions"); - expect(retry.endpoint).toEqual({ host: "127.0.0.1", port: 9999 }); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "functions") - ?.state, - ).toBe("ready"); - expect( - (yield* Ref.get(fixture.calls)).filter((call) => call === "start:functions:edge-runtime"), - ).toHaveLength(2); - }), - ), - ); - - it.live("rolls back a lazy launch when ingress opening fails", () => - run( - Effect.gen(function* () { - const openCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }), - open: () => - Ref.updateAndGet(openCalls, (count) => count + 1).pipe( - Effect.flatMap((count) => - count === 2 - ? Effect.fail(new GatewayActivationError({ message: "injected open failure" })) - : Effect.void, - ), - ), - close: Effect.void, - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const failed = yield* fixture.supervisor.activate("functions").pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect(yield* Ref.get(fixture.resources)).toEqual([ - expect.objectContaining({ workloadId: "database:database", state: "ready" }), - ]); - yield* fixture.supervisor.activate("functions"); - expect( - (yield* Ref.get(fixture.calls)).filter((call) => call === "start:functions:edge-runtime"), - ).toHaveLength(2); - }), - ), - ); - - it.live("fences fresh ingress when open failure cleanup is not proven", () => - run( - Effect.gen(function* () { - const openCalls = yield* Ref.make(0); - const closeFailures = yield* Ref.make(0); - const fixture = yield* makeFixture({ - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }), - open: () => - Ref.updateAndGet(openCalls, (count) => count + 1).pipe( - Effect.flatMap((count) => - count === 2 - ? Effect.fail(new GatewayActivationError({ message: "injected open failure" })) - : Effect.void, - ), - ), - close: Effect.gen(function* () { - const remaining = yield* Ref.get(closeFailures); - if (remaining > 0) { - yield* Ref.set(closeFailures, remaining - 1); - return yield* new StackCleanupError({ message: "injected close failure" }); - } - }), - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - yield* Ref.set(closeFailures, 1); - expect( - Exit.isFailure(yield* fixture.supervisor.activate("functions").pipe(Effect.exit)), - ).toBe(true); - expect(yield* Ref.get(fixture.resources)).toEqual([ - expect.objectContaining({ workloadId: "database:database", state: "ready" }), - ]); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - }), - ), - ); - - it.live("rolls back fresh ingress after activation failure and retries", () => - run( - Effect.gen(function* () { - const activationFailFirst = yield* Ref.make(true); - const openCalls = yield* Ref.make(0); - const closeCalls = yield* Ref.make(0); - const fixture = yield* makeFixture({ - activationFailFirst, - ingress: { - acquire: () => - Effect.succeed({ - assignments: {}, - privateAssignments: [], - hostListeners: [], - fresh: true, - ownershipToken: Symbol(), - }), - open: () => Ref.update(openCalls, (count) => count + 1), - close: Ref.update(closeCalls, (count) => count + 1), - }, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - expect( - Exit.isFailure(yield* fixture.supervisor.activate("functions").pipe(Effect.exit)), - ).toBe(true); - expect(yield* Ref.get(closeCalls)).toBe(2); - expect(yield* Ref.get(fixture.resources)).toEqual([ - expect.objectContaining({ workloadId: "database:database", state: "ready" }), - ]); - const retry = yield* fixture.supervisor.activate("functions"); - expect(retry.endpoint).toEqual({ host: "127.0.0.1", port: 9999 }); - expect(yield* Ref.get(openCalls)).toBe(3); - }), - ), - ); - - it.live("fences the owner when lazy rollback cannot be proven", () => - run( - Effect.gen(function* () { - const activationFailFirst = yield* Ref.make(true); - const workloadStopFailFirst = yield* Ref.make(true); - const fixture = yield* makeFixture({ activationFailFirst, workloadStopFailFirst }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const failed = yield* fixture.supervisor.activate("functions").pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect( - errorOf(yield* fixture.supervisor.activate("functions").pipe(Effect.exit)), - ).toBeInstanceOf(StackLifecycleConflictError); - expect(errorOf(yield* fixture.supervisor.start().pipe(Effect.exit))).toBeInstanceOf( - StackLifecycleConflictError, - ); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - yield* fixture.supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdown; - const successor = yield* makeSupervisor({ - stackId: fixture.id, - ownerSessionId: "successor-session", - stateStore: fixture.store, - context: fixture.context, - runtime: fixture.runtime, - }); - yield* Ref.set(fixture.calls, []); - const restarted = yield* successor.start().pipe(Effect.exit); - expect(Exit.isSuccess(restarted)).toBe(true); - if (Exit.isSuccess(restarted)) expect(restarted.value.lifecycle).toBe("running"); - expect(yield* Ref.get(fixture.calls)).toContain("start:database:database"); - expect((yield* successor.maintenanceHandlers.stop).ok).toBe(true); - }), - ), - ); - - it.live("fences a lazy activation when endpoint rollback removal fails", () => - run( - Effect.gen(function* () { - const activationFailFirst = yield* Ref.make(true); - const workloadRemoveFailFirst = yield* Ref.make(true); - const activationStarted = yield* Deferred.make(); - const activationGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - activationFailFirst, - activationStarted, - activationGate, - workloadRemoveFailFirst, - workloadRemoveFailWorkload: "rest:rest", - }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - rest: { activation: "lazy" }, - studio: { activation: "lazy" }, - }, - }, - }); - const activation = yield* Effect.forkChild(fixture.supervisor.activate("studio"), { - startImmediately: true, - }); - yield* Deferred.await(activationStarted); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "rest:rest", state: "ready" }), - ); - yield* Deferred.succeed(activationGate, undefined); - const failed = yield* Fiber.join(activation).pipe(Effect.exit); - expect(errorOf(failed)).toBeInstanceOf(GatewayActivationError); - expect(Exit.isFailure(failed)).toBe(true); - expect(yield* Ref.get(fixture.resources)).toContainEqual( - expect.objectContaining({ workloadId: "rest:rest", state: "stopped" }), - ); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect( - (yield* fixture.supervisor.status).capabilities.find(({ name }) => name === "rest") - ?.state, - ).toBe("failed"); - expect( - errorOf(yield* fixture.supervisor.activate("rest").pipe(Effect.exit)), - ).toBeInstanceOf(StackLifecycleConflictError); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - }), - ), - ); - - it.live("fences the owner when lazy rollback remove fails", () => - run( - Effect.gen(function* () { - const activationFailFirst = yield* Ref.make(true); - const workloadRemoveFailFirst = yield* Ref.make(true); - const fixture = yield* makeFixture({ activationFailFirst, workloadRemoveFailFirst }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const failed = yield* fixture.supervisor.activate("functions").pipe(Effect.exit); - expect(Exit.isFailure(failed)).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopped"); - }), - ), - ); - - it.live("validates lifecycle before cached lazy activation results", () => - run( - Effect.gen(function* () { - const activationFailFirst = yield* Ref.make(false); - const workloadRemoveFailFirst = yield* Ref.make(false); - const fixture = yield* makeFixture({ activationFailFirst, workloadRemoveFailFirst }); - yield* fixture.supervisor.start({ - config: { - capabilities: { - functions: { activation: "lazy" }, - auth: { activation: "lazy" }, - }, - }, - }); - yield* fixture.supervisor.activate("functions"); - yield* Ref.set(activationFailFirst, true); - yield* Ref.set(workloadRemoveFailFirst, true); - expect(Exit.isFailure(yield* fixture.supervisor.activate("auth").pipe(Effect.exit))).toBe( - true, - ); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - expect( - errorOf(yield* fixture.supervisor.activate("functions").pipe(Effect.exit)), - ).toBeInstanceOf(StackLifecycleConflictError); - expect((yield* fixture.supervisor.maintenanceHandlers.stop).ok).toBe(true); - expect( - errorOf(yield* fixture.supervisor.activate("functions").pipe(Effect.exit)), - ).toBeInstanceOf(StackNotRunningError); - }), - ), - ); - - it.live("rejects activation while an explicit lifecycle transition is active", () => - run( - Effect.gen(function* () { - const stopGate = yield* Deferred.make(); - const stopStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ stopGate, stopStarted }); - yield* fixture.supervisor.start({ - config: { capabilities: { rest: {}, functions: { activation: "lazy" } } }, - }); - - const stopping = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop); - yield* Deferred.await(stopStarted); - const activation = yield* fixture.supervisor.activate("functions").pipe(Effect.exit); - expect(errorOf(activation)).toBeInstanceOf(StackLifecycleConflictError); - - yield* Deferred.succeed(stopGate, undefined); - yield* Fiber.join(stopping); - }), - ), - ); - - it.live("fences activation against a concurrent stop", () => - run( - Effect.gen(function* () { - const activationGate = yield* Deferred.make(); - const activationStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ activationGate, activationStarted }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const activation = yield* Effect.forkChild(fixture.supervisor.activate("functions")); - yield* Deferred.await(activationStarted); - const stop = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop); - yield* Deferred.succeed(activationGate, undefined); - yield* Fiber.join(activation); - yield* Fiber.join(stop); - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopped"); - expect(status.capabilities.find(({ name }) => name === "functions")?.state).toBe("stopped"); - }), - ), - ); - - it.live("stops cleanly when stop is queued during activation", () => - run( - Effect.gen(function* () { - const activationStarted = yield* Deferred.make(); - const activationGate = yield* Deferred.make(); - const fixture = yield* makeFixture({ - activationStarted, - activationGate, - }); - yield* fixture.supervisor.start({ - config: { capabilities: { functions: { activation: "lazy" } } }, - }); - const activation = yield* Effect.forkChild(fixture.supervisor.activate("functions"), { - startImmediately: true, - }); - yield* Deferred.await(activationStarted).pipe(Effect.timeout("5 seconds"), Effect.orDie); - const stopping = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop, { - startImmediately: true, - }); - expect((yield* fixture.supervisor.status).lifecycle).toBe("stopping"); - - yield* Deferred.succeed(activationGate, undefined); - expect(Exit.isSuccess(yield* Fiber.join(activation).pipe(Effect.exit))).toBe(true); - expect((yield* Fiber.join(stopping)).ok).toBe(true); - - const status = yield* fixture.supervisor.status; - expect(status.lifecycle).toBe("stopped"); - expect(status.capabilities.find(({ name }) => name === "functions")?.state).toBe("stopped"); - }), - ), - ); - - it.live("completes accepted stop after its waiter is interrupted", () => - run( - Effect.gen(function* () { - const stopGate = yield* Deferred.make(); - const stopStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ stopGate, stopStarted }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - const waiter = yield* Effect.forkChild(fixture.supervisor.maintenanceHandlers.stop); - yield* Deferred.await(stopStarted); - yield* Fiber.interrupt(waiter); - yield* Deferred.succeed(stopGate, undefined); - yield* Effect.yieldNow; - yield* fixture.supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdown; - expect((yield* fixture.store.read(fixture.id))?.desiredLifecycle).toBe("stopped"); - }), - ), - ); - - it.live("completes accepted destroy after its waiter is interrupted", () => - run( - Effect.gen(function* () { - const destroyGate = yield* Deferred.make(); - const destroyStarted = yield* Deferred.make(); - const fixture = yield* makeFixture({ destroyGate, destroyStarted }); - yield* fixture.supervisor.start({ config: { capabilities: { rest: {} } } }); - const waiter = yield* Effect.forkChild(fixture.supervisor.destroy); - yield* Deferred.await(destroyStarted); - yield* Fiber.interrupt(waiter); - yield* Deferred.succeed(destroyGate, undefined); - yield* Effect.yieldNow; - yield* fixture.supervisor.shutdownIfIdle; - yield* fixture.supervisor.shutdown; - expect(yield* fixture.store.read(fixture.id)).toBeUndefined(); - }), - ), - ); -}); diff --git a/packages/stack/src/testing.ts b/packages/stack/src/testing.ts deleted file mode 100644 index 8bcd5c7b95..0000000000 --- a/packages/stack/src/testing.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { createTestStack } from "./public/Testing.ts"; -export type { CreateTestStackOptions, TestStack } from "./public/Testing.ts"; diff --git a/packages/stack/src/whole-stack.docker.e2e.test.ts b/packages/stack/src/whole-stack.docker.e2e.test.ts new file mode 100644 index 0000000000..2cbaf21608 --- /dev/null +++ b/packages/stack/src/whole-stack.docker.e2e.test.ts @@ -0,0 +1,21 @@ +import { it } from "@effect/vitest"; +import { Effect } from "effect"; +import { allEager, defaultLifecycle, parallel } from "../tests/whole-stack/scenarios.ts"; +import { idleWake } from "../tests/whole-stack/idle.ts"; +import { servicesLayer } from "../tests/whole-stack/fixture.ts"; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(servicesLayer)); + +it.live( + "Docker: default database eager lifecycle and reopen", + () => run(defaultLifecycle("docker")), + { + timeout: 15 * 60_000, + }, +); +it.live("Docker: all services eager", () => run(allEager("docker")), { timeout: 15 * 60_000 }); +it.live("Docker: idle and wake", () => run(idleWake("docker")), { timeout: 10 * 60_000 }); +it.live("Docker: parallel stack isolation", () => run(parallel("docker")), { + timeout: 30 * 60_000, +}); diff --git a/packages/stack/src/whole-stack.native.e2e.test.ts b/packages/stack/src/whole-stack.native.e2e.test.ts new file mode 100644 index 0000000000..70ca2a6835 --- /dev/null +++ b/packages/stack/src/whole-stack.native.e2e.test.ts @@ -0,0 +1,21 @@ +import { it } from "@effect/vitest"; +import { Effect } from "effect"; +import { allEager, defaultLifecycle, parallel } from "../tests/whole-stack/scenarios.ts"; +import { idleWake } from "../tests/whole-stack/idle.ts"; +import { servicesLayer } from "../tests/whole-stack/fixture.ts"; + +const run = (effect: Effect.Effect) => + Effect.scoped(effect).pipe(Effect.provide(servicesLayer)); + +it.live( + "native: default database eager lifecycle and reopen", + () => run(defaultLifecycle("native")), + { + timeout: 15 * 60_000, + }, +); +it.live("native: all services eager", () => run(allEager("native")), { timeout: 15 * 60_000 }); +it.live("native: idle and wake", () => run(idleWake("native")), { timeout: 10 * 60_000 }); +it.live("native: parallel stack isolation", () => run(parallel("native")), { + timeout: 30 * 60_000, +}); diff --git a/packages/stack/tests/compiled-dispatch-fixture.ts b/packages/stack/tests/compiled-dispatch-fixture.ts new file mode 100644 index 0000000000..7f3b1d60d6 --- /dev/null +++ b/packages/stack/tests/compiled-dispatch-fixture.ts @@ -0,0 +1,91 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { Data, Effect, Layer, Option, Stream } from "effect"; +import { RpcClient, RpcSerialization } from "effect/unstable/rpc"; +import { + runHostProcessIfDispatched, + runNativeProcessIfDispatched, +} from "../src/internal/dispatch.ts"; +import { connectHost, launchHost, type HostEndpoint } from "../src/HostProcess.ts"; +import { spawnNativeProcess } from "../src/runtime/NativeProcess.ts"; +import { StackRpc } from "../src/Rpc.ts"; +import * as State from "../src/State.ts"; + +const argv = process.argv.slice(2); +class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} +const clientFor = (port: number) => + RpcClient.make(StackRpc).pipe( + Effect.provide( + RpcClient.layerProtocolHttp({ url: `http://127.0.0.1:${port}/rpc` }).pipe( + Layer.provide(RpcSerialization.layerNdjson), + ), + ), + ); +if (!(await runHostProcessIfDispatched(argv)) && !(await runNativeProcessIfDispatched(argv))) { + const [mode, ...modeArgs] = argv; + const output = (endpoint: HostEndpoint) => process.stdout.write(`${JSON.stringify(endpoint)}\n`); + + if (mode === "owner" || mode === "stop") { + const [stateRoot, cacheRoot, stackId] = modeArgs; + if (stateRoot === undefined || stackId === undefined) + throw new Error("Owner fixture arguments missing"); + const program = Effect.scoped( + Effect.gen(function* () { + const state = yield* State.Service; + const endpoint = yield* mode === "owner" + ? launchHost(state, { stateRoot, cacheRoot: cacheRoot ?? stateRoot, stackId }) + : connectHost(state, stackId); + if (mode === "stop") { + const client = yield* clientFor(endpoint.port); + yield* client.shutdown({ destroy: false }); + } + output(endpoint); + }), + ).pipe( + Effect.provide( + Layer.mergeAll( + NodeServices.layer, + NodeHttpClient.layerNodeHttp, + State.layer({ root: stateRoot }).pipe(Layer.provide(NodeServices.layer)), + ), + ), + ); + await Effect.runPromise(program); + } else if (mode === "native") { + const [pidMarker] = modeArgs; + if (pidMarker === undefined) throw new Error("Native fixture arguments missing"); + const program = Effect.scoped( + Effect.gen(function* () { + const native = yield* spawnNativeProcess({ + executable: "/bin/sh", + args: [ + "-c", + "trap 'exit 0' TERM; printf '%s' \"$$\" > \"$1\"; printf 'native-child-ready\\n'; while :; do sleep 1; done", + "sh", + pidMarker, + ], + gracefulStopSignal: "SIGTERM", + gracefulStopTimeout: "2 seconds", + }); + yield* native.stdout.pipe( + Stream.decodeText, + Stream.splitLines, + Stream.runHead, + Effect.flatMap( + Option.match({ + onNone: () => + Effect.fail(new FixtureError({ message: "Native child exited before readiness" })), + onSome: Effect.succeed, + }), + ), + ); + process.stdout.write("native-ready\n"); + yield* native.kill; + yield* Effect.exit(native.exitCode); + process.stdout.write("native-stopped\n"); + }), + ).pipe(Effect.provide(NodeServices.layer)); + await Effect.runPromise(program); + } else { + throw new Error(`Unknown fixture mode: ${mode}`); + } +} diff --git a/packages/stack/tests/docker-cleanup.ts b/packages/stack/tests/docker-cleanup.ts new file mode 100644 index 0000000000..cf64d569b6 --- /dev/null +++ b/packages/stack/tests/docker-cleanup.ts @@ -0,0 +1,36 @@ +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { Effect, Stream } from "effect"; + +export const cleanupDockerRoot = Effect.fn("DockerTest.cleanupRoot")((root: string) => + Effect.scoped( + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const child = yield* spawner.spawn( + ChildProcess.make( + "docker", + [ + "run", + "--rm", + "--user", + "0", + "--mount", + `type=bind,src=${root},dst=/mnt`, + "busybox:1.36", + "find", + "/mnt", + "-mindepth", + "1", + "-delete", + ], + { stdout: "pipe", stderr: "pipe" }, + ), + ); + const [output, code] = yield* Effect.all( + [Stream.mkString(Stream.decodeText(child.all)), child.exitCode], + { concurrency: "unbounded" }, + ); + if (Number(code) !== 0) + return yield* Effect.die(`Docker test cleanup exited with ${code}: ${output}`); + }), + ).pipe(Effect.catchCause(Effect.die)), +); diff --git a/packages/stack/tests/docker-relay.ts b/packages/stack/tests/docker-relay.ts new file mode 100644 index 0000000000..975de1ae79 --- /dev/null +++ b/packages/stack/tests/docker-relay.ts @@ -0,0 +1,36 @@ +import { Effect, Predicate } from "effect"; +import { bindTcp, serveTcp, ProxyError } from "../src/Proxy.ts"; +import { makeHttpProxy } from "../src/HttpProxy.ts"; +import type { ServiceEndpoint } from "../src/services/Recipe.ts"; + +const address = (endpoint: Effect.Effect) => + endpoint.pipe( + Effect.flatMap((value) => + value.kind === "tcp" && value.host !== undefined + ? Effect.succeed({ host: value.host, port: value.port }) + : Effect.fail(new ProxyError({ message: "Service endpoint is not TCP" })), + ), + Effect.mapError((cause) => + cause instanceof ProxyError ? cause : new ProxyError({ message: String(cause), cause }), + ), + ); + +export const makeDockerTcpRelay = Effect.fn("DockerRelay.makeTcp")( + (endpoint: Effect.Effect) => + Effect.gen(function* () { + const listener = yield* bindTcp("0.0.0.0", 0); + if (!Predicate.isTagged(listener.address, "TcpAddress")) + return yield* Effect.die("Expected TCP relay listener"); + yield* serveTcp(listener, address(endpoint)).pipe(Effect.forkScoped); + return { host: "host.docker.internal", port: listener.address.port }; + }), +); + +export const makeDockerHttpRelay = Effect.fn("DockerRelay.makeHttp")( + (endpoint: Effect.Effect) => + Effect.gen(function* () { + const proxy = yield* makeHttpProxy({ host: "0.0.0.0", port: 0 }); + yield* proxy.setRoutes([{ id: "service", prefix: "/", target: address(endpoint) }]); + return { host: "host.docker.internal", port: proxy.port }; + }), +); diff --git a/packages/stack/tests/host-process-fixture.ts b/packages/stack/tests/host-process-fixture.ts new file mode 100644 index 0000000000..32e8f58705 --- /dev/null +++ b/packages/stack/tests/host-process-fixture.ts @@ -0,0 +1,139 @@ +import { NodeHttpClient, NodeServices, NodeStream } from "@effect/platform-node"; +import { Context, Data, Deferred, Effect, FileSystem, Layer, Path, Schema, Stream } from "effect"; +import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; +import { Rpc, RpcGroup, RpcSerialization, RpcServer } from "effect/unstable/rpc"; +// oxlint-disable-next-line effecttsgo/node-builtin-import -- test fixture owns inherited readiness and release descriptors. +import { closeSync, createReadStream, writeSync } from "node:fs"; +import { acquireHost, HostEndpoint, launchHost } from "../src/HostProcess.ts"; +import * as State from "../src/State.ts"; + +class FixtureError extends Data.TaggedError("FixtureError")<{ readonly message: string }> {} + +const FixtureRpc = RpcGroup.make( + Rpc.make("shutdown", { + payload: { destroy: Schema.Boolean }, + error: Schema.Never, + }), +); + +const causeCode = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null) return undefined; + if ("code" in cause && typeof cause.code === "string") return cause.code; + if ("cause" in cause) return causeCode(cause.cause); + return undefined; +}; + +const makeState = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); + +const awaitBarrier = NodeStream.fromReadable({ + evaluate: () => createReadStream("", { fd: 4 }), + onError: (cause) => new FixtureError({ message: String(cause) }), +}).pipe(Stream.take(1), Stream.runDrain); + +const [stateRoot, cacheRoot, stackId, mode = "owner", ownerEntrypoint] = process.argv.slice(2); +if (stateRoot === undefined || stackId === undefined) + throw new FixtureError({ message: "fixture arguments missing" }); + +const owner = Effect.scoped( + Effect.gen(function* () { + const state = yield* makeState(stateRoot); + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stack = yield* state.read(stackId); + if (stack === undefined) return yield* new FixtureError({ message: "stack is not registered" }); + const host = yield* acquireHost(state, stackId); + if (stack.identity.stackName === "slow-handshake") { + const marker = path.join(stateRoot, "slow-handshake.pid"); + yield* fs.writeFileString(`${marker}.tmp`, String(process.pid)); + yield* fs.rename(`${marker}.tmp`, marker); + return yield* Effect.never; + } + const shutdown = yield* Deferred.make(); + const endpoint: HostEndpoint = { + stackId, + identity: stack.identity, + pid: process.pid, + port: host.port, + }; + const rpc = yield* RpcServer.toHttpEffect(FixtureRpc, { streamBufferSize: 4 }).pipe( + Effect.provide( + Layer.merge( + FixtureRpc.toLayer({ + shutdown: () => Deferred.succeed(shutdown, undefined).pipe(Effect.asVoid), + }), + RpcSerialization.layerNdjson, + ), + ), + ); + const serving = host.server.serve( + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + if (request.method === "GET" && request.url === "/identity") + return yield* HttpServerResponse.json(endpoint).pipe( + Effect.map(HttpServerResponse.setHeader("connection", "close")), + ); + if (request.method === "POST" && request.url === "/shutdown") { + yield* Deferred.succeed(shutdown, undefined); + return HttpServerResponse.empty({ status: 202 }).pipe( + HttpServerResponse.setHeader("connection", "close"), + ); + } + if (mode === "rpc-held" && request.method === "POST" && request.url.startsWith("/rpc")) + return yield* rpc; + return HttpServerResponse.empty({ status: 404 }); + }), + ); + yield* serving.pipe(Effect.forkScoped); + try { + const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ + type: "ready", + endpoint, + }); + writeSync(3, Buffer.from(`${encoded}\n`)); + } finally { + closeSync(3); + } + yield* Deferred.await(shutdown); + if (mode === "rpc-held") yield* awaitBarrier; + }), +); + +const launcher = Effect.gen(function* () { + const state = yield* makeState(stateRoot); + const endpoint = yield* launchHost(state, { + stateRoot, + cacheRoot: cacheRoot ?? stateRoot, + stackId, + entrypoint: ownerEntrypoint ?? new URL(import.meta.url).pathname, + }); + const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))(endpoint); + process.stdout.write(`${encoded}\n`); +}); + +const program = mode === "launcher" ? launcher : owner; +try { + await Effect.runPromise( + Effect.scoped(program).pipe( + Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp)), + ), + ); +} catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + const reason = causeCode(cause) === "EADDRINUSE" ? "bind-conflict" : undefined; + const error = `${Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))({ + type: "error", + message, + ...(reason === undefined ? {} : { reason }), + })}\n`; + if (mode === "owner") { + try { + writeSync(3, Buffer.from(error)); + } finally { + closeSync(3); + } + } else process.stderr.write(error); + process.exitCode = 1; +} diff --git a/packages/stack/tests/owner-exit-client.ts b/packages/stack/tests/owner-exit-client.ts new file mode 100644 index 0000000000..d542af630f --- /dev/null +++ b/packages/stack/tests/owner-exit-client.ts @@ -0,0 +1,80 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { Data, Effect, Layer, Predicate } from "effect"; +import * as Stack from "../src/index.ts"; + +class FixtureError extends Data.TaggedError("OwnerExitFixtureError")<{ + readonly message: string; + readonly cause?: unknown; +}> {} + +const promise = (run: () => Promise) => + Effect.tryPromise({ + try: run, + catch: (cause) => new FixtureError({ message: String(cause), cause }), + }); + +const assertExited = Effect.fn("Fixture.assertExited")(function* (pid: number) { + const alive = yield* Effect.try({ + try: () => process.kill(pid, 0), + catch: (cause) => new FixtureError({ message: String(cause), cause }), + }).pipe( + Effect.as(true), + Effect.catchIf( + ({ cause }) => Predicate.hasProperty(cause, "code") && cause.code === "ESRCH", + () => Effect.succeed(false), + ), + ); + if (alive) return yield* new FixtureError({ message: `Owner ${pid} survived teardown` }); +}); + +const program = Effect.gen(function* () { + const [root, cacheRoot] = process.argv.slice(2); + if (root === undefined || cacheRoot === undefined) + return yield* new FixtureError({ message: "Missing fixture roots" }); + const locations = { stateRoot: `${root}/state`, cacheRoot }; + const ownerPid = (id: string) => + promise(() => Stack.discover(locations)).pipe( + Effect.flatMap((entries) => { + const owner = entries.find((entry) => entry.definition.id === id)?.host; + return owner === undefined + ? Effect.fail(new FixtureError({ message: `No owner for ${id}` })) + : Effect.succeed(owner.pid); + }), + ); + yield* Effect.acquireUseRelease( + promise(() => Stack.create({ ...locations, projectRoot: root, runtime: "native" })), + (stack) => + Effect.acquireUseRelease( + Effect.succeed(stack), + (client) => + Effect.gen(function* () { + const mail = yield* promise(() => + client.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }), + ); + yield* promise(() => mail.start()); + yield* promise(() => mail.ready()); + const firstPid = yield* ownerPid(client.id); + yield* promise(() => client.stop()); + yield* assertExited(firstPid); + yield* promise(() => mail.start()); + yield* promise(() => mail.ready()); + }), + (client) => + Effect.gen(function* () { + const pid = yield* ownerPid(client.id); + yield* promise(() => client.destroy()); + yield* assertExited(pid); + }), + ), + (client) => promise(() => client.close()), + ); + process.stdout.write("owner-exit-confirmed\n"); +}); + +await Effect.runPromise( + program.pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/tests/owner.ts b/packages/stack/tests/owner.ts new file mode 100644 index 0000000000..a295b375ab --- /dev/null +++ b/packages/stack/tests/owner.ts @@ -0,0 +1,29 @@ +import { expect } from "@effect/vitest"; +import { Effect, Predicate } from "effect"; +import { discover, type StackLocations } from "../src/effect.ts"; +import { HostProcessError } from "../src/HostProcess.ts"; + +export const captureOwnerPid = Effect.fn("Test.captureOwnerPid")(function* ( + locations: StackLocations, + id: string, +) { + const owner = (yield* discover(locations)).find((entry) => entry.definition.id === id)?.host; + if (owner === undefined) + return yield* new HostProcessError({ operation: "test-owner", message: `No owner for ${id}` }); + return owner.pid; +}); + +export const assertOwnerExited = Effect.fn("Test.assertOwnerExited")(function* (pid: number) { + const exists = yield* Effect.try({ + try: () => process.kill(pid, 0), + catch: (cause) => + new HostProcessError({ operation: "test-owner", message: String(cause), cause }), + }).pipe( + Effect.as(true), + Effect.catchIf( + ({ cause }) => Predicate.hasProperty(cause, "code") && cause.code === "ESRCH", + () => Effect.succeed(false), + ), + ); + expect(exists, `Owner process ${pid} survived successful teardown`).toBe(false); +}); diff --git a/packages/stack/tests/public-client-fixture.ts b/packages/stack/tests/public-client-fixture.ts new file mode 100644 index 0000000000..46bb2df010 --- /dev/null +++ b/packages/stack/tests/public-client-fixture.ts @@ -0,0 +1,24 @@ +import { create } from "../src/index.ts"; +const [root, cacheRoot] = process.argv.slice(2); +if (root === undefined || cacheRoot === undefined) throw new Error("Client fixture roots missing"); +const stack = await create({ + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot, + runtime: "native", +}); +try { + const mail = await stack.services.create({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + await mail.start(); + await mail.ready(); + process.stdout.write(`${JSON.stringify({ stackId: stack.id, instanceId: mail.id })}\n`); +} catch (cause) { + await stack.destroy(); + throw cause; +} finally { + await stack.close(); +} diff --git a/packages/stack/src/functions/serve-main-bundler.ts b/packages/stack/tests/serve-main-bundler.ts similarity index 84% rename from packages/stack/src/functions/serve-main-bundler.ts rename to packages/stack/tests/serve-main-bundler.ts index 3837734286..01f6bc3394 100644 --- a/packages/stack/src/functions/serve-main-bundler.ts +++ b/packages/stack/tests/serve-main-bundler.ts @@ -7,10 +7,10 @@ class ServeMainBundleError extends Data.TaggedError("ServeMainBundleError")<{ readonly cause?: unknown; }> {} -/** Absolute path to the stack-owned Edge Runtime main service template. */ -export const serveMainEntrypoint = fileURLToPath(new URL("./serve.main.ts", import.meta.url)); +const serveMainEntrypoint = fileURLToPath( + new URL("../src/functions/serve.main.ts", import.meta.url), +); -/** Produces one offline ES module with jose and path helpers inlined. */ export const bundleServeMainTemplate = Effect.gen(function* () { const result = yield* Effect.tryPromise({ try: () => diff --git a/packages/stack/tests/state-lock-fixture.ts b/packages/stack/tests/state-lock-fixture.ts new file mode 100644 index 0000000000..deed86a674 --- /dev/null +++ b/packages/stack/tests/state-lock-fixture.ts @@ -0,0 +1,58 @@ +import { NodeServices, NodeSocketServer } from "@effect/platform-node"; +import { Console, Effect } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import * as State from "../src/State.ts"; +import { isBunVirtualPath } from "../src/internal/dispatch-markers.ts"; +import { fileURLToPath } from "node:url"; + +const [mode, root, id] = process.argv.slice(2); +if (root === undefined) throw new Error("Registry root missing"); +if (mode !== "write" && mode !== "hold" && mode !== "child") + throw new Error(`Unknown fixture mode: ${mode ?? ""}`); +const fixturePath = fileURLToPath(import.meta.url); + +const program = Effect.scoped( + Effect.gen(function* () { + if (mode === "child") { + yield* NodeSocketServer.make({ host: "127.0.0.1", port: 0 }); + yield* Console.log(`child:${process.pid}`); + return yield* Effect.never; + } + yield* Effect.gen(function* () { + const state = yield* State.Service; + yield* state.withLock( + Effect.gen(function* () { + if (mode === "write") { + const saved = yield* state.read("stack-main"); + if (saved === undefined || id === undefined) + return yield* Effect.die("Missing writer state"); + yield* state.save({ + ...saved, + instances: [...saved.instances, { id, creation: {} }], + }); + return; + } + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + yield* spawner.spawn( + ChildProcess.make( + process.execPath, + [...(isBunVirtualPath(fixturePath) ? [] : [fixturePath]), "child", root], + { + // Keep the child independent on Windows so holder termination tests lock recovery. + detached: process.platform === "win32", + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", + forceKillAfter: "1 second", + }, + ), + ); + yield* Console.log("locked"); + return yield* Effect.never; + }), + ); + }).pipe(Effect.provide(State.layer({ root }))); + }), +).pipe(Effect.provide(NodeServices.layer)); + +await Effect.runPromise(program); diff --git a/packages/stack/tests/whole-stack/analytics.ts b/packages/stack/tests/whole-stack/analytics.ts new file mode 100644 index 0000000000..0d15891c8e --- /dev/null +++ b/packages/stack/tests/whole-stack/analytics.ts @@ -0,0 +1,173 @@ +import { Clock, DateTime, Effect, Schedule, Schema } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; + +const AnalyticsLog = Schema.Struct({ + event_message: Schema.String, +}); + +const AnalyticsQueryResponse = Schema.Struct({ + result: Schema.Array(AnalyticsLog), +}); + +const AnalyticsLogResponse = Schema.Struct({ + message: Schema.String, +}); + +class AnalyticsFlowError extends Schema.TaggedError()( + "WholeStack.AnalyticsFlowError", + { + operation: Schema.String, + cause: Schema.Unknown, + }, +) {} + +/** Builds the Vector config used by the whole-stack Analytics flow. */ +export const vectorAnalyticsConfig = (marker: string): string => + `sources: + demo_logs: + type: demo_logs + format: json + interval: 1 +transforms: + fixture_marker: + type: remap + inputs: [demo_logs] + source: | + .event_message = ${JSON.stringify(marker)} + .message = ${JSON.stringify(marker)} +sinks: + analytics: + type: http + inputs: [fixture_marker] + uri: "${"${LOGFLARE_URL}"}/api/logs?source_name=postgres.logs" + method: post + encoding: + codec: json + request: + headers: + x-api-key: "${"${LOGFLARE_PRIVATE_ACCESS_TOKEN}"}" + batch: + max_events: 1 + timeout_secs: 1 + healthcheck: false +api: + enabled: true + address: "${"${VECTOR_API_ADDRESS}"}" +`; + +const request = Effect.fn("WholeStack.analyticsRequest")( + (request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + return yield* client.execute(request); + }).pipe(Effect.mapError((cause) => new AnalyticsFlowError({ operation: "request", cause }))), +); + +/** Sends one marker directly to Logflare, for separating API ingest from Vector ingest. */ +const postAnalyticsMarker = Effect.fn("WholeStack.postAnalyticsMarker")( + (analyticsUrl: string, apiKey: string, marker: string, timestampMillis: number) => + Effect.gen(function* () { + const body = [{ event_message: marker, timestamp: timestampMillis * 1000 }]; + const bodyRequest = yield* HttpClientRequest.bodyJson(body)( + HttpClientRequest.post(`${analyticsUrl}/api/logs?source_name=postgres.logs`), + ).pipe( + Effect.mapError((cause) => new AnalyticsFlowError({ operation: "encode-log", cause })), + ); + const response = yield* request( + HttpClientRequest.setHeader("x-api-key", apiKey)(bodyRequest), + ); + if (response.status >= 400) + return yield* new AnalyticsFlowError({ + operation: "post-log", + cause: { + status: response.status, + body: yield* response.text, + url: `${analyticsUrl}/api/logs?source_name=postgres.logs`, + }, + }); + const decoded = yield* Schema.decodeUnknownEffect(AnalyticsLogResponse)( + yield* response.json, + ).pipe( + Effect.mapError( + (cause) => new AnalyticsFlowError({ operation: "decode-log-response", cause }), + ), + ); + if (decoded.message !== "Logged!") + return yield* new AnalyticsFlowError({ operation: "post-log", cause: decoded.message }); + }), +); + +/** Performs one exact-marker SQL query against Logflare's Postgres backend. */ +export const queryAnalyticsMarker = Effect.fn("WholeStack.queryAnalyticsMarker")( + (analyticsUrl: string, apiKey: string, marker: string, phaseStartedAtMillis: number) => + Effect.gen(function* () { + const escapedMarker = marker.replaceAll("'", "''"); + const timestamp = DateTime.formatIso(DateTime.fromEpochSeconds(phaseStartedAtMillis / 1000)) + .replace("T", " ") + .replace("Z", ""); + const sql = + `select event_message from postgres.logs where event_message = '${escapedMarker}' ` + + `and timestamp >= '${timestamp}'`; + const response = yield* request( + HttpClientRequest.get(`${analyticsUrl}/api/query?pg_sql=${encodeURIComponent(sql)}`).pipe( + HttpClientRequest.setHeader("x-api-key", apiKey), + ), + ); + if (response.status >= 400) + return yield* new AnalyticsFlowError({ operation: "query-log", cause: response.status }); + const rows = yield* Schema.decodeUnknownEffect(AnalyticsQueryResponse)( + yield* response.json, + ).pipe( + Effect.mapError( + (cause) => new AnalyticsFlowError({ operation: "decode-query-response", cause }), + ), + Effect.map((result) => result.result), + ); + return rows; + }), +); + +const queryMarkerUntilVisible = Effect.fn("WholeStack.queryMarkerUntilVisible")( + (analyticsUrl: string, apiKey: string, marker: string, phaseStartedAtMillis: number) => + queryAnalyticsMarker(analyticsUrl, apiKey, marker, phaseStartedAtMillis).pipe( + Effect.flatMap((rows) => + rows.some((row) => row.event_message === marker) + ? Effect.void + : Effect.fail(new AnalyticsFlowError({ operation: "query-empty", cause: { marker } })), + ), + Effect.retry({ + schedule: Schedule.spaced("1 second"), + while: (error) => Schema.is(AnalyticsFlowError)(error) && error.operation === "query-empty", + }), + Effect.timeoutOrElse({ + duration: "60 seconds", + orElse: () => + Effect.fail(new AnalyticsFlowError({ operation: "query-timeout", cause: { marker } })), + }), + ), +); + +/** Exercises direct and Vector Analytics ingestion for one whole-stack phase. */ +export const exerciseAnalytics = Effect.fn("WholeStack.exerciseAnalytics")( + (analyticsUrl: string, vectorUrl: string, apiKey: string, stackId: string, phase: string) => + Effect.gen(function* () { + const phaseStartedAtMillis = yield* Clock.currentTimeMillis; + const directMarker = `direct-${stackId}-${phase}-${phaseStartedAtMillis}`; + const vectorMarker = `vector-${stackId}`; + yield* postAnalyticsMarker(analyticsUrl, apiKey, directMarker, phaseStartedAtMillis); + const vectorHealth = yield* request(HttpClientRequest.get(`${vectorUrl}/health`)); + yield* vectorHealth.text; + if (vectorHealth.status >= 400) + return yield* new AnalyticsFlowError({ + operation: "vector-health", + cause: vectorHealth.status, + }); + yield* Effect.all( + [ + queryMarkerUntilVisible(analyticsUrl, apiKey, directMarker, phaseStartedAtMillis), + queryMarkerUntilVisible(analyticsUrl, apiKey, vectorMarker, phaseStartedAtMillis), + ], + { concurrency: 2, discard: true }, + ); + }), +); diff --git a/packages/stack/tests/whole-stack/fixture.ts b/packages/stack/tests/whole-stack/fixture.ts new file mode 100644 index 0000000000..4bcefe6d60 --- /dev/null +++ b/packages/stack/tests/whole-stack/fixture.ts @@ -0,0 +1,356 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { + Cause, + Crypto, + Effect, + Exit, + FileSystem, + Layer, + Option, + Redacted, + Ref, + Stream, +} from "effect"; +import { postgres } from "../../src/Tools.ts"; +import { homedir, tmpdir } from "node:os"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { bundleServeMainTemplate } from "../serve-main-bundler.ts"; +import { create, type Stack } from "../../src/effect.ts"; +import type { Observation } from "../../src/Rpc.ts"; +import { vectorAnalyticsConfig } from "./analytics.ts"; +import { cleanupDockerRoot } from "../docker-cleanup.ts"; + +type AnyService = Effect.Success[number]; + +export const serviceNames = [ + "database", + "rest", + "auth", + "realtime", + "storage", + "imgproxy", + "functions", + "studio", + "pgmeta", + "mail", + "analytics", + "vector", + "pooler", +] as const; + +export type Runtime = "native" | "docker"; +export type WholeStack = Readonly<{ + readonly stack: Stack; + readonly services: ReadonlyArray; + readonly root: string; + readonly locations: { + readonly stateRoot: string; + readonly cacheRoot: string; + }; + readonly secret: string; + readonly owner: Ref.Ref>; + readonly logTails: Ref.Ref>; +}>; + +const watchServiceLogs = Effect.fn("WholeStack.watchServiceLogs")( + ( + services: ReadonlyArray, + logTails: Ref.Ref>, + ) => + Effect.forEach( + services, + (instance) => + Effect.forkScoped( + instance.logs.pipe( + Stream.runForEach(({ bytes }) => + Ref.update(logTails, (tails) => { + const text = new TextDecoder().decode(bytes); + const existing = tails.find(([name]) => name === instance.service)?.[1] ?? ""; + const updated = `${existing}${text}`.slice(-8192); + const without = tails.filter(([name]) => name !== instance.service); + return [...without, [instance.service, updated] as const]; + }), + ), + Effect.ignoreCause, + ), + ), + { discard: true }, + ), +); + +const endpoint = (port: "auto") => ({ port }); + +export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ + prefix: `stack-whole-${runtime}-`, + ...(runtime === "native" && process.platform === "linux" ? { directory: homedir() } : {}), + }); + const functionsRoot = `${root}/functions`; + const storageRoot = `${root}/storage`; + const vectorConfigPath = `${root}/vector.yaml`; + yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); + yield* fs.makeDirectory(storageRoot, { recursive: true }); + yield* fs.writeFileString( + `${functionsRoot}/hello/index.ts`, + "Deno.serve(async (request) => { const authorization = request.headers.get('authorization') ?? ''; const input = await request.json(); const response = await fetch(`${Deno.env.get('SUPABASE_URL')}/rest/v1/whole_stack_items?id=eq.${input.id}`, { headers: { authorization, apikey: authorization.replace('Bearer ', '') } }); return new Response(await response.text(), { status: response.status, headers: { 'content-type': 'application/json' } }); });", + ); + const bootstrap = yield* bundleServeMainTemplate; + const crypto = yield* Crypto.Crypto; + const secret = `whole-stack-${yield* crypto.randomUUIDv4}-secret`; + const locations = { + stateRoot: `${root}/state`, + cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + }; + const stack = yield* create({ + projectRoot: root, + ...locations, + runtime, + name: `whole-${runtime}`, + }); + yield* fs.writeFileString(vectorConfigPath, vectorAnalyticsConfig(`vector-${stack.id}`)); + const owner = yield* Ref.make>(Option.some(stack)); + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + const current = yield* Ref.get(owner); + const destroy = Option.isSome(current) + ? current.value.destroy.pipe(Effect.catchCause(Effect.die)) + : Effect.void; + yield* runtime === "docker" + ? destroy.pipe(Effect.ensuring(cleanupDockerRoot(storageRoot))) + : destroy; + }), + ); + const created = yield* stack.composition.supabase([ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("postgres"), + jwtSecret: Redacted.make(secret), + jwtExpiry: 3600, + }, + endpoints: { sql: endpoint("auto") }, + }, + { + service: "rest", + config: { databaseUrl: "postgresql://placeholder", jwtSecret: secret }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "auth", + config: { databaseUrl: "postgresql://placeholder", jwtSecret: secret }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "realtime", + config: { databaseUrl: "postgresql://placeholder", jwtSecret: secret }, + endpoints: { http: endpoint("auto"), rpc: endpoint("auto") }, + }, + { + service: "storage", + config: { + databaseUrl: "postgresql://placeholder", + filePath: storageRoot, + jwtSecret: secret, + }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "imgproxy", + config: { filePath: storageRoot }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "functions", + config: { functionsRoot, bootstrap, verifyJwt: true, jwtSecret: secret }, + endpoints: { http: endpoint("auto") }, + }, + { service: "studio", config: { jwtSecret: secret }, endpoints: { http: endpoint("auto") } }, + { + service: "pgmeta", + config: { databaseUrl: "postgresql://placeholder" }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "mail", + config: {}, + endpoints: { http: endpoint("auto"), smtp: endpoint("auto"), pop3: endpoint("auto") }, + }, + { + service: "analytics", + config: { databaseUrl: "postgresql://placeholder", backend: "postgres", apiKey: secret }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "vector", + config: { + analyticsUrl: "http://placeholder", + apiKey: secret, + configPath: vectorConfigPath, + }, + endpoints: { http: endpoint("auto") }, + }, + { + service: "pooler", + config: { + databaseUrl: "postgresql://placeholder", + jwtSecret: secret, + tenant: "whole", + poolMode: "transaction", + }, + endpoints: { http: endpoint("auto"), sql: endpoint("auto") }, + }, + ]); + const logTails = yield* Ref.make>([]); + yield* watchServiceLogs(created, logTails); + yield* Effect.addFinalizer((exit) => + Exit.isFailure(exit) + ? Effect.gen(function* () { + const current = yield* Ref.get(owner); + const observed = Option.isSome(current) + ? yield* current.value.services.list.pipe(Effect.orElseSucceed(() => created)) + : created; + const statuses = yield* Effect.forEach(observed, (instance) => + instance.status.pipe( + Effect.map((status) => `${instance.service}=${status.lifecycle}`), + Effect.catchCause(() => Effect.succeed(`${instance.service}=unknown`)), + ), + ); + const tails = yield* Ref.get(logTails); + yield* Effect.logError( + `Whole-stack failure diagnostics: cause=${Cause.pretty(exit.cause)} statuses=${statuses.join(",")} logs=${tails.map(([name, value]) => `${name}: ${value}`).join("\n")}`, + ); + }).pipe(Effect.ignoreCause) + : Effect.void, + ); + return { + stack, + services: created, + root, + locations, + secret, + owner, + logTails, + } satisfies WholeStack; + }), +); + +export const refreshLogTails = Effect.fn("WholeStack.refreshLogTails")((fixture: WholeStack) => + watchServiceLogs(fixture.services, fixture.logTails), +); + +export const setStackOwner = Effect.fn("WholeStack.setStackOwner")( + (fixture: WholeStack, stack: Stack) => Ref.set(fixture.owner, Option.some(stack)), +); + +export const clearStackOwner = Effect.fn("WholeStack.clearStackOwner")((fixture: WholeStack) => + Ref.set(fixture.owner, Option.none()), +); + +export const service = (fixture: WholeStack, name: (typeof serviceNames)[number]): AnyService => { + const found = fixture.services.find((candidate) => candidate.service === name); + if (found === undefined) throw new Error(`Missing ${name} service`); + return found; +}; + +export const allMembers = (stack: WholeStack["stack"]) => + Effect.gen(function* () { + const configuration = yield* stack.composition.describe; + return configuration.members; + }); + +export const setActivation = Effect.fn("WholeStack.setActivation")( + (stack: WholeStack["stack"], activation: "eager" | "lazy", idleMillis?: number) => + Effect.gen(function* () { + const configuration = yield* stack.composition.describe; + yield* stack.composition.configure({ + ...configuration, + members: configuration.members.map((member) => ({ + id: member.id, + activation, + ...(idleMillis === undefined ? {} : { idleMillis }), + })), + }); + }), +); + +export const statusByService = Effect.fn("WholeStack.statusByService")((fixture: WholeStack) => + Effect.forEach(serviceNames, (name) => + service(fixture, name).status.pipe(Effect.map((status) => [name, status] as const)), + ).pipe(Effect.map((entries) => entries)), +); + +export const requestWithHeaders = Effect.fn("WholeStack.requestWithHeaders")( + (url: string, headers: Readonly>) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const request = Object.entries(headers).reduce( + (current, [name, value]) => HttpClientRequest.setHeader(name, value)(current), + HttpClientRequest.get(url), + ); + const response = yield* client.execute(request); + return { status: response.status, body: yield* response.text }; + }), +); + +export const jsonRequest = Effect.fn("WholeStack.jsonRequest")( + ( + method: "POST" | "PATCH", + url: string, + body: unknown, + headers?: Readonly>, + ) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const request = yield* HttpClientRequest.bodyJson(body)( + method === "POST" ? HttpClientRequest.post(url) : HttpClientRequest.patch(url), + ); + const response = yield* client.execute( + Object.entries(headers ?? {}).reduce( + (current, [name, value]) => HttpClientRequest.setHeader(name, value)(current), + request, + ), + ); + return { status: response.status, body: yield* response.text }; + }), +); + +export const sql = Effect.fn("WholeStack.sql")((fixture: WholeStack, statement: string) => + Effect.gen(function* () { + const database = service(fixture, "database"); + const credentials = yield* database.credentials({ from: "runtime" }); + const databaseUrl = credentials.databaseUrl; + if (databaseUrl === undefined) return yield* Effect.die("Database URL missing"); + const output: Array = []; + const errors: Array = []; + const result = yield* fixture.stack.tools.run(postgres.psql({ major: 17 }), { + args: ["--set", "ON_ERROR_STOP=1", "--dbname", databaseUrl, "-At"], + stdin: Stream.make(new TextEncoder().encode(`${statement}\n`)), + stdout: (bytes) => Effect.sync(() => output.push(bytes)), + stderr: (bytes) => Effect.sync(() => errors.push(bytes)), + }); + if (result.exitCode !== 0) + return yield* Effect.die( + `SQL failed (${result.exitCode}): ${errors.map((bytes) => new TextDecoder().decode(bytes)).join("")}`, + ); + return output + .map((bytes) => new TextDecoder().decode(bytes)) + .join("") + .trim(); + }), +); + +export const waitForLifecycle = Effect.fn("WholeStack.waitForLifecycle")( + (service: AnyService, lifecycle: Observation["lifecycle"]) => + service.followStatus.pipe( + Stream.filter((status) => status.lifecycle === lifecycle), + Stream.runHead, + Effect.flatMap((status) => + Option.isSome(status) ? Effect.void : Effect.die("status stream ended"), + ), + ), +); + +export const servicesLayer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); diff --git a/packages/stack/tests/whole-stack/idle.ts b/packages/stack/tests/whole-stack/idle.ts new file mode 100644 index 0000000000..b781328220 --- /dev/null +++ b/packages/stack/tests/whole-stack/idle.ts @@ -0,0 +1,187 @@ +import { expect } from "@effect/vitest"; +import { Context, Effect, Fiber, Layer, Redacted, Schema } from "effect"; +import { PgClient } from "@effect/sql-pg"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { SignJWT } from "jose"; +import { + jsonRequest, + requestWithHeaders, + service, + sql, + waitForLifecycle, + wholeStack, + type Runtime, +} from "./fixture.ts"; +import { subscribeRealtime } from "./websocket.ts"; + +class IdleProbeError extends Schema.TaggedError()("IdleProbeError", { + message: Schema.String, + cause: Schema.optionalKey(Schema.Unknown), +}) {} + +const signServiceToken = Effect.fn("WholeStack.signServiceToken")((secret: string) => + Effect.tryPromise({ + try: () => + new SignJWT({ role: "service_role" }) + .setProtectedHeader({ alg: "HS256" }) + .setIssuedAt() + .setExpirationTime("1h") + .sign(new TextEncoder().encode(secret)), + catch: (cause) => new IdleProbeError({ message: String(cause), cause }), + }), +); + +export const idleWake = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* wholeStack(runtime); + const configuration = yield* fixture.stack.composition.describe; + const database = service(fixture, "database"); + const timedServices = new Set([ + service(fixture, "rest").id, + service(fixture, "realtime").id, + service(fixture, "pooler").id, + ]); + yield* fixture.stack.composition.configure({ + ...configuration, + members: configuration.members.map((member) => + member.id === database.id + ? { id: member.id, activation: "eager" as const } + : timedServices.has(member.id) + ? { id: member.id, activation: "lazy" as const, idleMillis: 15_000 } + : { id: member.id, activation: "lazy" as const }, + ), + }); + yield* fixture.stack.composition.start; + yield* sql( + fixture, + "create table if not exists public.idle_items (id text primary key, value text not null); alter table public.idle_items enable row level security; grant select, insert on public.idle_items to anon, authenticated, service_role; alter publication supabase_realtime add table public.idle_items;", + ); + + const token = yield* signServiceToken(fixture.secret); + const restUrl = (yield* service(fixture, "rest").credentials()).url; + const authUrl = (yield* service(fixture, "auth").credentials()).url; + const realtimeUrl = (yield* service(fixture, "realtime").credentials()).url; + const poolerCredentials = yield* service(fixture, "pooler").credentials(); + const poolerUrl = poolerCredentials.sqlUrl; + if ( + restUrl === undefined || + authUrl === undefined || + realtimeUrl === undefined || + poolerUrl === undefined + ) + return yield* Effect.die("Idle scenario credentials are incomplete"); + const headers = { authorization: `Bearer ${token}`, apikey: token }; + yield* Effect.scoped( + Effect.gen(function* () { + const realtime = yield* subscribeRealtime(realtimeUrl, token, "idle_items", "INSERT"); + const poolerAddress = new URL(poolerUrl); + const poolerLayer = yield* Layer.build( + PgClient.layer({ + host: poolerAddress.hostname, + port: Number(poolerAddress.port), + database: "postgres", + username: "supabase_admin.whole", + password: Redacted.make("postgres"), + connectTimeout: "60 seconds", + }), + ); + const pooler = Context.get(poolerLayer, PgClient.PgClient); + const poolerConnection = yield* pooler.reserve; + yield* poolerConnection.executeRaw("SELECT 1", []); + + const client = yield* HttpClient.HttpClient; + const keepAlive = yield* client.execute( + HttpClientRequest.get(restUrl).pipe( + HttpClientRequest.setHeader("connection", "keep-alive"), + ), + ); + yield* keepAlive.text; + const firstChange = yield* realtime.nextChange.pipe(Effect.forkScoped); + const insert = yield* jsonRequest( + "POST", + `${restUrl}/idle_items`, + { id: fixture.stack.id, value: "awake" }, + { ...headers, "content-type": "application/json", prefer: "return=minimal" }, + ); + expect(insert.status).toBe(201); + yield* Fiber.join(firstChange); + const rest = service(fixture, "rest"); + yield* waitForLifecycle(rest, "stopped"); + expect((yield* rest.status).wakeEnabled).toBe(true); + const authHealth = yield* requestWithHeaders(`${authUrl}/health`, {}); + expect(authHealth.status).toBe(200); + expect((yield* rest.status).lifecycle).toBe("stopped"); + expect((yield* database.status).lifecycle).toBe("running"); + expect((yield* service(fixture, "realtime").status).lifecycle).toBe("running"); + expect((yield* service(fixture, "pooler").status).lifecycle).toBe("running"); + yield* poolerConnection.executeRaw("SELECT 1", []); + + const nextChange = yield* realtime.nextChange.pipe(Effect.forkScoped); + const wake = yield* requestWithHeaders( + `${restUrl}/idle_items?id=eq.${fixture.stack.id}`, + headers, + ); + expect(wake.status).toBe(200); + expect((yield* rest.status).lifecycle).toBe("running"); + const marker = `${fixture.stack.id}-change`; + const changed = yield* jsonRequest( + "POST", + `${restUrl}/idle_items`, + { id: marker, value: "wake" }, + { ...headers, "content-type": "application/json", prefer: "return=minimal" }, + ); + expect(changed.status).toBe(201); + const change = yield* Fiber.join(nextChange); + const changeRow = yield* Schema.decodeUnknownEffect( + Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), + )(change.payload.data); + expect(changeRow.record).toEqual({ id: marker, value: "wake" }); + yield* waitForLifecycle(rest, "stopped"); + expect((yield* rest.status).wakeEnabled).toBe(true); + }), + ); + yield* waitForLifecycle(service(fixture, "realtime"), "stopped"); + yield* waitForLifecycle(service(fixture, "pooler"), "stopped"); + yield* Effect.scoped( + Effect.gen(function* () { + const realtime = yield* subscribeRealtime(realtimeUrl, token, "idle_items", "INSERT"); + const poolerAddress = new URL(poolerUrl); + const poolerLayer = yield* Layer.build( + PgClient.layer({ + host: poolerAddress.hostname, + port: Number(poolerAddress.port), + database: "postgres", + username: "supabase_admin.whole", + password: Redacted.make("postgres"), + connectTimeout: "60 seconds", + }), + ); + const pooler = Context.get(poolerLayer, PgClient.PgClient); + const poolerConnection = yield* pooler.reserve; + yield* poolerConnection.executeRaw("SELECT 1", []); + const nextChange = yield* realtime.nextChange.pipe(Effect.forkScoped); + const wake = yield* requestWithHeaders( + `${restUrl}/idle_items?id=eq.${fixture.stack.id}`, + headers, + ); + expect(wake.status).toBe(200); + const marker = `${fixture.stack.id}-reconnect`; + const changed = yield* jsonRequest( + "POST", + `${restUrl}/idle_items`, + { id: marker, value: "reconnect" }, + { ...headers, "content-type": "application/json", prefer: "return=minimal" }, + ); + expect(changed.status).toBe(201); + const change = yield* Fiber.join(nextChange); + const changeRow = yield* Schema.decodeUnknownEffect( + Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), + )(change.payload.data); + expect(changeRow.record).toEqual({ id: marker, value: "reconnect" }); + }), + ); + yield* waitForLifecycle(service(fixture, "realtime"), "stopped"); + yield* waitForLifecycle(service(fixture, "pooler"), "stopped"); + }), + ); diff --git a/packages/stack/tests/whole-stack/realtime-mail.ts b/packages/stack/tests/whole-stack/realtime-mail.ts new file mode 100644 index 0000000000..8a6e25d7f3 --- /dev/null +++ b/packages/stack/tests/whole-stack/realtime-mail.ts @@ -0,0 +1,86 @@ +import { Effect, Option, Schema, Scope, Stream } from "effect"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { openWebSocket } from "./websocket.ts"; + +const MailRecipient = Schema.Struct({ Address: Schema.String }); +const MailEvent = Schema.Struct({ + Type: Schema.Literal("new"), + Data: Schema.Struct({ ID: Schema.String, To: Schema.Array(MailRecipient) }), +}); +const MailEventEnvelope = Schema.Struct({ Type: Schema.String, Data: Schema.Unknown }); + +const MailMessage = Schema.Struct({ + ID: Schema.String, + To: Schema.Array(MailRecipient), + Subject: Schema.optionalKey(Schema.String), + Text: Schema.optionalKey(Schema.String), + HTML: Schema.optionalKey(Schema.String), +}); + +type MailMessage = Schema.Schema.Type; + +class MailProbeError extends Schema.TaggedError()("MailProbeError", { + message: Schema.String, + cause: Schema.optionalKey(Schema.Unknown), +}) {} + +const errorFor = (cause: unknown) => + new MailProbeError({ + message: cause instanceof Error ? cause.message : String(cause), + cause, + }); + +const eventsUrl = (url: string) => { + const value = new URL(url); + value.protocol = value.protocol === "https:" ? "wss:" : "ws:"; + value.pathname = "/api/events"; + value.search = ""; + return value.toString(); +}; + +export interface RecoveryMailWatcher { + readonly awaitFullMail: Effect.Effect; +} + +export const watchRecoveryMail = Effect.fn("WholeStack.watchRecoveryMail")( + ( + mailUrl: string, + email: string, + ): Effect.Effect => + Effect.gen(function* () { + const channel = yield* openWebSocket(eventsUrl(mailUrl)).pipe(Effect.mapError(errorFor)); + return { + awaitFullMail: Stream.fromQueue(channel.messages).pipe( + Stream.mapError(errorFor), + Stream.mapEffect((value) => + Schema.decodeEffect(Schema.fromJsonString(MailEventEnvelope))(value).pipe( + Effect.mapError(errorFor), + ), + ), + Stream.filter((event) => event.Type === "new"), + Stream.mapEffect((event) => + Schema.decodeUnknownEffect(MailEvent)(event).pipe(Effect.mapError(errorFor)), + ), + Stream.filter((event) => event.Data.To.some((recipient) => recipient.Address === email)), + Stream.runHead, + Effect.flatMap((result) => + Option.isSome(result) + ? Effect.succeed(result.value) + : Effect.fail(new MailProbeError({ message: "Mail event stream ended" })), + ), + Effect.flatMap((event) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + const response = yield* client + .execute(HttpClientRequest.get(`${mailUrl}/api/v1/message/${event.Data.ID}`)) + .pipe(Effect.mapError(errorFor)); + return yield* response.json.pipe( + Effect.flatMap(Schema.decodeUnknownEffect(MailMessage)), + Effect.mapError(errorFor), + ); + }), + ), + ), + }; + }), +); diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts new file mode 100644 index 0000000000..9a8c93a4c4 --- /dev/null +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -0,0 +1,522 @@ +import { expect } from "@effect/vitest"; +import { Cause, Effect, FileSystem, Ref, Schema } from "effect"; +import { open } from "../../src/effect.ts"; +import { + allMembers, + clearStackOwner, + jsonRequest, + requestWithHeaders, + refreshLogTails, + sql, + serviceNames, + setActivation, + service, + setStackOwner, + statusByService, + wholeStack, + type Runtime, + type WholeStack, +} from "./fixture.ts"; +import { + assertStoredImage, + assertStoredMarker, + exerciseMetadataAndPooler, + exerciseStorageAndFunctions, +} from "./service-flows.ts"; +import { exerciseAnalytics, queryAnalyticsMarker } from "./analytics.ts"; +import { subscribeRealtime } from "./websocket.ts"; +import { watchRecoveryMail } from "./realtime-mail.ts"; +import { assertWorkloadsGone, captureWorkloads } from "./workloads.ts"; +import { assertOwnerExited, captureOwnerPid } from "../owner.ts"; + +const withFixture = ( + runtime: Runtime, + use: (fixture: WholeStack) => Effect.Effect, +) => + Effect.scoped( + Effect.gen(function* () { + const fixture = yield* wholeStack(runtime); + return yield* use(fixture); + }), + ); + +const assertDefaultPolicy = (fixture: WholeStack) => + Effect.gen(function* () { + const members = yield* allMembers(fixture.stack); + const byId = new Map(members.map((member) => [member.id, member])); + for (const name of serviceNames) { + const member = byId.get(service(fixture, name).id); + expect(member).toBeDefined(); + if (member === undefined) continue; + if (name === "database") { + expect(member.activation).toBe("eager"); + expect(member.idleMillis).toBeUndefined(); + } else if (name === "functions") { + expect(member.activation).toBe("lazy"); + expect(member.idleMillis).toBeUndefined(); + } else { + expect(member.activation).toBe("lazy"); + expect(member.idleMillis).toBe(60_000); + } + } + }); + +const assertPersistedPolicy = (fixture: WholeStack) => + Effect.gen(function* () { + const members = yield* allMembers(fixture.stack); + const byId = new Map(members.map((member) => [member.id, member])); + for (const name of serviceNames) { + const member = byId.get(service(fixture, name).id); + expect(member).toBeDefined(); + if (member === undefined) continue; + expect(member.idleMillis).toBeUndefined(); + expect(member.activation).toBe(name === "database" ? "eager" : "lazy"); + } + }); + +const assertLifecycle = (fixture: WholeStack, lifecycle: "running" | "stopped") => + Effect.gen(function* () { + const statuses = yield* statusByService(fixture); + for (const name of serviceNames) { + const status = statuses.find(([candidate]) => candidate === name)?.[1]; + expect(status?.lifecycle).toBe(lifecycle); + if (lifecycle === "running") expect(status?.health).toBe("healthy"); + } + }); + +const assertColdStart = (fixture: WholeStack) => + Effect.gen(function* () { + const statuses = yield* statusByService(fixture); + const database = statuses.find(([name]) => name === "database")?.[1]; + expect(database?.lifecycle).toBe("running"); + expect(database?.health).toBe("healthy"); + for (const name of serviceNames.filter((candidate) => candidate !== "database")) { + const status = statuses.find(([candidate]) => candidate === name)?.[1]; + expect(status?.lifecycle).toBe("stopped"); + expect(status?.wakeEnabled).toBe(true); + } + }); + +const prepareSchema = Effect.fn("WholeStack.prepareSchema")((fixture: WholeStack) => + sql( + fixture, + "create table if not exists public.whole_stack_items (id text primary key, owner_id uuid not null, value text not null); alter table public.whole_stack_items enable row level security; grant select, insert, update on public.whole_stack_items to authenticated; drop policy if exists whole_stack_owner on public.whole_stack_items; create policy whole_stack_owner on public.whole_stack_items using (auth.uid() = owner_id) with check (auth.uid() = owner_id); do $$ begin alter publication supabase_realtime add table public.whole_stack_items; exception when duplicate_object then null; end $$;", + ), +); + +const clearIdleTimers = Effect.fn("WholeStack.clearIdleTimers")((fixture: WholeStack) => + Effect.gen(function* () { + const configuration = yield* fixture.stack.composition.describe; + yield* fixture.stack.composition.configure({ + ...configuration, + members: configuration.members.map((member) => ({ + id: member.id, + activation: member.activation, + })), + }); + }), +); + +const assertOwnedPathsGone = Effect.fn("WholeStack.assertOwnedPathsGone")((fixture: WholeStack) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dataRoot = `${fixture.locations.stateRoot}/${fixture.stack.id}/data`; + const databasePath = `${dataRoot}/${service(fixture, "database").id}`; + const functionsPath = `${dataRoot}/${service(fixture, "functions").id}/runtime/functions`; + const statePath = `${fixture.locations.stateRoot}/${fixture.stack.id}/state.json`; + expect(yield* fs.exists(databasePath)).toBe(false); + expect(yield* fs.exists(functionsPath)).toBe(false); + expect(yield* fs.exists(statePath)).toBe(false); + expect(yield* fs.exists(`${fixture.locations.stateRoot}/${fixture.stack.id}`)).toBe(false); + }), +); + +const assertOwnedPathsPresent = Effect.fn("WholeStack.assertOwnedPathsPresent")( + (fixture: WholeStack) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const dataRoot = `${fixture.locations.stateRoot}/${fixture.stack.id}/data`; + const databasePath = `${dataRoot}/${service(fixture, "database").id}`; + const functionsPath = `${dataRoot}/${service(fixture, "functions").id}/runtime/functions`; + expect(yield* fs.exists(databasePath)).toBe(true); + expect(yield* fs.exists(functionsPath)).toBe(true); + }), +); + +const stopWithDiagnostics = Effect.fn("WholeStack.stopWithDiagnostics")((fixture: WholeStack) => + fixture.stack.stop.pipe( + Effect.catchCause((cause) => + Effect.gen(function* () { + const statuses = yield* statusByService(fixture).pipe( + Effect.catchCause(() => Effect.succeed([])), + ); + const logs = yield* Ref.get(fixture.logTails); + const serviceIds = serviceNames + .map((name) => `${name}=${service(fixture, name).id}`) + .join(","); + yield* Effect.logError( + `Whole-stack stop diagnostics: cause=${Cause.pretty(cause)} serviceIds=${serviceIds} statuses=${statuses + .map(([name, status]) => `${name}=${status.lifecycle}`) + .join(",")} logs=${logs.map(([name, value]) => `${name}: ${value}`).join("\n")}`, + ); + return yield* Effect.failCause(cause); + }), + ), + ), +); + +const exerciseStack = Effect.fn("WholeStack.exerciseStack")( + (fixture: WholeStack, phase: string, resourceKey?: string) => + Effect.scoped( + Effect.gen(function* () { + yield* prepareSchema(fixture); + const authUrl = (yield* service(fixture, "auth").credentials()).url; + const restUrl = (yield* service(fixture, "rest").credentials()).url; + if (authUrl === undefined || restUrl === undefined) + return yield* Effect.die("Auth/REST URL missing"); + const email = `whole-${phase}-${fixture.stack.id}@example.test`; + const password = "whole-stack-password"; + const signup = yield* jsonRequest("POST", `${authUrl}/signup`, { email, password }); + expect(signup.status).toBe(200); + const signupBody = yield* Schema.decodeUnknownEffect( + Schema.Struct({ + access_token: Schema.String, + user: Schema.Struct({ id: Schema.String }), + }), + )(yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(signup.body)); + const login = yield* jsonRequest("POST", `${authUrl}/token?grant_type=password`, { + email, + password, + }); + const loginBody = yield* Schema.decodeUnknownEffect( + Schema.Struct({ access_token: Schema.String }), + )(yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(login.body)); + const headers = { + authorization: `Bearer ${loginBody.access_token}`, + apikey: loginBody.access_token, + "content-type": "application/json", + }; + const rowId = resourceKey ?? `${fixture.stack.id}-${phase}`; + const insert = yield* jsonRequest( + "POST", + `${restUrl}/whole_stack_items`, + { id: rowId, owner_id: signupBody.user.id, value: phase }, + { ...headers, prefer: "return=representation" }, + ); + expect(insert.status).toBe(201); + const read = yield* requestWithHeaders( + `${restUrl}/whole_stack_items?id=eq.${rowId}`, + headers, + ); + expect(read.status).toBe(200); + const rows = yield* Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array(Schema.Struct({ id: Schema.String, value: Schema.String })), + ), + )(read.body); + expect(rows).toEqual([{ id: rowId, value: phase }]); + const secondEmail = `whole-other-${phase}-${fixture.stack.id}@example.test`; + const secondSignup = yield* jsonRequest("POST", `${authUrl}/signup`, { + email: secondEmail, + password, + }); + expect(secondSignup.status).toBe(200); + const secondSignupBody = yield* Schema.decodeUnknownEffect( + Schema.Struct({ access_token: Schema.String }), + )(yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(secondSignup.body)); + const otherRead = yield* requestWithHeaders(`${restUrl}/whole_stack_items?id=eq.${rowId}`, { + authorization: `Bearer ${secondSignupBody.access_token}`, + apikey: secondSignupBody.access_token, + }); + expect(otherRead.status).toBe(200); + expect( + yield* Schema.decodeEffect( + Schema.fromJsonString(Schema.Array(Schema.Struct({ id: Schema.String }))), + )(otherRead.body), + ).toEqual([]); + yield* exerciseStorageAndFunctions( + fixture, + signupBody.access_token, + rowId, + phase, + resourceKey ?? phase, + ); + yield* Effect.scoped( + Effect.gen(function* () { + const mailUrl = (yield* service(fixture, "mail").credentials()).url; + const realtimeUrl = (yield* service(fixture, "realtime").credentials()).url; + if (mailUrl === undefined || realtimeUrl === undefined) + return yield* Effect.die("Mail or Realtime URL missing"); + const watcher = yield* watchRecoveryMail(mailUrl, email); + const recovery = yield* jsonRequest("POST", `${authUrl}/recover`, { email }); + expect(recovery.status).toBe(200); + const mail = yield* watcher.awaitFullMail; + expect(mail.To.map((recipient) => recipient.Address)).toContain(email); + const authOrigin = new URL(authUrl).origin; + const mailBody = `${mail.Text ?? ""}\n${mail.HTML ?? ""}`; + expect(mailBody).toContain(authOrigin); + expect(mailBody).toContain("/verify?token="); + const realtime = yield* subscribeRealtime( + realtimeUrl, + loginBody.access_token, + "whole_stack_items", + "UPDATE", + ); + yield* exerciseMetadataAndPooler(fixture, loginBody.access_token, rowId, phase); + const poolerChange = yield* realtime.nextChange; + const poolerEvent = yield* Schema.decodeUnknownEffect( + Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), + )(poolerChange.payload.data); + expect(poolerEvent.record).toEqual({ id: rowId, value: `${phase}-pooler` }); + const update = yield* jsonRequest( + "PATCH", + `${restUrl}/whole_stack_items?id=eq.${rowId}`, + { value: `${phase}-realtime` }, + { ...headers, prefer: "return=representation" }, + ); + expect(update.status).toBe(200); + const change = yield* realtime.nextChange; + const event = yield* Schema.decodeUnknownEffect( + Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), + )(change.payload.data); + expect(event.record).toEqual({ id: rowId, value: `${phase}-realtime` }); + }), + ); + const analyticsUrl = (yield* service(fixture, "analytics").credentials()).url; + const vectorUrl = (yield* service(fixture, "vector").credentials()).url; + if (analyticsUrl === undefined || vectorUrl === undefined) + return yield* Effect.die("Analytics or Vector URL missing"); + yield* exerciseAnalytics(analyticsUrl, vectorUrl, fixture.secret, fixture.stack.id, phase); + yield* assertLifecycle(fixture, "running"); + return { + accessToken: loginBody.access_token, + rowId, + value: `${phase}-realtime`, + credentials: yield* Effect.forEach(serviceNames, (name) => + service(fixture, name).credentials(), + ), + }; + }), + ), +); + +export const defaultLifecycle = (runtime: Runtime) => + withFixture(runtime, (fixture) => + Effect.gen(function* () { + yield* assertDefaultPolicy(fixture); + yield* clearIdleTimers(fixture); + yield* fixture.stack.composition.start; + yield* assertColdStart(fixture); + yield* prepareSchema(fixture); + const coldWorkloads = yield* captureWorkloads(runtime, fixture); + expect(coldWorkloads.identities.length).toBeGreaterThan(0); + const databaseMarker = service(fixture, "database").id; + for (const identity of coldWorkloads.identities) + expect( + runtime === "native" + ? identity.includes(`supabase-workload-id=${databaseMarker}`) + : identity.endsWith(databaseMarker), + ).toBe(true); + const before = yield* exerciseStack(fixture, "default"); + const workloads = yield* captureWorkloads(runtime, fixture); + const ownerPid = yield* captureOwnerPid(fixture.locations, fixture.stack.id); + yield* stopWithDiagnostics(fixture); + yield* assertOwnerExited(ownerPid); + yield* assertWorkloadsGone(runtime, fixture, workloads); + const reopenedStack = yield* open({ ...fixture.locations, id: fixture.stack.id }); + yield* setStackOwner(fixture, reopenedStack); + const reopened = { + ...fixture, + stack: reopenedStack, + services: yield* reopenedStack.services.list, + }; + yield* refreshLogTails(reopened); + yield* assertPersistedPolicy(reopened); + yield* clearIdleTimers(reopened); + yield* reopened.stack.composition.start; + yield* assertColdStart(reopened); + const reopenedColdWorkloads = yield* captureWorkloads(runtime, reopened); + expect(reopenedColdWorkloads.identities.length).toBeGreaterThan(0); + for (const identity of reopenedColdWorkloads.identities) + expect( + runtime === "native" + ? identity.includes(`supabase-workload-id=${databaseMarker}`) + : identity.endsWith(databaseMarker), + ).toBe(true); + expect( + yield* sql( + reopened, + `select id from public.whole_stack_items where id = '${fixture.stack.id}-default'`, + ), + ).toBe(`${fixture.stack.id}-default`); + yield* assertStoredImage(reopened, "default"); + const after = yield* exerciseStack(reopened, "reopened"); + expect(after.credentials).toEqual(before.credentials); + yield* assertOwnedPathsPresent(reopened); + const reopenedOwnerPid = yield* captureOwnerPid(reopened.locations, reopened.stack.id); + yield* reopened.stack.destroy; + yield* assertOwnerExited(reopenedOwnerPid); + yield* clearStackOwner(reopened); + yield* assertOwnedPathsGone(reopened); + }), + ); + +export const allEager = (runtime: Runtime) => + withFixture(runtime, (fixture) => + Effect.gen(function* () { + const preStartWorkloads = yield* captureWorkloads(runtime, fixture); + expect(preStartWorkloads.identities).toEqual([]); + yield* fixture.stack.composition.start; + yield* prepareSchema(fixture); + const stoppedWorkloads = yield* captureWorkloads(runtime, fixture); + yield* fixture.stack.composition.stop; + yield* assertWorkloadsGone(runtime, fixture, stoppedWorkloads); + yield* setActivation(fixture.stack, "eager"); + yield* fixture.stack.composition.start; + yield* assertLifecycle(fixture, "running"); + const eagerWorkloads = yield* captureWorkloads(runtime, fixture); + expect(eagerWorkloads.identities.length).toBeGreaterThanOrEqual(serviceNames.length); + yield* exerciseStack(fixture, "eager"); + const finalWorkloads = yield* captureWorkloads(runtime, fixture); + yield* fixture.stack.composition.stop; + yield* assertLifecycle(fixture, "stopped"); + yield* assertWorkloadsGone(runtime, fixture, finalWorkloads); + yield* assertOwnedPathsPresent(fixture); + yield* fixture.stack.destroy; + yield* clearStackOwner(fixture); + yield* assertOwnedPathsGone(fixture); + }), + ); + +export const parallel = (runtime: Runtime) => + Effect.scoped( + Effect.gen(function* () { + const left = yield* wholeStack(runtime); + const right = yield* wholeStack(runtime); + yield* Effect.all([assertDefaultPolicy(left), assertDefaultPolicy(right)], { + concurrency: "unbounded", + }); + yield* Effect.all([clearIdleTimers(left), clearIdleTimers(right)], { + concurrency: "unbounded", + }); + yield* Effect.all([left.stack.composition.start, right.stack.composition.start], { + concurrency: "unbounded", + }); + expect(left.stack.id).not.toBe(right.stack.id); + const [leftLedger, rightLedger] = yield* Effect.all( + [ + exerciseStack(left, "parallel-left", "parallel-shared"), + exerciseStack(right, "parallel-right", "parallel-shared"), + ], + { concurrency: "unbounded" }, + ); + expect(leftLedger.rowId).toBe("parallel-shared"); + expect(rightLedger.rowId).toBe(leftLedger.rowId); + expect(leftLedger.value).not.toBe(rightLedger.value); + expect( + yield* sql(left, "select value from public.whole_stack_items where id = 'parallel-shared'"), + ).toBe(leftLedger.value); + expect( + yield* sql( + right, + "select value from public.whole_stack_items where id = 'parallel-shared'", + ), + ).toBe(rightLedger.value); + const leftEndpoints = leftLedger.credentials.flatMap((credentials) => + credentials.url === undefined ? [] : [credentials.url], + ); + const rightEndpoints = rightLedger.credentials.flatMap((credentials) => + credentials.url === undefined ? [] : [credentials.url], + ); + expect(new Set(leftEndpoints).size).toBe(leftEndpoints.length); + expect(new Set(rightEndpoints).size).toBe(rightEndpoints.length); + expect(new Set([...leftEndpoints, ...rightEndpoints]).size).toBe( + leftEndpoints.length + rightEndpoints.length, + ); + const leftAnalyticsUrl = (yield* service(left, "analytics").credentials()).url; + const rightAnalyticsUrl = (yield* service(right, "analytics").credentials()).url; + if (leftAnalyticsUrl === undefined || rightAnalyticsUrl === undefined) + return yield* Effect.die("Parallel Analytics URL missing"); + expect( + (yield* queryAnalyticsMarker(leftAnalyticsUrl, left.secret, `vector-${right.stack.id}`, 0)) + .length, + ).toBe(0); + expect( + (yield* queryAnalyticsMarker(rightAnalyticsUrl, right.secret, `vector-${left.stack.id}`, 0)) + .length, + ).toBe(0); + const rightRest = service(right, "rest"); + const rightRestUrl = (yield* rightRest.credentials()).url; + const rightRealtimeUrl = (yield* service(right, "realtime").credentials()).url; + if (rightRestUrl === undefined || rightRealtimeUrl === undefined) + return yield* Effect.die("Parallel REST or Realtime URL missing"); + const crossStack = yield* requestWithHeaders(`${rightRestUrl}/whole_stack_items`, { + authorization: `Bearer ${leftLedger.accessToken}`, + apikey: leftLedger.accessToken, + }); + expect(crossStack.status).toBe(401); + yield* Effect.scoped( + Effect.gen(function* () { + const rightRealtime = yield* subscribeRealtime( + rightRealtimeUrl, + rightLedger.accessToken, + "whole_stack_items", + "UPDATE", + ); + yield* stopWithDiagnostics(left); + const reopenedLeft = yield* open({ ...left.locations, id: left.stack.id }); + yield* setStackOwner(left, reopenedLeft); + const reopenedLeftFixture = { + ...left, + stack: reopenedLeft, + services: yield* reopenedLeft.services.list, + }; + yield* refreshLogTails(reopenedLeftFixture); + yield* clearIdleTimers(reopenedLeftFixture); + yield* reopenedLeft.composition.start; + expect( + yield* sql( + reopenedLeftFixture, + `select id from public.whole_stack_items where id = 'parallel-shared'`, + ), + ).toBe("parallel-shared"); + yield* assertStoredImage(reopenedLeftFixture, "parallel-shared"); + const reopenedLedger = yield* exerciseStack( + reopenedLeftFixture, + "parallel-left-reopened", + ); + expect(reopenedLedger.credentials).toEqual(leftLedger.credentials); + const reopenedWorkloads = yield* captureWorkloads(runtime, reopenedLeftFixture); + yield* assertOwnedPathsPresent(reopenedLeftFixture); + const ownerPid = yield* captureOwnerPid(reopenedLeftFixture.locations, reopenedLeft.id); + yield* reopenedLeft.destroy; + yield* assertOwnerExited(ownerPid); + yield* clearStackOwner(left); + yield* assertWorkloadsGone(runtime, reopenedLeftFixture, reopenedWorkloads, true); + yield* assertOwnedPathsGone(reopenedLeftFixture); + expect((yield* rightRest.status).lifecycle).toBe("running"); + yield* assertStoredMarker(right, "parallel-shared"); + const update = yield* jsonRequest( + "PATCH", + `${rightRestUrl}/whole_stack_items?id=eq.${rightLedger.rowId}`, + { value: "parallel-right-survived" }, + { + authorization: `Bearer ${rightLedger.accessToken}`, + apikey: rightLedger.accessToken, + prefer: "return=representation", + }, + ); + expect(update.status).toBe(200); + const change = yield* rightRealtime.nextChange; + const event = yield* Schema.decodeUnknownEffect( + Schema.Struct({ record: Schema.Struct({ id: Schema.String, value: Schema.String }) }), + )(change.payload.data); + expect(event.record).toEqual({ + id: rightLedger.rowId, + value: "parallel-right-survived", + }); + }), + ); + yield* exerciseStack(right, "parallel-right-after"); + }), + ); diff --git a/packages/stack/tests/whole-stack/service-flows.ts b/packages/stack/tests/whole-stack/service-flows.ts new file mode 100644 index 0000000000..df884e15d7 --- /dev/null +++ b/packages/stack/tests/whole-stack/service-flows.ts @@ -0,0 +1,292 @@ +import { expect } from "@effect/vitest"; +import { Context, Effect, Layer, Redacted, Schema } from "effect"; +import { PgClient } from "@effect/sql-pg"; +import { HttpClient, HttpClientRequest } from "effect/unstable/http"; +import { SignJWT } from "jose"; +import { requestWithHeaders, service, type WholeStack } from "./fixture.ts"; + +const image = Uint8Array.from( + atob( + "iVBORw0KGgoAAAANSUhEUgAAAAgAAAAECAIAAAA8r+mnAAAAQUlEQVR4nBWLURUAQAiDlsQkS2ISkphkSUx0J5/wkEQJixaIERErJFPGpg1mTMz6B6gzNMdAYP+sUMGhc25CwoYHii4nYbjsDOUAAAAASUVORK5CYII=", + ), + (value) => value.charCodeAt(0), +); + +const serviceToken = Effect.fn("WholeStack.serviceToken")((fixture: WholeStack) => + Effect.tryPromise(() => + new SignJWT({ role: "service_role" }) + .setProtectedHeader({ alg: "HS256", typ: "JWT" }) + .setSubject("whole-stack-service") + .setIssuedAt() + .setExpirationTime("1h") + .sign(new TextEncoder().encode(fixture.secret)), + ), +); + +const execute = Effect.fn("WholeStack.serviceFlowRequest")( + (request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + const client = yield* HttpClient.HttpClient; + return yield* client.execute(request); + }), +); + +export const exerciseStorageAndFunctions = Effect.fn("WholeStack.exerciseStorageAndFunctions")( + (fixture: WholeStack, accessToken: string, rowId: string, expectedValue: string, phase: string) => + Effect.gen(function* () { + const storageUrl = (yield* service(fixture, "storage").credentials()).url; + const functionsUrl = (yield* service(fixture, "functions").credentials()).url; + if (storageUrl === undefined || functionsUrl === undefined) + return yield* Effect.die("Storage or Functions URL missing"); + const token = yield* serviceToken(fixture); + const auth = { + authorization: `Bearer ${token}`, + apikey: token, + }; + const bucketName = `whole-stack-${phase}`; + const objectName = `${bucketName}/source.png`; + const bucketRequest = yield* HttpClientRequest.bodyJson({ name: bucketName, public: true })( + HttpClientRequest.setHeader( + "apikey", + auth.apikey, + )( + HttpClientRequest.setHeader( + "authorization", + auth.authorization, + )(HttpClientRequest.post(`${storageUrl}/bucket`)), + ), + ); + const bucket = yield* execute(bucketRequest); + if (bucket.status >= 400) return yield* Effect.die(`Storage bucket failed: ${bucket.status}`); + const uploadRequest = HttpClientRequest.bodyUint8Array( + HttpClientRequest.setHeader( + "apikey", + auth.apikey, + )( + HttpClientRequest.setHeader( + "authorization", + auth.authorization, + )(HttpClientRequest.post(`${storageUrl}/object/${objectName}`)), + ), + image, + "image/png", + ); + const upload = yield* execute(uploadRequest); + if (upload.status >= 400) return yield* Effect.die(`Storage upload failed: ${upload.status}`); + const download = yield* execute( + HttpClientRequest.get(`${storageUrl}/object/public/${objectName}`).pipe( + HttpClientRequest.setHeader("authorization", auth.authorization), + HttpClientRequest.setHeader("apikey", auth.apikey), + ), + ); + if (download.status !== 200) + return yield* Effect.die(`Storage download failed: ${download.status}`); + const downloaded = new Uint8Array(yield* download.arrayBuffer); + if ( + downloaded.length !== image.length || + downloaded.some((value, index) => value !== image[index]) + ) + return yield* Effect.die("Storage download differs from uploaded bytes"); + const transformed = yield* execute( + HttpClientRequest.get( + `${storageUrl}/render/image/public/${objectName}?width=4&height=2&format=origin`, + ).pipe( + HttpClientRequest.setHeader("authorization", auth.authorization), + HttpClientRequest.setHeader("apikey", auth.apikey), + ), + ); + if (transformed.status !== 200 || !transformed.headers["content-type"]?.includes("image/")) + return yield* Effect.die(`Imgproxy transform failed: ${transformed.status}`); + const transformedBytes = new Uint8Array(yield* transformed.arrayBuffer); + const view = new DataView( + transformedBytes.buffer, + transformedBytes.byteOffset, + transformedBytes.byteLength, + ); + if (transformedBytes.length < 24 || view.getUint32(16) !== 4 || view.getUint32(20) !== 2) + return yield* Effect.die("Imgproxy did not return a 4x2 PNG"); + const markerName = `${bucketName}/marker.txt`; + const marker = new TextEncoder().encode(fixture.stack.id); + const markerUpload = yield* execute( + HttpClientRequest.bodyUint8Array( + HttpClientRequest.setHeader( + "apikey", + auth.apikey, + )( + HttpClientRequest.setHeader( + "authorization", + auth.authorization, + )(HttpClientRequest.post(`${storageUrl}/object/${markerName}`)), + ), + marker, + "text/plain", + ), + ); + if (markerUpload.status >= 400) + return yield* Effect.die(`Storage marker upload failed: ${markerUpload.status}`); + const markerDownload = yield* execute( + HttpClientRequest.get(`${storageUrl}/object/public/${markerName}`).pipe( + HttpClientRequest.setHeader("authorization", auth.authorization), + HttpClientRequest.setHeader("apikey", auth.apikey), + ), + ); + if (markerDownload.status !== 200 || (yield* markerDownload.text) !== fixture.stack.id) + return yield* Effect.die("Storage marker differs from stack identity"); + const functionRequest = yield* HttpClientRequest.bodyJson({ id: rowId })( + HttpClientRequest.setHeader( + "content-type", + "application/json", + )( + HttpClientRequest.setHeader( + "authorization", + `Bearer ${accessToken}`, + )(HttpClientRequest.post(`${functionsUrl}/hello`)), + ), + ); + const functionResponse = yield* execute(functionRequest); + if (functionResponse.status !== 200) + return yield* Effect.die( + `Function failed at ${functionsUrl}/hello: ${functionResponse.status}: ${yield* functionResponse.text}`, + ); + const functionBody = yield* Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array(Schema.Struct({ id: Schema.String, value: Schema.String })), + ), + )(yield* functionResponse.text); + if ( + functionBody.length !== 1 || + functionBody[0]?.id !== rowId || + functionBody[0]?.value !== expectedValue + ) + return yield* Effect.die("Function did not return the managed REST row"); + }), +); + +export const assertStoredImage = Effect.fn("WholeStack.assertStoredImage")( + (fixture: WholeStack, phase: string) => + Effect.gen(function* () { + const storageUrl = (yield* service(fixture, "storage").credentials()).url; + if (storageUrl === undefined) return yield* Effect.die("Storage URL missing"); + const token = yield* serviceToken(fixture); + const objectName = `whole-stack-${phase}/source.png`; + const response = yield* execute( + HttpClientRequest.get(`${storageUrl}/object/public/${objectName}`).pipe( + HttpClientRequest.setHeader("authorization", `Bearer ${token}`), + HttpClientRequest.setHeader("apikey", token), + ), + ); + if (response.status !== 200) + return yield* Effect.die(`Stored image read failed: ${response.status}`); + const downloaded = new Uint8Array(yield* response.arrayBuffer); + if ( + downloaded.length !== image.length || + downloaded.some((value, index) => value !== image[index]) + ) + return yield* Effect.die("Stored image differs from original bytes"); + }), +); + +export const assertStoredMarker = Effect.fn("WholeStack.assertStoredMarker")( + (fixture: WholeStack, phase: string) => + Effect.gen(function* () { + const storageUrl = (yield* service(fixture, "storage").credentials()).url; + if (storageUrl === undefined) return yield* Effect.die("Storage URL missing"); + const token = yield* serviceToken(fixture); + const response = yield* execute( + HttpClientRequest.get(`${storageUrl}/object/public/whole-stack-${phase}/marker.txt`).pipe( + HttpClientRequest.setHeader("authorization", `Bearer ${token}`), + HttpClientRequest.setHeader("apikey", token), + ), + ); + if (response.status !== 200 || (yield* response.text) !== fixture.stack.id) + return yield* Effect.die("Stored marker differs from stack identity"); + }), +); + +export const exerciseMetadataAndPooler = Effect.fn("WholeStack.exerciseMetadataAndPooler")( + (fixture: WholeStack, token: string, rowId: string, expectedValue: string) => + Effect.gen(function* () { + const pgmetaUrl = (yield* service(fixture, "pgmeta").credentials()).url; + const studioUrl = (yield* service(fixture, "studio").credentials()).url; + const restUrl = (yield* service(fixture, "rest").credentials()).url; + const databaseSql = (yield* service(fixture, "database").credentials({ from: "host" })) + .databaseUrl; + const poolerSql = (yield* service(fixture, "pooler").credentials()).sqlUrl; + if ( + pgmetaUrl === undefined || + studioUrl === undefined || + restUrl === undefined || + databaseSql === undefined || + poolerSql === undefined + ) + return yield* Effect.die("Metadata or Pooler credentials missing"); + const metadata = yield* execute( + HttpClientRequest.get(`${pgmetaUrl}/tables?included_schemas=public`), + ); + const metadataBody = yield* metadata.text; + if (metadata.status !== 200 || !metadataBody.includes("whole_stack_items")) + return yield* Effect.die(`Pgmeta did not return the fixture table: ${metadata.status}`); + const studioRequest = HttpClientRequest.setHeader( + "authorization", + `Bearer ${token}`, + )(HttpClientRequest.get(`${studioUrl}/api/platform/pg-meta/default/tables`)); + const studio = yield* execute(studioRequest); + const studioBody = yield* studio.text; + if (studio.status >= 400 || !studioBody.includes("whole_stack_items")) + return yield* Effect.die( + `Studio metadata did not return the fixture table: ${studio.status}: ${studioBody.slice(0, 1000)}`, + ); + const databaseAddress = new URL(databaseSql); + const databaseLayer = yield* Layer.build( + PgClient.layer({ + host: databaseAddress.hostname, + port: Number(databaseAddress.port), + database: databaseAddress.pathname.slice(1) || "postgres", + username: decodeURIComponent(databaseAddress.username) || "supabase_admin", + password: Redacted.make(decodeURIComponent(databaseAddress.password) || "postgres"), + connectTimeout: "120 seconds", + }), + ); + const database = Context.get(databaseLayer, PgClient.PgClient); + const databaseRows = + yield* database`select id, value from public.whole_stack_items where id = ${rowId}`; + if ( + databaseRows.length !== 1 || + databaseRows[0]?.id !== rowId || + databaseRows[0]?.value !== expectedValue + ) + return yield* Effect.die( + `Public database SQL did not return the fixture row at ${databaseAddress.hostname}:${databaseAddress.port}/${databaseAddress.pathname}: rows=${databaseRows.map((row) => `${String(row.id)}=${String(row.value)}`).join(",")}`, + ); + const poolerAddress = new URL(poolerSql); + const poolerLayer = yield* Layer.build( + PgClient.layer({ + host: poolerAddress.hostname, + port: Number(poolerAddress.port), + database: "postgres", + username: "supabase_admin.whole", + password: Redacted.make("postgres"), + connectTimeout: "120 seconds", + }), + ); + const pooler = Context.get(poolerLayer, PgClient.PgClient); + const poolerValue = `${expectedValue}-pooler`; + yield* pooler`update public.whole_stack_items set value = ${poolerValue} where id = ${rowId}`; + const rows = + yield* pooler`select id, value from public.whole_stack_items where id = ${rowId}`; + if (rows.length !== 1 || rows[0]?.id !== rowId || rows[0]?.value !== poolerValue) + return yield* Effect.die("Pooler did not read back the fixture row"); + const rest = yield* requestWithHeaders(`${restUrl}/whole_stack_items?id=eq.${rowId}`, { + authorization: `Bearer ${token}`, + apikey: token, + }); + expect(rest.status).toBe(200); + const restRows = yield* Schema.decodeEffect( + Schema.fromJsonString( + Schema.Array(Schema.Struct({ id: Schema.String, value: Schema.String })), + ), + )(rest.body); + if (restRows.length !== 1 || restRows[0]?.id !== rowId || restRows[0]?.value !== poolerValue) + return yield* Effect.die("REST did not observe the Pooler write"); + }), +); diff --git a/packages/stack/tests/whole-stack/websocket.ts b/packages/stack/tests/whole-stack/websocket.ts new file mode 100644 index 0000000000..744cb6018c --- /dev/null +++ b/packages/stack/tests/whole-stack/websocket.ts @@ -0,0 +1,199 @@ +import { Deferred, Effect, Queue, Schedule, Schema, Scope } from "effect"; +import { NodeSocket } from "@effect/platform-node"; +import * as Socket from "effect/unstable/socket/Socket"; + +class RealtimeProbeError extends Schema.TaggedError()("RealtimeProbeError", { + message: Schema.String, + cause: Schema.optionalKey(Schema.Unknown), +}) {} + +export class WebSocketChannelError extends Schema.TaggedError()( + "WebSocketChannelError", + { message: Schema.String, cause: Schema.optionalKey(Schema.Unknown) }, +) {} + +export interface WebSocketChannel { + readonly messages: Queue.Queue; + readonly send: (message: string) => Effect.Effect; +} + +const RealtimeMessage = Schema.Struct({ + event: Schema.String, + payload: Schema.Unknown, + topic: Schema.optionalKey(Schema.String), +}); +type RealtimeMessage = Schema.Schema.Type; + +const RealtimeChange = Schema.Struct({ + event: Schema.Literal("postgres_changes"), + payload: Schema.Struct({ data: Schema.Unknown }), + topic: Schema.String, +}); + +type RealtimeChange = Schema.Schema.Type; + +interface RealtimeSubscription { + readonly nextChange: Effect.Effect; +} + +const errorFor = (cause: unknown) => + new RealtimeProbeError({ + message: String(cause), + cause, + }); + +const channelError = (cause: unknown) => + new WebSocketChannelError({ + message: String(cause), + cause, + }); + +export const openWebSocket = Effect.fn("WholeStack.openWebSocket")( + (url: string): Effect.Effect => + Effect.gen(function* () { + const messages = yield* Queue.unbounded(); + const ready = yield* Deferred.make(); + const socket = yield* Socket.makeWebSocket(url, { openTimeout: "60 seconds" }).pipe( + Effect.provide(NodeSocket.layerWebSocketConstructorWS), + ); + yield* socket + .runString((message) => Queue.offer(messages, message), { + onOpen: Deferred.succeed(ready, undefined), + }) + .pipe( + Effect.catch((cause: Socket.SocketError) => + Effect.all([ + Deferred.fail(ready, cause), + Queue.fail(messages, channelError(cause)), + ]).pipe(Effect.asVoid), + ), + Effect.andThen( + Queue.fail(messages, new WebSocketChannelError({ message: "WebSocket closed" })).pipe( + Effect.asVoid, + ), + ), + Effect.forkScoped, + ); + yield* Deferred.await(ready).pipe(Effect.mapError(channelError)); + const writer = yield* socket.writer; + return { + messages, + send: (message) => writer(message).pipe(Effect.mapError(channelError)), + }; + }), +); + +const websocketUrl = (url: string, token: string) => { + const value = new URL(url); + value.protocol = value.protocol === "https:" ? "wss:" : "ws:"; + value.pathname = "/realtime/v1/websocket"; + value.search = new URLSearchParams({ apikey: token, vsn: "1.0.0" }).toString(); + return value.toString(); +}; + +const decodeMessage = (value: string) => + Schema.decodeEffect(Schema.fromJsonString(RealtimeMessage))(value).pipe( + Effect.mapError(errorFor), + ); + +const waitsFor = ( + messages: Queue.Queue, + predicate: (message: RealtimeMessage) => message is A, +) => + Effect.gen(function* () { + while (true) { + const message = yield* Queue.take(messages).pipe( + Effect.mapError(errorFor), + Effect.flatMap(decodeMessage), + ); + if (predicate(message)) return message; + } + }); + +const waitForHandshake = ( + messages: Queue.Queue, +): Effect.Effect => + Effect.gen(function* () { + let joined = false; + let subscribed = false; + while (!joined || !subscribed) { + const message = yield* Queue.take(messages).pipe( + Effect.mapError(errorFor), + Effect.flatMap(decodeMessage), + ); + if ( + message.event === "phx_reply" && + typeof message.payload === "object" && + message.payload !== null + ) + joined = "status" in message.payload && message.payload.status === "ok"; + if ( + message.event === "system" && + typeof message.payload === "object" && + message.payload !== null + ) + subscribed = + "status" in message.payload && + message.payload.status === "ok" && + "extension" in message.payload && + message.payload.extension === "postgres_changes"; + } + }); + +const isRealtimeChange = (message: RealtimeMessage): message is RealtimeChange => + Schema.is(RealtimeChange)(message); + +export const subscribeRealtime = Effect.fn("WholeStack.subscribeRealtime")( + ( + url: string, + token: string, + table: string, + event: "INSERT" | "UPDATE", + ): Effect.Effect => + Effect.gen(function* () { + const channel = yield* openWebSocket(websocketUrl(url, token)).pipe( + Effect.mapError(errorFor), + ); + const topic = `realtime:public:${table}`; + const joinMessage = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ + topic, + event: "phx_join", + payload: { + config: { + broadcast: { ack: false, self: false }, + presence: { key: "" }, + postgres_changes: [{ event, schema: "public", table }], + }, + access_token: token, + }, + ref: "1", + }).pipe(Effect.mapError(errorFor)); + yield* channel.send(joinMessage).pipe(Effect.mapError(errorFor)); + yield* waitForHandshake(channel.messages).pipe( + Effect.timeout("30 seconds"), + Effect.mapError(errorFor), + ); + const heartbeat = yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ + topic: "phoenix", + event: "heartbeat", + payload: {}, + ref: "heartbeat", + }).pipe(Effect.mapError(errorFor)); + yield* channel + .send(heartbeat) + .pipe( + Effect.mapError(errorFor), + Effect.repeat(Schedule.spaced("20 seconds")), + Effect.forkScoped, + ); + return { + nextChange: waitsFor(channel.messages, isRealtimeChange).pipe( + Effect.flatMap((message) => + Schema.decodeEffect(RealtimeChange)(message).pipe(Effect.mapError(errorFor)), + ), + Effect.timeout("30 seconds"), + Effect.mapError(errorFor), + ), + }; + }), +); diff --git a/packages/stack/tests/whole-stack/workloads.ts b/packages/stack/tests/whole-stack/workloads.ts new file mode 100644 index 0000000000..2193493cfa --- /dev/null +++ b/packages/stack/tests/whole-stack/workloads.ts @@ -0,0 +1,68 @@ +import { expect } from "@effect/vitest"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { Effect, Schema } from "effect"; +import { service, type Runtime, type WholeStack } from "./fixture.ts"; + +export const WorkloadSnapshot = Schema.Struct({ + identities: Schema.Array(Schema.String), + markers: Schema.Array(Schema.String), +}); +export type WorkloadSnapshot = Schema.Schema.Type; + +const commandOutput = Effect.fn("WholeStack.commandOutput")( + (command: string, args: ReadonlyArray) => + Effect.gen(function* () { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + return yield* spawner.string( + ChildProcess.make(command, args, { stdout: "pipe", stderr: "pipe" }), + ); + }), +); + +const snapshotOutput = (runtime: Runtime, fixture: WholeStack, includeStopped: boolean) => + Effect.gen(function* () { + if (runtime === "native") { + const stackMarker = `supabase-stack-id=${fixture.stack.id}`; + const databaseMarker = `supabase-workload-id=${service(fixture, "database").id}`; + const output = yield* commandOutput("ps", ["-axo", "pid=,command="]); + const identities = output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.includes(stackMarker) || line.includes(databaseMarker)); + return { identities, markers: [stackMarker, databaseMarker] } satisfies WorkloadSnapshot; + } + const marker = `com.supabase.stack=${fixture.stack.id}`; + const output = yield* commandOutput("docker", [ + "ps", + ...(includeStopped ? ["--all"] : []), + "--filter", + `label=${marker}`, + "--format", + '{{.ID}} {{.Label "com.supabase.instance"}}', + ]); + return { + identities: output + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0), + markers: [marker], + } satisfies WorkloadSnapshot; + }); + +export const captureWorkloads = Effect.fn("WholeStack.captureWorkloads")( + (runtime: Runtime, fixture: WholeStack, includeStopped = false) => + snapshotOutput(runtime, fixture, includeStopped), +); + +export const assertWorkloadsGone = Effect.fn("WholeStack.assertWorkloadsGone")( + (runtime: Runtime, fixture: WholeStack, snapshot: WorkloadSnapshot, includeStopped = false) => + Effect.gen(function* () { + expect(snapshot.identities.length).toBeGreaterThan(0); + const current = yield* snapshotOutput(runtime, fixture, includeStopped); + if (runtime === "docker") expect(current.identities).toEqual([]); + for (const marker of snapshot.markers) + expect(current.identities.some((identity) => identity.includes(marker))).toBe(false); + for (const identity of snapshot.identities) + expect(current.identities).not.toContain(identity); + }), +); diff --git a/packages/stack/tsconfig.json b/packages/stack/tsconfig.json index ba396eb057..ad72c13ea1 100644 --- a/packages/stack/tsconfig.json +++ b/packages/stack/tsconfig.json @@ -1,3 +1,4 @@ { - "extends": "@tsconfig/bun/tsconfig.json" + "extends": "@tsconfig/bun/tsconfig.json", + "exclude": ["node_modules", "tmp"] }