From 6907ccdf6ac1f9911ca003d842056b75b774490c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 08:41:30 +0000 Subject: [PATCH 1/9] ci(repo): upload slim natives to S3 without waiting on the ECR mirror `upload-natives-s3` needed the `mirror` job to succeed, so any ECR Public failure (missing `cli/*` repositories or push permissions) also kept every native out of the S3 bucket. The job now runs alongside the ECR copy and verifies the source image digest itself, so nothing reaches the bucket from a dispatch whose claimed image digest does not match GHCR. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- .github/workflows/mirror-slim-image.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/mirror-slim-image.yml b/.github/workflows/mirror-slim-image.yml index 482290062b..4e9584f160 100644 --- a/.github/workflows/mirror-slim-image.yml +++ b/.github/workflows/mirror-slim-image.yml @@ -11,11 +11,12 @@ name: Mirror Slim Image # `.github/scripts/mirror-slim-image.ts` rather than trusted from the payload, and the source # must resolve to the claimed digest before anything is copied. # -# `upload-natives-s3` then copies the same native triplets (fetched from GHCR by digest and +# `upload-natives-s3` copies the same native triplets (fetched from GHCR by digest and # re-verified) to the public-read S3 bucket in the cli-artifacts-prod account, for sandboxes that -# allow `*.amazonaws.com` but block registry blob CDNs. It runs only after the image mirror and its -# digest checks succeeded, is best-effort like the native ECR copy, and uses the PutObject-only -# `slim-artifacts-publisher` role (infra/cli-artifacts/README.md). +# allow `*.amazonaws.com` but block registry blob CDNs. It runs in parallel with the ECR copy so an +# ECR Public outage or missing permission never holds back the S3 mirror, is best-effort like the +# native ECR copy, and uses the PutObject-only `slim-artifacts-publisher` role +# (infra/cli-artifacts/README.md). # # Replay locally: `bun .github/scripts/mirror-slim-image.ts --help` # Full contract: docs/design/ecr-mirror-dispatch.md in supabase/slim-services. @@ -145,12 +146,10 @@ jobs: run: bun .github/scripts/mirror-slim-image.ts copy-natives upload-natives-s3: - needs: mirror # Natives only ride in repository_dispatch payloads. Nothing reaches the public bucket from a - # dispatch whose image digest check failed; a release whose mirror failed for other reasons is - # re-dispatched by the sender's backfill. The job is skipped until the cli-artifacts-prod - # account exists and its ID is set as a repository variable. - if: ${{ success() && github.event_name == 'repository_dispatch' && vars.CLI_ARTIFACTS_AWS_ACCOUNT_ID != '' }} + # dispatch whose source image digest check fails. The job is skipped until the + # cli-artifacts-prod account exists and its ID is set as a repository variable. + if: ${{ github.event_name == 'repository_dispatch' && vars.CLI_ARTIFACTS_AWS_ACCOUNT_ID != '' }} runs-on: ubuntu-latest timeout-minutes: 15 continue-on-error: true @@ -196,6 +195,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Verify source digest + env: + REFERENCE: ${{ steps.validate.outputs.source }} + DIGEST: ${{ steps.validate.outputs.digest }} + run: bun .github/scripts/mirror-slim-image.ts verify-digest + - name: Fetch and verify native artifacts id: fetch env: From ced2c01454dc17c5c21ed8538f15814c083472a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 08:46:24 +0000 Subject: [PATCH 2/9] feat(stack): fall back to the S3 mirror for slim native artifacts Hosts that block GitHub release downloads, such as agent sandboxes that only allow `*.amazonaws.com`, could not prepare native service artifacts. Each artifact now lists its hosts in order, the GitHub Release first and the public S3 bucket second. The checksum comes from the first host that serves one, and the archive from the first host whose copy hashes to that checksum, so falling back never accepts different bytes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- docs/adr/0026-slim-artifact-mirrors.md | 4 +- packages/stack/src/Artifacts.ts | 25 ++- .../src/preparation/SlimServicesSource.ts | 184 +++++++++++------- .../slim-services.integration.test.ts | 173 +++++++++++++++- 4 files changed, 302 insertions(+), 84 deletions(-) diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 59fa5be703..a18402e62d 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -22,7 +22,7 @@ Client fail-through (ordered GHCR / ECR / GitHub candidates in the CLI and stack ## Follow-up - Consume the mirrored image tags and `:version-native-` artifacts from the CLI and local stack, with host-aware order and digest pins preserved ([ADR 0017](0017-simplified-managed-stack-architecture.md)). -- Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Later options: S3 HTTPS on `*.amazonaws.com`, or Anthropic adding blob hosts. +- Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Natives are therefore also copied to a public S3 bucket on `*.amazonaws.com` ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)), independently of the ECR copy. The local stack tries the GitHub Release first and falls back to that bucket, accepting an archive from either host only when it matches the checksum. - Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. ## Rationale @@ -32,7 +32,7 @@ ECR Public mirroring already exists for other CLI images and needs no new vendor ## Consequences - Same-version overwrite works on GHCR, GitHub Releases, and ECR Public. -- Until the follow-up client PR, local stack and CLI still pull the GHCR catalog pin and GitHub Releases for natives. +- Until the follow-up client PR, local stack and CLI still pull the GHCR catalog pin for images. Natives come from GitHub Releases with the S3 bucket as fallback. - A GitHub native can be live while the ECR native is stale. - Old CLI releases keep pulling the previous image digest. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 1f7ed92735..81f6e9212c 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -157,6 +157,13 @@ const platformText = (platform: { readonly os: string; readonly arch: string }): const errorMessage = (cause: unknown): string => cause instanceof Error ? cause.message : typeof cause === "string" ? cause : String(cause); +/** + * Public S3 copy of the release assets for hosts that block GitHub release downloads, such as + * agent sandboxes that allow `*.amazonaws.com`. + * @see ../../../infra/cli-artifacts/README.md + */ +const SLIM_ARTIFACTS_BUCKET_URL = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; + const artifactFor = ( service: ServiceKind, resolved: ArtifactResolution, @@ -165,7 +172,8 @@ const artifactFor = ( const sourceService = definitions[service].sourceService; const releaseTag = `${sourceService}-${resolved.version}`; const assetName = `${releaseTag}-${target}`; - const base = `https://github.com/supabase/slim-services/releases/download/${releaseTag}`; + const release = `https://github.com/supabase/slim-services/releases/download/${releaseTag}`; + const bucket = `${SLIM_ARTIFACTS_BUCKET_URL}/${sourceService}/${resolved.version}`; return { provider: "supabase/slim-services", service: sourceService, @@ -174,9 +182,18 @@ const artifactFor = ( target, archive: "tar.zst", assetName, - downloadUrl: `${base}/${assetName}.tar.zst`, - manifestUrl: `${base}/${assetName}.manifest.json`, - checksumUrl: `${base}/SHA256SUMS`, + mirrors: [ + { + downloadUrl: `${release}/${assetName}.tar.zst`, + manifestUrl: `${release}/${assetName}.manifest.json`, + checksumUrl: `${release}/SHA256SUMS`, + }, + { + downloadUrl: `${bucket}/${assetName}.tar.zst`, + manifestUrl: `${bucket}/${assetName}.manifest.json`, + checksumUrl: `${bucket}/${assetName}.SHA256SUMS`, + }, + ], requiredRuntimePaths: resolved.requiredRuntimePaths, executablePath: resolved.executablePath, }; diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index faddbd0e3f..70232d1278 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -7,6 +7,13 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; import { PreparationError } from "./Errors.ts"; +/** One host serving the archive, manifest, and checksum file of a slim-services release asset. */ +interface SlimServicesMirror { + readonly downloadUrl: string; + readonly manifestUrl: string; + readonly checksumUrl: string; +} + export interface SlimServicesArtifact { readonly provider: "supabase/slim-services"; readonly service: string; @@ -15,9 +22,8 @@ export interface SlimServicesArtifact { readonly target: "darwin-arm64" | "linux-amd64" | "linux-arm64"; readonly archive: "tar.zst"; readonly assetName: string; - readonly downloadUrl: string; - readonly manifestUrl: string; - readonly checksumUrl: string; + /** Hosts carrying the same release assets, tried in order until one serves them. */ + readonly mirrors: readonly [SlimServicesMirror, ...ReadonlyArray]; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath: string; } @@ -200,6 +206,29 @@ const downloadToFile = Effect.fn("SlimServicesSource.downloadToFile")(function* }); }); +/** + * Runs `attempt` against each mirror until one succeeds. Mixing hosts across the checksum and + * materialize phases is safe because an archive is only accepted when it hashes to the checksum. + */ +const fromMirrors = ( + artifact: SlimServicesArtifact, + attempt: (mirror: SlimServicesMirror) => Effect.Effect, +): Effect.Effect => { + const [first, ...fallbacks] = artifact.mirrors; + return fallbacks.reduce( + (previous, mirror) => + previous.pipe( + Effect.catch((cause) => + Effect.logWarning( + `Falling back to ${mirror.downloadUrl} for slim-services ${artifact.assetName}`, + cause, + ).pipe(Effect.andThen(attempt(mirror))), + ), + ), + attempt(first), + ); +}; + const checksumFor = (contents: string, archiveName: string): string | undefined => contents .split(/\r?\n/u) @@ -210,23 +239,72 @@ export const slimServicesChecksum = Effect.fn("SlimServicesSource.checksum")(fun artifact: SlimServicesArtifact, backoff: Schedule.Schedule = transferBackoff, ) { - return yield* fetchBytes(artifact.checksumUrl, backoff).pipe( - Effect.map((bytes) => new TextDecoder().decode(bytes)), - Effect.flatMap((contents) => { - const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); - return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) - ? Effect.fail( - new PreparationError({ - message: "Slim-services checksum is missing", - service: artifact.service, - version: artifact.version, - }), - ) - : Effect.succeed(checksum.toLowerCase()); - }), + return yield* fromMirrors(artifact, (mirror) => + fetchBytes(mirror.checksumUrl, backoff).pipe( + Effect.map((bytes) => new TextDecoder().decode(bytes)), + Effect.flatMap((contents) => { + const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); + return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) + ? Effect.fail( + new PreparationError({ + message: "Slim-services checksum is missing", + service: artifact.service, + version: artifact.version, + }), + ) + : Effect.succeed(checksum.toLowerCase()); + }), + ), ); }); +const manifestSchema = Schema.Struct({ + service: Schema.String, + version: Schema.String, + target: Schema.Literals(["darwin-arm64", "linux-amd64", "linux-arm64"]), + entrypoint: Schema.optionalKey(Schema.Array(Schema.String)), + cmd: Schema.optionalKey(Schema.Array(Schema.String)), +}); + +const verifiedManifest = Effect.fn("SlimServicesSource.verifiedManifest")(function* ( + artifact: SlimServicesArtifact, + mirror: SlimServicesMirror, + backoff: Schedule.Schedule, +) { + const manifestBytes = yield* fetchBytes(mirror.manifestUrl, backoff); + const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( + new TextDecoder().decode(manifestBytes), + ).pipe( + Effect.mapError( + (cause) => + new PreparationError({ + message: "Slim-services manifest is invalid", + cause, + }), + ), + ); + if ( + manifest.service !== artifact.service || + manifest.version !== artifact.version || + manifest.target !== artifact.target + ) + return yield* new PreparationError({ + message: "Slim-services manifest does not match the catalog artifact", + service: artifact.service, + version: artifact.version, + target: artifact.target, + }); + if ( + (manifest.entrypoint !== undefined && manifest.entrypoint.some(unsafeManifestCommand)) || + (manifest.cmd !== undefined && manifest.cmd.some(unsafeManifestCommand)) + ) + return yield* new PreparationError({ + message: "Slim-services manifest command is invalid", + service: artifact.service, + version: artifact.version, + }); +}); + const unsafeArchivePath = (value: string): boolean => { const normalized = value.trim(); if (normalized.length === 0) return false; @@ -332,60 +410,24 @@ export const makeSlimServicesSource = ( ]); return yield* Effect.gen(function* () { const artifact = yield* resolveArtifact(request); - const manifestBytes = yield* fetchBytes(artifact.manifestUrl, backoff); - const manifestText = new TextDecoder().decode(manifestBytes); - const manifestSchema = Schema.Struct({ - service: Schema.String, - version: Schema.String, - target: Schema.Literals(["darwin-arm64", "linux-amd64", "linux-arm64"]), - entrypoint: Schema.optionalKey(Schema.Array(Schema.String)), - cmd: Schema.optionalKey(Schema.Array(Schema.String)), - }); - const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( - manifestText, - ).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: "Slim-services manifest is invalid", - cause, - }), - ), - ); - if ( - manifest.service !== artifact.service || - manifest.version !== artifact.version || - manifest.target !== artifact.target - ) - return yield* new PreparationError({ - message: "Slim-services manifest does not match the catalog artifact", - service: artifact.service, - version: artifact.version, - target: artifact.target, - }); - const entrypoint = manifest.entrypoint; - const command = manifest.cmd; - if ( - (entrypoint !== undefined && entrypoint.some(unsafeManifestCommand)) || - (command !== undefined && command.some(unsafeManifestCommand)) - ) - return yield* new PreparationError({ - message: "Slim-services manifest command is invalid", - service: artifact.service, - version: artifact.version, - }); - yield* Effect.sync(() => onProgress?.("downloading")).pipe( - Effect.andThen( - downloadToFile(artifact.downloadUrl, compressedPath, expectedSha256, backoff), - ), - Effect.mapError( - (cause) => - new PreparationError({ - message: "Unable to download slim-services archive", - service: artifact.service, - version: artifact.version, - cause, - }), + yield* fromMirrors(artifact, (mirror) => + verifiedManifest(artifact, mirror, backoff).pipe( + Effect.andThen( + Effect.sync(() => onProgress?.("downloading")).pipe( + Effect.andThen( + downloadToFile(mirror.downloadUrl, compressedPath, expectedSha256, backoff), + ), + Effect.mapError( + (cause) => + new PreparationError({ + message: "Unable to download slim-services archive", + service: artifact.service, + version: artifact.version, + cause, + }), + ), + ), + ), ), ); yield* Effect.sync(() => onProgress?.("preparing")); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 919588ebba..39c13f37fa 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -17,6 +17,7 @@ import { import { createServer, type Server } from "node:http"; import { zstdCompress } from "node:zlib"; import { FetchHttpClient } from "effect/unstable/http"; +import { prepareNativeArtifact, resolveArtifact } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest } from "./ArtifactStore.ts"; import { digestHex } from "./Integrity.ts"; import { @@ -41,9 +42,13 @@ const artifact: SlimServicesArtifact = { target: "linux-amd64", archive: "tar.zst", assetName: "demo-v1.0.0-linux-amd64", - downloadUrl: "https://example.test/demo.tar.zst", - manifestUrl: "https://example.test/demo.manifest.json", - checksumUrl: "https://example.test/SHA256SUMS", + mirrors: [ + { + downloadUrl: "https://example.test/demo.tar.zst", + manifestUrl: "https://example.test/demo.manifest.json", + checksumUrl: "https://example.test/SHA256SUMS", + }, + ], requiredRuntimePaths: ["bin/demo"], executablePath: "bin/demo", }; @@ -164,9 +169,11 @@ describe("slim-services artifact source", () => { const address = server.address(); if (address === null || typeof address === "string") return yield* Effect.die("redirect server did not expose a port"); - const redirected = { + const redirected: SlimServicesArtifact = { ...artifact, - checksumUrl: `http://127.0.0.1:${address.port}/redirect`, + mirrors: [ + { ...artifact.mirrors[0], checksumUrl: `http://127.0.0.1:${address.port}/redirect` }, + ], }; const checksum = yield* slimServicesChecksum(redirected); expect(checksum).toBe("a".repeat(64)); @@ -243,10 +250,10 @@ describe("slim-services artifact source", () => { const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-retry-" }); const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); expect(yield* withFetch(flaky, source.checksum(request))).toBe(expected); - expect(attempts.get(artifact.checksumUrl)).toBe(2); + expect(attempts.get(artifact.mirrors[0].checksumUrl)).toBe(2); yield* withFetch(flaky, source.materialize(request, destination, expected)); expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); - expect(attempts.get(artifact.downloadUrl)).toBe(2); + expect(attempts.get(artifact.mirrors[0].downloadUrl)).toBe(2); }).pipe(Effect.provide(NodeServices.layer)), ), ); @@ -271,6 +278,115 @@ describe("slim-services artifact source", () => { }), ); + it.live("prepares the artifact from the next mirror when the release host is blocked", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const mirrored: SlimServicesArtifact = { + ...artifact, + mirrors: [ + { + downloadUrl: "https://release.test/demo-v1.0.0-linux-amd64.tar.zst", + manifestUrl: "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", + checksumUrl: "https://release.test/SHA256SUMS", + }, + { + downloadUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", + manifestUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", + checksumUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.SHA256SUMS", + }, + ], + }; + const blocked = new Set(); + const fetcher: FetchLike = (input) => { + const url = requestUrl(input); + if (url.startsWith("https://release.test/")) { + blocked.add(url); + return Promise.resolve(new Response("", { status: 403 })); + } + if (url.endsWith("SHA256SUMS")) + return Promise.resolve(new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`)); + if (url.endsWith("manifest.json")) + return Promise.resolve( + new Response( + JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), + ), + ); + return Promise.resolve(new Response(archive)); + }; + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-fallback-" }); + const store = yield* makeArtifactStore({ + cacheRoot: root, + source: makeSlimServicesSource(() => mirrored, { backoff: immediate }), + }); + const prepared = yield* withFetch(fetcher, store.prepare(request)); + expect(prepared.outcome).toBe("downloaded"); + expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); + expect([...blocked]).toEqual([ + "https://release.test/SHA256SUMS", + "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live("accepts a fallback archive only when it matches the checksum", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const tampered = yield* compress(tar("bin/demo", "evil")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const mirrored = (fallback: Uint8Array): SlimServicesArtifact => ({ + ...artifact, + mirrors: [ + { + downloadUrl: "https://release.test/demo.tar.zst", + manifestUrl: "https://release.test/demo.manifest.json", + checksumUrl: "https://release.test/SHA256SUMS", + }, + { + downloadUrl: `https://bucket.test/${fallback === archive ? "good" : "bad"}.tar.zst`, + manifestUrl: "https://bucket.test/demo.manifest.json", + checksumUrl: "https://bucket.test/SHA256SUMS", + }, + ], + }); + const fetcher: FetchLike = (input) => { + const url = requestUrl(input); + if (url.endsWith("manifest.json")) + return Promise.resolve( + new Response( + JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), + ), + ); + return Promise.resolve(new Response(url.endsWith("good.tar.zst") ? archive : tampered)); + }; + const fs = yield* FileSystem.FileSystem; + + const recovered = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-recover-" }); + yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored(archive)).materialize(request, recovered, expected), + ); + expect(yield* fs.readFileString(`${recovered}/bin/demo`)).toBe("demo"); + + const rejected = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-reject-" }); + const failed = yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored(tampered)) + .materialize(request, rejected, expected) + .pipe(Effect.exit), + ); + expect(errorOf(failed)?.message).toBe("Unable to download slim-services archive"); + expect(yield* fs.exists(`${rejected}/bin/demo`)).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + it.live("rejects archive members that escape the artifact root", () => Effect.scoped( Effect.gen(function* () { @@ -658,3 +774,46 @@ describe("slim-services artifact source", () => { ), ); }); + +describe("native artifact catalog", () => { + it.live("downloads from the public S3 mirror when GitHub release assets are blocked", () => + Effect.scoped( + Effect.gen(function* () { + const { version } = yield* resolveArtifact({ service: "rest" }); + const asset = `postgrest-${version}-linux-amd64`; + const archive = yield* compress(tar("bin/postgrest", "postgrest")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const served: string[] = []; + const fetcher: FetchLike = (input) => { + const url = requestUrl(input); + if (url.startsWith("https://github.com/")) + return Promise.resolve(new Response("", { status: 403 })); + served.push(url); + if (url.endsWith(".SHA256SUMS")) + return Promise.resolve(new Response(`${expected} ${asset}.tar.zst\n`)); + if (url.endsWith(".manifest.json")) + return Promise.resolve( + new Response( + JSON.stringify({ service: "postgrest", version, target: "linux-amd64" }), + ), + ); + return Promise.resolve(new Response(archive)); + }; + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-catalog-" }); + const prepared = yield* withFetch( + fetcher, + prepareNativeArtifact({ service: "rest" }, cacheRoot, { os: "linux", arch: "x64" }), + ); + expect(yield* fs.readFileString(prepared.executable)).toBe("postgrest"); + const bucket = `https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com/postgrest/${version}`; + expect(served).toEqual([ + `${bucket}/${asset}.SHA256SUMS`, + `${bucket}/${asset}.manifest.json`, + `${bucket}/${asset}.tar.zst`, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); +}); From c5907596b8d013e70009c721ed7ae05717e35a34 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:05:51 +0000 Subject: [PATCH 3/9] feat(stack): fall back to ECR Public for slim images, silently on failure The container runtime now pulls a catalog image from its ECR Public mirror when the GHCR pull fails, and launches the mirror reference it pulled. Both this and the S3 fallback for natives now report the primary's original error when the mirror fails too, so behavior only changes when a mirror rescues a failed download. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- docs/adr/0026-slim-artifact-mirrors.md | 3 +- packages/stack/src/Artifacts.ts | 9 ++ packages/stack/src/host/ToolRunner.ts | 9 +- .../src/preparation/SlimServicesSource.ts | 29 +++--- .../slim-services.integration.test.ts | 22 +++++ packages/stack/src/runtime/Container.ts | 32 +++++- ...container-image-mirror.integration.test.ts | 99 +++++++++++++++++++ packages/stack/src/services/Catalog.ts | 4 +- packages/stack/src/services/Database.ts | 3 +- .../stack/src/services/DatabaseSnapshot.ts | 4 +- 10 files changed, 189 insertions(+), 25 deletions(-) create mode 100644 packages/stack/src/runtime/container-image-mirror.integration.test.ts diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index a18402e62d..5b9c965f08 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -23,6 +23,7 @@ Client fail-through (ordered GHCR / ECR / GitHub candidates in the CLI and stack - Consume the mirrored image tags and `:version-native-` artifacts from the CLI and local stack, with host-aware order and digest pins preserved ([ADR 0017](0017-simplified-managed-stack-architecture.md)). - Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Natives are therefore also copied to a public S3 bucket on `*.amazonaws.com` ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)), independently of the ECR copy. The local stack tries the GitHub Release first and falls back to that bucket, accepting an archive from either host only when it matches the checksum. +- The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. For images and natives alike, a failing fallback reports the primary's original error, so the mirrors only change behavior when they rescue a failed download. - Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. ## Rationale @@ -32,7 +33,7 @@ ECR Public mirroring already exists for other CLI images and needs no new vendor ## Consequences - Same-version overwrite works on GHCR, GitHub Releases, and ECR Public. -- Until the follow-up client PR, local stack and CLI still pull the GHCR catalog pin for images. Natives come from GitHub Releases with the S3 bucket as fallback. +- Images come from the GHCR catalog pin with ECR Public as fallback, and natives from GitHub Releases with the S3 bucket as fallback. - A GitHub native can be live while the ECR native is stale. - Old CLI releases keep pulling the previous image digest. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 81f6e9212c..9aaeff7d3c 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -164,6 +164,15 @@ const errorMessage = (cause: unknown): string => */ const SLIM_ARTIFACTS_BUCKET_URL = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; +const SLIM_IMAGE_REGISTRY = "ghcr.io/supabase/cli/"; +const SLIM_IMAGE_MIRROR = "public.ecr.aws/supabase/cli/"; + +/** ECR Public copy of a catalog slim image under the same tag and digest, if it has one. */ +export const imageMirror = (image: string): string | undefined => + image.startsWith(SLIM_IMAGE_REGISTRY) + ? `${SLIM_IMAGE_MIRROR}${image.slice(SLIM_IMAGE_REGISTRY.length)}` + : undefined; + const artifactFor = ( service: ServiceKind, resolved: ArtifactResolution, diff --git a/packages/stack/src/host/ToolRunner.ts b/packages/stack/src/host/ToolRunner.ts index 017c93677f..8be3aa342a 100644 --- a/packages/stack/src/host/ToolRunner.ts +++ b/packages/stack/src/host/ToolRunner.ts @@ -14,7 +14,12 @@ import { import { rmdir } from "node:fs/promises"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; -import { prepareNativeArtifact, postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { + imageMirror, + prepareNativeArtifact, + postgresVersion, + resolveArtifact, +} from "../Artifacts.ts"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { PostgresTool, type PgProveOptions } from "../Tools.ts"; @@ -67,7 +72,7 @@ const makeToolRunner = (options: { const container = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); const jobsRoot = path.join(options.root, "jobs"); yield* fs .makeDirectory(jobsRoot, { recursive: true, mode: 0o700 }) diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index 70232d1278..292731576b 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -207,26 +207,29 @@ const downloadToFile = Effect.fn("SlimServicesSource.downloadToFile")(function* }); /** - * Runs `attempt` against each mirror until one succeeds. Mixing hosts across the checksum and + * Runs `attempt` against each mirror until one succeeds. Fallback failures are not surfaced: when + * every mirror fails, the error is the primary host's. Mixing hosts across the checksum and * materialize phases is safe because an archive is only accepted when it hashes to the checksum. */ const fromMirrors = ( artifact: SlimServicesArtifact, attempt: (mirror: SlimServicesMirror) => Effect.Effect, ): Effect.Effect => { - const [first, ...fallbacks] = artifact.mirrors; - return fallbacks.reduce( - (previous, mirror) => - previous.pipe( - Effect.catch((cause) => - Effect.logWarning( - `Falling back to ${mirror.downloadUrl} for slim-services ${artifact.assetName}`, - cause, - ).pipe(Effect.andThen(attempt(mirror))), - ), + const [primary, ...fallbacks] = artifact.mirrors; + const fallback = ( + primaryError: PreparationError, + remaining: ReadonlyArray, + ): Effect.Effect => { + const [mirror, ...rest] = remaining; + if (mirror === undefined) return Effect.fail(primaryError); + return attempt(mirror).pipe( + Effect.tapError((cause) => + Effect.logDebug(`Slim-services mirror ${mirror.downloadUrl} failed`, cause), ), - attempt(first), - ); + Effect.catch(() => fallback(primaryError, rest)), + ); + }; + return attempt(primary).pipe(Effect.catch((primaryError) => fallback(primaryError, fallbacks))); }; const checksumFor = (contents: string, archiveName: string): string | undefined => diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 39c13f37fa..1c1630abba 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -333,6 +333,28 @@ describe("slim-services artifact source", () => { ), ); + it.live("reports the primary host's failure when every mirror fails", () => + Effect.gen(function* () { + const mirrored: SlimServicesArtifact = { + ...artifact, + mirrors: [ + { ...artifact.mirrors[0], checksumUrl: "https://release.test/SHA256SUMS" }, + { ...artifact.mirrors[0], checksumUrl: "https://bucket.test/demo.SHA256SUMS" }, + ], + }; + const requested: string[] = []; + const failed = yield* withFetch((input) => { + requested.push(requestUrl(input)); + return Promise.resolve(new Response("", { status: 403 })); + }, slimServicesChecksum(mirrored, immediate).pipe(Effect.exit)); + expect(errorOf(failed)?.message).toBe("Unable to download https://release.test/SHA256SUMS"); + expect(requested).toEqual([ + "https://release.test/SHA256SUMS", + "https://bucket.test/demo.SHA256SUMS", + ]); + }), + ); + it.live("accepts a fallback archive only when it matches the checksum", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 82bed8b63a..4f28a9c48f 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -91,9 +91,13 @@ const mountField = (key: string, value: string) => { return /[,"\n\r]/u.test(field) ? `"${field.replaceAll('"', '""')}"` : field; }; -/** Captures the selected local engine; each launch owns one exact container. */ +/** + * Captures the selected local engine; each launch owns one exact container. An image whose pull + * fails is pulled from `imageMirror` instead, and launches of it then use the mirror reference. + */ export const makeContainerRuntime = (options: { readonly engine: "docker" | "podman"; + readonly imageMirror?: (image: string) => string | undefined; }): Effect.Effect< ContainerRuntime, never, @@ -142,9 +146,28 @@ export const makeContainerRuntime = (options: { ); }); + const mirrored = yield* Ref.make>(new Map()); + const present = (image: string) => + run(["image", "ls", "--quiet", "--no-trunc", image]).pipe( + Effect.map((ids) => ids.length > 0), + ); + const pull = (image: string) => run(["pull", image], { timeout: "5 minutes" }); + const prepare = Effect.fn("Container.prepare")(function* (image: string) { - const present = yield* run(["image", "ls", "--quiet", "--no-trunc", image]); - if (present.length === 0) yield* run(["pull", image], { timeout: "5 minutes" }); + if (yield* present(image)) return; + const mirror = options.imageMirror?.(image); + if (mirror === undefined) return yield* pull(image); + yield* pull(image).pipe( + Effect.catch((primaryError) => + Effect.gen(function* () { + if (!(yield* present(mirror))) yield* pull(mirror); + yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); + }).pipe( + Effect.tapError((cause) => Effect.logDebug(`Image mirror ${mirror} failed`, cause)), + Effect.mapError(() => primaryError), + ), + ), + ); }); const launch = Effect.fn("Container.launch")(function* ( @@ -152,6 +175,7 @@ export const makeContainerRuntime = (options: { interactive = false, ) { const owner = yield* Scope.Scope; + const image = (yield* Ref.get(mirrored)).get(spec.image) ?? spec.image; for (const [key, value] of Object.entries(spec.env)) { if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key) || /[\0\r\n]/u.test(value)) { return yield* errorFor( @@ -206,7 +230,7 @@ export const makeContainerRuntime = (options: { ...(spec.workingDir === undefined ? [] : ["--workdir", spec.workingDir]), ...(spec.ports ?? []).flatMap((port) => ["--publish", `127.0.0.1::${port}`]), ...(spec.entrypoint === undefined ? [] : ["--entrypoint", spec.entrypoint]), - spec.image, + image, ...(spec.args ?? []), ]; diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts new file mode 100644 index 0000000000..6a6fcbc798 --- /dev/null +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -0,0 +1,99 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, Layer, Option, Sink, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { makeContainerRuntime } from "./Container.ts"; + +const primary = "ghcr.io/supabase/cli/postgrest:v16.2"; +const mirror = "public.ecr.aws/supabase/cli/postgrest:v16.2"; + +/** Docker stand-in: `image ls` reports `local`, `pull` succeeds for `pullable`, `create` refuses. */ +const fakeEngine = (options: { readonly pullable: ReadonlyArray }) => { + const local = new Set(); + const commands: string[][] = []; + const handle = (exitCode: number, stdout = "", stderr = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(4242), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.succeed(new TextEncoder().encode(stderr)), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + const args = [...command.args]; + commands.push(args); + const ref = args.at(-1) ?? ""; + if (args[0] === "image") return Effect.succeed(handle(0, local.has(ref) ? "sha256:1" : "")); + if (args[0] === "pull") { + if (!options.pullable.includes(ref)) return Effect.succeed(handle(1, "", `denied: ${ref}`)); + local.add(ref); + return Effect.succeed(handle(0)); + } + return Effect.succeed(handle(1, "", "create refused")); + }); + return { + commands, + layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + }; +}; + +const launchImage = (runtime: Effect.Success>) => + Effect.scoped( + runtime.launch({ image: primary, stackId: "a".repeat(64), instanceId: "mirror", env: {} }), + ).pipe(Effect.exit); + +describe("container image mirror", () => { + it.live("pulls and launches the mirror image when the primary pull fails", () => { + const engine = fakeEngine({ pullable: [mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirror: (image) => (image === primary ? mirror : undefined), + }); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.filter((args) => args[0] === "pull")).toEqual([ + ["pull", primary], + ["pull", mirror], + ]); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(mirror); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("reports the primary pull failure when the mirror also fails", () => { + const engine = fakeEngine({ pullable: [] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirror: (image) => (image === primary ? mirror : undefined), + }); + const failed = yield* runtime.prepare(primary).pipe(Effect.exit); + const error = Exit.isFailure(failed) + ? Option.getOrUndefined(Cause.findErrorOption(failed.cause)) + : undefined; + expect(error?.message).toContain(`denied: ${primary}`); + yield* launchImage(runtime); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("never contacts the mirror when the primary pull succeeds", () => { + const engine = fakeEngine({ pullable: [primary, mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirror: (image) => (image === primary ? mirror : undefined), + }); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.flat()).not.toContain(mirror); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); +}); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index a7823c4266..ca44142cbe 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -29,7 +29,7 @@ import { type ProcessRecipeResult, } from "./Recipe.ts"; import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; -import type { ServiceKind } from "../Artifacts.ts"; +import { imageMirror, type ServiceKind } from "../Artifacts.ts"; import type { ServiceInstanceContext } from "../Service.ts"; export type { CatalogLog } from "./Recipe.ts"; @@ -260,7 +260,7 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( const container = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); const deps: ProcessDependencies = { fs, path, crypto, client, spawner, container }; switch (creation.service) { case "rest": diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 2b2dab5a92..e10b8be69a 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -20,6 +20,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient } from "effect/unstable/http"; import { + imageMirror, prepareNativeArtifact, postgresVersion, resolveArtifact, @@ -467,7 +468,7 @@ export const makeDatabase = ( const container: ContainerRuntime | undefined = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); const dataCommand = Effect.fn("Database.containerFiles")( function* (version: string, args: ReadonlyArray) { diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index 6f512a54c7..5631d4dc7c 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -1,6 +1,6 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Crypto, Data, Effect, FileSystem, Path, Schema, Stream } from "effect"; -import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { imageMirror, postgresVersion, resolveArtifact } from "../Artifacts.ts"; import type { DatabaseRuntime } from "./Database.ts"; import { makeContainerRuntime, type ContainerRuntime } from "../runtime/Container.ts"; @@ -61,7 +61,7 @@ export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function const container: ContainerRuntime | undefined = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); const helperImage = options.runtime === "native" ? undefined From f16785cdb851066dd39168d4a9f10d971d6acf62 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:11:22 +0000 Subject: [PATCH 4/9] refactor(stack): name each slim mirror after its host Mirror constants now say which host they point at (`SLIM_NATIVE_SUPABASE_S3_MIRROR`, `SLIM_IMAGE_SUPABASE_ECR_MIRROR`, and the GHCR / GitHub Releases primaries), and images take an ordered mirror list like natives do, so another host can be added next to the existing ones without renaming or reshaping anything. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- packages/stack/src/Artifacts.ts | 34 ++++++++++--------- packages/stack/src/host/ToolRunner.ts | 4 +-- packages/stack/src/runtime/Container.ts | 34 ++++++++++++------- ...container-image-mirror.integration.test.ts | 25 ++++++++++++-- packages/stack/src/services/Catalog.ts | 7 ++-- packages/stack/src/services/Database.ts | 4 +-- .../stack/src/services/DatabaseSnapshot.ts | 4 +-- 7 files changed, 72 insertions(+), 40 deletions(-) diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 9aaeff7d3c..11a20cf813 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -157,21 +157,23 @@ const platformText = (platform: { readonly os: string; readonly arch: string }): const errorMessage = (cause: unknown): string => cause instanceof Error ? cause.message : typeof cause === "string" ? cause : String(cause); +const SLIM_NATIVE_GITHUB_RELEASES = "https://github.com/supabase/slim-services/releases/download"; + /** * Public S3 copy of the release assets for hosts that block GitHub release downloads, such as * agent sandboxes that allow `*.amazonaws.com`. * @see ../../../infra/cli-artifacts/README.md */ -const SLIM_ARTIFACTS_BUCKET_URL = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; +const SLIM_NATIVE_SUPABASE_S3_MIRROR = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; -const SLIM_IMAGE_REGISTRY = "ghcr.io/supabase/cli/"; -const SLIM_IMAGE_MIRROR = "public.ecr.aws/supabase/cli/"; +const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; +const SLIM_IMAGE_SUPABASE_ECR_MIRROR = "public.ecr.aws/supabase/cli/"; -/** ECR Public copy of a catalog slim image under the same tag and digest, if it has one. */ -export const imageMirror = (image: string): string | undefined => - image.startsWith(SLIM_IMAGE_REGISTRY) - ? `${SLIM_IMAGE_MIRROR}${image.slice(SLIM_IMAGE_REGISTRY.length)}` - : undefined; +/** Mirrors carrying a catalog slim image under the same tag and digest, in fallback order. */ +export const slimImageMirrors = (image: string): ReadonlyArray => + image.startsWith(SLIM_IMAGE_GHCR_REGISTRY) + ? [`${SLIM_IMAGE_SUPABASE_ECR_MIRROR}${image.slice(SLIM_IMAGE_GHCR_REGISTRY.length)}`] + : []; const artifactFor = ( service: ServiceKind, @@ -181,8 +183,8 @@ const artifactFor = ( const sourceService = definitions[service].sourceService; const releaseTag = `${sourceService}-${resolved.version}`; const assetName = `${releaseTag}-${target}`; - const release = `https://github.com/supabase/slim-services/releases/download/${releaseTag}`; - const bucket = `${SLIM_ARTIFACTS_BUCKET_URL}/${sourceService}/${resolved.version}`; + const githubRelease = `${SLIM_NATIVE_GITHUB_RELEASES}/${releaseTag}`; + const supabaseS3 = `${SLIM_NATIVE_SUPABASE_S3_MIRROR}/${sourceService}/${resolved.version}`; return { provider: "supabase/slim-services", service: sourceService, @@ -193,14 +195,14 @@ const artifactFor = ( assetName, mirrors: [ { - downloadUrl: `${release}/${assetName}.tar.zst`, - manifestUrl: `${release}/${assetName}.manifest.json`, - checksumUrl: `${release}/SHA256SUMS`, + downloadUrl: `${githubRelease}/${assetName}.tar.zst`, + manifestUrl: `${githubRelease}/${assetName}.manifest.json`, + checksumUrl: `${githubRelease}/SHA256SUMS`, }, { - downloadUrl: `${bucket}/${assetName}.tar.zst`, - manifestUrl: `${bucket}/${assetName}.manifest.json`, - checksumUrl: `${bucket}/${assetName}.SHA256SUMS`, + downloadUrl: `${supabaseS3}/${assetName}.tar.zst`, + manifestUrl: `${supabaseS3}/${assetName}.manifest.json`, + checksumUrl: `${supabaseS3}/${assetName}.SHA256SUMS`, }, ], requiredRuntimePaths: resolved.requiredRuntimePaths, diff --git a/packages/stack/src/host/ToolRunner.ts b/packages/stack/src/host/ToolRunner.ts index 8be3aa342a..b98cb4fdda 100644 --- a/packages/stack/src/host/ToolRunner.ts +++ b/packages/stack/src/host/ToolRunner.ts @@ -15,7 +15,7 @@ import { rmdir } from "node:fs/promises"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; import { - imageMirror, + slimImageMirrors, prepareNativeArtifact, postgresVersion, resolveArtifact, @@ -72,7 +72,7 @@ const makeToolRunner = (options: { const container = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirrors: slimImageMirrors }); const jobsRoot = path.join(options.root, "jobs"); yield* fs .makeDirectory(jobsRoot, { recursive: true, mode: 0o700 }) diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 4f28a9c48f..b940a66404 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -93,11 +93,12 @@ const mountField = (key: string, value: string) => { /** * Captures the selected local engine; each launch owns one exact container. An image whose pull - * fails is pulled from `imageMirror` instead, and launches of it then use the mirror reference. + * fails is pulled from the first of its `imageMirrors` that succeeds, and launches of it then use + * that mirror reference. When every mirror fails, the primary pull error is reported. */ export const makeContainerRuntime = (options: { readonly engine: "docker" | "podman"; - readonly imageMirror?: (image: string) => string | undefined; + readonly imageMirrors?: (image: string) => ReadonlyArray; }): Effect.Effect< ContainerRuntime, never, @@ -153,20 +154,27 @@ export const makeContainerRuntime = (options: { ); const pull = (image: string) => run(["pull", image], { timeout: "5 minutes" }); + const fromMirror = ( + image: string, + mirrors: ReadonlyArray, + primaryError: ContainerError, + ): Effect.Effect => { + const [mirror, ...rest] = mirrors; + if (mirror === undefined) return Effect.fail(primaryError); + return Effect.gen(function* () { + if (!(yield* present(mirror))) yield* pull(mirror); + yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); + }).pipe( + Effect.tapError((cause) => Effect.logDebug(`Image mirror ${mirror} failed`, cause)), + Effect.catch(() => fromMirror(image, rest, primaryError)), + ); + }; + const prepare = Effect.fn("Container.prepare")(function* (image: string) { if (yield* present(image)) return; - const mirror = options.imageMirror?.(image); - if (mirror === undefined) return yield* pull(image); + const mirrors = options.imageMirrors?.(image) ?? []; yield* pull(image).pipe( - Effect.catch((primaryError) => - Effect.gen(function* () { - if (!(yield* present(mirror))) yield* pull(mirror); - yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); - }).pipe( - Effect.tapError((cause) => Effect.logDebug(`Image mirror ${mirror} failed`, cause)), - Effect.mapError(() => primaryError), - ), - ), + Effect.catch((primaryError) => fromMirror(image, mirrors, primaryError)), ); }); diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts index 6a6fcbc798..2573266d59 100644 --- a/packages/stack/src/runtime/container-image-mirror.integration.test.ts +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -6,6 +6,7 @@ import { makeContainerRuntime } from "./Container.ts"; const primary = "ghcr.io/supabase/cli/postgrest:v16.2"; const mirror = "public.ecr.aws/supabase/cli/postgrest:v16.2"; +const secondMirror = "registry.test/supabase/cli/postgrest:v16.2"; /** Docker stand-in: `image ls` reports `local`, `pull` succeeds for `pullable`, `create` refuses. */ const fakeEngine = (options: { readonly pullable: ReadonlyArray }) => { @@ -55,7 +56,7 @@ describe("container image mirror", () => { return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ engine: "docker", - imageMirror: (image) => (image === primary ? mirror : undefined), + imageMirrors: (image) => (image === primary ? [mirror] : []), }); yield* runtime.prepare(primary); yield* launchImage(runtime); @@ -67,12 +68,30 @@ describe("container image mirror", () => { }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); + it.live("tries each mirror in order until one pull succeeds", () => { + const engine = fakeEngine({ pullable: [secondMirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []), + }); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.filter((args) => args[0] === "pull")).toEqual([ + ["pull", primary], + ["pull", mirror], + ["pull", secondMirror], + ]); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(secondMirror); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + it.live("reports the primary pull failure when the mirror also fails", () => { const engine = fakeEngine({ pullable: [] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ engine: "docker", - imageMirror: (image) => (image === primary ? mirror : undefined), + imageMirrors: (image) => (image === primary ? [mirror] : []), }); const failed = yield* runtime.prepare(primary).pipe(Effect.exit); const error = Exit.isFailure(failed) @@ -89,7 +108,7 @@ describe("container image mirror", () => { return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ engine: "docker", - imageMirror: (image) => (image === primary ? mirror : undefined), + imageMirrors: (image) => (image === primary ? [mirror] : []), }); yield* runtime.prepare(primary); yield* launchImage(runtime); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index ca44142cbe..eabeb7e70e 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -29,7 +29,7 @@ import { type ProcessRecipeResult, } from "./Recipe.ts"; import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; -import { imageMirror, type ServiceKind } from "../Artifacts.ts"; +import { slimImageMirrors, type ServiceKind } from "../Artifacts.ts"; import type { ServiceInstanceContext } from "../Service.ts"; export type { CatalogLog } from "./Recipe.ts"; @@ -260,7 +260,10 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( const container = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); + : yield* makeContainerRuntime({ + engine: options.runtime, + imageMirrors: slimImageMirrors, + }); const deps: ProcessDependencies = { fs, path, crypto, client, spawner, container }; switch (creation.service) { case "rest": diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index e10b8be69a..3a26a66f80 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -20,7 +20,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient } from "effect/unstable/http"; import { - imageMirror, + slimImageMirrors, prepareNativeArtifact, postgresVersion, resolveArtifact, @@ -468,7 +468,7 @@ export const makeDatabase = ( const container: ContainerRuntime | undefined = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirrors: slimImageMirrors }); const dataCommand = Effect.fn("Database.containerFiles")( function* (version: string, args: ReadonlyArray) { diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index 5631d4dc7c..805b08b017 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -1,6 +1,6 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { Crypto, Data, Effect, FileSystem, Path, Schema, Stream } from "effect"; -import { imageMirror, postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { slimImageMirrors, postgresVersion, resolveArtifact } from "../Artifacts.ts"; import type { DatabaseRuntime } from "./Database.ts"; import { makeContainerRuntime, type ContainerRuntime } from "../runtime/Container.ts"; @@ -61,7 +61,7 @@ export const makeDatabaseSnapshots = Effect.fn("DatabaseSnapshot.make")(function const container: ContainerRuntime | undefined = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime, imageMirror }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirrors: slimImageMirrors }); const helperImage = options.runtime === "native" ? undefined From e3a358aef55e5d1669ddd74a4708106d9b06b0ce Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:16:51 +0000 Subject: [PATCH 5/9] fix(stack): take slim native checksums from GitHub or GHCR, never S3 The S3 fallback also served the checksum for its own archive, making the bucket a second trust root. Checksum authorities are now listed apart from download mirrors: the GitHub Release SHA256SUMS first, then the archive layer digest of the native OCI artifact on GHCR, read through an anonymous pull token. The S3 bucket only serves the archive and manifest, which must match that checksum. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- docs/adr/0026-slim-artifact-mirrors.md | 2 +- packages/stack/src/Artifacts.ts | 14 +- .../src/preparation/SlimServicesSource.ts | 171 +++++++++++++----- .../slim-services.integration.test.ts | 94 +++++++--- 4 files changed, 208 insertions(+), 73 deletions(-) diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 5b9c965f08..bc08b4cda1 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -22,7 +22,7 @@ Client fail-through (ordered GHCR / ECR / GitHub candidates in the CLI and stack ## Follow-up - Consume the mirrored image tags and `:version-native-` artifacts from the CLI and local stack, with host-aware order and digest pins preserved ([ADR 0017](0017-simplified-managed-stack-architecture.md)). -- Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Natives are therefore also copied to a public S3 bucket on `*.amazonaws.com` ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)), independently of the ECR copy. The local stack tries the GitHub Release first and falls back to that bucket, accepting an archive from either host only when it matches the checksum. +- Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Natives are therefore also copied to a public S3 bucket on `*.amazonaws.com` ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)), independently of the ECR copy. The local stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR; the bucket is never a checksum authority, and an archive from either host is only accepted when it matches. - The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. For images and natives alike, a failing fallback reports the primary's original error, so the mirrors only change behavior when they rescue a failed download. - Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 11a20cf813..50ba106eaf 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -166,6 +166,9 @@ const SLIM_NATIVE_GITHUB_RELEASES = "https://github.com/supabase/slim-services/r */ const SLIM_NATIVE_SUPABASE_S3_MIRROR = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; +const SLIM_NATIVE_GHCR_REGISTRY = "ghcr.io"; +const SLIM_NATIVE_GHCR_REPOSITORY = "supabase/cli"; + const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; const SLIM_IMAGE_SUPABASE_ECR_MIRROR = "public.ecr.aws/supabase/cli/"; @@ -193,16 +196,23 @@ const artifactFor = ( target, archive: "tar.zst", assetName, + checksums: [ + { kind: "sha256sums", url: `${githubRelease}/SHA256SUMS` }, + { + kind: "oci", + registry: SLIM_NATIVE_GHCR_REGISTRY, + repository: `${SLIM_NATIVE_GHCR_REPOSITORY}/${sourceService}`, + tag: `${resolved.version}-native-${target}`, + }, + ], mirrors: [ { downloadUrl: `${githubRelease}/${assetName}.tar.zst`, manifestUrl: `${githubRelease}/${assetName}.manifest.json`, - checksumUrl: `${githubRelease}/SHA256SUMS`, }, { downloadUrl: `${supabaseS3}/${assetName}.tar.zst`, manifestUrl: `${supabaseS3}/${assetName}.manifest.json`, - checksumUrl: `${supabaseS3}/${assetName}.SHA256SUMS`, }, ], requiredRuntimePaths: resolved.requiredRuntimePaths, diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index 292731576b..4751e747bb 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -7,13 +7,25 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; import { PreparationError } from "./Errors.ts"; -/** One host serving the archive, manifest, and checksum file of a slim-services release asset. */ +/** One host serving the archive and manifest of a slim-services release asset. */ interface SlimServicesMirror { readonly downloadUrl: string; readonly manifestUrl: string; - readonly checksumUrl: string; } +/** + * Where the expected archive digest comes from: a release `SHA256SUMS` file, or the archive layer + * of the native OCI artifact in a registry that allows anonymous pulls. + */ +type SlimServicesChecksumSource = + | { readonly kind: "sha256sums"; readonly url: string } + | { + readonly kind: "oci"; + readonly registry: string; + readonly repository: string; + readonly tag: string; + }; + export interface SlimServicesArtifact { readonly provider: "supabase/slim-services"; readonly service: string; @@ -22,6 +34,11 @@ export interface SlimServicesArtifact { readonly target: "darwin-arm64" | "linux-amd64" | "linux-arm64"; readonly archive: "tar.zst"; readonly assetName: string; + /** Checksum authorities, tried in order. Download mirrors never supply their own checksum. */ + readonly checksums: readonly [ + SlimServicesChecksumSource, + ...ReadonlyArray, + ]; /** Hosts carrying the same release assets, tried in order until one serves them. */ readonly mirrors: readonly [SlimServicesMirror, ...ReadonlyArray]; readonly requiredRuntimePaths: ReadonlyArray; @@ -79,9 +96,9 @@ const systemTarBoundary: TarBoundary = { ); }), }; -const responseFor = (url: string) => +const responseFor = (url: string, headers?: Readonly>) => Effect.flatMap(HttpClient.HttpClient, (client) => - HttpClient.followRedirects(client).get(url), + HttpClient.followRedirects(client).get(url, { headers }), ).pipe( Effect.flatMap((response) => Effect.gen(function* () { @@ -127,8 +144,9 @@ const withTransferRetry = const fetchBytes = Effect.fn("SlimServicesSource.fetchBytes")(function* ( url: string, backoff: Schedule.Schedule, + headers?: Readonly>, ) { - return yield* responseFor(url).pipe( + return yield* responseFor(url, headers).pipe( Effect.flatMap((response) => response.arrayBuffer), Effect.map((bytes) => new Uint8Array(bytes)), withTransferRetry(url, backoff), @@ -207,24 +225,25 @@ const downloadToFile = Effect.fn("SlimServicesSource.downloadToFile")(function* }); /** - * Runs `attempt` against each mirror until one succeeds. Fallback failures are not surfaced: when - * every mirror fails, the error is the primary host's. Mixing hosts across the checksum and + * Runs `attempt` against each candidate until one succeeds. Fallback failures are not surfaced: + * when every candidate fails, the error is the primary's. Mixing hosts across the checksum and * materialize phases is safe because an archive is only accepted when it hashes to the checksum. */ -const fromMirrors = ( - artifact: SlimServicesArtifact, - attempt: (mirror: SlimServicesMirror) => Effect.Effect, +const firstSuccess = ( + candidates: readonly [T, ...ReadonlyArray], + describe: (candidate: T) => string, + attempt: (candidate: T) => Effect.Effect, ): Effect.Effect => { - const [primary, ...fallbacks] = artifact.mirrors; + const [primary, ...fallbacks] = candidates; const fallback = ( primaryError: PreparationError, - remaining: ReadonlyArray, + remaining: ReadonlyArray, ): Effect.Effect => { - const [mirror, ...rest] = remaining; - if (mirror === undefined) return Effect.fail(primaryError); - return attempt(mirror).pipe( + const [candidate, ...rest] = remaining; + if (candidate === undefined) return Effect.fail(primaryError); + return attempt(candidate).pipe( Effect.tapError((cause) => - Effect.logDebug(`Slim-services mirror ${mirror.downloadUrl} failed`, cause), + Effect.logDebug(`Slim-services fallback ${describe(candidate)} failed`, cause), ), Effect.catch(() => fallback(primaryError, rest)), ); @@ -238,26 +257,79 @@ const checksumFor = (contents: string, archiveName: string): string | undefined .map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu)) .find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]; +const ARCHIVE_MEDIA_TYPE = "application/vnd.supabase.slim.archive.v1.tar+zstd"; + +const RegistryToken = Schema.Struct({ token: Schema.String }); + +const NativeOciManifest = Schema.Struct({ + layers: Schema.Array(Schema.Struct({ mediaType: Schema.String, digest: Schema.String })), +}); + +const decodeJson = (schema: S, bytes: Uint8Array, message: string) => + Schema.decodeEffect(Schema.fromJsonString(schema))(new TextDecoder().decode(bytes)).pipe( + Effect.mapError((cause) => new PreparationError({ message, cause })), + ); + +/** Reads the archive layer digest of a native OCI artifact through an anonymous pull token. */ +const ociArchiveDigest = Effect.fn("SlimServicesSource.ociArchiveDigest")(function* ( + source: Extract, + backoff: Schedule.Schedule, +) { + const tokenUrl = `https://${source.registry}/token?scope=repository:${source.repository}:pull&service=${source.registry}`; + const { token } = yield* decodeJson( + RegistryToken, + yield* fetchBytes(tokenUrl, backoff), + "Slim-services registry token is invalid", + ); + const manifest = yield* decodeJson( + NativeOciManifest, + yield* fetchBytes( + `https://${source.registry}/v2/${source.repository}/manifests/${source.tag}`, + backoff, + { + accept: "application/vnd.oci.image.manifest.v1+json", + authorization: `Bearer ${token}`, + }, + ), + "Slim-services native OCI manifest is invalid", + ); + const digest = manifest.layers + .find((layer) => layer.mediaType === ARCHIVE_MEDIA_TYPE) + ?.digest.match(/^sha256:([a-f0-9]{64})$/u)?.[1]; + if (digest === undefined) + return yield* new PreparationError({ + message: "Slim-services native OCI manifest has no archive layer", + }); + return digest; +}); + +const describeChecksumSource = (source: SlimServicesChecksumSource): string => + source.kind === "sha256sums" + ? source.url + : `${source.registry}/${source.repository}:${source.tag}`; + export const slimServicesChecksum = Effect.fn("SlimServicesSource.checksum")(function* ( artifact: SlimServicesArtifact, backoff: Schedule.Schedule = transferBackoff, ) { - return yield* fromMirrors(artifact, (mirror) => - fetchBytes(mirror.checksumUrl, backoff).pipe( - Effect.map((bytes) => new TextDecoder().decode(bytes)), - Effect.flatMap((contents) => { - const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); - return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) - ? Effect.fail( - new PreparationError({ - message: "Slim-services checksum is missing", - service: artifact.service, - version: artifact.version, - }), - ) - : Effect.succeed(checksum.toLowerCase()); - }), - ), + return yield* firstSuccess(artifact.checksums, describeChecksumSource, (source) => + source.kind === "oci" + ? ociArchiveDigest(source, backoff) + : fetchBytes(source.url, backoff).pipe( + Effect.map((bytes) => new TextDecoder().decode(bytes)), + Effect.flatMap((contents) => { + const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); + return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) + ? Effect.fail( + new PreparationError({ + message: "Slim-services checksum is missing", + service: artifact.service, + version: artifact.version, + }), + ) + : Effect.succeed(checksum.toLowerCase()); + }), + ), ); }); @@ -413,25 +485,28 @@ export const makeSlimServicesSource = ( ]); return yield* Effect.gen(function* () { const artifact = yield* resolveArtifact(request); - yield* fromMirrors(artifact, (mirror) => - verifiedManifest(artifact, mirror, backoff).pipe( - Effect.andThen( - Effect.sync(() => onProgress?.("downloading")).pipe( - Effect.andThen( - downloadToFile(mirror.downloadUrl, compressedPath, expectedSha256, backoff), - ), - Effect.mapError( - (cause) => - new PreparationError({ - message: "Unable to download slim-services archive", - service: artifact.service, - version: artifact.version, - cause, - }), + yield* firstSuccess( + artifact.mirrors, + (mirror) => mirror.downloadUrl, + (mirror) => + verifiedManifest(artifact, mirror, backoff).pipe( + Effect.andThen( + Effect.sync(() => onProgress?.("downloading")).pipe( + Effect.andThen( + downloadToFile(mirror.downloadUrl, compressedPath, expectedSha256, backoff), + ), + Effect.mapError( + (cause) => + new PreparationError({ + message: "Unable to download slim-services archive", + service: artifact.service, + version: artifact.version, + cause, + }), + ), ), ), ), - ), ); yield* Effect.sync(() => onProgress?.("preparing")); yield* decompressor.decompress(compressedPath, archivePath); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 1c1630abba..467c31b253 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -42,11 +42,11 @@ const artifact: SlimServicesArtifact = { target: "linux-amd64", archive: "tar.zst", assetName: "demo-v1.0.0-linux-amd64", + checksums: [{ kind: "sha256sums", url: "https://example.test/SHA256SUMS" }], mirrors: [ { downloadUrl: "https://example.test/demo.tar.zst", manifestUrl: "https://example.test/demo.manifest.json", - checksumUrl: "https://example.test/SHA256SUMS", }, ], requiredRuntimePaths: ["bin/demo"], @@ -138,6 +138,44 @@ const withFetch = (fetcher: FetchLike, effect: Effect.Effect) Effect.provideService(FetchHttpClient.Fetch, fetcher), ); +const registryChecksum = { + kind: "oci", + registry: "registry.test", + repository: "supabase/cli/demo", + tag: "v1.0.0-native-linux-amd64", +} as const; + +/** Anonymous registry token and native manifest; the manifest requires the issued token. */ +const registryResponse = ( + input: Parameters[0], + init: Parameters[1], + archiveSha256: string, + registry = "registry.test", + repository = "supabase/cli/demo", + tag = "v1.0.0-native-linux-amd64", +): Response | undefined => { + const url = requestUrl(input); + if (url === `https://${registry}/token?scope=repository:${repository}:pull&service=${registry}`) + return new Response(JSON.stringify({ token: "anonymous" })); + if (url !== `https://${registry}/v2/${repository}/manifests/${tag}`) return undefined; + if (new Headers(init?.headers).get("authorization") !== "Bearer anonymous") + return new Response("", { status: 401 }); + return new Response( + JSON.stringify({ + layers: [ + { + mediaType: "application/vnd.supabase.slim.archive.v1.tar+zstd", + digest: `sha256:${archiveSha256}`, + }, + { + mediaType: "application/vnd.supabase.slim.checksum.v1", + digest: `sha256:${"f".repeat(64)}`, + }, + ], + }), + ); +}; + describe("slim-services artifact source", () => { it.live("follows release redirects through the supplied Node HTTP client", () => Effect.scoped( @@ -171,9 +209,7 @@ describe("slim-services artifact source", () => { return yield* Effect.die("redirect server did not expose a port"); const redirected: SlimServicesArtifact = { ...artifact, - mirrors: [ - { ...artifact.mirrors[0], checksumUrl: `http://127.0.0.1:${address.port}/redirect` }, - ], + checksums: [{ kind: "sha256sums", url: `http://127.0.0.1:${address.port}/redirect` }], }; const checksum = yield* slimServicesChecksum(redirected); expect(checksum).toBe("a".repeat(64)); @@ -250,7 +286,7 @@ describe("slim-services artifact source", () => { const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-retry-" }); const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); expect(yield* withFetch(flaky, source.checksum(request))).toBe(expected); - expect(attempts.get(artifact.mirrors[0].checksumUrl)).toBe(2); + expect(attempts.get("https://example.test/SHA256SUMS")).toBe(2); yield* withFetch(flaky, source.materialize(request, destination, expected)); expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); expect(attempts.get(artifact.mirrors[0].downloadUrl)).toBe(2); @@ -286,28 +322,32 @@ describe("slim-services artifact source", () => { const expected = digestHex(yield* crypto.digest("SHA-256", archive)); const mirrored: SlimServicesArtifact = { ...artifact, + checksums: [ + { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, + registryChecksum, + ], mirrors: [ { downloadUrl: "https://release.test/demo-v1.0.0-linux-amd64.tar.zst", manifestUrl: "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", - checksumUrl: "https://release.test/SHA256SUMS", }, { downloadUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", manifestUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", - checksumUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.SHA256SUMS", }, ], }; const blocked = new Set(); - const fetcher: FetchLike = (input) => { + const bucket: string[] = []; + const fetcher: FetchLike = (input, init) => { const url = requestUrl(input); if (url.startsWith("https://release.test/")) { blocked.add(url); return Promise.resolve(new Response("", { status: 403 })); } - if (url.endsWith("SHA256SUMS")) - return Promise.resolve(new Response(`${expected} demo-v1.0.0-linux-amd64.tar.zst\n`)); + const registry = registryResponse(input, init, expected); + if (registry !== undefined) return Promise.resolve(registry); + bucket.push(url); if (url.endsWith("manifest.json")) return Promise.resolve( new Response( @@ -329,17 +369,21 @@ describe("slim-services artifact source", () => { "https://release.test/SHA256SUMS", "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", ]); + expect(bucket).toEqual([ + "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", + "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", + ]); }).pipe(Effect.provide(NodeServices.layer)), ), ); - it.live("reports the primary host's failure when every mirror fails", () => + it.live("reports the primary checksum failure when every checksum source fails", () => Effect.gen(function* () { const mirrored: SlimServicesArtifact = { ...artifact, - mirrors: [ - { ...artifact.mirrors[0], checksumUrl: "https://release.test/SHA256SUMS" }, - { ...artifact.mirrors[0], checksumUrl: "https://bucket.test/demo.SHA256SUMS" }, + checksums: [ + { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, + registryChecksum, ], }; const requested: string[] = []; @@ -350,7 +394,7 @@ describe("slim-services artifact source", () => { expect(errorOf(failed)?.message).toBe("Unable to download https://release.test/SHA256SUMS"); expect(requested).toEqual([ "https://release.test/SHA256SUMS", - "https://bucket.test/demo.SHA256SUMS", + "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test", ]); }), ); @@ -368,12 +412,10 @@ describe("slim-services artifact source", () => { { downloadUrl: "https://release.test/demo.tar.zst", manifestUrl: "https://release.test/demo.manifest.json", - checksumUrl: "https://release.test/SHA256SUMS", }, { downloadUrl: `https://bucket.test/${fallback === archive ? "good" : "bad"}.tar.zst`, manifestUrl: "https://bucket.test/demo.manifest.json", - checksumUrl: "https://bucket.test/SHA256SUMS", }, ], }); @@ -798,7 +840,7 @@ describe("slim-services artifact source", () => { }); describe("native artifact catalog", () => { - it.live("downloads from the public S3 mirror when GitHub release assets are blocked", () => + it.live("verifies an S3 download against GHCR when GitHub release assets are blocked", () => Effect.scoped( Effect.gen(function* () { const { version } = yield* resolveArtifact({ service: "rest" }); @@ -807,13 +849,20 @@ describe("native artifact catalog", () => { const crypto = yield* Crypto.Crypto; const expected = digestHex(yield* crypto.digest("SHA-256", archive)); const served: string[] = []; - const fetcher: FetchLike = (input) => { + const fetcher: FetchLike = (input, init) => { const url = requestUrl(input); if (url.startsWith("https://github.com/")) return Promise.resolve(new Response("", { status: 403 })); served.push(url); - if (url.endsWith(".SHA256SUMS")) - return Promise.resolve(new Response(`${expected} ${asset}.tar.zst\n`)); + const registry = registryResponse( + input, + init, + expected, + "ghcr.io", + "supabase/cli/postgrest", + `${version}-native-linux-amd64`, + ); + if (registry !== undefined) return Promise.resolve(registry); if (url.endsWith(".manifest.json")) return Promise.resolve( new Response( @@ -831,7 +880,8 @@ describe("native artifact catalog", () => { expect(yield* fs.readFileString(prepared.executable)).toBe("postgrest"); const bucket = `https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com/postgrest/${version}`; expect(served).toEqual([ - `${bucket}/${asset}.SHA256SUMS`, + "https://ghcr.io/token?scope=repository:supabase/cli/postgrest:pull&service=ghcr.io", + `https://ghcr.io/v2/supabase/cli/postgrest/manifests/${version}-native-linux-amd64`, `${bucket}/${asset}.manifest.json`, `${bucket}/${asset}.tar.zst`, ]); From 7ab81a78fd6dd3fcef2b9b7ea7a1e524e65c3a7e Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 16:26:08 +0200 Subject: [PATCH 6/9] fix(stack): refuse root and report mirror fallback failures Native Postgres cannot initialize as root, so startup fails before any artifact download or process launch. A failed mirror fallback, including a checksum mismatch, is logged while the returned error stays the primary's. Unresolved env() placeholders are ignored only for the experimental S3 fields. --- apps/cli/src/command-internal/stack-config.ts | 9 +++ .../stack/stack-config.integration.test.ts | 27 ++++++++ .../experimental/stack/start/start.handler.ts | 8 ++- docs/adr/0026-slim-artifact-mirrors.md | 9 ++- packages/stack/src/Artifacts.unit.test.ts | 16 +++++ packages/stack/src/Service.ts | 25 ++++--- packages/stack/src/effect.ts | 1 + .../src/preparation/SlimServicesSource.ts | 18 +++-- packages/stack/src/runtime/Container.ts | 3 +- packages/stack/src/services/Database.ts | 68 ++++++++++++++----- .../stack/src/services/Database.unit.test.ts | 10 +++ 11 files changed, 159 insertions(+), 35 deletions(-) create mode 100644 packages/stack/src/Artifacts.unit.test.ts create mode 100644 packages/stack/src/services/Database.unit.test.ts diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index 3d5e438188..93fdaafdb8 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -717,6 +717,14 @@ const unsupportedConfigPaths = [ "experimental.s3_secret_key", ] as const; +/** An unset `env(NAME)` stays as that literal, which is not a configured value. */ +const unresolvedEnvLiteral = (value: unknown): boolean => { + if (typeof value === "string") return /^env\(([A-Z_][A-Z0-9_]*)\)$/.test(value); + if (!Redacted.isRedacted(value)) return false; + const inner = Redacted.value(value); + return typeof inner === "string" && /^env\(([A-Z_][A-Z0-9_]*)\)$/.test(inner); +}; + const pathValue = (value: unknown, path: string): unknown => { let current = value; for (const segment of path.split(".")) { @@ -779,6 +787,7 @@ const configValidationError = (config: CliConfig): string | undefined => { for (const path of unsupportedConfigPaths) { if (path.startsWith("auth.") && !config.auth.enabled) continue; const value = pathValue(config, path); + if (path.startsWith("experimental.s3_") && unresolvedEnvLiteral(value)) continue; const baseline = pathValue(defaults, path); const difference = firstDifference(value, baseline, path); if (difference !== undefined) return `${difference} is unsupported by the experimental stack`; diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index 6637d1398c..bd179e32ad 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -172,6 +172,33 @@ major_version = 14 }).pipe(Effect.provide(BunServices.layer)), ); + it.live("ignores unresolved experimental S3 env placeholders", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-s3-placeholder" +[experimental] +s3_host = "env(S3_HOST)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" +`); + const config = yield* load(root); + const services = yield* config.creations("stack-s3-placeholder"); + expect(services.some((service) => service.service === "database")).toBe(true); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.live("still rejects an unresolved auth service role key", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-auth-key" +[auth] +service_role_key = "env(SUPABASE_AUTH_SERVICE_ROLE_KEY)" +`); + const exit = yield* load(root).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("auth.service_role_key"); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("rejects unsupported OrioleDB and experimental S3 settings", () => Effect.gen(function* () { const orioledb = yield* project(`project_id = "stack-config-orioledb" diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 33c2e3164b..1fba17b5df 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -2,7 +2,7 @@ import { endpointReports } from "../stack-endpoints.format.ts"; import { readStackFunctionsEnv } from "../../../../command-internal/stack-functions-env.ts"; import { defaultStackRuntime } from "../../../../command-internal/stack-runtime.ts"; import { Effect, Equal, FileSystem, Fiber, Option, Path, Redacted, Ref } from "effect"; -import type { ServiceCreation, Stack } from "@supabase/stack/effect"; +import { nativePostgresRootError, type ServiceCreation, type Stack } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag, @@ -207,6 +207,10 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }), ), ); + const selectedRuntime = target.runtime ?? defaultStackRuntime(runtime); + const rootError = nativePostgresRootError(selectedRuntime, process.getuid?.()); + if (rootError !== undefined) + return yield* new StackCommandStartError({ reason: "lifecycle", message: rootError }); const stateRoot = path.join(settings.supabaseHome, "stacks"); const cacheRoot = path.join(settings.supabaseHome, "cache", "stack"); const stack = @@ -216,7 +220,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags projectRoot: target.projectRoot, stateRoot, cacheRoot, - runtime: target.runtime ?? defaultStackRuntime(runtime), + runtime: selectedRuntime, ...(target.name === undefined ? {} : { name: target.name }), }) .pipe(Effect.mapError(stackError)) diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index bc08b4cda1..0b392e06f9 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -15,9 +15,12 @@ Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase `force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Catalog sync consumes the image `service` / `version` / `digest` only; native tags ride in a separate `natives[]` payload field. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until a catalog PR ships. Natives follow the moved tag immediately. -ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. Do not add a public S3 bucket or a second cloud vendor for this cut. +ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. -Client fail-through (ordered GHCR / ECR / GitHub candidates in the CLI and stack) is out of scope here and lands in a follow-up PR. +The two decisions below are superseded by the Follow-up, which is the behavior this repository now ships: + +- A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases. It is not a checksum authority. +- The stack falls through to that bucket for natives and to ECR Public for images. A fallback that also fails still reports the primary's original error. ## Follow-up @@ -40,7 +43,7 @@ ECR Public mirroring already exists for other CLI images and needs no new vendor ## Alternatives considered 1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one major sandbox Trusted list. Rejected for this cut: new company-wide vendor. -2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Rejected for this cut: new public bucket, IAM, and security review on `PROD_AWS_ROLE`. +2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Superseded by the Follow-up for native archives only. The bucket is not a checksum authority, and image pulls stay on GHCR and ECR Public. 3. **npm packages** — allowlisted widely, but a published version cannot be replaced. 4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and names collide with upstream `supabase/`. 5. **Unpin slim images** — would make old CLIs see a moved tag. Rejected: conflicts with ADR 0017. diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts new file mode 100644 index 0000000000..0e0ecf510d --- /dev/null +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -0,0 +1,16 @@ +import { expect, it } from "@effect/vitest"; +import { slimImageMirrors } from "./Artifacts.ts"; + +it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { + expect( + slimImageMirrors( + "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + ), + ).toEqual([ + "public.ecr.aws/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + ]); +}); + +it("returns no mirror for an image outside the slim catalog registry", () => { + expect(slimImageMirrors("public.ecr.aws/supabase/postgres:17.6.1.173")).toEqual([]); +}); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 7e64997c7e..6d7e3e4470 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -56,6 +56,7 @@ class ServiceNotStopped extends Data.TaggedError("ServiceNotStopped")<{ export class ServiceStaleLaunch extends Data.TaggedError("ServiceStaleLaunch")<{ readonly id: string; readonly launchId: number; + readonly message: string; }> {} /** The exact runtime resources owned by one service launch. */ @@ -551,6 +552,14 @@ export const makeService = ( ); }); + const staleLaunch = (launchId: number, message: string) => + new ServiceStaleLaunch({ id: options.id, launchId, message }); + const diedBeforeReady = (launchId: number) => + Effect.gen(function* () { + const error = (yield* SubscriptionRef.get(observations)).error; + if (error !== undefined) return yield* error; + return yield* staleLaunch(launchId, `Service ${options.id} stopped before it was ready`); + }); const ready = Effect.fn("Service.ready")(function* () { const initial = yield* SubscriptionRef.get(observations); if (!initial.registered) return yield* new ServiceDestroyed({ id: options.id }); @@ -563,26 +572,26 @@ export const makeService = ( } if (record === undefined) { if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceStaleLaunch({ - id: options.id, - launchId: yield* Ref.get(launchCounter), - }); + return yield* staleLaunch( + yield* Ref.get(launchCounter), + `Service ${options.id} changed before it was ready`, + ); return yield* new ServiceNotRunning({ id: options.id }); } const launchId = record.launchId; if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); if (expectedLaunchId !== undefined && record.launchId !== expectedLaunchId) - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); if ((yield* SubscriptionRef.get(observations)).lifecycle === "stopping") { - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* diedBeforeReady(launchId); } const healthResult = yield* Deferred.await(record.health); if ( (yield* Ref.get(current)) !== record || (yield* SubscriptionRef.get(observations)).lifecycle !== "running" ) { - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* diedBeforeReady(launchId); } yield* healthResult; }); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 1e581bc2ff..d862b0be62 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -39,6 +39,7 @@ import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "./Defaults. export { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "./Defaults.ts"; export { postgres } from "./Tools.ts"; +export { nativePostgresRootError } from "./services/Database.ts"; export { StackError } from "./Rpc.ts"; export type { ServiceCreation } from "./services/Catalog.ts"; type DatabaseCreation = Extract; diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index 4751e747bb..90b57330a5 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -224,10 +224,17 @@ const downloadToFile = Effect.fn("SlimServicesSource.downloadToFile")(function* }); }); +const fallbackDetail = (error: PreparationError): string => { + const nested = error.cause instanceof PreparationError ? error.cause.message : undefined; + return nested === undefined || nested.length === 0 + ? error.message + : `${error.message} (${nested})`; +}; + /** - * Runs `attempt` against each candidate until one succeeds. Fallback failures are not surfaced: - * when every candidate fails, the error is the primary's. Mixing hosts across the checksum and - * materialize phases is safe because an archive is only accepted when it hashes to the checksum. + * Runs `attempt` against each candidate until one succeeds. When every candidate fails, the + * returned error is still the primary's. A fallback failure, including a checksum mismatch, is + * logged as a warning, and a fallback that succeeds names the host it used. */ const firstSuccess = ( candidates: readonly [T, ...ReadonlyArray], @@ -242,8 +249,11 @@ const firstSuccess = ( const [candidate, ...rest] = remaining; if (candidate === undefined) return Effect.fail(primaryError); return attempt(candidate).pipe( + Effect.tap(() => Effect.logInfo(`Slim-services used ${describe(candidate)}`)), Effect.tapError((cause) => - Effect.logDebug(`Slim-services fallback ${describe(candidate)} failed`, cause), + Effect.logWarning( + `Slim-services fallback ${describe(candidate)} failed: ${fallbackDetail(cause)}`, + ), ), Effect.catch(() => fallback(primaryError, rest)), ); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index b940a66404..fe456a717c 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -165,7 +165,8 @@ export const makeContainerRuntime = (options: { if (!(yield* present(mirror))) yield* pull(mirror); yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); }).pipe( - Effect.tapError((cause) => Effect.logDebug(`Image mirror ${mirror} failed`, cause)), + Effect.tap(() => Effect.logInfo(`Pulled image from mirror ${mirror}`)), + Effect.tapError((cause) => Effect.logWarning(`Image mirror ${mirror} failed`, cause)), Effect.catch(() => fromMirror(image, rest, primaryError)), ); }; diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 3a26a66f80..823c5e43ce 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -5,6 +5,7 @@ import { Data, Effect, Exit, + Fiber, FileSystem, Layer, Path, @@ -111,6 +112,13 @@ export interface DatabaseComponent { readonly logs: Stream.Stream; } +/** initdb and the server both refuse uid 0, so native startup fails before any process is spawned. */ +export const nativePostgresRootError = ( + runtime: string, + uid: number | undefined, +): string | undefined => + runtime === "native" && uid === 0 ? "PostgreSQL cannot be run as root" : undefined; + const errorFor = (operation: string, cause: unknown): ServiceError => cause instanceof ServiceError ? cause @@ -129,15 +137,30 @@ const databaseError = (operation: string, cause: unknown): DatabaseError => const processExit = ( exitCode: Effect.Effect, + stderr?: { + readonly tail: Ref.Ref; + readonly drained: Fiber.Fiber; + }, ): Effect.Effect> => - exitCode.pipe( - Effect.flatMap((code) => + Effect.all( + [stderr === undefined ? Effect.void : Fiber.join(stderr.drained).pipe(Effect.ignore), exitCode], + { concurrency: "unbounded" }, + ).pipe( + Effect.flatMap(([, code]) => Number(code) === 0 ? Effect.void - : Effect.fail( - new ServiceError({ - operation: "exit", - message: `PostgreSQL exited with code ${String(code)}`, + : (stderr === undefined ? Effect.succeed("") : Ref.get(stderr.tail)).pipe( + Effect.flatMap((text) => { + const detail = text.trim(); + return Effect.fail( + new ServiceError({ + operation: "exit", + message: + detail.length === 0 + ? `PostgreSQL exited with code ${String(code)}` + : `PostgreSQL exited with code ${String(code)}: ${detail}`, + }), + ); }), ), ), @@ -305,19 +328,25 @@ const publishLogs = Effect.fn("Database.publishLogs")(( }, logs: PubSub.PubSub, scope: Scope.Closeable, -): Effect.Effect => { + stderrTail?: Ref.Ref, +) => { const drain = (stream: Stream.Stream, name: DatabaseLog["stream"]) => stream.pipe( - Stream.runForEach((bytes) => PubSub.publish(logs, { stream: name, bytes })), + Stream.runForEach((bytes) => + Effect.gen(function* () { + if (name === "stderr" && stderrTail !== undefined) { + const text = new TextDecoder().decode(bytes); + yield* Ref.update(stderrTail, (current) => (current + text).slice(-4096)); + } + yield* PubSub.publish(logs, { stream: name, bytes }); + }), + ), Effect.catch((cause) => Effect.logError(cause)), ); - return Effect.all( - [ - Effect.forkIn(drain(process.stdout, "stdout"), scope), - Effect.forkIn(drain(process.stderr, "stderr"), scope), - ], - { concurrency: "unbounded", discard: true }, - ); + return Effect.gen(function* () { + yield* Effect.forkIn(drain(process.stdout, "stdout"), scope); + return yield* Effect.forkIn(drain(process.stderr, "stderr"), scope); + }); }); const runtimeFromContainer = (process: ContainerProcess): RuntimeSession => ({ @@ -572,6 +601,8 @@ export const makeDatabase = ( const prepare = Effect.fn("Database.prepare")( function* (input: DatabaseConfig) { + const rootError = nativePostgresRootError(options.runtime, process.getuid?.()); + if (rootError !== undefined) return yield* errorFor("prepare", rootError); const markerPath = path.join(instanceRoot, ".supabase-database-ready.json"); const hasMarker = yield* fs.exists(markerPath); if (hasMarker) { @@ -618,6 +649,8 @@ export const makeDatabase = ( context: ServiceInstanceContext, ): Effect.Effect => Effect.gen(function* () { + const rootError = nativePostgresRootError(options.runtime, process.getuid?.()); + if (rootError !== undefined) return yield* errorFor("launch", rootError); const config = { ...context.config, version: postgresVersion(context.config.version) }; const dataPath = path.join(instanceRoot, "data"); yield* fs @@ -667,7 +700,8 @@ export const makeDatabase = ( port: 5432, }; yield* Ref.set(endpoint, selectedEndpoint); - yield* publishLogs(process, logs, context.scope); + const stderrTail = yield* Ref.make(""); + const stderrDrained = yield* publishLogs(process, logs, context.scope, stderrTail); return { health: health(selectedEndpoint, config, Effect.void, { fs, @@ -675,7 +709,7 @@ export const makeDatabase = ( version: config.version, runtime: options.runtime, }), - exit: processExit(process.exitCode), + exit: processExit(process.exitCode, { tail: stderrTail, drained: stderrDrained }), stop: process.kill.pipe(Effect.mapError((cause) => errorFor("stop", cause))), remove: fs.remove(socketPath, { recursive: true, force: true }).pipe( Effect.mapError((cause) => errorFor("remove", cause)), diff --git a/packages/stack/src/services/Database.unit.test.ts b/packages/stack/src/services/Database.unit.test.ts new file mode 100644 index 0000000000..b64a07e101 --- /dev/null +++ b/packages/stack/src/services/Database.unit.test.ts @@ -0,0 +1,10 @@ +import { expect, it } from "@effect/vitest"; +import { nativePostgresRootError } from "./Database.ts"; + +it("refuses native PostgreSQL for uid 0 and allows every other runtime", () => { + expect(nativePostgresRootError("native", 0)).toBe("PostgreSQL cannot be run as root"); + expect(nativePostgresRootError("native", 501)).toBeUndefined(); + expect(nativePostgresRootError("native", undefined)).toBeUndefined(); + expect(nativePostgresRootError("docker", 0)).toBeUndefined(); + expect(nativePostgresRootError("podman", 0)).toBeUndefined(); +}); From 4b3f12a043eac6a5210f356dc73363f6efccceab Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 17:37:51 +0200 Subject: [PATCH 7/9] fix(stack): bound native exit and keep launch errors together A held stderr pipe no longer stalls PostgreSQL shutdown, and a readiness waiter only reports the error from its own launch. Unresolved env() placeholders use the config loader's matcher. --- apps/cli/src/command-internal/stack-config.ts | 6 +-- .../stack/stack-config.integration.test.ts | 23 +++++++-- docs/adr/0026-slim-artifact-mirrors.md | 12 ++--- packages/stack/src/Service.ts | 47 ++++++++++++++----- packages/stack/src/services/Database.ts | 38 +++++++++++---- 5 files changed, 87 insertions(+), 39 deletions(-) diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index 93fdaafdb8..ed7fed08b4 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -1,5 +1,5 @@ import { getDefaultCliConfig, type CliConfig } from "@supabase/config"; -import { resolveCliConfigSubtree } from "@supabase/config/internal"; +import { ENV_CAPTURE_REGEX, resolveCliConfigSubtree } from "@supabase/config/internal"; import { validateCliConfig } from "@supabase/config/effect"; import { DEFAULT_LOCAL_JWT_SECRET, @@ -719,10 +719,10 @@ const unsupportedConfigPaths = [ /** An unset `env(NAME)` stays as that literal, which is not a configured value. */ const unresolvedEnvLiteral = (value: unknown): boolean => { - if (typeof value === "string") return /^env\(([A-Z_][A-Z0-9_]*)\)$/.test(value); + if (typeof value === "string") return ENV_CAPTURE_REGEX.test(value); if (!Redacted.isRedacted(value)) return false; const inner = Redacted.value(value); - return typeof inner === "string" && /^env\(([A-Z_][A-Z0-9_]*)\)$/.test(inner); + return typeof inner === "string" && ENV_CAPTURE_REGEX.test(inner); }; const pathValue = (value: unknown, path: string): unknown => { diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index bd179e32ad..a93011eb1c 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -172,11 +172,14 @@ major_version = 14 }).pipe(Effect.provide(BunServices.layer)), ); - it.live("ignores unresolved experimental S3 env placeholders", () => - Effect.gen(function* () { + it.live("ignores unresolved experimental S3 env placeholders", () => { + const names = ["s3_host", "S3_REGION", "S3_ACCESS_KEY", "S3_SECRET_KEY"] as const; + const saved = names.map((name) => [name, process.env[name]] as const); + for (const name of names) process.env[name] = ""; + return Effect.gen(function* () { const root = yield* project(`project_id = "stack-config-s3-placeholder" [experimental] -s3_host = "env(S3_HOST)" +s3_host = "env(s3_host)" s3_region = "env(S3_REGION)" s3_access_key = "env(S3_ACCESS_KEY)" s3_secret_key = "env(S3_SECRET_KEY)" @@ -184,8 +187,18 @@ s3_secret_key = "env(S3_SECRET_KEY)" const config = yield* load(root); const services = yield* config.creations("stack-s3-placeholder"); expect(services.some((service) => service.service === "database")).toBe(true); - }).pipe(Effect.provide(BunServices.layer)), - ); + }).pipe( + Effect.ensuring( + Effect.sync(() => { + for (const [name, value] of saved) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }), + ), + Effect.provide(BunServices.layer), + ); + }); it.live("still rejects an unresolved auth service role key", () => Effect.gen(function* () { diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 0b392e06f9..e46a852c8b 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -7,7 +7,7 @@ Slim service images publish to `ghcr.io/supabase/cli/:`. Native archives publish to GitHub Releases. A broken build must be replaceable under the **same** upstream version string (`force=true`); bumping the tag is not an option. -Some consumers cannot reach GitHub release assets or a single registry blob CDN. Publishing one digest to more than one host is the prerequisite for later client fail-through; this ADR covers the publish path only. +Some consumers cannot reach GitHub release assets or a single registry blob CDN. This ADR covers publishing those copies and the local stack's fail-through. ## Decision @@ -17,16 +17,12 @@ Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. -The two decisions below are superseded by the Follow-up, which is the behavior this repository now ships: +A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR. An archive from either host is accepted only when it matches. -- A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases. It is not a checksum authority. -- The stack falls through to that bucket for natives and to ECR Public for images. A fallback that also fails still reports the primary's original error. +The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing fallback, for an image or a native archive, still reports the primary's original error. ## Follow-up -- Consume the mirrored image tags and `:version-native-` artifacts from the CLI and local stack, with host-aware order and digest pins preserved ([ADR 0017](0017-simplified-managed-stack-architecture.md)). -- Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Natives are therefore also copied to a public S3 bucket on `*.amazonaws.com` ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)), independently of the ECR copy. The local stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR; the bucket is never a checksum authority, and an archive from either host is only accepted when it matches. -- The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. For images and natives alike, a failing fallback reports the primary's original error, so the mirrors only change behavior when they rescue a failed download. - Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. ## Rationale @@ -43,7 +39,7 @@ ECR Public mirroring already exists for other CLI images and needs no new vendor ## Alternatives considered 1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one major sandbox Trusted list. Rejected for this cut: new company-wide vendor. -2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Superseded by the Follow-up for native archives only. The bucket is not a checksum authority, and image pulls stay on GHCR and ECR Public. +2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Adopted for native archives. The bucket is not a checksum authority, and image pulls stay on GHCR and ECR Public. 3. **npm packages** — allowlisted widely, but a published version cannot be replaced. 4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and names collide with upstream `supabase/`. 5. **Unpin slim images** — would make old CLIs see a moved tag. Rejected: conflicts with ADR 0017. diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 6d7e3e4470..22671eb590 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -182,6 +182,10 @@ export const makeService = ( const revision = yield* Ref.make(0); const launchCounter = yield* Ref.make(0); const current = yield* Ref.make(undefined); + // stopNow clears observation.launchId while keeping the error, so readiness must not read that field. + const launchError = yield* Ref.make< + { readonly launchId: number; readonly error: ServiceError } | undefined + >(undefined); const config = yield* Ref.make(options.config); const observations = yield* SubscriptionRef.make>({ id: options.id, @@ -297,15 +301,19 @@ export const makeService = ( "start", guard.pipe( Effect.andThen( - update({ - lifecycle: "starting", - health: "starting", - error: undefined, - cleanupError: undefined, - exit: undefined, - wakeEnabled: observation.wakeEnabled, - launchId, - }), + Ref.set(launchError, undefined).pipe( + Effect.andThen( + update({ + lifecycle: "starting", + health: "starting", + error: undefined, + cleanupError: undefined, + exit: undefined, + wakeEnabled: observation.wakeEnabled, + launchId, + }), + ), + ), ), ), ); @@ -320,11 +328,13 @@ export const makeService = ( ); if (Exit.isFailure(launchExit)) { yield* Scope.close(runtimeScope, launchExit); + const error = exitError("launch", launchExit); + if (error !== undefined) yield* Ref.set(launchError, { launchId, error }); yield* update({ lifecycle: "stopped", health: undefined, wakeEnabled: observation.wakeEnabled, - error: exitError("launch", launchExit), + error, launchId: undefined, }); return yield* Effect.failCause(launchExit.cause); @@ -341,6 +351,8 @@ export const makeService = ( removed: yield* Ref.make(false), }; yield* Ref.set(current, record); + if (launchFailure !== undefined) + yield* Ref.set(launchError, { launchId, error: launchFailure }); yield* update({ lifecycle: launchFailure === undefined ? "running" : "stopping", health: launchFailure === undefined ? "starting" : undefined, @@ -352,12 +364,19 @@ export const makeService = ( const observeHealth = runtime.health.pipe( Effect.onExit((exit) => Effect.gen(function* () { + const error = exitError("health", exit); + const currentObservation = yield* SubscriptionRef.get(observations); + if ( + error !== undefined && + currentObservation.launchId === launchId && + currentObservation.lifecycle === "running" + ) + yield* Ref.set(launchError, { launchId, error }); yield* SubscriptionRef.update( observations, (observation): ServiceObservation => { if (observation.launchId !== launchId || observation.lifecycle !== "running") return observation; - const error = exitError("health", exit); return { ...observation, health: Exit.isSuccess(exit) ? "healthy" : "unhealthy", @@ -382,6 +401,8 @@ export const makeService = ( operation: "exit", message: "Runtime exited unexpectedly", })); + if (error !== undefined) + yield* Ref.set(launchError, { launchId: record.launchId, error }); yield* update({ lifecycle: "stopping", health: undefined, @@ -556,8 +577,8 @@ export const makeService = ( new ServiceStaleLaunch({ id: options.id, launchId, message }); const diedBeforeReady = (launchId: number) => Effect.gen(function* () { - const error = (yield* SubscriptionRef.get(observations)).error; - if (error !== undefined) return yield* error; + const owned = yield* Ref.get(launchError); + if (owned !== undefined && owned.launchId === launchId) return yield* owned.error; return yield* staleLaunch(launchId, `Service ${options.id} stopped before it was ready`); }); const ready = Effect.fn("Service.ready")(function* () { diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index aeb6afc719..75d7426797 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -148,6 +148,14 @@ const databaseError = (operation: string, cause: unknown): DatabaseError => cause, }); +/** A descendant outside the process group can keep stderr open after the launcher exits. */ +const stderrTailReady = (drained: Fiber.Fiber) => + Fiber.await(drained).pipe( + Effect.asVoid, + Effect.raceFirst(Effect.sleep("1 second")), + Effect.ignore, + ); + const processExit = ( exitCode: Effect.Effect, stderr?: { @@ -155,11 +163,13 @@ const processExit = ( readonly drained: Fiber.Fiber; }, ): Effect.Effect> => - Effect.all( - [stderr === undefined ? Effect.void : Fiber.join(stderr.drained).pipe(Effect.ignore), exitCode], - { concurrency: "unbounded" }, + (stderr === undefined + ? exitCode + : Effect.all([exitCode, stderrTailReady(stderr.drained)], { concurrency: "unbounded" }).pipe( + Effect.map(([code]) => code), + ) ).pipe( - Effect.flatMap(([, code]) => + Effect.flatMap((code) => Number(code) === 0 ? Effect.void : (stderr === undefined ? Effect.succeed("") : Ref.get(stderr.tail)).pipe( @@ -345,19 +355,27 @@ const publishLogs = Effect.fn("Database.publishLogs")(( scope: Scope.Closeable, stderrTail?: Ref.Ref, ) => { - const drain = (stream: Stream.Stream, name: DatabaseLog["stream"]) => - stream.pipe( + const drain = (stream: Stream.Stream, name: DatabaseLog["stream"]) => { + const decoder = name === "stderr" && stderrTail !== undefined ? new TextDecoder() : undefined; + const appendTail = (text: string) => + text.length === 0 || stderrTail === undefined + ? Effect.void + : Ref.update(stderrTail, (current) => (current + text).slice(-4096)); + return stream.pipe( Stream.runForEach((bytes) => Effect.gen(function* () { - if (name === "stderr" && stderrTail !== undefined) { - const text = new TextDecoder().decode(bytes); - yield* Ref.update(stderrTail, (current) => (current + text).slice(-4096)); - } + if (decoder !== undefined) yield* appendTail(decoder.decode(bytes, { stream: true })); yield* PubSub.publish(logs, { stream: name, bytes }); }), ), + Effect.andThen( + decoder === undefined + ? Effect.void + : Effect.sync(() => decoder.decode()).pipe(Effect.flatMap(appendTail)), + ), Effect.catch((cause) => Effect.logError(cause)), ); + }; return Effect.gen(function* () { yield* Effect.forkIn(drain(process.stdout, "stdout"), scope); return yield* Effect.forkIn(drain(process.stderr, "stderr"), scope); From ad7bf027fe0dc2d87a70708b7f3dc7857a4a519c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 15:41:18 +0000 Subject: [PATCH 8/9] test(cli): stub S3 placeholder env with vi.stubEnv The S3 placeholder test blanked its env vars by writing `process.env` directly, which the Effect lint rejects in CI. `vi.stubEnv` with an `afterEach` `vi.unstubAllEnvs()` keeps the same isolation. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- .../stack/stack-config.integration.test.ts | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index a93011eb1c..3edd257096 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -1,5 +1,5 @@ import { BunServices } from "@effect/platform-bun"; -import { describe, expect, it } from "@effect/vitest"; +import { afterEach, describe, expect, it, vi } from "@effect/vitest"; import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "@supabase/stack"; import { Effect, Exit, Layer, Redacted, Schema } from "effect"; import { ServiceCreation } from "../../../../../../packages/stack/src/services/Catalog.ts"; @@ -23,6 +23,10 @@ const byService = (services: ReadonlyArray [service.service, service])); describe("loadStackConfig", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it.live("decodes the default recipe and leaves listeners automatic", () => Effect.gen(function* () { const root = yield* project(`project_id = "stack-config-defaults" @@ -173,9 +177,8 @@ major_version = 14 ); it.live("ignores unresolved experimental S3 env placeholders", () => { - const names = ["s3_host", "S3_REGION", "S3_ACCESS_KEY", "S3_SECRET_KEY"] as const; - const saved = names.map((name) => [name, process.env[name]] as const); - for (const name of names) process.env[name] = ""; + for (const name of ["s3_host", "S3_REGION", "S3_ACCESS_KEY", "S3_SECRET_KEY"]) + vi.stubEnv(name, ""); return Effect.gen(function* () { const root = yield* project(`project_id = "stack-config-s3-placeholder" [experimental] @@ -187,17 +190,7 @@ s3_secret_key = "env(S3_SECRET_KEY)" const config = yield* load(root); const services = yield* config.creations("stack-s3-placeholder"); expect(services.some((service) => service.service === "database")).toBe(true); - }).pipe( - Effect.ensuring( - Effect.sync(() => { - for (const [name, value] of saved) { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; - } - }), - ), - Effect.provide(BunServices.layer), - ); + }).pipe(Effect.provide(BunServices.layer)); }); it.live("still rejects an unresolved auth service role key", () => From 86a72a0d4e4065c82466a6cb28e32090f41bc290 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 16:17:19 +0000 Subject: [PATCH 9/9] fix(stack): return to the primary image and wait for stderr after exit After a mirror rescue, a later prepare that found or pulled the GHCR image kept the remembered ECR reference, so a pruned mirror image failed the launch. Prepare now forgets the mirror whenever the primary is ready. The native PostgreSQL stderr-drain guard ran alongside the exit wait, so it had already expired for any process that lived past one second. It now starts after the exit code resolves. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016ctyhVP3n3xNy7YRtGYp4p --- packages/stack/src/runtime/Container.ts | 10 ++++- ...container-image-mirror.integration.test.ts | 39 ++++++++++++++++++- packages/stack/src/services/Database.ts | 7 ++-- .../stack/src/services/Database.unit.test.ts | 22 ++++++++++- 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index a8692db74d..7cf8c1ae24 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -179,9 +179,17 @@ export const makeContainerRuntime = (options: { }; const prepare = Effect.fn("Container.prepare")(function* (image: string) { - if (yield* present(image)) return; + // A mirror chosen earlier may have been pruned since; launches follow the primary again. + const usePrimary = Ref.update(mirrored, (map) => { + if (!map.has(image)) return map; + const next = new Map(map); + next.delete(image); + return next; + }); + if (yield* present(image)) return yield* usePrimary; const mirrors = options.imageMirrors?.(image) ?? []; yield* pull(image).pipe( + Effect.andThen(usePrimary), Effect.catch((primaryError) => fromMirror(image, mirrors, primaryError)), ); }); diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts index 2573266d59..0680053bd4 100644 --- a/packages/stack/src/runtime/container-image-mirror.integration.test.ts +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -8,8 +8,9 @@ const primary = "ghcr.io/supabase/cli/postgrest:v16.2"; const mirror = "public.ecr.aws/supabase/cli/postgrest:v16.2"; const secondMirror = "registry.test/supabase/cli/postgrest:v16.2"; -/** Docker stand-in: `image ls` reports `local`, `pull` succeeds for `pullable`, `create` refuses. */ +/** Docker stand-in: `image inspect` reports `local`, `pull` succeeds for `pullable`, `create` refuses. */ const fakeEngine = (options: { readonly pullable: ReadonlyArray }) => { + const pullable = new Set(options.pullable); const local = new Set(); const commands: string[][] = []; const handle = (exitCode: number, stdout = "", stderr = "") => @@ -33,7 +34,7 @@ const fakeEngine = (options: { readonly pullable: ReadonlyArray }) => { const ref = args.at(-1) ?? ""; if (args[0] === "image") return Effect.succeed(handle(0, local.has(ref) ? "sha256:1" : "")); if (args[0] === "pull") { - if (!options.pullable.includes(ref)) return Effect.succeed(handle(1, "", `denied: ${ref}`)); + if (!pullable.has(ref)) return Effect.succeed(handle(1, "", `denied: ${ref}`)); local.add(ref); return Effect.succeed(handle(0)); } @@ -41,6 +42,8 @@ const fakeEngine = (options: { readonly pullable: ReadonlyArray }) => { }); return { commands, + pullable, + local, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), }; }; @@ -86,6 +89,38 @@ describe("container image mirror", () => { }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); + it.live("launches the primary again once it pulls after an earlier mirror rescue", () => { + const engine = fakeEngine({ pullable: [mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + yield* runtime.prepare(primary); + engine.local.delete(mirror); + engine.pullable.add(primary); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("launches the primary again once it is present after an earlier mirror rescue", () => { + const engine = fakeEngine({ pullable: [mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + yield* runtime.prepare(primary); + engine.local.delete(mirror); + engine.local.add(primary); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + it.live("reports the primary pull failure when the mirror also fails", () => { const engine = fakeEngine({ pullable: [] }); return Effect.gen(function* () { diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 75d7426797..f6e89b365c 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -156,7 +156,8 @@ const stderrTailReady = (drained: Fiber.Fiber) => Effect.ignore, ); -const processExit = ( +/** Settles a native PostgreSQL exit, waiting briefly after it for the stderr tail to drain. */ +export const processExit = ( exitCode: Effect.Effect, stderr?: { readonly tail: Ref.Ref; @@ -165,9 +166,7 @@ const processExit = ( ): Effect.Effect> => (stderr === undefined ? exitCode - : Effect.all([exitCode, stderrTailReady(stderr.drained)], { concurrency: "unbounded" }).pipe( - Effect.map(([code]) => code), - ) + : exitCode.pipe(Effect.tap(() => stderrTailReady(stderr.drained))) ).pipe( Effect.flatMap((code) => Number(code) === 0 diff --git a/packages/stack/src/services/Database.unit.test.ts b/packages/stack/src/services/Database.unit.test.ts index b64a07e101..106c559758 100644 --- a/packages/stack/src/services/Database.unit.test.ts +++ b/packages/stack/src/services/Database.unit.test.ts @@ -1,5 +1,7 @@ import { expect, it } from "@effect/vitest"; -import { nativePostgresRootError } from "./Database.ts"; +import { Cause, Effect, Exit, Fiber, Option, Ref } from "effect"; +import * as TestClock from "effect/testing/TestClock"; +import { nativePostgresRootError, processExit } from "./Database.ts"; it("refuses native PostgreSQL for uid 0 and allows every other runtime", () => { expect(nativePostgresRootError("native", 0)).toBe("PostgreSQL cannot be run as root"); @@ -8,3 +10,21 @@ it("refuses native PostgreSQL for uid 0 and allows every other runtime", () => { expect(nativePostgresRootError("docker", 0)).toBeUndefined(); expect(nativePostgresRootError("podman", 0)).toBeUndefined(); }); + +it.effect("waits for the stderr tail after a long-running native PostgreSQL exits", () => + Effect.gen(function* () { + const tail = yield* Ref.make(""); + const drained = yield* Effect.forkChild( + Effect.sleep("2500 millis").pipe(Effect.andThen(Ref.set(tail, "FATAL: data directory"))), + ); + const settled = yield* Effect.forkChild( + processExit(Effect.sleep("2 seconds").pipe(Effect.as(1)), { tail, drained }), + ); + yield* TestClock.adjust("3 seconds"); + const exit = yield* Fiber.join(settled); + const error = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) + : undefined; + expect(error?.message).toBe("PostgreSQL exited with code 1: FATAL: data directory"); + }), +);