diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 809dce551f..33af901949 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -92,6 +92,24 @@ FROM postgrest/postgrest:v16.3 AS postgrest expect(oriole.source).toContain(`"${POSTGRES_17}"`); }); + test("an image the ECR mirror does not serve under the GHCR digest blocks the commit", async () => { + const plan = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM supabase/postgres:17.6.1.168 AS pg\n", + dockerfile: "FROM supabase/postgres:17.6.1.171 AS pg\n", + catalog: fixture, + publication: async () => ({ status: "unmirrored" }), + }); + expect(plan.source).toBe(fixture); + expect(plan.skipped).toEqual([ + { + alias: "pg", + reason: + "postgres:17.6.1.171 on public.ecr.aws/supabase/cli does not match the GHCR digest.", + blocking: true, + }, + ]); + }); + test("refuses a backward pin and a tag that would escape the catalog string", async () => { const backward = await planArtifactCatalogUpdate({ baseDockerfile: "FROM supabase/postgres:17.6.1.170 AS pg\n", diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 838f2c5b75..2fe0bef1dc 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -3,6 +3,8 @@ * `apps/cli/src/shared/services/Dockerfile`. Native archive URLs are derived * from service + version. An entry that already carries a digest keeps one: * the published `ghcr.io/supabase/cli/:` manifest digest. + * A pin is refused until ECR Public serves the same digest, so the stack's + * image fallback can pull it. * * Run: `bun .github/scripts/sync-artifacts-catalog.ts [base-dockerfile]` */ @@ -10,6 +12,7 @@ import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; import { + DEST_REGISTRY, DIGEST_PATTERN, InvalidPayloadError, SOURCE_REGISTRY, @@ -71,7 +74,7 @@ export interface CatalogPlan { export type ReleasePublication = | { readonly status: "published"; readonly digest?: string } - | { readonly status: "missing" | "lookup-failed" }; + | { readonly status: "missing" | "unmirrored" | "lookup-failed" }; /** `definition("", "", ""`. */ function defaultEntryPattern(service: string): RegExp { @@ -267,7 +270,9 @@ export async function planArtifactCatalogUpdate(input: { reason: release.status === "missing" ? `${pin.service}:${pin.version} has no published slim image and native release.` - : `${pin.service}:${pin.version} publication check failed.`, + : release.status === "unmirrored" + ? `${pin.service}:${pin.version} on ${DEST_REGISTRY} does not match the GHCR digest.` + : `${pin.service}:${pin.version} publication check failed.`, blocking: true, }); continue; @@ -347,15 +352,17 @@ async function probeNativeRelease(service: string, version: string): Promise { - const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; - const [manifest, native] = await Promise.all([ - probeManifest(reference), + const [manifest, mirror, native] = await Promise.all([ + probeManifest(`${SLIM_IMAGE_PREFIX}${service}:${version}`), + probeManifest(`${DEST_REGISTRY}/${service}:${version}`), probeNativeRelease(service, version), ]); if (manifest.probe === "lookup-failed" || native === "lookup-failed") { return { status: "lookup-failed" }; } if (manifest.probe === "missing" || native === "missing") return { status: "missing" }; + if (mirror.probe === "lookup-failed") return { status: "lookup-failed" }; + if (mirror.digest !== manifest.digest) return { status: "unmirrored" }; return { status: "published", digest: manifest.digest }; } diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index 3ddf1f803c..de4c4298e4 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -75,8 +75,9 @@ stack = true The selected backend determines accepted flags, help, and completion before the command is parsed. Set the flag to `false`, or remove it, to restore the legacy top-level commands. The explicit -`supabase stack` namespace is available only when this flag is enabled. `supabase status` is routed -the same way as `supabase start` and `supabase stop`. +`supabase stack` namespace is available only when this flag is enabled; otherwise `supabase stack` +exits with an error that names both ways to enable it. `supabase status` is routed the same way as +`supabase start` and `supabase stop`. Root help and root completion resolve the same feature flag from the environment or project configuration. Help and completion for `start`, `status`, and `stop` resolve the same backend as the diff --git a/apps/cli/src/command-internal/stack-backend.ts b/apps/cli/src/command-internal/stack-backend.ts index 9c12d7023a..8475f575dd 100644 --- a/apps/cli/src/command-internal/stack-backend.ts +++ b/apps/cli/src/command-internal/stack-backend.ts @@ -36,9 +36,12 @@ const isFunctionsServePath = (path: ReadonlyArray): boolean => export class StackRoutingError extends Data.TaggedError("StackRoutingError")<{ readonly message: string; readonly cause?: unknown; + readonly disabledNamespace?: boolean; }> { get suggestion(): string { - return "Set SUPABASE_EXPERIMENTAL_STACK=1 to enable stack commands, or 0 to use legacy start/stop/status."; + return this.disabledNamespace === true + ? "Set SUPABASE_EXPERIMENTAL_STACK=1, or leave it unset and add `stack = true` under [experimental] in supabase/config.toml." + : "Set SUPABASE_EXPERIMENTAL_STACK=1 to enable stack commands, or 0 to use legacy start/stop/status."; } get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { @@ -69,10 +72,8 @@ export const resolveStackBackend = (input: { Effect.gen(function* () { // Completion passes the final token as the cursor word, so it is not part of // the resolved command path. - const routingArgs = - input.args[0] === "__complete" || input.args[0] === "__completeNoDesc" - ? input.args.slice(0, -1) - : input.args; + const completing = input.args[0] === "__complete" || input.args[0] === "__completeNoDesc"; + const routingArgs = completing ? input.args.slice(0, -1) : input.args; if (hasRootVersionFlag(routingArgs)) return "legacy"; const commandPath = extractCommandPath(routingArgs); @@ -96,5 +97,10 @@ export const resolveStackBackend = (input: { }).pipe( Effect.mapError((error) => new StackRoutingError({ message: error.message, cause: error })), ); + if (!enabled && command === "stack" && !completing) + return yield* new StackRoutingError({ + message: "`supabase stack` requires the experimental stack backend.", + disabledNamespace: true, + }); return enabled ? "stack" : "legacy"; }); diff --git a/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts index 2dfb685dd2..a522e6d7a5 100644 --- a/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts @@ -29,9 +29,23 @@ const completionFlags = (backend: "legacy" | "stack", command: string) => ])?.candidates.map(({ name }) => name); describe("resolveStackBackend", () => { - it.effect("keeps the explicit stack namespace disabled without the feature flag", () => + it.effect("explains how to enable the explicit stack namespace without the feature flag", () => Effect.gen(function* () { - expect(yield* resolve({ args: ["stack", "start"], cwd: "/missing", env: {} })).toBe("legacy"); + for (const args of [ + ["stack", "start"], + ["help", "stack"], + ]) { + const error = yield* resolve({ args, cwd: "/missing", env: {} }).pipe(Effect.flip); + expect(error).toBeInstanceOf(StackRoutingError); + expect(error.message).toBe("`supabase stack` requires the experimental stack backend."); + expect(error.suggestion).toContain("SUPABASE_EXPERIMENTAL_STACK=1"); + expect(error.suggestion).toContain( + "leave it unset and add `stack = true` under [experimental]", + ); + } + expect(yield* resolve({ args: ["__complete", "stack", ""], cwd: "/missing", env: {} })).toBe( + "legacy", + ); }), ); diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 31b429dcc1..6eb39e5abc 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -19,7 +19,7 @@ ECR Public tags are always mutable; `ecr-public create-repository` has no immuta A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR. An archive from either host is accepted only when it matches. -The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing fallback, for an image or a native archive, still reports the primary's original error. +The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. When every source fails, for an image or a native archive, the error leads with the primary's failure and then lists each fallback's. The catalog sync refuses a pin until ECR Public serves the GHCR digest, because a digest pin that only GHCR carries cannot fall back. ## Follow-up diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 78b6d5032c..1b2680329e 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -111,13 +111,13 @@ const definitions: Readonly> = { studio: definition( "studio", "2026.09.04-sha-5a67366", - "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9823a31668028f1846e87331bc21598d9cd74bcaa1466c72dab58c33c9c82720", + "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9362a66196c1622bb90e1b8014ff5376505969906fac38e58ffee0ca178daf07", "bin/studio", ), pgmeta: definition( "pgmeta", "v0.99.0", - "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:90de2dcf03ac548ae2d1d3e71b3cd10bde4c627572720a42e4c3946b7090292e", + "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:4a49f779f78b66af42a20425d66b0cc3083ffe09fb42da05e62c9d771bc309b0", "bin/pgmeta", ), mail: definition("mailpit", "v1.30.2", "ghcr.io/supabase/cli/mailpit:v1.30.2", "bin/mailpit"), @@ -135,7 +135,7 @@ const definitions: Readonly> = { pooler: definition( "pooler", "v2.9.12", - "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:12bb9dcb7ddace79bee173ccb7327c6646af2236679f3bd932a86b3a06479aac", + "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:a0278f60f7bc73b8fabc28d8d144eeb326afea566ea81012826c9a869d62664a", "bin/server", ["bin/server", "bin/prepare", "bin/provision-tenant"], ), diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts index 0e0ecf510d..c418fe490d 100644 --- a/packages/stack/src/Artifacts.unit.test.ts +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -4,10 +4,10 @@ import { slimImageMirrors } from "./Artifacts.ts"; it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { expect( slimImageMirrors( - "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a", ), ).toEqual([ - "public.ecr.aws/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + "public.ecr.aws/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a", ]); }); diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index 90b57330a5..71289e591d 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -233,8 +233,9 @@ const fallbackDetail = (error: PreparationError): string => { /** * Runs `attempt` against each candidate until one succeeds. When every candidate fails, the - * returned error is still the primary's. A fallback failure, including a checksum mismatch, is - * logged as a warning, and a fallback that succeeds names the host it used. + * returned error is the primary's, with each fallback failure appended to its message. A fallback + * failure, including a checksum mismatch, is logged as a warning, and a fallback that succeeds + * names the host it used. */ const firstSuccess = ( candidates: readonly [T, ...ReadonlyArray], @@ -255,7 +256,17 @@ const firstSuccess = ( `Slim-services fallback ${describe(candidate)} failed: ${fallbackDetail(cause)}`, ), ), - Effect.catch(() => fallback(primaryError, rest)), + Effect.catch((cause) => + fallback( + new PreparationError({ + ...primaryError, + // `cause` is a non-enumerable Error property, so the spread drops it. + cause: primaryError.cause, + message: `${primaryError.message}\nFallback ${describe(candidate)} also failed: ${fallbackDetail(cause)}`, + }), + rest, + ), + ), ); }; return attempt(primary).pipe(Effect.catch((primaryError) => fallback(primaryError, fallbacks))); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 467c31b253..be72958c19 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -377,7 +377,7 @@ describe("slim-services artifact source", () => { ), ); - it.live("reports the primary checksum failure when every checksum source fails", () => + it.live("reports the primary checksum failure, then the fallback's", () => Effect.gen(function* () { const mirrored: SlimServicesArtifact = { ...artifact, @@ -391,7 +391,11 @@ describe("slim-services artifact source", () => { requested.push(requestUrl(input)); return Promise.resolve(new Response("", { status: 403 })); }, slimServicesChecksum(mirrored, immediate).pipe(Effect.exit)); - expect(errorOf(failed)?.message).toBe("Unable to download https://release.test/SHA256SUMS"); + expect(errorOf(failed)?.message.split("\n")).toEqual([ + "Unable to download https://release.test/SHA256SUMS", + "Fallback registry.test/supabase/cli/demo:v1.0.0-native-linux-amd64 also failed: Unable to download https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test (HTTP 403)", + ]); + expect(errorOf(failed)?.cause).toMatchObject({ message: "HTTP 403", status: 403 }); expect(requested).toEqual([ "https://release.test/SHA256SUMS", "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test", @@ -445,7 +449,9 @@ describe("slim-services artifact source", () => { .materialize(request, rejected, expected) .pipe(Effect.exit), ); - expect(errorOf(failed)?.message).toBe("Unable to download slim-services archive"); + expect(errorOf(failed)?.message.split("\n")[0]).toBe( + "Unable to download slim-services archive", + ); expect(yield* fs.exists(`${rejected}/bin/demo`)).toBe(false); }).pipe(Effect.provide(NodeServices.layer)), ), diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 5d97f9c483..c56831dfeb 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -109,8 +109,8 @@ const mountField = (key: string, value: string) => { /** * Captures the selected local engine; each launch owns one exact container. An image whose pull * fails is pulled from the first of its `imageMirrors` that succeeds, and launches of it then use - * that mirror reference. When every mirror fails, the primary pull error is reported; a - * rate-limited primary retries the whole chain with backoff. + * that mirror reference. When every mirror fails, the primary pull error is reported first, + * followed by each mirror's failure; a rate-limited source retries the whole chain with backoff. */ export const makeContainerRuntime = (options: { readonly engine: "docker" | "podman"; @@ -189,7 +189,17 @@ export const makeContainerRuntime = (options: { }).pipe( Effect.tap(() => Effect.logInfo(`Pulled image from mirror ${mirror}`)), Effect.tapError((cause) => Effect.logWarning(`Image mirror ${mirror} failed`, cause)), - Effect.catch(() => fromMirror(image, rest, primaryError)), + Effect.catch((mirrorError) => + fromMirror( + image, + rest, + new ContainerError({ + operation: primaryError.operation, + message: `${primaryError.message}\nMirror ${mirror} also failed: ${mirrorError.message}`, + cause: primaryError.cause, + }), + ), + ), ); }; diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts index ce8eb85b40..7a582b9812 100644 --- a/packages/stack/src/runtime/container-image-mirror.integration.test.ts +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -134,18 +134,22 @@ describe("container image mirror", () => { }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); - it.live("reports the primary pull failure when the mirror also fails", () => { + it.live("reports the primary pull failure, then each mirror's, when every mirror fails", () => { const engine = fakeEngine({ pullable: [] }); return Effect.gen(function* () { const runtime = yield* makeContainerRuntime({ engine: "docker", - imageMirrors: (image) => (image === primary ? [mirror] : []), + imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []), }); const failed = yield* runtime.prepare(primary).pipe(Effect.exit); const error = Exit.isFailure(failed) ? Option.getOrUndefined(Cause.findErrorOption(failed.cause)) : undefined; - expect(error?.message).toContain(`denied: ${primary}`); + expect(error?.message.split("\n")).toEqual([ + `denied: ${primary}`, + `Mirror ${mirror} also failed: denied: ${mirror}`, + `Mirror ${secondMirror} also failed: denied: ${secondMirror}`, + ]); yield* launchImage(runtime); expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); @@ -204,6 +208,28 @@ describe("container image mirror", () => { }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); }); + it.effect( + "retries the chain when the primary is unreachable and the mirror is rate-limited", + () => { + const engine = fakeEngine({ pullable: [mirror], throttled: { [mirror]: 1 } }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + const prepared = yield* runtime.prepare(primary).pipe(Effect.forkChild); + yield* TestClock.adjust("1 minute"); + yield* Fiber.join(prepared); + expect(engine.commands.filter((args) => args[0] === "pull")).toEqual([ + ["pull", primary], + ["pull", mirror], + ["pull", primary], + ["pull", mirror], + ]); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }, + ); + it.effect("falls back to a mirror before backing off on a rate-limited primary", () => { const engine = fakeEngine({ pullable: [mirror], throttled: { [primary]: 10 } }); return Effect.gen(function* () {