fix(stack): pin catalog images that ECR Public mirrors and report every mirror failure

.github/scripts/sync-artifacts-catalog.test.ts

@@ -92,6 +92,24 @@ FROM postgrest/postgrest:v16.3 AS postgrest
     expect(oriole.source).toContain(`"${POSTGRES_17}"`);
   });
 
+  test("an image the ECR mirror does not serve under the GHCR digest blocks the commit", async () => {
+    const plan = await planArtifactCatalogUpdate({
+      baseDockerfile: "FROM supabase/postgres:17.6.1.168 AS pg\n",
+      dockerfile: "FROM supabase/postgres:17.6.1.171 AS pg\n",
+      catalog: fixture,
+      publication: async () => ({ status: "unmirrored" }),
+    });
+    expect(plan.source).toBe(fixture);
+    expect(plan.skipped).toEqual([
+      {
+        alias: "pg",
+        reason:
+          "postgres:17.6.1.171 on public.ecr.aws/supabase/cli does not match the GHCR digest.",
+        blocking: true,
+      },
+    ]);
+  });
+
   test("refuses a backward pin and a tag that would escape the catalog string", async () => {
     const backward = await planArtifactCatalogUpdate({
       baseDockerfile: "FROM supabase/postgres:17.6.1.170 AS pg\n",

.github/scripts/sync-artifacts-catalog.ts

@@ -3,13 +3,16 @@
  * `apps/cli/src/shared/services/Dockerfile`. Native archive URLs are derived
  * from service + version. An entry that already carries a digest keeps one:
  * the published `ghcr.io/supabase/cli/<service>:<version>` manifest digest.
+ * A pin is refused until ECR Public serves the same digest, so the stack's
+ * image fallback can pull it.
  *
  * Run: `bun .github/scripts/sync-artifacts-catalog.ts <dockerfile> <catalog> [base-dockerfile]`
  */
 
 import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts";
 import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts";
 import {
+  DEST_REGISTRY,
   DIGEST_PATTERN,
   InvalidPayloadError,
   SOURCE_REGISTRY,
@@ -71,7 +74,7 @@ export interface CatalogPlan {
 
 export type ReleasePublication =
   | { readonly status: "published"; readonly digest?: string }
-  | { readonly status: "missing" | "lookup-failed" };
+  | { readonly status: "missing" | "unmirrored" | "lookup-failed" };
 
 /** `definition("<service>", "<version>", "<image>"`. */
 function defaultEntryPattern(service: string): RegExp {
@@ -267,7 +270,9 @@ export async function planArtifactCatalogUpdate(input: {
         reason:
           release.status === "missing"
             ? `${pin.service}:${pin.version} has no published slim image and native release.`
-            : `${pin.service}:${pin.version} publication check failed.`,
+            : release.status === "unmirrored"
+              ? `${pin.service}:${pin.version} on ${DEST_REGISTRY} does not match the GHCR digest.`
+              : `${pin.service}:${pin.version} publication check failed.`,
         blocking: true,
       });
       continue;
@@ -347,15 +352,17 @@ async function probeNativeRelease(service: string, version: string): Promise<Pro
 }
 
 async function lookupPublication(service: string, version: string): Promise<ReleasePublication> {
-  const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`;
-  const [manifest, native] = await Promise.all([
-    probeManifest(reference),
+  const [manifest, mirror, native] = await Promise.all([
+    probeManifest(`${SLIM_IMAGE_PREFIX}${service}:${version}`),
+    probeManifest(`${DEST_REGISTRY}/${service}:${version}`),
     probeNativeRelease(service, version),
   ]);
   if (manifest.probe === "lookup-failed" || native === "lookup-failed") {
     return { status: "lookup-failed" };
   }
   if (manifest.probe === "missing" || native === "missing") return { status: "missing" };
+  if (mirror.probe === "lookup-failed") return { status: "lookup-failed" };
+  if (mirror.digest !== manifest.digest) return { status: "unmirrored" };
   return { status: "published", digest: manifest.digest };
 }
 

apps/cli/docs/stack-commands.md

@@ -75,8 +75,9 @@ stack = true
 
 The selected backend determines accepted flags, help, and completion before the command is parsed.
 Set the flag to `false`, or remove it, to restore the legacy top-level commands. The explicit
-`supabase stack` namespace is available only when this flag is enabled. `supabase status` is routed
-the same way as `supabase start` and `supabase stop`.
+`supabase stack` namespace is available only when this flag is enabled; otherwise `supabase stack`
+exits with an error that names both ways to enable it. `supabase status` is routed the same way as
+`supabase start` and `supabase stop`.
 
 Root help and root completion resolve the same feature flag from the environment or project
 configuration. Help and completion for `start`, `status`, and `stop` resolve the same backend as the

apps/cli/src/command-internal/stack-backend.ts

@@ -36,9 +36,12 @@ const isFunctionsServePath = (path: ReadonlyArray<string>): boolean =>
 export class StackRoutingError extends Data.TaggedError("StackRoutingError")<{
   readonly message: string;
   readonly cause?: unknown;
+  readonly disabledNamespace?: boolean;
 }> {
   get suggestion(): string {
-    return "Set SUPABASE_EXPERIMENTAL_STACK=1 to enable stack commands, or 0 to use legacy start/stop/status.";
+    return this.disabledNamespace === true
+      ? "Set SUPABASE_EXPERIMENTAL_STACK=1, or leave it unset and add `stack = true` under [experimental] in supabase/config.toml."
+      : "Set SUPABASE_EXPERIMENTAL_STACK=1 to enable stack commands, or 0 to use legacy start/stop/status.";
   }
 
   get [ErrorActionabilityId](): CliErrorActionabilityDeclaration {
@@ -69,10 +72,8 @@ export const resolveStackBackend = (input: {
   Effect.gen(function* () {
     // Completion passes the final token as the cursor word, so it is not part of
     // the resolved command path.
-    const routingArgs =
-      input.args[0] === "__complete" || input.args[0] === "__completeNoDesc"
-        ? input.args.slice(0, -1)
-        : input.args;
+    const completing = input.args[0] === "__complete" || input.args[0] === "__completeNoDesc";
+    const routingArgs = completing ? input.args.slice(0, -1) : input.args;
     if (hasRootVersionFlag(routingArgs)) return "legacy";
 
     const commandPath = extractCommandPath(routingArgs);
@@ -96,5 +97,10 @@ export const resolveStackBackend = (input: {
     }).pipe(
       Effect.mapError((error) => new StackRoutingError({ message: error.message, cause: error })),
     );
+    if (!enabled && command === "stack" && !completing)
+      return yield* new StackRoutingError({
+        message: "`supabase stack` requires the experimental stack backend.",
+        disabledNamespace: true,
+      });
     return enabled ? "stack" : "legacy";
   });

apps/cli/src/commands/experimental/stack/stack-backend.integration.test.ts

@@ -29,9 +29,23 @@ const completionFlags = (backend: "legacy" | "stack", command: string) =>
   ])?.candidates.map(({ name }) => name);
 
 describe("resolveStackBackend", () => {
-  it.effect("keeps the explicit stack namespace disabled without the feature flag", () =>
+  it.effect("explains how to enable the explicit stack namespace without the feature flag", () =>
     Effect.gen(function* () {
-      expect(yield* resolve({ args: ["stack", "start"], cwd: "/missing", env: {} })).toBe("legacy");
+      for (const args of [
+        ["stack", "start"],
+        ["help", "stack"],
+      ]) {
+        const error = yield* resolve({ args, cwd: "/missing", env: {} }).pipe(Effect.flip);
+        expect(error).toBeInstanceOf(StackRoutingError);
+        expect(error.message).toBe("`supabase stack` requires the experimental stack backend.");
+        expect(error.suggestion).toContain("SUPABASE_EXPERIMENTAL_STACK=1");
+        expect(error.suggestion).toContain(
+          "leave it unset and add `stack = true` under [experimental]",
+        );
+      }
+      expect(yield* resolve({ args: ["__complete", "stack", ""], cwd: "/missing", env: {} })).toBe(
+        "legacy",
+      );
     }),
   );
 

docs/adr/0026-slim-artifact-mirrors.md

@@ -19,7 +19,7 @@ ECR Public tags are always mutable; `ecr-public create-repository` has no immuta
 
 A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-<target>` artifact on GHCR. An archive from either host is accepted only when it matches.
 
-The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing fallback, for an image or a native archive, still reports the primary's original error.
+The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. When every source fails, for an image or a native archive, the error leads with the primary's failure and then lists each fallback's. The catalog sync refuses a pin until ECR Public serves the GHCR digest, because a digest pin that only GHCR carries cannot fall back.
 
 ## Follow-up
 

packages/stack/src/Artifacts.ts

@@ -111,13 +111,13 @@ const definitions: Readonly<Record<ServiceKind, ArtifactDefinition>> = {
   studio: definition(
     "studio",
     "2026.09.04-sha-5a67366",
-    "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9823a31668028f1846e87331bc21598d9cd74bcaa1466c72dab58c33c9c82720",
+    "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:9362a66196c1622bb90e1b8014ff5376505969906fac38e58ffee0ca178daf07",
     "bin/studio",
   ),
   pgmeta: definition(
     "pgmeta",
     "v0.99.0",
-    "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:90de2dcf03ac548ae2d1d3e71b3cd10bde4c627572720a42e4c3946b7090292e",
+    "ghcr.io/supabase/cli/pgmeta:v0.99.0@sha256:4a49f779f78b66af42a20425d66b0cc3083ffe09fb42da05e62c9d771bc309b0",
     "bin/pgmeta",
   ),
   mail: definition("mailpit", "v1.30.2", "ghcr.io/supabase/cli/mailpit:v1.30.2", "bin/mailpit"),
@@ -135,7 +135,7 @@ const definitions: Readonly<Record<ServiceKind, ArtifactDefinition>> = {
   pooler: definition(
     "pooler",
     "v2.9.12",
-    "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:12bb9dcb7ddace79bee173ccb7327c6646af2236679f3bd932a86b3a06479aac",
+    "ghcr.io/supabase/cli/pooler:v2.9.12@sha256:a0278f60f7bc73b8fabc28d8d144eeb326afea566ea81012826c9a869d62664a",
     "bin/server",
     ["bin/server", "bin/prepare", "bin/provision-tenant"],
   ),

packages/stack/src/Artifacts.unit.test.ts

@@ -4,10 +4,10 @@ import { slimImageMirrors } from "./Artifacts.ts";
 it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => {
   expect(
     slimImageMirrors(
-      "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf",
+      "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a",
     ),
   ).toEqual([
-    "public.ecr.aws/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf",
+    "public.ecr.aws/supabase/cli/postgres:17.6.1.173@sha256:9d6e542382946cad5eb1f11f1c8108a51297902ee42fe5098358816d3784ba5a",
   ]);
 });
 

packages/stack/src/preparation/SlimServicesSource.ts

@@ -233,8 +233,9 @@ const fallbackDetail = (error: PreparationError): string => {
 
 /**
  * Runs `attempt` against each candidate until one succeeds. When every candidate fails, the
- * returned error is still the primary's. A fallback failure, including a checksum mismatch, is
- * logged as a warning, and a fallback that succeeds names the host it used.
+ * returned error is the primary's, with each fallback failure appended to its message. A fallback
+ * failure, including a checksum mismatch, is logged as a warning, and a fallback that succeeds
+ * names the host it used.
  */
 const firstSuccess = <T, A, R>(
   candidates: readonly [T, ...ReadonlyArray<T>],
@@ -255,7 +256,17 @@ const firstSuccess = <T, A, R>(
           `Slim-services fallback ${describe(candidate)} failed: ${fallbackDetail(cause)}`,
         ),
       ),
-      Effect.catch(() => fallback(primaryError, rest)),
+      Effect.catch((cause) =>
+        fallback(
+          new PreparationError({
+            ...primaryError,
+            // `cause` is a non-enumerable Error property, so the spread drops it.
+            cause: primaryError.cause,
+            message: `${primaryError.message}\nFallback ${describe(candidate)} also failed: ${fallbackDetail(cause)}`,
+          }),
+          rest,
+        ),
+      ),
     );
   };
   return attempt(primary).pipe(Effect.catch((primaryError) => fallback(primaryError, fallbacks)));

packages/stack/src/preparation/slim-services.integration.test.ts

@@ -377,7 +377,7 @@ describe("slim-services artifact source", () => {
     ),
   );
 
-  it.live("reports the primary checksum failure when every checksum source fails", () =>
+  it.live("reports the primary checksum failure, then the fallback's", () =>
     Effect.gen(function* () {
       const mirrored: SlimServicesArtifact = {
         ...artifact,
@@ -391,7 +391,11 @@ describe("slim-services artifact source", () => {
         requested.push(requestUrl(input));
         return Promise.resolve(new Response("", { status: 403 }));
       }, slimServicesChecksum(mirrored, immediate).pipe(Effect.exit));
-      expect(errorOf(failed)?.message).toBe("Unable to download https://release.test/SHA256SUMS");
+      expect(errorOf(failed)?.message.split("\n")).toEqual([
+        "Unable to download https://release.test/SHA256SUMS",
+        "Fallback registry.test/supabase/cli/demo:v1.0.0-native-linux-amd64 also failed: Unable to download https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test (HTTP 403)",
+      ]);
+      expect(errorOf(failed)?.cause).toMatchObject({ message: "HTTP 403", status: 403 });
       expect(requested).toEqual([
         "https://release.test/SHA256SUMS",
         "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test",
@@ -445,7 +449,9 @@ describe("slim-services artifact source", () => {
             .materialize(request, rejected, expected)
             .pipe(Effect.exit),
         );
-        expect(errorOf(failed)?.message).toBe("Unable to download slim-services archive");
+        expect(errorOf(failed)?.message.split("\n")[0]).toBe(
+          "Unable to download slim-services archive",
+        );
         expect(yield* fs.exists(`${rejected}/bin/demo`)).toBe(false);
       }).pipe(Effect.provide(NodeServices.layer)),
     ),

packages/stack/src/runtime/Container.ts

@@ -109,8 +109,8 @@ const mountField = (key: string, value: string) => {
 /**
  * Captures the selected local engine; each launch owns one exact container. An image whose pull
  * fails is pulled from the first of its `imageMirrors` that succeeds, and launches of it then use
- * that mirror reference. When every mirror fails, the primary pull error is reported; a
- * rate-limited primary retries the whole chain with backoff.
+ * that mirror reference. When every mirror fails, the primary pull error is reported first,
+ * followed by each mirror's failure; a rate-limited source retries the whole chain with backoff.
  */
 export const makeContainerRuntime = (options: {
   readonly engine: "docker" | "podman";
@@ -189,7 +189,17 @@ export const makeContainerRuntime = (options: {
       }).pipe(
         Effect.tap(() => Effect.logInfo(`Pulled image from mirror ${mirror}`)),
         Effect.tapError((cause) => Effect.logWarning(`Image mirror ${mirror} failed`, cause)),
-        Effect.catch(() => fromMirror(image, rest, primaryError)),
+        Effect.catch((mirrorError) =>
+          fromMirror(
+            image,
+            rest,
+            new ContainerError({
+              operation: primaryError.operation,
+              message: `${primaryError.message}\nMirror ${mirror} also failed: ${mirrorError.message}`,
+              cause: primaryError.cause,
+            }),
+          ),
+        ),
       );
     };
 

packages/stack/src/runtime/container-image-mirror.integration.test.ts

@@ -134,18 +134,22 @@ describe("container image mirror", () => {
     }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer)));
   });
 
-  it.live("reports the primary pull failure when the mirror also fails", () => {
+  it.live("reports the primary pull failure, then each mirror's, when every mirror fails", () => {
     const engine = fakeEngine({ pullable: [] });
     return Effect.gen(function* () {
       const runtime = yield* makeContainerRuntime({
         engine: "docker",
-        imageMirrors: (image) => (image === primary ? [mirror] : []),
+        imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []),
       });
       const failed = yield* runtime.prepare(primary).pipe(Effect.exit);
       const error = Exit.isFailure(failed)
         ? Option.getOrUndefined(Cause.findErrorOption(failed.cause))
         : undefined;
-      expect(error?.message).toContain(`denied: ${primary}`);
+      expect(error?.message.split("\n")).toEqual([
+        `denied: ${primary}`,
+        `Mirror ${mirror} also failed: denied: ${mirror}`,
+        `Mirror ${secondMirror} also failed: denied: ${secondMirror}`,
+      ]);
       yield* launchImage(runtime);
       expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary);
     }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer)));
@@ -204,6 +208,28 @@ describe("container image mirror", () => {
     }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer)));
   });
 
+  it.effect(
+    "retries the chain when the primary is unreachable and the mirror is rate-limited",
+    () => {
+      const engine = fakeEngine({ pullable: [mirror], throttled: { [mirror]: 1 } });
+      return Effect.gen(function* () {
+        const runtime = yield* makeContainerRuntime({
+          engine: "docker",
+          imageMirrors: (image) => (image === primary ? [mirror] : []),
+        });
+        const prepared = yield* runtime.prepare(primary).pipe(Effect.forkChild);
+        yield* TestClock.adjust("1 minute");
+        yield* Fiber.join(prepared);
+        expect(engine.commands.filter((args) => args[0] === "pull")).toEqual([
+          ["pull", primary],
+          ["pull", mirror],
+          ["pull", primary],
+          ["pull", mirror],
+        ]);
+      }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer)));
+    },
+  );
+
   it.effect("falls back to a mirror before backing off on a rate-limited primary", () => {
     const engine = fakeEngine({ pullable: [mirror], throttled: { [primary]: 10 } });
     return Effect.gen(function* () {