ci(repo): re-read the slim mirror destination tag before failing

.github/scripts/mirror-slim-image.test.ts

@@ -120,6 +120,77 @@ describe("verifyDigest", () => {
       }),
     ).rejects.toThrow(InvalidPayloadError);
   });
+
+  test("re-reads a stale tag until it converges", async () => {
+    const heads = [fail("not found [http 404]"), ok(`${OTHER}\n`), ok(`${DIGEST}\n`)];
+    let reads = 0;
+    const run: RunCommand = async () => heads[reads++] ?? fail("unexpected read");
+    await verifyDigest({
+      reference: "public.ecr.aws/supabase/cli/postgrest:v16.2",
+      digest: DIGEST,
+      run,
+      attempts: 5,
+      sleep: async () => undefined,
+      log: () => undefined,
+    });
+    expect(reads).toBe(3);
+  });
+
+  test("reports the last mismatch once attempts are exhausted", async () => {
+    let reads = 0;
+    const run: RunCommand = async () => {
+      reads++;
+      return ok(`${OTHER}\n`);
+    };
+    await expect(
+      verifyDigest({
+        reference: "public.ecr.aws/supabase/cli/postgrest:v16.2",
+        digest: DIGEST,
+        run,
+        attempts: 3,
+        sleep: async () => undefined,
+        log: () => undefined,
+      }),
+    ).rejects.toThrow(`resolves to ${OTHER}, expected ${DIGEST}`);
+    expect(reads).toBe(3);
+  });
+
+  test("fails on the first read when the head errors for another reason", async () => {
+    let reads = 0;
+    const run: RunCommand = async () => {
+      reads++;
+      return fail("unauthorized [http 401]");
+    };
+    await expect(
+      verifyDigest({
+        reference: "public.ecr.aws/supabase/cli/postgrest:v16.2",
+        digest: DIGEST,
+        run,
+        attempts: 5,
+        sleep: async () => undefined,
+        log: () => undefined,
+      }),
+    ).rejects.toThrow("unauthorized [http 401]");
+    expect(reads).toBe(1);
+  });
+
+  test("rejects a non-integer attempt count before reading", async () => {
+    let reads = 0;
+    const run: RunCommand = async () => {
+      reads++;
+      return ok(`${OTHER}\n`);
+    };
+    await expect(
+      verifyDigest({
+        reference: "public.ecr.aws/supabase/cli/postgrest:v16.2",
+        digest: DIGEST,
+        run,
+        attempts: Number.NaN,
+        log: () => undefined,
+      }),
+    ).rejects.toThrow(InvalidPayloadError);
+    expect(reads).toBe(0);
+  });
 });
 
 describe("ensureEcrPublicRepo", () => {
@@ -266,6 +337,23 @@ describe("main", () => {
     expect(fields["destination"]).toBe("public.ecr.aws/supabase/cli/postgrest:v16.2");
   });
 
+  test("verify-digest re-reads up to ATTEMPTS times", async () => {
+    const heads = [ok(`${OTHER}\n`), ok(`${DIGEST}\n`)];
+    let reads = 0;
+    const code = await main(["verify-digest"], {
+      env: {
+        REFERENCE: "public.ecr.aws/supabase/cli/postgrest:v16.2",
+        DIGEST,
+        ATTEMPTS: "2",
+      },
+      run: async () => heads[reads++] ?? fail("unexpected read"),
+      sleep: async () => undefined,
+      log: () => undefined,
+    });
+    expect(code).toBe(0);
+    expect(reads).toBe(2);
+  });
+
   test("copy-natives no-ops without a dispatch payload", async () => {
     const logs: string[] = [];
     const code = await main(["copy-natives"], {

.github/scripts/mirror-slim-image.ts

@@ -54,6 +54,7 @@ export type MirrorIo = {
   readonly run: RunCommand;
   readonly runToFile?: RunCommandToFile;
   readonly log?: (message: string) => void;
+  readonly sleep?: (ms: number) => Promise<void>;
   readonly readText?: (path: string) => Promise<string>;
   readonly sha256File?: (path: string) => Promise<string>;
   readonly httpStatus?: (url: string, method: "HEAD" | "GET") => Promise<number>;
@@ -95,23 +96,43 @@ const writeGithubOutput = (
   else process.stdout.write(body);
 };
 
+const VERIFY_RETRY_DELAY_MS = 10_000;
+
+const NOT_FOUND = /\bnot found\b|\bhttp 404\b/i;
+
+/**
+ * Checks that `reference` resolves to `digest`, re-reading a mismatch or 404 up to
+ * `attempts` times. ECR Public can keep serving a tag's previous digest (or a 404 for a
+ * new tag) for a short while after a push, so the destination check needs more than one read.
+ */
 export const verifyDigest = async (options: {
   readonly reference: string;
   readonly digest: string;
   readonly run: RunCommand;
+  readonly attempts?: number;
+  readonly sleep?: (ms: number) => Promise<void>;
   readonly log?: (message: string) => void;
 }): Promise<void> => {
   const log = options.log ?? console.log;
-  const head = await options.run(["regctl", "manifest", "head", options.reference]);
-  const live = head.stdout.trim();
-  if (!head.ok || live !== options.digest) {
+  const attempts = options.attempts ?? 1;
+  if (!Number.isInteger(attempts) || attempts < 1)
+    throw new InvalidPayloadError(`attempts must be a positive integer, got ${attempts}`);
+  const sleep = options.sleep ?? Bun.sleep;
+  for (let attempt = 1; ; attempt++) {
+    const head = await options.run(["regctl", "manifest", "head", options.reference]);
+    const live = head.stdout.trim();
+    if (head.ok && live === options.digest) {
+      log(`${options.reference} resolves to ${options.digest}`);
+      return;
+    }
     const resolved = head.ok ? live : live || "missing";
     const detail = !head.ok && head.stderr.trim() !== "" ? `: ${head.stderr.trim()}` : "";
-    throw new InvalidPayloadError(
-      `${options.reference} resolves to ${resolved}, expected ${options.digest}${detail}`,
-    );
+    const mismatch = `${options.reference} resolves to ${resolved}, expected ${options.digest}${detail}`;
+    const stale = head.ok || NOT_FOUND.test(head.stderr);
+    if (!stale || attempt >= attempts) throw new InvalidPayloadError(mismatch);
+    log(`${mismatch}; re-reading (attempt ${attempt + 1}/${attempts})`);
+    await sleep(VERIFY_RETRY_DELAY_MS);
   }
-  log(`${options.reference} resolves to ${options.digest}`);
 };
 
 export const ensureEcrPublicRepo = async (options: {
@@ -479,6 +500,8 @@ export const main = async (argv: ReadonlyArray<string>, io: MirrorIo): Promise<n
       reference: requireEnv(io.env, "REFERENCE"),
       digest: requireEnv(io.env, "DIGEST"),
       run: io.run,
+      attempts: Number(envValue(io.env, "ATTEMPTS").trim() || "1"),
+      sleep: io.sleep,
       log,
     });
     return 0;

.github/workflows/mirror-slim-image.yml

@@ -133,6 +133,9 @@ jobs:
         env:
           REFERENCE: ${{ steps.validate.outputs.destination }}
           DIGEST: ${{ steps.validate.outputs.digest }}
+          # ECR Public can serve the tag's previous digest (or a 404) for ~30 s after the push;
+          # 13 reads 10 s apart allow ~2 min.
+          ATTEMPTS: "13"
         run: bun .github/scripts/mirror-slim-image.ts verify-digest
 
       - name: Mirror native artifacts