From 89dfbbe94abefc5682fb69470dd9ccba9fec5536 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 12:25:39 +0000 Subject: [PATCH 1/6] ci(repo): accumulate slim-service pins in a single catalog PR The workload catalog sync opened one branch and PR per service, so a slim-services release train produced a PR per workload. Every dispatch now rebuilds one shared `sync/stack-workload-catalog` branch as develop plus the pins already pending on it plus its own pin, pushes with a lease on the head it read, and recomposes when a concurrent run for another service moved it first. The PR is opened once and then only updated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011xm6NLFBFUSFXYrEc7Cp5c --- .github/scripts/compose-workload-catalog.sh | 26 ++++ .github/scripts/sync-workload-catalog.test.ts | 52 +++++++ .github/scripts/sync-workload-catalog.ts | 99 ++++++++++++- .../workflows/sync-stack-workload-catalog.yml | 135 +++++++++++++----- 4 files changed, 277 insertions(+), 35 deletions(-) create mode 100755 .github/scripts/compose-workload-catalog.sh diff --git a/.github/scripts/compose-workload-catalog.sh b/.github/scripts/compose-workload-catalog.sh new file mode 100755 index 0000000000..6c07abf811 --- /dev/null +++ b/.github/scripts/compose-workload-catalog.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Rebuilds the workload catalog as BASE_SHA + the pins pending on SYNC_BRANCH + +# this dispatch's pin (SLIM_SERVICE/SLIM_VERSION/SLIM_DIGEST). Prints the +# SYNC_BRANCH head it read, empty when the branch does not exist, so the push +# can lease against it. Needs full history for the merge base. +set -euo pipefail + +: "${BASE_SHA:?}" "${SYNC_BRANCH:?}" +catalog=packages/stack/src/Artifacts.ts + +git checkout "$BASE_SHA" -- "$catalog" + +head=$(git ls-remote --heads origin "refs/heads/$SYNC_BRANCH" | cut -f1) +if [ -n "$head" ]; then + git fetch --quiet --no-tags origin "$head" >&2 + merge_base=$(git merge-base "$BASE_SHA" "$head") + scratch=$(mktemp -d) + git show "$merge_base:$catalog" > "$scratch/merge-base.ts" + git show "$head:$catalog" > "$scratch/branch.ts" + bun .github/scripts/sync-workload-catalog.ts carry "$scratch/merge-base.ts" "$scratch/branch.ts" >&2 +fi + +bun .github/scripts/sync-workload-catalog.ts >&2 +pnpm exec oxfmt --config .oxfmtrc.json "$catalog" >&2 + +printf '%s\n' "$head" diff --git a/.github/scripts/sync-workload-catalog.test.ts b/.github/scripts/sync-workload-catalog.test.ts index 63419fa8da..21618705ce 100644 --- a/.github/scripts/sync-workload-catalog.test.ts +++ b/.github/scripts/sync-workload-catalog.test.ts @@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test"; import { CATALOG_PATH, InvalidPayloadError, + carryPins, planCatalogUpdate, releaseLine, validatePayload, @@ -250,6 +251,57 @@ describe("planCatalogUpdate", () => { }); }); +function pin(source: string, service: string, version: string, digest: string = DIGEST_A): string { + const plan = planCatalogUpdate({ source, service, version, digest }); + if (plan.kind !== "updated") + throw new Error(`fixture pin ${service} ${version} was ${plan.kind}`); + return plan.source; +} + +describe("carryPins", () => { + test("reapplies every pin the sync branch added onto a newer develop", () => { + const branch = pin(pin(fixture, "auth", "v2.197.0"), "postgrest", "v16.3"); + const develop = pin(fixture, "studio", "2026.09.21-sha-512201d", DIGEST_B); + + const result = carryPins({ source: develop, mergeBase: fixture, branch }); + + expect(result.carried.map((carried) => carried.service)).toEqual(["postgrest", "auth"]); + expect(result.superseded).toEqual([]); + expect(result.source).toContain(`"ghcr.io/supabase/cli/auth:v2.197.0@${DIGEST_A}"`); + expect(result.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3@${DIGEST_A}"`); + expect(result.source).toContain( + `"ghcr.io/supabase/cli/studio:2026.09.21-sha-512201d@${DIGEST_B}"`, + ); + }); + + test("carries a postgres pin onto the release line the branch moved", () => { + const branch = pin(fixture, "postgres", "15.14.1.175"); + + const result = carryPins({ source: fixture, mergeBase: fixture, branch }); + + expect(result.source).toContain(`"15.14.1.175"`); + expect(result.source).toContain(`"17.6.1.168"`); + }); + + test("keeps develop's entry when develop moved it after the branch was cut", () => { + const branch = pin(fixture, "auth", "v2.197.0"); + const develop = pin(fixture, "auth", "v2.198.0", DIGEST_B); + + const result = carryPins({ source: develop, mergeBase: fixture, branch }); + + expect(result.source).toBe(develop); + expect(result.superseded.map((dropped) => dropped.version)).toEqual(["v2.197.0"]); + }); + + test("a pin develop already merged is neither carried nor reported", () => { + const branch = pin(fixture, "auth", "v2.197.0"); + + const result = carryPins({ source: branch, mergeBase: fixture, branch }); + + expect(result).toEqual({ source: branch, carried: [], superseded: [] }); + }); +}); + describe("against the real catalog", () => { test("every modelled service is addressable and idempotent", () => { // Derived from the catalog, so a new workload is covered without editing this. diff --git a/.github/scripts/sync-workload-catalog.ts b/.github/scripts/sync-workload-catalog.ts index d6a290712b..70b973140f 100644 --- a/.github/scripts/sync-workload-catalog.ts +++ b/.github/scripts/sync-workload-catalog.ts @@ -11,6 +11,10 @@ * Run: `bun .github/scripts/sync-workload-catalog.ts` with SLIM_SERVICE, * SLIM_VERSION, SLIM_DIGEST. Exit 1 on an invalid payload; an unmodelled * service or release line is a successful no-op. + * + * `bun .github/scripts/sync-workload-catalog.ts carry ` + * reapplies the pins the shared sync branch holds over its merge base, so one + * PR accumulates every pending release. */ import { @@ -45,6 +49,7 @@ export type CatalogUpdatePlan = readonly kind: "updated"; readonly source: string; readonly previousVersion: string; + readonly previousImage: string; /** `default` bumped the entry's defaultVersion; `additional` bumped one of its extra lines. */ readonly target: "default" | "additional"; } @@ -111,6 +116,7 @@ export function planCatalogUpdate(input: CatalogUpdateInput): CatalogUpdatePlan `${prefix}${version}${mid}${desiredImage}${suffix}`, ), previousVersion: currentDefaultVersion, + previousImage: currentDefaultImage, target: "default", }; } @@ -134,10 +140,74 @@ export function planCatalogUpdate(input: CatalogUpdateInput): CatalogUpdatePlan (_full, _key: string, separator: string) => `"${version}"${separator}"${desiredImage}"`, ), previousVersion: sameLine.version, + previousImage: sameLine.image, target: "additional", }; } +export interface CatalogPin { + readonly service: string; + readonly version: string; + readonly digest: string; +} + +const PIN_PATTERN = new RegExp( + `"${escapeRegExp(SLIM_IMAGE_PREFIX)}([a-z][a-z0-9-]*):([A-Za-z0-9._-]+)@(sha256:[0-9a-f]{64})"`, + "g", +); + +function pinsIn(source: string): ReadonlyArray { + return [...source.matchAll(PIN_PATTERN)].map((match) => ({ + service: match[1] ?? "", + version: match[2] ?? "", + digest: match[3] ?? "", + })); +} + +export interface CarryInput { + readonly source: string; + readonly mergeBase: string; + readonly branch: string; +} + +export interface CarryResult { + readonly source: string; + readonly carried: ReadonlyArray; + /** Pins whose entry `source` moved since the merge base; `source` wins. */ + readonly superseded: ReadonlyArray; +} + +/** Applies the pins `branch` added over `mergeBase` onto `source`. */ +export function carryPins(input: CarryInput): CarryResult { + const baseRefs = new Set(pinsIn(input.mergeBase).map(slimRefOf)); + const pending = pinsIn(input.branch).filter((pin) => !baseRefs.has(slimRefOf(pin))); + + let source = input.source; + const carried: CatalogPin[] = []; + const superseded: CatalogPin[] = []; + for (const pin of pending) { + const onBase = planCatalogUpdate({ source: input.mergeBase, ...pin }); + const onSource = planCatalogUpdate({ source, ...pin }); + if (onSource.kind === "unchanged") continue; + if ( + onBase.kind === "updated" && + onSource.kind === "updated" && + onSource.previousVersion === onBase.previousVersion && + onSource.previousImage === onBase.previousImage + ) { + source = onSource.source; + carried.push(pin); + } else { + superseded.push(pin); + } + } + return { source, carried, superseded }; +} + +function slimRefOf(pin: CatalogPin): string { + return slimImageRef(pin.service, pin.version, pin.digest); +} + function requireEnv(name: string): string { const value = process.env[name]; if (value === undefined || value.trim() === "") { @@ -146,7 +216,32 @@ function requireEnv(name: string): string { return value.trim(); } -async function main(): Promise { +async function carry(mergeBasePath: string, branchPath: string): Promise { + const result = carryPins({ + source: await Bun.file(CATALOG_PATH).text(), + mergeBase: await Bun.file(mergeBasePath).text(), + branch: await Bun.file(branchPath).text(), + }); + await Bun.write(CATALOG_PATH, result.source); + for (const pin of result.carried) { + console.log(`Carried pending ${pin.service} ${pin.version} (${pin.digest}).`); + } + for (const pin of result.superseded) { + console.log( + `::warning ::Dropped pending ${pin.service} ${pin.version}: develop changed that entry since the sync branch was cut.`, + ); + } +} + +async function main(argv: ReadonlyArray): Promise { + if (argv[0] === "carry") { + const [, mergeBasePath, branchPath] = argv; + if (mergeBasePath === undefined || branchPath === undefined) { + throw new Error("usage: sync-workload-catalog.ts carry "); + } + return carry(mergeBasePath, branchPath); + } + const service = requireEnv("SLIM_SERVICE"); const version = requireEnv("SLIM_VERSION"); const digest = requireEnv("SLIM_DIGEST"); @@ -176,7 +271,7 @@ async function main(): Promise { } if (import.meta.main) { - main().catch((error: unknown) => { + main(process.argv.slice(2)).catch((error: unknown) => { console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); process.exit(1); }); diff --git a/.github/workflows/sync-stack-workload-catalog.yml b/.github/workflows/sync-stack-workload-catalog.yml index 02cb67690c..971acde8f5 100644 --- a/.github/workflows/sync-stack-workload-catalog.yml +++ b/.github/workflows/sync-stack-workload-catalog.yml @@ -8,8 +8,12 @@ name: Sync Stack Workload Catalog # Separate from `mirror-slim-image.yml` on purpose — that mirror runs against # the sender's 15-minute verification poll and must not be delayed or reddened # by a catalog PR. Opens a PR rather than pushing: `develop` is protected and a -# version pin is reviewable. Re-dispatches are no-ops, so retries do not open -# duplicate PRs. +# version pin is reviewable. +# +# Every release lands on one shared branch and PR. Each run rebuilds the branch +# as `develop` + the pins already pending on it + its own pin, and pushes with a +# lease on the head it read, recomposing when a concurrent run for another +# service moved it first. Re-dispatches are no-ops. on: repository_dispatch: @@ -34,11 +38,21 @@ permissions: contents: read concurrency: - # Per service, so two releases of one service cannot race onto the shared - # branch. Not cancel-in-progress: a superseded run may already have opened it. + # Per service, so a newer release of one service supersedes its queued run. + # Runs for different services race on the shared branch; the push lease + # serializes them. Not cancel-in-progress: a run may be mid-push. group: sync-stack-workload-catalog-${{ github.event.client_payload.service || inputs.service }} cancel-in-progress: false +env: + SYNC_BRANCH: sync/stack-workload-catalog + # Payload is untrusted; the script revalidates it. Passed via env, never + # interpolated into the shell. Extra payload fields such as natives[] are + # ignored here; this workflow pins the image digest only. + SLIM_SERVICE: ${{ github.event.client_payload.service || inputs.service }} + SLIM_VERSION: ${{ github.event.client_payload.version || inputs.version }} + SLIM_DIGEST: ${{ github.event.client_payload.digest || inputs.digest }} + jobs: sync: name: Sync workload catalog @@ -48,37 +62,41 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + # The merge base with the sync branch decides which of its pins are pending. + fetch-depth: 0 + filter: blob:none - name: Setup uses: ./.github/actions/setup with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - # Payload is untrusted; the script revalidates it. Passed via env, never - # interpolated into the shell. Extra payload fields such as natives[] are - # ignored here; this workflow pins the image digest only. - - name: Sync workload catalog - env: - SLIM_SERVICE: ${{ github.event.client_payload.service || inputs.service }} - SLIM_VERSION: ${{ github.event.client_payload.version || inputs.version }} - SLIM_DIGEST: ${{ github.event.client_payload.digest || inputs.digest }} - run: bun .github/scripts/sync-workload-catalog.ts - - - name: Format catalog - run: pnpm run fmt:fix + - name: Compose workload catalog + id: compose + run: | + set -euo pipefail + base_sha=$(git rev-parse HEAD) + lease=$(BASE_SHA="$base_sha" bash .github/scripts/compose-workload-catalog.sh) + echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" + echo "lease=$lease" >> "$GITHUB_OUTPUT" - name: Check for catalog changes id: check + env: + LEASE: ${{ steps.compose.outputs.lease }} run: | if git diff --exit-code --quiet packages/stack/src/Artifacts.ts; then echo "Workload catalog is already in sync." echo "has_changes=false" >> "$GITHUB_OUTPUT" + elif [ -n "$LEASE" ] && git diff --quiet "$LEASE" -- packages/stack/src/Artifacts.ts; then + echo "${SYNC_BRANCH} already carries this catalog." + echo "has_changes=false" >> "$GITHUB_OUTPUT" else echo "Workload catalog updated." echo "has_changes=true" >> "$GITHUB_OUTPUT" fi - # Prove the rewritten pin compiles and the affected catalog surfaces pass. + # Prove the rewritten pins compile and the affected catalog surfaces pass. - name: Type-check stack if: steps.check.outputs.has_changes == 'true' run: pnpm types:check @@ -106,20 +124,71 @@ jobs: permission-pull-requests: write permission-contents: write - - name: Create Pull Request + # A retry recomposes onto the branch head another run pushed; those pins + # were validated by their own runs and the PR's CI covers the combination. + - name: Push sync branch if: steps.check.outputs.has_changes == 'true' - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 - with: - token: ${{ steps.app-token.outputs.token }} - commit-message: "chore(stack): pin ${{ github.event.client_payload.service || inputs.service }} to ${{ github.event.client_payload.version || inputs.version }}" - title: "chore(stack): pin ${{ github.event.client_payload.service || inputs.service }} to ${{ github.event.client_payload.version || inputs.version }}" - body: | - Pins `packages/stack/src/Artifacts.ts` to the - `${{ github.event.client_payload.service || inputs.service }}` `${{ github.event.client_payload.version || inputs.version }}` - slim-services release and its image digest. - - Opened from the `mirror-slim-image` dispatch. `artifactFor` derives the - native release tag, asset names, and URLs from service + version, so this - pin is the whole change. - branch: sync/stack-workload-catalog-${{ github.event.client_payload.service || inputs.service }} - base: develop + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + BASE_SHA: ${{ steps.compose.outputs.base_sha }} + LEASE: ${{ steps.compose.outputs.lease }} + run: | + set -euo pipefail + catalog=packages/stack/src/Artifacts.ts + bot="${APP_SLUG}[bot]" + bot_id=$(gh api "/users/${bot}" --jq .id) + git config user.name "$bot" + git config user.email "${bot_id}+${bot}@users.noreply.github.com" + remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + + lease="$LEASE" + for attempt in 1 2 3 4 5; do + if [ "$attempt" -gt 1 ]; then + git reset --quiet "$BASE_SHA" + lease=$(bash .github/scripts/compose-workload-catalog.sh) + fi + if [ -n "$lease" ] && git diff --quiet "$lease" -- "$catalog"; then + echo "${SYNC_BRANCH} already carries this catalog." + exit 0 + fi + pins=$(git diff "$BASE_SHA" -- "$catalog" \ + | grep -oE '^\+.*"ghcr\.io/supabase/cli/[a-z0-9-]+:[A-Za-z0-9._-]+@' \ + | sed -E 's|.*ghcr\.io/supabase/cli/([a-z0-9-]+):([^@]+)@|- \1 \2|' || true) + git commit --quiet -m "chore(stack): pin slim-services workloads" -m "$pins" -- "$catalog" + if git push --force-with-lease="refs/heads/${SYNC_BRANCH}:${lease}" "$remote" "HEAD:refs/heads/${SYNC_BRANCH}"; then + exit 0 + fi + echo "::notice ::${SYNC_BRANCH} moved under attempt ${attempt}; recomposing." + done + echo "::error ::Could not push ${SYNC_BRANCH} after 5 attempts." + exit 1 + + - name: Open or update pull request + if: steps.check.outputs.has_changes == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + existing() { + gh pr list --repo "$GITHUB_REPOSITORY" --head "$SYNC_BRANCH" --base develop \ + --state open --json number --jq '.[0].number // empty' + } + if [ -n "$(existing)" ]; then + echo "PR #$(existing) already tracks ${SYNC_BRANCH}." + exit 0 + fi + body=$(cat <<'EOF' + Pins `packages/stack/src/Artifacts.ts` to the latest slim-services releases + and their image digests. The commit message lists every pin in this PR. + + Opened from the `mirror-slim-image` dispatch. Each new release rebuilds this + branch onto `develop` with all pending pins, so this PR stays the only one. + `artifactFor` derives the native release tag, asset names, and URLs from + service + version, so these pins are the whole change. + EOF + ) + # A concurrent run may open the PR between the check and the create. + gh pr create --repo "$GITHUB_REPOSITORY" --base develop --head "$SYNC_BRANCH" \ + --title "chore(stack): pin slim-services workloads" --body "$body" \ + || [ -n "$(existing)" ] From 14a0bff2882b819d06e69adebc9aa2288d41b9f6 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 15:14:00 +0200 Subject: [PATCH 2/6] ci(repo): drop the artifacts catalog PR from slim mirroring The mirror dispatch should not open a pull request that pins Artifacts.ts. A later workflow will own that catalog update. --- .github/scripts/compose-workload-catalog.sh | 26 -- .github/scripts/sync-workload-catalog.test.ts | 341 ------------------ .github/scripts/sync-workload-catalog.ts | 278 -------------- .github/workflows/github-scripts-ci.yml | 1 - .../workflows/sync-stack-workload-catalog.yml | 194 ---------- docs/adr/0026-slim-artifact-mirrors.md | 2 +- 6 files changed, 1 insertion(+), 841 deletions(-) delete mode 100755 .github/scripts/compose-workload-catalog.sh delete mode 100644 .github/scripts/sync-workload-catalog.test.ts delete mode 100644 .github/scripts/sync-workload-catalog.ts delete mode 100644 .github/workflows/sync-stack-workload-catalog.yml diff --git a/.github/scripts/compose-workload-catalog.sh b/.github/scripts/compose-workload-catalog.sh deleted file mode 100755 index 6c07abf811..0000000000 --- a/.github/scripts/compose-workload-catalog.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Rebuilds the workload catalog as BASE_SHA + the pins pending on SYNC_BRANCH + -# this dispatch's pin (SLIM_SERVICE/SLIM_VERSION/SLIM_DIGEST). Prints the -# SYNC_BRANCH head it read, empty when the branch does not exist, so the push -# can lease against it. Needs full history for the merge base. -set -euo pipefail - -: "${BASE_SHA:?}" "${SYNC_BRANCH:?}" -catalog=packages/stack/src/Artifacts.ts - -git checkout "$BASE_SHA" -- "$catalog" - -head=$(git ls-remote --heads origin "refs/heads/$SYNC_BRANCH" | cut -f1) -if [ -n "$head" ]; then - git fetch --quiet --no-tags origin "$head" >&2 - merge_base=$(git merge-base "$BASE_SHA" "$head") - scratch=$(mktemp -d) - git show "$merge_base:$catalog" > "$scratch/merge-base.ts" - git show "$head:$catalog" > "$scratch/branch.ts" - bun .github/scripts/sync-workload-catalog.ts carry "$scratch/merge-base.ts" "$scratch/branch.ts" >&2 -fi - -bun .github/scripts/sync-workload-catalog.ts >&2 -pnpm exec oxfmt --config .oxfmtrc.json "$catalog" >&2 - -printf '%s\n' "$head" diff --git a/.github/scripts/sync-workload-catalog.test.ts b/.github/scripts/sync-workload-catalog.test.ts deleted file mode 100644 index 21618705ce..0000000000 --- a/.github/scripts/sync-workload-catalog.test.ts +++ /dev/null @@ -1,341 +0,0 @@ -import { describe, expect, test } from "bun:test"; - -import { - CATALOG_PATH, - InvalidPayloadError, - carryPins, - planCatalogUpdate, - releaseLine, - validatePayload, -} from "./sync-workload-catalog.ts"; - -const DIGEST_A = `sha256:${"a".repeat(64)}`; -const DIGEST_B = `sha256:${"b".repeat(64)}`; - -/** The real catalog, so a shape drift fails here instead of passing on a stale fixture. */ -const realCatalog = await Bun.file(CATALOG_PATH).text(); - -/** A trimmed catalog carrying both entry shapes the script has to rewrite. */ -const fixture = `const workloadCatalog = { - "database:database": definition( - "postgres", - "17.6.1.168", - "ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}", - "bin/supabase-postgres-start", - ["bin/supabase-postgres-start"], - { - "15.14.1.168": - "ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}", - }, - ), - "rest:rest": definition( - "postgrest", - "v16.2", - "ghcr.io/supabase/cli/postgrest:v16.2", - "bin/postgrest", - ["bin/postgrest"], - ), - "auth:auth": definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth", [ - "bin/auth", - ]), - "studio:studio": definition( - "studio", - "2026.09.04-sha-5a67366", - "ghcr.io/supabase/cli/studio:2026.09.04-sha-5a67366@sha256:${"c".repeat(64)}", - "bin/studio", - ["bin/studio"], - ), -} satisfies Readonly>; -`; - -describe("releaseLine", () => { - test.each([ - ["17.6.1.168", "17"], - ["15.14.1.168", "15"], - ["v2.196.0", "2"], - ["v16.2", "16"], - ["2026.09.04-sha-5a67366", "2026"], - ["0.53.0", "0"], - ])("reads %s as line %s", (version, expected) => { - expect(releaseLine(version)).toBe(expected); - }); -}); - -describe("validatePayload", () => { - test("rejects a version that would escape the string literal", () => { - expect(() => - validatePayload({ service: "auth", version: 'v1",\n "pwned', digest: DIGEST_A }), - ).toThrow(InvalidPayloadError); - }); - - test("rejects a non-sha256 digest", () => { - expect(() => validatePayload({ service: "auth", version: "v1.0.0", digest: "latest" })).toThrow( - InvalidPayloadError, - ); - }); - - test("rejects an uppercase or path-traversing service name", () => { - expect(() => - validatePayload({ service: "../../etc", version: "v1.0.0", digest: DIGEST_A }), - ).toThrow(InvalidPayloadError); - expect(() => validatePayload({ service: "Auth", version: "v1.0.0", digest: DIGEST_A })).toThrow( - InvalidPayloadError, - ); - }); -}); - -describe("planCatalogUpdate", () => { - test("bumps a single-line service and pins the digest", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "auth", - version: "v2.197.0", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.previousVersion).toBe("v2.196.0"); - expect(plan.target).toBe("default"); - expect(plan.source).toContain( - `definition("auth", "v2.197.0", "ghcr.io/supabase/cli/auth:v2.197.0@${DIGEST_A}", "bin/auth"`, - ); - expect(plan.source).not.toContain("v2.196.0"); - }); - - test("adds a digest to a previously tag-only pin", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "postgrest", - version: "v16.2", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.2@${DIGEST_A}"`); - }); - - test("bumps a date-versioned service across a year boundary", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "studio", - version: "2027.01.02-sha-abc1234", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.source).toContain( - `"ghcr.io/supabase/cli/studio:2027.01.02-sha-abc1234@${DIGEST_A}"`, - ); - }); - - test("a 17.x postgres release moves the default and leaves the 15.x line alone", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "postgres", - version: "17.6.1.169", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.previousVersion).toBe("17.6.1.168"); - expect(plan.target).toBe("default"); - expect(plan.source).toContain(`"postgres",\n "17.6.1.169",`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:17.6.1.169@${DIGEST_A}"`); - expect(plan.source).toContain(`"15.14.1.168":`); - expect(plan.source).toContain(`sha256:${"f".repeat(64)}`); - }); - - test("a 15.x postgres release moves the additional line, never the 17.x default", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "postgres", - version: "15.14.1.169", - digest: DIGEST_B, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.previousVersion).toBe("15.14.1.168"); - expect(plan.target).toBe("additional"); - expect(plan.source).toContain(`"15.14.1.169":`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:15.14.1.169@${DIGEST_B}"`); - expect(plan.source).toContain(`"postgres",\n "17.6.1.168",`); - expect(plan.source).toContain(`sha256:${"9".repeat(64)}`); - }); - - test("skips a postgres release line the catalog does not carry", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "postgres", - version: "16.4.1.001", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("unmodelled-release-line"); - if (plan.kind !== "unmodelled-release-line") return; - expect(plan.known).toEqual(["17.6.1.168", "15.14.1.168"]); - }); - - test("re-dispatching the same release is a no-op", () => { - const first = planCatalogUpdate({ - source: fixture, - service: "auth", - version: "v2.197.0", - digest: DIGEST_A, - }); - expect(first.kind).toBe("updated"); - if (first.kind !== "updated") return; - - expect( - planCatalogUpdate({ - source: first.source, - service: "auth", - version: "v2.197.0", - digest: DIGEST_A, - }).kind, - ).toBe("unchanged"); - }); - - test("a digest change on the same version still syncs", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "auth", - version: "v2.196.0", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.source).toContain(`auth:v2.196.0@${DIGEST_A}`); - }); - - test("reports a service the catalog does not model", () => { - expect( - planCatalogUpdate({ - source: fixture, - service: "kong", - version: "v3.0.0", - digest: DIGEST_A, - }).kind, - ).toBe("unmodelled-service"); - }); - - test("does not confuse postgres with postgrest", () => { - const plan = planCatalogUpdate({ - source: fixture, - service: "postgrest", - version: "v17.0", - digest: DIGEST_A, - }); - - expect(plan.kind).toBe("updated"); - if (plan.kind !== "updated") return; - expect(plan.previousVersion).toBe("v16.2"); - expect(plan.source).toContain(`"17.6.1.168",`); - expect(plan.source).toContain(`"15.14.1.168":`); - }); - - test("rejects an invalid payload instead of rewriting the catalog", () => { - expect(() => - planCatalogUpdate({ - source: fixture, - service: "auth", - version: "v1.0.0", - digest: "sha256:not-a-digest", - }), - ).toThrow(InvalidPayloadError); - }); -}); - -function pin(source: string, service: string, version: string, digest: string = DIGEST_A): string { - const plan = planCatalogUpdate({ source, service, version, digest }); - if (plan.kind !== "updated") - throw new Error(`fixture pin ${service} ${version} was ${plan.kind}`); - return plan.source; -} - -describe("carryPins", () => { - test("reapplies every pin the sync branch added onto a newer develop", () => { - const branch = pin(pin(fixture, "auth", "v2.197.0"), "postgrest", "v16.3"); - const develop = pin(fixture, "studio", "2026.09.21-sha-512201d", DIGEST_B); - - const result = carryPins({ source: develop, mergeBase: fixture, branch }); - - expect(result.carried.map((carried) => carried.service)).toEqual(["postgrest", "auth"]); - expect(result.superseded).toEqual([]); - expect(result.source).toContain(`"ghcr.io/supabase/cli/auth:v2.197.0@${DIGEST_A}"`); - expect(result.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3@${DIGEST_A}"`); - expect(result.source).toContain( - `"ghcr.io/supabase/cli/studio:2026.09.21-sha-512201d@${DIGEST_B}"`, - ); - }); - - test("carries a postgres pin onto the release line the branch moved", () => { - const branch = pin(fixture, "postgres", "15.14.1.175"); - - const result = carryPins({ source: fixture, mergeBase: fixture, branch }); - - expect(result.source).toContain(`"15.14.1.175"`); - expect(result.source).toContain(`"17.6.1.168"`); - }); - - test("keeps develop's entry when develop moved it after the branch was cut", () => { - const branch = pin(fixture, "auth", "v2.197.0"); - const develop = pin(fixture, "auth", "v2.198.0", DIGEST_B); - - const result = carryPins({ source: develop, mergeBase: fixture, branch }); - - expect(result.source).toBe(develop); - expect(result.superseded.map((dropped) => dropped.version)).toEqual(["v2.197.0"]); - }); - - test("a pin develop already merged is neither carried nor reported", () => { - const branch = pin(fixture, "auth", "v2.197.0"); - - const result = carryPins({ source: branch, mergeBase: fixture, branch }); - - expect(result).toEqual({ source: branch, carried: [], superseded: [] }); - }); -}); - -describe("against the real catalog", () => { - test("every modelled service is addressable and idempotent", () => { - // Derived from the catalog, so a new workload is covered without editing this. - const services = [ - ...new Set( - [...realCatalog.matchAll(/definition\(\s*"([a-z][a-z0-9-]*)",/g)].map( - (match) => match[1] ?? "", - ), - ), - ]; - expect(services.length).toBeGreaterThan(10); - - for (const service of services) { - const bumped = planCatalogUpdate({ - source: realCatalog, - service, - version: "99.99.99", - digest: DIGEST_A, - }); - // 99.x is a line no service carries: multi-line postgres skips, single-line services bump. - expect( - bumped.kind === "updated" || bumped.kind === "unmodelled-release-line", - `${service} was not addressable in the real catalog`, - ).toBe(true); - - if (bumped.kind !== "updated") continue; - expect( - planCatalogUpdate({ - source: bumped.source, - service, - version: "99.99.99", - digest: DIGEST_A, - }).kind, - ).toBe("unchanged"); - } - }); -}); diff --git a/.github/scripts/sync-workload-catalog.ts b/.github/scripts/sync-workload-catalog.ts deleted file mode 100644 index 70b973140f..0000000000 --- a/.github/scripts/sync-workload-catalog.ts +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Pins one workload in `packages/stack/src/Artifacts.ts` to a - * slim-services release, driven by the same `mirror-slim-image` dispatch as the - * ECR mirror (`mirror-slim-image.yml`). - * - * Dependabot owns the Dockerfile and cannot own this table: these pins carry - * image digests, which tag resolution never produces (ADR 0017). The dispatch - * payload is untrusted and revalidated here — `slim-mirror-payload.ts` - * keeps `version`/`digest` inside the string literals they are written into. - * - * Run: `bun .github/scripts/sync-workload-catalog.ts` with SLIM_SERVICE, - * SLIM_VERSION, SLIM_DIGEST. Exit 1 on an invalid payload; an unmodelled - * service or release line is a successful no-op. - * - * `bun .github/scripts/sync-workload-catalog.ts carry ` - * reapplies the pins the shared sync branch holds over its merge base, so one - * PR accumulates every pending release. - */ - -import { - InvalidPayloadError, - SOURCE_REGISTRY, - escapeRegExp, - validatePayload, -} from "./slim-mirror-payload.ts"; - -export { InvalidPayloadError, validatePayload } from "./slim-mirror-payload.ts"; - -export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; - -const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; - -/** Leading numeric component, `v` stripped. Only postgres carries >1 line. */ -export function releaseLine(version: string): string { - const withoutPrefix = version.replace(/^[vV]/, ""); - const separator = withoutPrefix.indexOf("."); - return separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); -} - -export interface CatalogUpdateInput { - readonly source: string; - readonly service: string; - readonly version: string; - readonly digest: string; -} - -export type CatalogUpdatePlan = - | { - readonly kind: "updated"; - readonly source: string; - readonly previousVersion: string; - readonly previousImage: string; - /** `default` bumped the entry's defaultVersion; `additional` bumped one of its extra lines. */ - readonly target: "default" | "additional"; - } - | { readonly kind: "unchanged" } - | { readonly kind: "unmodelled-service" } - | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; - -/** `definition("", "", ""` — image anchored so postgres != postgrest. */ -function defaultEntryPattern(service: string): RegExp { - const s = escapeRegExp(service); - return new RegExp( - `(definition\\(\\s*"${s}",\\s*")([^"]+)("\\s*,\\s*")(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)(")`, - ); -} - -/** Additional release entries: `"": ""`. The `:` is what distinguishes them. */ -function additionalEntryPattern(service: string, version?: string): RegExp { - const s = escapeRegExp(service); - const key = version === undefined ? `[^"]+` : escapeRegExp(version); - // Groups: 1 key, 2 separator (kept, to preserve wrapping), 3 image. - return new RegExp( - `"(${key})"(\\s*:\\s*)"(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)"`, - version === undefined ? "g" : "", - ); -} - -function slimImageRef(service: string, version: string, digest: string): string { - return `${SLIM_IMAGE_PREFIX}${service}:${version}@${digest}`; -} - -/** Rewrites `service`'s entry onto `version`/`digest`, or says why there was nothing to do. */ -export function planCatalogUpdate(input: CatalogUpdateInput): CatalogUpdatePlan { - validatePayload(input); - const { source, service, version, digest } = input; - - const defaultMatch = defaultEntryPattern(service).exec(source); - if (defaultMatch === null) { - return { kind: "unmodelled-service" }; - } - - const currentDefaultVersion = defaultMatch[2] ?? ""; - const currentDefaultImage = defaultMatch[4] ?? ""; - const desiredImage = slimImageRef(service, version, digest); - - const additional = [...source.matchAll(additionalEntryPattern(service))].map((match) => ({ - version: match[1] ?? "", - image: match[3] ?? "", - })); - - // One line: always the default. Several (postgres): the release line picks, - // so a 15.x release can never overwrite the 17.x default. - const bumpsDefault = - additional.length === 0 || releaseLine(version) === releaseLine(currentDefaultVersion); - - if (bumpsDefault) { - if (currentDefaultVersion === version && currentDefaultImage === desiredImage) { - return { kind: "unchanged" }; - } - return { - kind: "updated", - source: source.replace( - defaultEntryPattern(service), - (_full, prefix: string, _version: string, mid: string, _image: string, suffix: string) => - `${prefix}${version}${mid}${desiredImage}${suffix}`, - ), - previousVersion: currentDefaultVersion, - previousImage: currentDefaultImage, - target: "default", - }; - } - - const sameLine = additional.find((entry) => releaseLine(entry.version) === releaseLine(version)); - if (sameLine === undefined) { - return { - kind: "unmodelled-release-line", - known: [currentDefaultVersion, ...additional.map((entry) => entry.version)], - }; - } - - if (sameLine.version === version && sameLine.image === desiredImage) { - return { kind: "unchanged" }; - } - - return { - kind: "updated", - source: source.replace( - additionalEntryPattern(service, sameLine.version), - (_full, _key: string, separator: string) => `"${version}"${separator}"${desiredImage}"`, - ), - previousVersion: sameLine.version, - previousImage: sameLine.image, - target: "additional", - }; -} - -export interface CatalogPin { - readonly service: string; - readonly version: string; - readonly digest: string; -} - -const PIN_PATTERN = new RegExp( - `"${escapeRegExp(SLIM_IMAGE_PREFIX)}([a-z][a-z0-9-]*):([A-Za-z0-9._-]+)@(sha256:[0-9a-f]{64})"`, - "g", -); - -function pinsIn(source: string): ReadonlyArray { - return [...source.matchAll(PIN_PATTERN)].map((match) => ({ - service: match[1] ?? "", - version: match[2] ?? "", - digest: match[3] ?? "", - })); -} - -export interface CarryInput { - readonly source: string; - readonly mergeBase: string; - readonly branch: string; -} - -export interface CarryResult { - readonly source: string; - readonly carried: ReadonlyArray; - /** Pins whose entry `source` moved since the merge base; `source` wins. */ - readonly superseded: ReadonlyArray; -} - -/** Applies the pins `branch` added over `mergeBase` onto `source`. */ -export function carryPins(input: CarryInput): CarryResult { - const baseRefs = new Set(pinsIn(input.mergeBase).map(slimRefOf)); - const pending = pinsIn(input.branch).filter((pin) => !baseRefs.has(slimRefOf(pin))); - - let source = input.source; - const carried: CatalogPin[] = []; - const superseded: CatalogPin[] = []; - for (const pin of pending) { - const onBase = planCatalogUpdate({ source: input.mergeBase, ...pin }); - const onSource = planCatalogUpdate({ source, ...pin }); - if (onSource.kind === "unchanged") continue; - if ( - onBase.kind === "updated" && - onSource.kind === "updated" && - onSource.previousVersion === onBase.previousVersion && - onSource.previousImage === onBase.previousImage - ) { - source = onSource.source; - carried.push(pin); - } else { - superseded.push(pin); - } - } - return { source, carried, superseded }; -} - -function slimRefOf(pin: CatalogPin): string { - return slimImageRef(pin.service, pin.version, pin.digest); -} - -function requireEnv(name: string): string { - const value = process.env[name]; - if (value === undefined || value.trim() === "") { - throw new InvalidPayloadError(`missing required environment variable: ${name}`); - } - return value.trim(); -} - -async function carry(mergeBasePath: string, branchPath: string): Promise { - const result = carryPins({ - source: await Bun.file(CATALOG_PATH).text(), - mergeBase: await Bun.file(mergeBasePath).text(), - branch: await Bun.file(branchPath).text(), - }); - await Bun.write(CATALOG_PATH, result.source); - for (const pin of result.carried) { - console.log(`Carried pending ${pin.service} ${pin.version} (${pin.digest}).`); - } - for (const pin of result.superseded) { - console.log( - `::warning ::Dropped pending ${pin.service} ${pin.version}: develop changed that entry since the sync branch was cut.`, - ); - } -} - -async function main(argv: ReadonlyArray): Promise { - if (argv[0] === "carry") { - const [, mergeBasePath, branchPath] = argv; - if (mergeBasePath === undefined || branchPath === undefined) { - throw new Error("usage: sync-workload-catalog.ts carry "); - } - return carry(mergeBasePath, branchPath); - } - - const service = requireEnv("SLIM_SERVICE"); - const version = requireEnv("SLIM_VERSION"); - const digest = requireEnv("SLIM_DIGEST"); - - const source = await Bun.file(CATALOG_PATH).text(); - const plan = planCatalogUpdate({ source, service, version, digest }); - - switch (plan.kind) { - case "unmodelled-service": - console.log(`::notice ::${CATALOG_PATH} models no '${service}' workload; nothing to sync.`); - return; - case "unmodelled-release-line": - console.log( - `::notice ::${service} ${version} is not on a release line ${CATALOG_PATH} carries (${plan.known.join(", ")}); nothing to sync.`, - ); - return; - case "unchanged": - console.log(`::notice ::${service} is already pinned to ${version} at ${digest}.`); - return; - case "updated": - await Bun.write(CATALOG_PATH, plan.source); - console.log( - `Updated ${service} ${plan.target} release ${plan.previousVersion} -> ${version} (${digest}).`, - ); - return; - } -} - -if (import.meta.main) { - main(process.argv.slice(2)).catch((error: unknown) => { - console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); - process.exit(1); - }); -} diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index 5d4af05b36..d7cefc301d 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -9,7 +9,6 @@ on: pull_request: paths: - ".github/scripts/**" - - "packages/stack/src/Artifacts.ts" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" - ".github/workflows/mirror-slim-image.yml" diff --git a/.github/workflows/sync-stack-workload-catalog.yml b/.github/workflows/sync-stack-workload-catalog.yml deleted file mode 100644 index 971acde8f5..0000000000 --- a/.github/workflows/sync-stack-workload-catalog.yml +++ /dev/null @@ -1,194 +0,0 @@ -name: Sync Stack Workload Catalog - -# Pins `packages/stack/src/Artifacts.ts` to slim-services releases, -# off the same `mirror-slim-image` dispatch that drives the ECR mirror. -# Dependabot owns the Dockerfile and cannot own this table: these pins carry -# image digests, which tag resolution never produces (ADR 0017). -# -# Separate from `mirror-slim-image.yml` on purpose — that mirror runs against -# the sender's 15-minute verification poll and must not be delayed or reddened -# by a catalog PR. Opens a PR rather than pushing: `develop` is protected and a -# version pin is reviewable. -# -# Every release lands on one shared branch and PR. Each run rebuilds the branch -# as `develop` + the pins already pending on it + its own pin, and pushes with a -# lease on the head it read, recomposing when a concurrent run for another -# service moved it first. Re-dispatches are no-ops. - -on: - repository_dispatch: - types: - - mirror-slim-image - workflow_dispatch: - inputs: - service: - description: "Service name (e.g. postgrest)" - required: true - type: string - version: - description: "Image tag (e.g. v16.2)" - required: true - type: string - digest: - description: "Expected index digest (sha256:<64 hex chars>)" - required: true - type: string - -permissions: - contents: read - -concurrency: - # Per service, so a newer release of one service supersedes its queued run. - # Runs for different services race on the shared branch; the push lease - # serializes them. Not cancel-in-progress: a run may be mid-push. - group: sync-stack-workload-catalog-${{ github.event.client_payload.service || inputs.service }} - cancel-in-progress: false - -env: - SYNC_BRANCH: sync/stack-workload-catalog - # Payload is untrusted; the script revalidates it. Passed via env, never - # interpolated into the shell. Extra payload fields such as natives[] are - # ignored here; this workflow pins the image digest only. - SLIM_SERVICE: ${{ github.event.client_payload.service || inputs.service }} - SLIM_VERSION: ${{ github.event.client_payload.version || inputs.version }} - SLIM_DIGEST: ${{ github.event.client_payload.digest || inputs.digest }} - -jobs: - sync: - name: Sync workload catalog - runs-on: blacksmith-2vcpu-ubuntu-2404 - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - # The merge base with the sync branch decides which of its pins are pending. - fetch-depth: 0 - filter: blob:none - - - name: Setup - uses: ./.github/actions/setup - with: - dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - - - name: Compose workload catalog - id: compose - run: | - set -euo pipefail - base_sha=$(git rev-parse HEAD) - lease=$(BASE_SHA="$base_sha" bash .github/scripts/compose-workload-catalog.sh) - echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" - echo "lease=$lease" >> "$GITHUB_OUTPUT" - - - name: Check for catalog changes - id: check - env: - LEASE: ${{ steps.compose.outputs.lease }} - run: | - if git diff --exit-code --quiet packages/stack/src/Artifacts.ts; then - echo "Workload catalog is already in sync." - echo "has_changes=false" >> "$GITHUB_OUTPUT" - elif [ -n "$LEASE" ] && git diff --quiet "$LEASE" -- packages/stack/src/Artifacts.ts; then - echo "${SYNC_BRANCH} already carries this catalog." - echo "has_changes=false" >> "$GITHUB_OUTPUT" - else - echo "Workload catalog updated." - echo "has_changes=true" >> "$GITHUB_OUTPUT" - fi - - # Prove the rewritten pins compile and the affected catalog surfaces pass. - - name: Type-check stack - if: steps.check.outputs.has_changes == 'true' - run: pnpm types:check - working-directory: packages/stack - - - name: Test catalog - if: steps.check.outputs.has_changes == 'true' - run: pnpm run test:github-scripts - - - name: Test affected stack preparation - if: steps.check.outputs.has_changes == 'true' - run: >- - pnpm --filter @supabase/stack test:integration:run - src/preparation/artifacts.integration.test.ts - src/preparation/slim-services.integration.test.ts - --passWithNoTests=false - - - name: Generate token - if: steps.check.outputs.has_changes == 'true' - id: app-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.GH_APP_CLIENT_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - permission-pull-requests: write - permission-contents: write - - # A retry recomposes onto the branch head another run pushed; those pins - # were validated by their own runs and the PR's CI covers the combination. - - name: Push sync branch - if: steps.check.outputs.has_changes == 'true' - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} - BASE_SHA: ${{ steps.compose.outputs.base_sha }} - LEASE: ${{ steps.compose.outputs.lease }} - run: | - set -euo pipefail - catalog=packages/stack/src/Artifacts.ts - bot="${APP_SLUG}[bot]" - bot_id=$(gh api "/users/${bot}" --jq .id) - git config user.name "$bot" - git config user.email "${bot_id}+${bot}@users.noreply.github.com" - remote="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - - lease="$LEASE" - for attempt in 1 2 3 4 5; do - if [ "$attempt" -gt 1 ]; then - git reset --quiet "$BASE_SHA" - lease=$(bash .github/scripts/compose-workload-catalog.sh) - fi - if [ -n "$lease" ] && git diff --quiet "$lease" -- "$catalog"; then - echo "${SYNC_BRANCH} already carries this catalog." - exit 0 - fi - pins=$(git diff "$BASE_SHA" -- "$catalog" \ - | grep -oE '^\+.*"ghcr\.io/supabase/cli/[a-z0-9-]+:[A-Za-z0-9._-]+@' \ - | sed -E 's|.*ghcr\.io/supabase/cli/([a-z0-9-]+):([^@]+)@|- \1 \2|' || true) - git commit --quiet -m "chore(stack): pin slim-services workloads" -m "$pins" -- "$catalog" - if git push --force-with-lease="refs/heads/${SYNC_BRANCH}:${lease}" "$remote" "HEAD:refs/heads/${SYNC_BRANCH}"; then - exit 0 - fi - echo "::notice ::${SYNC_BRANCH} moved under attempt ${attempt}; recomposing." - done - echo "::error ::Could not push ${SYNC_BRANCH} after 5 attempts." - exit 1 - - - name: Open or update pull request - if: steps.check.outputs.has_changes == 'true' - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - existing() { - gh pr list --repo "$GITHUB_REPOSITORY" --head "$SYNC_BRANCH" --base develop \ - --state open --json number --jq '.[0].number // empty' - } - if [ -n "$(existing)" ]; then - echo "PR #$(existing) already tracks ${SYNC_BRANCH}." - exit 0 - fi - body=$(cat <<'EOF' - Pins `packages/stack/src/Artifacts.ts` to the latest slim-services releases - and their image digests. The commit message lists every pin in this PR. - - Opened from the `mirror-slim-image` dispatch. Each new release rebuilds this - branch onto `develop` with all pending pins, so this PR stays the only one. - `artifactFor` derives the native release tag, asset names, and URLs from - service + version, so these pins are the whole change. - EOF - ) - # A concurrent run may open the PR between the check and the create. - gh pr create --repo "$GITHUB_REPOSITORY" --base develop --head "$SYNC_BRANCH" \ - --title "chore(stack): pin slim-services workloads" --body "$body" \ - || [ -n "$(existing)" ] diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 59fa5be703..4d4ff2f188 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -13,7 +13,7 @@ Some consumers cannot reach GitHub release assets or a single registry blob CDN. Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. Publish each slim **native** archive as an OCI artifact on the same two repositories under `:version-native-` (not `:version-linux-*`, which are image platform tags). GitHub Releases remain the human HTTPS copy and `--clobber` on force. -`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Catalog sync consumes the image `service` / `version` / `digest` only; native tags ride in a separate `natives[]` payload field. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until a catalog PR ships. Natives follow the moved tag immediately. +`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`; a later workflow owns that catalog update. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that catalog update ships. Natives follow the moved tag immediately. ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. Do not add a public S3 bucket or a second cloud vendor for this cut. From cde5400bbf390af0ea8667fe7d6ebb53a7828840 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 16:18:12 +0200 Subject: [PATCH 3/6] ci(repo): pin the artifacts catalog on docker image bumps Dependabot's Dockerfile pull request is the image bump. Commit the matching slim pins onto that branch so the docker tag, slim image, and native archive stay on one version. --- .github/scripts/slim-mirror-payload.ts | 4 +- .../scripts/sync-artifacts-catalog.test.ts | 163 +++++++++ .github/scripts/sync-artifacts-catalog.ts | 319 ++++++++++++++++++ .github/workflows/github-scripts-ci.yml | 2 + .github/workflows/sync-artifacts-catalog.yml | 89 +++++ apps/cli/src/shared/services/slim-images.ts | 39 ++- docs/adr/0026-slim-artifact-mirrors.md | 2 +- 7 files changed, 602 insertions(+), 16 deletions(-) create mode 100644 .github/scripts/sync-artifacts-catalog.test.ts create mode 100644 .github/scripts/sync-artifacts-catalog.ts create mode 100644 .github/workflows/sync-artifacts-catalog.yml diff --git a/.github/scripts/slim-mirror-payload.ts b/.github/scripts/slim-mirror-payload.ts index 72f9587e8e..fe516d4377 100644 --- a/.github/scripts/slim-mirror-payload.ts +++ b/.github/scripts/slim-mirror-payload.ts @@ -4,8 +4,8 @@ */ const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/; -const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; -const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; +export const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; +export const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; export const SOURCE_REGISTRY = "ghcr.io/supabase/cli"; export const DEST_REGISTRY = "public.ecr.aws/supabase/cli"; diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts new file mode 100644 index 0000000000..c984c3748f --- /dev/null +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, test } from "bun:test"; + +import { InvalidPayloadError } from "./slim-mirror-payload.ts"; +import { + CATALOG_PATH, + catalogDigestPins, + planArtifactCatalogUpdate, + releaseLine, +} from "./sync-artifacts-catalog.ts"; + +const DIGEST_A = `sha256:${"a".repeat(64)}`; +const DIGEST_B = `sha256:${"b".repeat(64)}`; + +const fixture = `const workloadCatalog = { + database: definition( + "postgres", + "17.6.1.168", + "ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}", + "bin/supabase-postgres-start", + ["bin/supabase-postgres-start"], + { + "15.14.1.168": + "ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}", + }, + ), + rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), + auth: definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth"), + analytics: definition( + "analytics", + "v1.50.9", + "ghcr.io/supabase/cli/analytics:v1.50.9@${DIGEST_A}", + "bin/logflare", + ), + vector: definition("vector", "0.49.0", "ghcr.io/supabase/cli/vector:0.49.0", "bin/vector"), + pooler: definition("pooler", "v2.9.10", "ghcr.io/supabase/cli/pooler:v2.9.10", "bin/server"), +}; +`; + +const dockerfile = `FROM supabase/postgres:17.6.1.171 AS pg +FROM library/kong:2.8.1 AS kong +FROM postgrest/postgrest:v16.3 AS postgrest +FROM timberio/vector:0.53.0-alpine AS vector +FROM supabase/supavisor:2.9.13 AS supavisor +FROM supabase/logflare:1.50.12 AS logflare +`; + +function digests( + overrides: Readonly> = {}, +): (service: string, version: string) => string { + return (service, version) => { + const digest = overrides[`${service}:${version}`]; + if (digest === undefined) throw new Error(`unexpected digest lookup ${service}:${version}`); + return digest; + }; +} + +describe("releaseLine", () => { + test.each([ + ["17.6.1.168", "17"], + ["15.14.1.168", "15"], + ["v2.196.0", "2"], + ["2026.09.04-sha-5a67366", "2026"], + ])("reads %s as line %s", (version, expected) => { + expect(releaseLine(version)).toBe(expected); + }); +}); + +describe("planArtifactCatalogUpdate", () => { + test("pins each slim Dockerfile image and skips aliases with no slim build", () => { + const plan = planArtifactCatalogUpdate({ + dockerfile, + catalog: fixture, + digestFor: digests({ "postgres:17.6.1.171": DIGEST_B, "analytics:v1.50.12": DIGEST_B }), + }); + + expect( + plan.updates.map( + (update) => `${update.service} ${update.previousVersion} -> ${update.version}`, + ), + ).toEqual([ + "postgres 17.6.1.168 -> 17.6.1.171", + "postgrest v16.2 -> v16.3", + "vector 0.49.0 -> 0.53.0", + "pooler v2.9.10 -> v2.9.13", + "analytics v1.50.9 -> v1.50.12", + ]); + expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:17.6.1.171@${DIGEST_B}"`); + expect(plan.source).toContain( + `"ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}"`, + ); + expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3"`); + expect(plan.source).toContain(`"ghcr.io/supabase/cli/vector:0.53.0"`); + expect(plan.source).toContain(`"ghcr.io/supabase/cli/pooler:v2.9.13"`); + expect(plan.source).not.toContain("kong"); + }); + + test("moves a postgres pin onto the matching additional release line", () => { + const plan = planArtifactCatalogUpdate({ + dockerfile: "FROM supabase/postgres:15.14.1.175 AS pg\n", + catalog: fixture, + digestFor: digests({ "postgres:15.14.1.175": DIGEST_B }), + }); + + expect(plan.updates).toEqual([ + { + service: "postgres", + version: "15.14.1.175", + previousVersion: "15.14.1.168", + target: "additional", + }, + ]); + expect(plan.source).toContain(`"17.6.1.168"`); + expect(plan.source).toContain(`"15.14.1.175"`); + expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:15.14.1.175@${DIGEST_B}"`); + }); + + test("does not look up a digest when the pin already matches", () => { + const plan = planArtifactCatalogUpdate({ + dockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", + catalog: fixture, + digestFor: () => { + throw new Error("digest lookup"); + }, + }); + + expect(plan).toEqual({ source: fixture, updates: [] }); + }); + + test("refuses a Dockerfile tag that would escape the catalog string", () => { + expect(() => + planArtifactCatalogUpdate({ + dockerfile: 'FROM supabase/gotrue:v1" AS gotrue\n', + catalog: fixture, + digestFor: digests(), + }), + ).toThrow(InvalidPayloadError); + }); + + test("refuses a release line the catalog does not carry", () => { + expect(() => + planArtifactCatalogUpdate({ + dockerfile: "FROM supabase/postgres:16.1.1.1 AS pg\n", + catalog: fixture, + digestFor: digests(), + }), + ).toThrow(InvalidPayloadError); + }); +}); + +describe("against the real catalog", () => { + test("every slim Dockerfile alias is a catalog entry", async () => { + const dockerfile = await Bun.file("apps/cli/src/shared/services/Dockerfile").text(); + const catalog = await Bun.file(CATALOG_PATH).text(); + + expect(() => catalogDigestPins(dockerfile, catalog)).not.toThrow(); + const plan = planArtifactCatalogUpdate({ + dockerfile, + catalog, + digestFor: () => DIGEST_A, + }); + expect(plan.source).toContain('definition(\n "postgres"'); + }); +}); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts new file mode 100644 index 0000000000..4703c316aa --- /dev/null +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -0,0 +1,319 @@ +/** + * Pins `packages/stack/src/Artifacts.ts` to the slim workloads named by + * `apps/cli/src/shared/services/Dockerfile`. Native archive URLs are derived + * from service + version. An entry that already carries a digest keeps one: + * the published `ghcr.io/supabase/cli/:` manifest digest. + * + * Run: `bun .github/scripts/sync-artifacts-catalog.ts ` + */ + +import { slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; +import { + DIGEST_PATTERN, + InvalidPayloadError, + SOURCE_REGISTRY, + VERSION_PATTERN, + escapeRegExp, +} from "./slim-mirror-payload.ts"; + +export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; +const DOCKERFILE_PATH = "apps/cli/src/shared/services/Dockerfile"; + +const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; + +// Same FROM shape as `parseDockerfileServiceImages` in dockerfile-images.ts. +const FROM_LINE_PATTERN = /^FROM\s+(.+):([^:\s]+)\s+AS\s+([^\s#]+)/i; + +function parseDockerfileServiceImages( + dockerfile: string, +): ReadonlyArray<{ readonly alias: string; readonly image: string }> { + return dockerfile + .split("\n") + .map((line) => line.trim()) + .flatMap((line) => { + const match = FROM_LINE_PATTERN.exec(line); + if (match === null) return []; + const [, repository, tag, alias] = match; + if (repository === undefined || tag === undefined || alias === undefined) return []; + return [{ alias, image: `${repository}:${tag}` }]; + }); +} + +/** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ +export function releaseLine(version: string): string { + const withoutPrefix = version.replace(/^[vV]/, ""); + const separator = withoutPrefix.indexOf("."); + return separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); +} + +export interface CatalogPinUpdate { + readonly service: string; + readonly version: string; + readonly previousVersion: string; + readonly target: "default" | "additional"; +} + +export interface CatalogPlan { + readonly source: string; + readonly updates: ReadonlyArray; +} + +interface DockerfilePin { + readonly alias: string; + readonly service: string; + readonly version: string; +} + +/** `definition("", "", ""`. */ +function defaultEntryPattern(service: string): RegExp { + const s = escapeRegExp(service); + return new RegExp( + `(definition\\(\\s*"${s}",\\s*")([^"]+)("\\s*,\\s*")(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)(")`, + ); +} + +/** Additional release entries: `"": ""`. */ +function additionalEntryPattern(service: string, version?: string): RegExp { + const s = escapeRegExp(service); + const key = version === undefined ? `[^"]+` : escapeRegExp(version); + return new RegExp( + `"(${key})"(\\s*:\\s*)"(${escapeRegExp(SLIM_IMAGE_PREFIX)}${s}:[^"]+)"`, + version === undefined ? "g" : "", + ); +} + +function imageHasDigest(image: string): boolean { + return /@sha256:[0-9a-f]{64}$/.test(image); +} + +function desiredImage( + service: string, + version: string, + currentImage: string, + digest: string | undefined, +): string { + const tagged = `${SLIM_IMAGE_PREFIX}${service}:${version}`; + if (!imageHasDigest(currentImage)) return tagged; + if (digest === undefined || !DIGEST_PATTERN.test(digest)) { + throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); + } + return `${tagged}@${digest}`; +} + +type SelectedEntry = + | { readonly kind: "default" | "additional"; readonly version: string; readonly image: string } + | { readonly kind: "unmodelled-service" } + | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; + +function selectEntry(source: string, service: string, version: string): SelectedEntry { + const defaultMatch = defaultEntryPattern(service).exec(source); + if (defaultMatch === null) return { kind: "unmodelled-service" }; + + const currentDefaultVersion = defaultMatch[2] ?? ""; + const currentDefaultImage = defaultMatch[4] ?? ""; + const additional = [...source.matchAll(additionalEntryPattern(service))].map((match) => ({ + version: match[1] ?? "", + image: match[3] ?? "", + })); + const bumpsDefault = + additional.length === 0 || releaseLine(version) === releaseLine(currentDefaultVersion); + if (bumpsDefault) { + return { kind: "default", version: currentDefaultVersion, image: currentDefaultImage }; + } + + const sameLine = additional.find((entry) => releaseLine(entry.version) === releaseLine(version)); + if (sameLine === undefined) { + return { + kind: "unmodelled-release-line", + known: [currentDefaultVersion, ...additional.map((entry) => entry.version)], + }; + } + return { kind: "additional", version: sameLine.version, image: sameLine.image }; +} + +function dockerfilePins(dockerfile: string): ReadonlyArray { + const pins: DockerfilePin[] = []; + for (const from of parseDockerfileServiceImages(dockerfile)) { + const pin = slimCatalogPin(from.alias, from.image); + if (pin === undefined) continue; + if (!VERSION_PATTERN.test(pin.version)) { + throw new InvalidPayloadError(`invalid version for ${from.alias}: '${pin.version}'`); + } + pins.push({ alias: from.alias, service: pin.service, version: pin.version }); + } + return pins; +} + +function rejectUnmodelled( + alias: string, + service: string, + version: string, + entry: SelectedEntry, +): void { + if (entry.kind === "unmodelled-service") { + throw new InvalidPayloadError( + `${CATALOG_PATH} has no ${service} entry for the ${alias} image.`, + ); + } + if (entry.kind === "unmodelled-release-line") { + throw new InvalidPayloadError( + `${service} ${version} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`, + ); + } +} + +/** Slim tags whose catalog entry stores a digest, so the rewrite can resolve them first. */ +export function catalogDigestPins( + dockerfile: string, + catalog: string, +): ReadonlyArray<{ readonly service: string; readonly version: string }> { + const pins: Array<{ service: string; version: string }> = []; + for (const pin of dockerfilePins(dockerfile)) { + const entry = selectEntry(catalog, pin.service, pin.version); + rejectUnmodelled(pin.alias, pin.service, pin.version, entry); + if (entry.kind !== "default" && entry.kind !== "additional") continue; + if (imageHasDigest(entry.image)) pins.push({ service: pin.service, version: pin.version }); + } + return pins; +} + +type PinResult = + | { + readonly kind: "updated"; + readonly source: string; + readonly previousVersion: string; + readonly target: "default" | "additional"; + } + | { readonly kind: "unchanged" }; + +function pinService( + source: string, + service: string, + version: string, + digest: string | undefined, +): PinResult { + const entry = selectEntry(source, service, version); + if (entry.kind !== "default" && entry.kind !== "additional") { + throw new InvalidPayloadError(`${service} ${version} is not a catalog entry.`); + } + const desired = desiredImage(service, version, entry.image, digest); + if (entry.version === version && entry.image === desired) return { kind: "unchanged" }; + + if (entry.kind === "default") { + return { + kind: "updated", + source: source.replace( + defaultEntryPattern(service), + (_full, prefix: string, _version: string, mid: string, _image: string, suffix: string) => + `${prefix}${version}${mid}${desired}${suffix}`, + ), + previousVersion: entry.version, + target: "default", + }; + } + + return { + kind: "updated", + source: source.replace( + additionalEntryPattern(service, entry.version), + (_full, _key: string, separator: string) => `"${version}"${separator}"${desired}"`, + ), + previousVersion: entry.version, + target: "additional", + }; +} + +/** + * Rewrites `catalog` so every slim Dockerfile pin matches. Aliases with no slim + * build are skipped. A modelled service on an unknown release line throws: + * the docker.io tag and the catalog would otherwise diverge. + */ +export function planArtifactCatalogUpdate(input: { + readonly dockerfile: string; + readonly catalog: string; + readonly digestFor: (service: string, version: string) => string; +}): CatalogPlan { + let source = input.catalog; + const updates: CatalogPinUpdate[] = []; + + for (const pin of dockerfilePins(input.dockerfile)) { + const entry = selectEntry(source, pin.service, pin.version); + rejectUnmodelled(pin.alias, pin.service, pin.version, entry); + if (entry.kind !== "default" && entry.kind !== "additional") continue; + const digest = imageHasDigest(entry.image) + ? input.digestFor(pin.service, pin.version) + : undefined; + const result = pinService(source, pin.service, pin.version, digest); + if (result.kind === "unchanged") continue; + source = result.source; + updates.push({ + service: pin.service, + version: pin.version, + previousVersion: result.previousVersion, + target: result.target, + }); + } + + return { source, updates }; +} + +async function publishedDigest(service: string, version: string): Promise { + const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; + const proc = Bun.spawn(["regctl", "manifest", "head", reference], { + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exit] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + const live = stdout.trim(); + if (exit !== 0 || !DIGEST_PATTERN.test(live)) { + const detail = stderr.trim(); + throw new InvalidPayloadError( + `${reference} has no slim manifest digest${detail === "" ? "" : `: ${detail}`}`, + ); + } + return live; +} + +async function main(argv: ReadonlyArray): Promise { + const [dockerfilePath = DOCKERFILE_PATH, catalogPath = CATALOG_PATH] = argv; + const dockerfile = await Bun.file(dockerfilePath).text(); + const catalog = await Bun.file(catalogPath).text(); + const digests = new Map(); + for (const pin of catalogDigestPins(dockerfile, catalog)) { + const key = `${pin.service}:${pin.version}`; + if (!digests.has(key)) digests.set(key, await publishedDigest(pin.service, pin.version)); + } + + const plan = planArtifactCatalogUpdate({ + dockerfile, + catalog, + digestFor: (service, version) => { + const digest = digests.get(`${service}:${version}`); + if (digest === undefined) { + throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); + } + return digest; + }, + }); + await Bun.write(catalogPath, plan.source); + if (plan.updates.length === 0) { + console.log("Workload catalog already matches the Dockerfile."); + return; + } + for (const update of plan.updates) { + console.log( + `Pinned ${update.service} ${update.target} ${update.previousVersion} -> ${update.version}.`, + ); + } +} + +if (import.meta.main) { + main(process.argv.slice(2)).catch((error: unknown) => { + console.log(`::error ::${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + }); +} diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index d7cefc301d..b9a8ec8d3a 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -9,6 +9,8 @@ on: pull_request: paths: - ".github/scripts/**" + - "apps/cli/src/shared/services/Dockerfile" + - "packages/stack/src/Artifacts.ts" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" - ".github/workflows/mirror-slim-image.yml" diff --git a/.github/workflows/sync-artifacts-catalog.yml b/.github/workflows/sync-artifacts-catalog.yml new file mode 100644 index 0000000000..f6d4abb362 --- /dev/null +++ b/.github/workflows/sync-artifacts-catalog.yml @@ -0,0 +1,89 @@ +name: Sync Artifacts catalog + +# Dependabot bumps apps/cli/src/shared/services/Dockerfile. This commits the +# matching slim pins into packages/stack/src/Artifacts.ts on that same branch, +# so the docker.io tag, the slim image, and the native archive stay on one +# version. Native URLs are derived from service + version; a digest already +# stored on the entry is replaced with the published slim manifest digest. +# +# The rewrite runs from the base revision. The pull request only supplies the +# Dockerfile and catalog text, and the job runs for Dependabot so a pull +# request cannot edit this workflow and then execute it with a write token. +# Dependabot's GITHUB_TOKEN is read-only unless this job raises contents. + +on: + pull_request: + paths: + - apps/cli/src/shared/services/Dockerfile + +permissions: + contents: read + +concurrency: + group: sync-artifacts-catalog-${{ github.event.pull_request.number }} + cancel-in-progress: false + +jobs: + sync: + name: Pin workload catalog + if: github.actor == 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: true + fetch-depth: 1 + + # Bun only. The script imports no workspace packages, and Dependabot + # runs cannot read Actions secrets the full setup action would use. + - name: Install toolchains + uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 + with: + version: 2026.9.0 + + - name: Install regctl + run: | + set -euo pipefail + install -d "${RUNNER_TEMP}/regctl-bin" + curl -fsSLo "${RUNNER_TEMP}/regctl-bin/regctl" \ + https://github.com/regclient/regclient/releases/download/v0.11.5/regctl-linux-amd64 + echo "c93aa7638749f5aaac1a8e01787321889c78f0101809bb2880343478d0ba0467 ${RUNNER_TEMP}/regctl-bin/regctl" | sha256sum -c - + chmod +x "${RUNNER_TEMP}/regctl-bin/regctl" + echo "${RUNNER_TEMP}/regctl-bin" >> "$GITHUB_PATH" + "${RUNNER_TEMP}/regctl-bin/regctl" version + + - name: Pin the catalog to the Dockerfile + env: + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + git fetch --no-tags origin "pull/${PR_NUMBER}/head" + fetched="$(git rev-parse FETCH_HEAD)" + if [ "$fetched" != "$HEAD_SHA" ]; then + echo "::error ::pull/${PR_NUMBER} head is ${fetched}, expected ${HEAD_SHA}." + exit 1 + fi + dockerfile="$(mktemp)" + catalog="$(mktemp)" + original="$(mktemp)" + git show "$HEAD_SHA:apps/cli/src/shared/services/Dockerfile" > "$dockerfile" + git show "$HEAD_SHA:packages/stack/src/Artifacts.ts" > "$catalog" + cp "$catalog" "$original" + bun .github/scripts/sync-artifacts-catalog.ts "$dockerfile" "$catalog" + if cmp -s "$original" "$catalog"; then + echo "Workload catalog already matches the Dockerfile." + exit 0 + fi + git checkout --detach "$HEAD_SHA" + cp "$catalog" packages/stack/src/Artifacts.ts + git add -- packages/stack/src/Artifacts.ts + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "chore(stack): pin the workload catalog to the Dockerfile image tags" + git push origin "HEAD:refs/heads/${HEAD_REF}" diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 9a27bc024e..cd34e92949 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -60,33 +60,46 @@ function slimTagForService(service: SlimServiceName, rawTag: string): string { return tag; } -function slimImageRef(service: SlimServiceName, rawTag: string): string { - return `${SLIM_IMAGE_PREFIX}${service}:${slimTagForService(service, rawTag)}`; +export interface SlimCatalogPin { + readonly service: SlimServiceName; + readonly version: string; } /** - * Rewrites a docker.io image reference to its `ghcr.io/supabase/cli` slim - * equivalent, keeping the pin's version. This helper owns tag normalization - * (`v`-prefixing, `tagPrefix`), so pins that differ only in prefix between the - * two registries (`supavisor`, `logflare`) land on the right slim tag. Vector's - * docker.io tags carry an `-alpine` variant suffix that the slim build does - * not publish, so the strip is scoped to `vector` only — an `-alpine`-suffixed - * pin on any other service is a real tag, not a variant marker. + * Slim service and tag for a Dockerfile alias. Absent when that alias has no + * slim build (kong and the one-shot job images). */ -export function toSlimImage(alias: string, image: string): string { +export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | undefined { const service = SLIM_SERVICE_LOOKUP[alias]; if (service === undefined) { - return image; + return undefined; } const tagSeparator = image.lastIndexOf(":"); if (tagSeparator === -1) { - return image; + return undefined; } const rawTag = image.slice(tagSeparator + 1); const tag = alias === "vector" ? rawTag.replace(/-alpine$/, "") : rawTag; - return slimImageRef(service, tag); + return { service, version: slimTagForService(service, tag) }; +} + +/** + * Rewrites a docker.io image reference to its `ghcr.io/supabase/cli` slim + * equivalent, keeping the pin's version. This helper owns tag normalization + * (`v`-prefixing, `tagPrefix`), so pins that differ only in prefix between the + * two registries (`supavisor`, `logflare`) land on the right slim tag. Vector's + * docker.io tags carry an `-alpine` variant suffix that the slim build does + * not publish, so the strip is scoped to `vector` only — an `-alpine`-suffixed + * pin on any other service is a real tag, not a variant marker. + */ +export function toSlimImage(alias: string, image: string): string { + const pin = slimCatalogPin(alias, image); + if (pin === undefined) { + return image; + } + return `${SLIM_IMAGE_PREFIX}${pin.service}:${pin.version}`; } /** `toSlimImage` behind the feature flag; a no-op while the flag is off. */ diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 4d4ff2f188..9b02b0dc2a 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -13,7 +13,7 @@ Some consumers cannot reach GitHub release assets or a single registry blob CDN. Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. Publish each slim **native** archive as an OCI artifact on the same two repositories under `:version-native-` (not `:version-linux-*`, which are image platform tags). GitHub Releases remain the human HTTPS copy and `--clobber` on force. -`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`; a later workflow owns that catalog update. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that catalog update ships. Natives follow the moved tag immediately. +`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`. A Dependabot Dockerfile bump commits that catalog pin onto the same pull request, so the docker.io tag, the slim image, and the native archive stay on one version. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that pull request merges. Natives follow the moved tag immediately. ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. Do not add a public S3 bucket or a second cloud vendor for this cut. From 0e235cd2873848d435db1590f372022b76a61aa0 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 16:29:39 +0200 Subject: [PATCH 4/6] ci(repo): skip OrioleDB when pinning the slim catalog OrioleDB tags have no slim image. Leave them on docker.io and keep pinning the other Dockerfile services. Read those services with the manifest parser instead of a second FROM reader. --- .../scripts/sync-artifacts-catalog.test.ts | 54 +++++-- .github/scripts/sync-artifacts-catalog.ts | 138 +++++++++--------- .github/workflows/sync-artifacts-catalog.yml | 11 +- .../src/shared/services/dockerfile-images.ts | 32 +--- .../parse-dockerfile-service-images.ts | 28 ++++ apps/cli/src/shared/services/slim-images.ts | 14 +- .../shared/services/slim-images.unit.test.ts | 11 ++ 7 files changed, 172 insertions(+), 116 deletions(-) create mode 100644 apps/cli/src/shared/services/parse-dockerfile-service-images.ts diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index c984c3748f..eb4632d8ad 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -123,7 +123,39 @@ describe("planArtifactCatalogUpdate", () => { }, }); - expect(plan).toEqual({ source: fixture, updates: [] }); + expect(plan).toEqual({ source: fixture, updates: [], skipped: [] }); + }); + + test("leaves an OrioleDB tag on docker.io and still pins the other images", () => { + const plan = planArtifactCatalogUpdate({ + dockerfile: `FROM supabase/postgres:16.0.0.1-orioledb AS pg +FROM postgrest/postgrest:v16.3 AS postgrest +FROM supabase/postgres:orioledb-15.1.0.55 AS orioledb +`, + catalog: fixture, + digestFor: digests(), + }); + + expect(plan.updates.map((update) => update.service)).toEqual(["postgrest"]); + expect(plan.source).toContain( + `"ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}"`, + ); + expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3"`); + expect(plan.skipped.map((skip) => skip.alias)).toEqual(["pg", "orioledb"]); + }); + + test("skips a pin with no slim manifest and still applies the next one", () => { + const plan = planArtifactCatalogUpdate({ + dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg +FROM postgrest/postgrest:v16.3 AS postgrest +`, + catalog: fixture, + digestFor: (service) => (service === "postgres" ? undefined : DIGEST_B), + }); + + expect(plan.updates.map((update) => update.service)).toEqual(["postgrest"]); + expect(plan.source).toContain(`"17.6.1.168"`); + expect(plan.skipped.map((skip) => skip.alias)).toEqual(["pg"]); }); test("refuses a Dockerfile tag that would escape the catalog string", () => { @@ -136,14 +168,18 @@ describe("planArtifactCatalogUpdate", () => { ).toThrow(InvalidPayloadError); }); - test("refuses a release line the catalog does not carry", () => { - expect(() => - planArtifactCatalogUpdate({ - dockerfile: "FROM supabase/postgres:16.1.1.1 AS pg\n", - catalog: fixture, - digestFor: digests(), - }), - ).toThrow(InvalidPayloadError); + test("skips a release line the catalog does not carry and still pins the rest", () => { + const plan = planArtifactCatalogUpdate({ + dockerfile: `FROM supabase/postgres:16.1.1.1 AS pg +FROM postgrest/postgrest:v16.3 AS postgrest +`, + catalog: fixture, + digestFor: digests(), + }); + + expect(plan.updates.map((update) => update.service)).toEqual(["postgrest"]); + expect(plan.skipped.map((skip) => skip.alias)).toEqual(["pg"]); + expect(plan.source).toContain(`"17.6.1.168"`); }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 4703c316aa..bf0f0448ff 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -7,7 +7,8 @@ * Run: `bun .github/scripts/sync-artifacts-catalog.ts ` */ -import { slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; +import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; +import { isOrioleImage, slimCatalogPin } from "../../apps/cli/src/shared/services/slim-images.ts"; import { DIGEST_PATTERN, InvalidPayloadError, @@ -21,24 +22,6 @@ const DOCKERFILE_PATH = "apps/cli/src/shared/services/Dockerfile"; const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; -// Same FROM shape as `parseDockerfileServiceImages` in dockerfile-images.ts. -const FROM_LINE_PATTERN = /^FROM\s+(.+):([^:\s]+)\s+AS\s+([^\s#]+)/i; - -function parseDockerfileServiceImages( - dockerfile: string, -): ReadonlyArray<{ readonly alias: string; readonly image: string }> { - return dockerfile - .split("\n") - .map((line) => line.trim()) - .flatMap((line) => { - const match = FROM_LINE_PATTERN.exec(line); - if (match === null) return []; - const [, repository, tag, alias] = match; - if (repository === undefined || tag === undefined || alias === undefined) return []; - return [{ alias, image: `${repository}:${tag}` }]; - }); -} - /** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ export function releaseLine(version: string): string { const withoutPrefix = version.replace(/^[vV]/, ""); @@ -53,15 +36,15 @@ export interface CatalogPinUpdate { readonly target: "default" | "additional"; } +export interface SkippedCatalogPin { + readonly alias: string; + readonly reason: string; +} + export interface CatalogPlan { readonly source: string; readonly updates: ReadonlyArray; -} - -interface DockerfilePin { - readonly alias: string; - readonly service: string; - readonly version: string; + readonly skipped: ReadonlyArray; } /** `definition("", "", ""`. */ @@ -90,11 +73,11 @@ function desiredImage( service: string, version: string, currentImage: string, - digest: string | undefined, + digest: string, ): string { const tagged = `${SLIM_IMAGE_PREFIX}${service}:${version}`; if (!imageHasDigest(currentImage)) return tagged; - if (digest === undefined || !DIGEST_PATTERN.test(digest)) { + if (!DIGEST_PATTERN.test(digest)) { throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); } return `${tagged}@${digest}`; @@ -131,35 +114,14 @@ function selectEntry(source: string, service: string, version: string): Selected return { kind: "additional", version: sameLine.version, image: sameLine.image }; } -function dockerfilePins(dockerfile: string): ReadonlyArray { - const pins: DockerfilePin[] = []; - for (const from of parseDockerfileServiceImages(dockerfile)) { - const pin = slimCatalogPin(from.alias, from.image); - if (pin === undefined) continue; - if (!VERSION_PATTERN.test(pin.version)) { - throw new InvalidPayloadError(`invalid version for ${from.alias}: '${pin.version}'`); - } - pins.push({ alias: from.alias, service: pin.service, version: pin.version }); - } - return pins; -} - -function rejectUnmodelled( - alias: string, - service: string, - version: string, - entry: SelectedEntry, -): void { +function skipReason(alias: string, version: string, entry: SelectedEntry): string | undefined { if (entry.kind === "unmodelled-service") { - throw new InvalidPayloadError( - `${CATALOG_PATH} has no ${service} entry for the ${alias} image.`, - ); + return `${CATALOG_PATH} has no slim entry for ${alias}.`; } if (entry.kind === "unmodelled-release-line") { - throw new InvalidPayloadError( - `${service} ${version} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`, - ); + return `${alias} ${version} is not on a release line ${CATALOG_PATH} carries (${entry.known.join(", ")}).`; } + return undefined; } /** Slim tags whose catalog entry stores a digest, so the rewrite can resolve them first. */ @@ -168,9 +130,10 @@ export function catalogDigestPins( catalog: string, ): ReadonlyArray<{ readonly service: string; readonly version: string }> { const pins: Array<{ service: string; version: string }> = []; - for (const pin of dockerfilePins(dockerfile)) { + for (const from of parseDockerfileServiceImages(dockerfile)) { + const pin = slimCatalogPin(from.alias, from.image); + if (pin === undefined || !VERSION_PATTERN.test(pin.version)) continue; const entry = selectEntry(catalog, pin.service, pin.version); - rejectUnmodelled(pin.alias, pin.service, pin.version, entry); if (entry.kind !== "default" && entry.kind !== "additional") continue; if (imageHasDigest(entry.image)) pins.push({ service: pin.service, version: pin.version }); } @@ -196,7 +159,10 @@ function pinService( if (entry.kind !== "default" && entry.kind !== "additional") { throw new InvalidPayloadError(`${service} ${version} is not a catalog entry.`); } - const desired = desiredImage(service, version, entry.image, digest); + if (imageHasDigest(entry.image) && (digest === undefined || !DIGEST_PATTERN.test(digest))) { + throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); + } + const desired = desiredImage(service, version, entry.image, digest ?? ""); if (entry.version === version && entry.image === desired) return { kind: "unchanged" }; if (entry.kind === "default") { @@ -224,25 +190,50 @@ function pinService( } /** - * Rewrites `catalog` so every slim Dockerfile pin matches. Aliases with no slim - * build are skipped. A modelled service on an unknown release line throws: - * the docker.io tag and the catalog would otherwise diverge. + * Rewrites `catalog` from the service-image Dockerfile. OrioleDB tags and other + * images with no slim build are left unchanged. A pin that cannot be applied + * is reported in `skipped` and does not stop the remaining pins. */ export function planArtifactCatalogUpdate(input: { readonly dockerfile: string; readonly catalog: string; - readonly digestFor: (service: string, version: string) => string; + readonly digestFor: (service: string, version: string) => string | undefined; }): CatalogPlan { let source = input.catalog; const updates: CatalogPinUpdate[] = []; + const skipped: SkippedCatalogPin[] = []; + + for (const from of parseDockerfileServiceImages(input.dockerfile)) { + if (isOrioleImage(from.image)) { + skipped.push({ + alias: from.alias, + reason: `${from.alias} ${from.image} has no slim image.`, + }); + continue; + } + const pin = slimCatalogPin(from.alias, from.image); + if (pin === undefined) continue; + if (!VERSION_PATTERN.test(pin.version)) { + throw new InvalidPayloadError(`invalid version for ${from.alias}: '${pin.version}'`); + } - for (const pin of dockerfilePins(input.dockerfile)) { const entry = selectEntry(source, pin.service, pin.version); - rejectUnmodelled(pin.alias, pin.service, pin.version, entry); + const reason = skipReason(from.alias, pin.version, entry); + if (reason !== undefined) { + skipped.push({ alias: from.alias, reason }); + continue; + } if (entry.kind !== "default" && entry.kind !== "additional") continue; const digest = imageHasDigest(entry.image) ? input.digestFor(pin.service, pin.version) : undefined; + if (imageHasDigest(entry.image) && digest === undefined) { + skipped.push({ + alias: from.alias, + reason: `${pin.service}:${pin.version} has no published slim manifest.`, + }); + continue; + } const result = pinService(source, pin.service, pin.version, digest); if (result.kind === "unchanged") continue; source = result.source; @@ -254,10 +245,10 @@ export function planArtifactCatalogUpdate(input: { }); } - return { source, updates }; + return { source, updates, skipped }; } -async function publishedDigest(service: string, version: string): Promise { +async function publishedDigest(service: string, version: string): Promise { const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; const proc = Bun.spawn(["regctl", "manifest", "head", reference], { stdout: "pipe", @@ -271,9 +262,10 @@ async function publishedDigest(service: string, version: string): Promise): Promise { const digests = new Map(); for (const pin of catalogDigestPins(dockerfile, catalog)) { const key = `${pin.service}:${pin.version}`; - if (!digests.has(key)) digests.set(key, await publishedDigest(pin.service, pin.version)); + if (!digests.has(key)) { + const digest = await publishedDigest(pin.service, pin.version); + if (digest !== undefined) digests.set(key, digest); + } } const plan = planArtifactCatalogUpdate({ dockerfile, catalog, - digestFor: (service, version) => { - const digest = digests.get(`${service}:${version}`); - if (digest === undefined) { - throw new InvalidPayloadError(`missing slim digest for ${service}:${version}`); - } - return digest; - }, + digestFor: (service, version) => digests.get(`${service}:${version}`), }); await Bun.write(catalogPath, plan.source); + for (const skip of plan.skipped) { + console.log(`::warning ::Left ${skip.alias} unchanged: ${skip.reason}`); + } if (plan.updates.length === 0) { console.log("Workload catalog already matches the Dockerfile."); return; diff --git a/.github/workflows/sync-artifacts-catalog.yml b/.github/workflows/sync-artifacts-catalog.yml index f6d4abb362..68d68db111 100644 --- a/.github/workflows/sync-artifacts-catalog.yml +++ b/.github/workflows/sync-artifacts-catalog.yml @@ -1,10 +1,11 @@ name: Sync Artifacts catalog -# Dependabot bumps apps/cli/src/shared/services/Dockerfile. This commits the -# matching slim pins into packages/stack/src/Artifacts.ts on that same branch, -# so the docker.io tag, the slim image, and the native archive stay on one -# version. Native URLs are derived from service + version; a digest already -# stored on the entry is replaced with the published slim manifest digest. +# Dependabot bumps apps/cli/src/shared/services/Dockerfile, the service-image +# manifest. This commits the matching slim pins into packages/stack/src/Artifacts.ts +# on that same branch. OrioleDB tags and any pin with no published slim image +# stay unchanged and do not block the other pins. Native URLs are derived from +# service + version; a digest already stored on the entry is replaced with the +# published slim manifest digest. # # The rewrite runs from the base revision. The pull request only supplies the # Dockerfile and catalog text, and the job runs for Dependabot so a pull diff --git a/apps/cli/src/shared/services/dockerfile-images.ts b/apps/cli/src/shared/services/dockerfile-images.ts index 4caf635140..70ca17a9b2 100644 --- a/apps/cli/src/shared/services/dockerfile-images.ts +++ b/apps/cli/src/shared/services/dockerfile-images.ts @@ -1,33 +1,11 @@ import serviceImagesDockerfile from "./Dockerfile" with { type: "text" }; +import { + parseDockerfileServiceImages, + type DockerfileImageSpec, +} from "./parse-dockerfile-service-images.ts"; import { slimImageForAlias } from "./slim-images.ts"; -export interface DockerfileImageSpec { - readonly alias: string; - readonly image: string; -} - -const FROM_LINE_PATTERN = /^FROM\s+(.+):([^:\s]+)\s+AS\s+([^\s#]+)/i; - -export function parseDockerfileServiceImages( - dockerfile: string, -): ReadonlyArray { - return dockerfile - .split("\n") - .map((line) => line.trim()) - .flatMap((line) => { - const match = FROM_LINE_PATTERN.exec(line); - if (match === null) { - return []; - } - - const [, repository, tag, alias] = match; - if (repository === undefined || tag === undefined || alias === undefined) { - return []; - } - - return [{ alias, image: `${repository}:${tag}` }]; - }); -} +export { parseDockerfileServiceImages, type DockerfileImageSpec }; export const dockerfileServiceImages = parseDockerfileServiceImages(serviceImagesDockerfile); diff --git a/apps/cli/src/shared/services/parse-dockerfile-service-images.ts b/apps/cli/src/shared/services/parse-dockerfile-service-images.ts new file mode 100644 index 0000000000..f2560d5822 --- /dev/null +++ b/apps/cli/src/shared/services/parse-dockerfile-service-images.ts @@ -0,0 +1,28 @@ +export interface DockerfileImageSpec { + readonly alias: string; + readonly image: string; +} + +const FROM_LINE_PATTERN = /^FROM\s+(.+):([^:\s]+)\s+AS\s+([^\s#]+)/i; + +/** Reads `FROM : AS ` lines from the service-image manifest. */ +export function parseDockerfileServiceImages( + dockerfile: string, +): ReadonlyArray { + return dockerfile + .split("\n") + .map((line) => line.trim()) + .flatMap((line) => { + const match = FROM_LINE_PATTERN.exec(line); + if (match === null) { + return []; + } + + const [, repository, tag, alias] = match; + if (repository === undefined || tag === undefined || alias === undefined) { + return []; + } + + return [{ alias, image: `${repository}:${tag}` }]; + }); +} diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index cd34e92949..57c658325a 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -4,7 +4,7 @@ const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; /** * Maps embedded-Dockerfile aliases onto the slim service catalog. Aliases with * no slim build (kong, the `differ`/`migra`/`pgprove` job images) are absent and - * keep their docker.io reference. + * keep their docker.io reference. OrioleDB tags are excluded in `slimCatalogPin`. */ const SLIM_SERVICE_BY_ALIAS = { pg: "postgres", @@ -65,11 +65,21 @@ export interface SlimCatalogPin { readonly version: string; } +/** OrioleDB tags are docker.io-only; slim-services does not publish them. */ +export function isOrioleImage(image: string): boolean { + const tagSeparator = image.lastIndexOf(":"); + const tag = tagSeparator === -1 ? image : image.slice(tagSeparator + 1); + return tag.toLowerCase().includes("orioledb"); +} + /** * Slim service and tag for a Dockerfile alias. Absent when that alias has no - * slim build (kong and the one-shot job images). + * slim build (kong, the one-shot job images, and OrioleDB tags). */ export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | undefined { + if (isOrioleImage(image)) { + return undefined; + } const service = SLIM_SERVICE_LOOKUP[alias]; if (service === undefined) { return undefined; diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index 2f1e8de860..4e082a3b7c 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; import { pinMatchesCurrentImage, + slimCatalogPin, slimImageForAlias, slimImageForCurrentPin, slimImagesEnabled, @@ -70,6 +71,16 @@ describe("toSlimImage", () => { ); }); + it("keeps OrioleDB tags on docker.io", () => { + expect(toSlimImage("pg", "supabase/postgres:16.0.0.1-orioledb")).toBe( + "supabase/postgres:16.0.0.1-orioledb", + ); + expect(toSlimImage("pg", "supabase/postgres:orioledb-15.1.0.55")).toBe( + "supabase/postgres:orioledb-15.1.0.55", + ); + expect(slimCatalogPin("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); + }); + it("strips vector's docker.io -alpine variant suffix", () => { expect(toSlimImage("vector", "timberio/vector:0.53.0-alpine")).toBe( "ghcr.io/supabase/cli/vector:0.53.0", From 9812ba061bb52509b213821feac9b5909a4440bd Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 17:22:29 +0200 Subject: [PATCH 5/6] ci(repo): pin only published Dockerfile tag changes A Dependabot bump was rewriting every slim pin, including a PostgreSQL downgrade, and the catalog push did not trigger CI. --- .../scripts/sync-artifacts-catalog.test.ts | 203 ++++++------------ .github/scripts/sync-artifacts-catalog.ts | 183 +++++++++++----- .github/workflows/github-scripts-ci.yml | 2 + .github/workflows/sync-artifacts-catalog.yml | 40 ++-- apps/cli/src/shared/services/slim-images.ts | 10 +- docs/adr/0026-slim-artifact-mirrors.md | 2 +- 6 files changed, 234 insertions(+), 206 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index eb4632d8ad..809dce551f 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -3,183 +3,115 @@ import { describe, expect, test } from "bun:test"; import { InvalidPayloadError } from "./slim-mirror-payload.ts"; import { CATALOG_PATH, - catalogDigestPins, planArtifactCatalogUpdate, - releaseLine, + type ReleasePublication, } from "./sync-artifacts-catalog.ts"; -const DIGEST_A = `sha256:${"a".repeat(64)}`; const DIGEST_B = `sha256:${"b".repeat(64)}`; +const POSTGRES_17 = `ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}`; +const POSTGRES_15 = `ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}`; const fixture = `const workloadCatalog = { database: definition( "postgres", "17.6.1.168", - "ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}", + "${POSTGRES_17}", "bin/supabase-postgres-start", ["bin/supabase-postgres-start"], - { - "15.14.1.168": - "ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}", - }, + { "15.14.1.168": "${POSTGRES_15}" }, ), rest: definition("postgrest", "v16.2", "ghcr.io/supabase/cli/postgrest:v16.2", "bin/postgrest"), - auth: definition("auth", "v2.196.0", "ghcr.io/supabase/cli/auth:v2.196.0", "bin/auth"), analytics: definition( "analytics", "v1.50.9", - "ghcr.io/supabase/cli/analytics:v1.50.9@${DIGEST_A}", + "ghcr.io/supabase/cli/analytics:v1.50.9@sha256:${"a".repeat(64)}", "bin/logflare", ), - vector: definition("vector", "0.49.0", "ghcr.io/supabase/cli/vector:0.49.0", "bin/vector"), - pooler: definition("pooler", "v2.9.10", "ghcr.io/supabase/cli/pooler:v2.9.10", "bin/server"), }; `; -const dockerfile = `FROM supabase/postgres:17.6.1.171 AS pg -FROM library/kong:2.8.1 AS kong -FROM postgrest/postgrest:v16.3 AS postgrest -FROM timberio/vector:0.53.0-alpine AS vector -FROM supabase/supavisor:2.9.13 AS supavisor -FROM supabase/logflare:1.50.12 AS logflare -`; - -function digests( - overrides: Readonly> = {}, -): (service: string, version: string) => string { - return (service, version) => { - const digest = overrides[`${service}:${version}`]; - if (digest === undefined) throw new Error(`unexpected digest lookup ${service}:${version}`); - return digest; - }; -} - -describe("releaseLine", () => { - test.each([ - ["17.6.1.168", "17"], - ["15.14.1.168", "15"], - ["v2.196.0", "2"], - ["2026.09.04-sha-5a67366", "2026"], - ])("reads %s as line %s", (version, expected) => { - expect(releaseLine(version)).toBe(expected); - }); -}); +const published = (digest?: string): ReleasePublication => ({ status: "published", digest }); describe("planArtifactCatalogUpdate", () => { - test("pins each slim Dockerfile image and skips aliases with no slim build", () => { - const plan = planArtifactCatalogUpdate({ - dockerfile, + test("pins Dockerfile tags that changed and leaves the other postgres line", async () => { + const plan = await planArtifactCatalogUpdate({ + baseDockerfile: `FROM supabase/postgres:17.6.1.168 AS pg +FROM postgrest/postgrest:v16.2 AS postgrest +FROM supabase/logflare:1.50.9 AS logflare +`, + dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg +FROM library/kong:2.8.1 AS kong +FROM postgrest/postgrest:v16.3 AS postgrest +FROM supabase/logflare:1.50.9 AS logflare +`, catalog: fixture, - digestFor: digests({ "postgres:17.6.1.171": DIGEST_B, "analytics:v1.50.12": DIGEST_B }), + publication: async (service) => published(service === "postgres" ? DIGEST_B : undefined), }); - expect( - plan.updates.map( - (update) => `${update.service} ${update.previousVersion} -> ${update.version}`, - ), - ).toEqual([ - "postgres 17.6.1.168 -> 17.6.1.171", - "postgrest v16.2 -> v16.3", - "vector 0.49.0 -> 0.53.0", - "pooler v2.9.10 -> v2.9.13", - "analytics v1.50.9 -> v1.50.12", + expect(plan.updates.map((update) => `${update.service} ${update.version}`)).toEqual([ + "postgres 17.6.1.171", + "postgrest v16.3", ]); expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:17.6.1.171@${DIGEST_B}"`); - expect(plan.source).toContain( - `"ghcr.io/supabase/cli/postgres:15.14.1.168@sha256:${"f".repeat(64)}"`, - ); + expect(plan.source).toContain(`"${POSTGRES_15}"`); expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3"`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/vector:0.53.0"`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/pooler:v2.9.13"`); - expect(plan.source).not.toContain("kong"); - }); - - test("moves a postgres pin onto the matching additional release line", () => { - const plan = planArtifactCatalogUpdate({ - dockerfile: "FROM supabase/postgres:15.14.1.175 AS pg\n", - catalog: fixture, - digestFor: digests({ "postgres:15.14.1.175": DIGEST_B }), - }); - - expect(plan.updates).toEqual([ - { - service: "postgres", - version: "15.14.1.175", - previousVersion: "15.14.1.168", - target: "additional", - }, - ]); - expect(plan.source).toContain(`"17.6.1.168"`); - expect(plan.source).toContain(`"15.14.1.175"`); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgres:15.14.1.175@${DIGEST_B}"`); + expect(plan.source).toContain(`"v1.50.9"`); + expect(plan.skipped).toEqual([]); }); - test("does not look up a digest when the pin already matches", () => { - const plan = planArtifactCatalogUpdate({ - dockerfile: "FROM postgrest/postgrest:v16.2 AS postgrest\n", + test("a missing release blocks the commit and OrioleDB does not", async () => { + const movesPostgres = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM supabase/postgres:17.6.1.168 AS pg\n", + dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg +FROM postgrest/postgrest:v16.3 AS postgrest +`, catalog: fixture, - digestFor: () => { - throw new Error("digest lookup"); - }, + publication: async () => ({ status: "missing" }), }); + expect(movesPostgres.updates).toEqual([]); + expect(movesPostgres.source).toBe(fixture); + expect(movesPostgres.skipped.every((skip) => skip.blocking)).toBe(true); - expect(plan).toEqual({ source: fixture, updates: [], skipped: [] }); - }); - - test("leaves an OrioleDB tag on docker.io and still pins the other images", () => { - const plan = planArtifactCatalogUpdate({ + const oriole = await planArtifactCatalogUpdate({ + baseDockerfile: + "FROM supabase/postgres:17.6.1.168 AS pg\nFROM postgrest/postgrest:v16.2 AS postgrest\n", dockerfile: `FROM supabase/postgres:16.0.0.1-orioledb AS pg FROM postgrest/postgrest:v16.3 AS postgrest -FROM supabase/postgres:orioledb-15.1.0.55 AS orioledb `, catalog: fixture, - digestFor: digests(), + publication: async () => published(), }); - - expect(plan.updates.map((update) => update.service)).toEqual(["postgrest"]); - expect(plan.source).toContain( - `"ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:${"9".repeat(64)}"`, - ); - expect(plan.source).toContain(`"ghcr.io/supabase/cli/postgrest:v16.3"`); - expect(plan.skipped.map((skip) => skip.alias)).toEqual(["pg", "orioledb"]); + expect(oriole.updates.map((update) => update.service)).toEqual(["postgrest"]); + expect(oriole.skipped).toEqual([ + { + alias: "pg", + reason: "pg supabase/postgres:16.0.0.1-orioledb has no slim image.", + blocking: false, + }, + ]); + expect(oriole.source).toContain(`"${POSTGRES_17}"`); }); - test("skips a pin with no slim manifest and still applies the next one", () => { - const plan = planArtifactCatalogUpdate({ - dockerfile: `FROM supabase/postgres:17.6.1.171 AS pg -FROM postgrest/postgrest:v16.3 AS postgrest -`, - catalog: fixture, - digestFor: (service) => (service === "postgres" ? undefined : DIGEST_B), + test("refuses a backward pin and a tag that would escape the catalog string", async () => { + const backward = await planArtifactCatalogUpdate({ + baseDockerfile: "FROM supabase/postgres:17.6.1.170 AS pg\n", + dockerfile: "FROM supabase/postgres:17.6.1.171 AS pg\n", + catalog: fixture.replaceAll("17.6.1.168", "17.6.1.173"), + publication: async () => { + throw new Error("publication lookup"); + }, }); + expect(backward.updates).toEqual([]); + expect(backward.skipped.map((skip) => skip.blocking)).toEqual([true]); + expect(backward.source).toContain(`"17.6.1.173"`); - expect(plan.updates.map((update) => update.service)).toEqual(["postgrest"]); - expect(plan.source).toContain(`"17.6.1.168"`); - expect(plan.skipped.map((skip) => skip.alias)).toEqual(["pg"]); - }); - - test("refuses a Dockerfile tag that would escape the catalog string", () => { - expect(() => + await expect( planArtifactCatalogUpdate({ dockerfile: 'FROM supabase/gotrue:v1" AS gotrue\n', catalog: fixture, - digestFor: digests(), + publication: async () => published(), }), - ).toThrow(InvalidPayloadError); - }); - - test("skips a release line the catalog does not carry and still pins the rest", () => { - const plan = planArtifactCatalogUpdate({ - dockerfile: `FROM supabase/postgres:16.1.1.1 AS pg -FROM postgrest/postgrest:v16.3 AS postgrest -`, - catalog: fixture, - digestFor: digests(), - }); - - expect(plan.updates.map((update) => update.service)).toEqual(["postgrest"]); - expect(plan.skipped.map((skip) => skip.alias)).toEqual(["pg"]); - expect(plan.source).toContain(`"17.6.1.168"`); + ).rejects.toThrow(InvalidPayloadError); }); }); @@ -187,13 +119,14 @@ describe("against the real catalog", () => { test("every slim Dockerfile alias is a catalog entry", async () => { const dockerfile = await Bun.file("apps/cli/src/shared/services/Dockerfile").text(); const catalog = await Bun.file(CATALOG_PATH).text(); - - expect(() => catalogDigestPins(dockerfile, catalog)).not.toThrow(); - const plan = planArtifactCatalogUpdate({ + const plan = await planArtifactCatalogUpdate({ dockerfile, catalog, - digestFor: () => DIGEST_A, + publication: async () => published(`sha256:${"a".repeat(64)}`), }); - expect(plan.source).toContain('definition(\n "postgres"'); + + expect( + plan.skipped.filter((skip) => skip.blocking && !skip.reason.includes("older than")), + ).toEqual([]); }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index bf0f0448ff..838f2c5b75 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -4,7 +4,7 @@ * from service + version. An entry that already carries a digest keeps one: * the published `ghcr.io/supabase/cli/:` manifest digest. * - * Run: `bun .github/scripts/sync-artifacts-catalog.ts ` + * Run: `bun .github/scripts/sync-artifacts-catalog.ts [base-dockerfile]` */ import { parseDockerfileServiceImages } from "../../apps/cli/src/shared/services/parse-dockerfile-service-images.ts"; @@ -19,16 +19,36 @@ import { export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; const DOCKERFILE_PATH = "apps/cli/src/shared/services/Dockerfile"; +const NATIVE_RELEASES = "https://api.github.com/repos/supabase/slim-services/releases/tags"; const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; /** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ -export function releaseLine(version: string): string { +function releaseLine(version: string): string { const withoutPrefix = version.replace(/^[vV]/, ""); const separator = withoutPrefix.indexOf("."); return separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); } +/** True when every numeric prefix of `next` is older than `current`. Equal prefixes are not older. */ +function isOlderRelease(next: string, current: string): boolean { + const nextParts = next.replace(/^[vV]/, "").split("."); + const currentParts = current.replace(/^[vV]/, "").split("."); + const count = Math.max(nextParts.length, currentParts.length); + for (let index = 0; index < count; index++) { + const nextValue = leadingInteger(nextParts[index] ?? "0"); + const currentValue = leadingInteger(currentParts[index] ?? "0"); + if (nextValue === undefined || currentValue === undefined) return false; + if (nextValue !== currentValue) return nextValue < currentValue; + } + return false; +} + +function leadingInteger(part: string): number | undefined { + const match = /^(\d+)/.exec(part); + return match === null ? undefined : Number(match[1]); +} + export interface CatalogPinUpdate { readonly service: string; readonly version: string; @@ -39,6 +59,8 @@ export interface CatalogPinUpdate { export interface SkippedCatalogPin { readonly alias: string; readonly reason: string; + /** OrioleDB has no slim image and must not fail the other pins. */ + readonly blocking: boolean; } export interface CatalogPlan { @@ -47,6 +69,10 @@ export interface CatalogPlan { readonly skipped: ReadonlyArray; } +export type ReleasePublication = + | { readonly status: "published"; readonly digest?: string } + | { readonly status: "missing" | "lookup-failed" }; + /** `definition("", "", ""`. */ function defaultEntryPattern(service: string): RegExp { const s = escapeRegExp(service); @@ -124,20 +150,13 @@ function skipReason(alias: string, version: string, entry: SelectedEntry): strin return undefined; } -/** Slim tags whose catalog entry stores a digest, so the rewrite can resolve them first. */ -export function catalogDigestPins( - dockerfile: string, - catalog: string, -): ReadonlyArray<{ readonly service: string; readonly version: string }> { - const pins: Array<{ service: string; version: string }> = []; +function slimVersions(dockerfile: string): ReadonlyMap { + const versions = new Map(); for (const from of parseDockerfileServiceImages(dockerfile)) { const pin = slimCatalogPin(from.alias, from.image); - if (pin === undefined || !VERSION_PATTERN.test(pin.version)) continue; - const entry = selectEntry(catalog, pin.service, pin.version); - if (entry.kind !== "default" && entry.kind !== "additional") continue; - if (imageHasDigest(entry.image)) pins.push({ service: pin.service, version: pin.version }); + if (pin !== undefined) versions.set(from.alias, pin.version); } - return pins; + return versions; } type PinResult = @@ -190,29 +209,36 @@ function pinService( } /** - * Rewrites `catalog` from the service-image Dockerfile. OrioleDB tags and other - * images with no slim build are left unchanged. A pin that cannot be applied - * is reported in `skipped` and does not stop the remaining pins. + * Rewrites `catalog` from Dockerfile tags that differ from `baseDockerfile`. + * With no base, every slim tag is in scope. A pin that cannot be applied is + * reported in `skipped`. OrioleDB is the only non-blocking skip. */ -export function planArtifactCatalogUpdate(input: { +export async function planArtifactCatalogUpdate(input: { readonly dockerfile: string; + readonly baseDockerfile?: string; readonly catalog: string; - readonly digestFor: (service: string, version: string) => string | undefined; -}): CatalogPlan { + readonly publication: (service: string, version: string) => Promise; +}): Promise { let source = input.catalog; const updates: CatalogPinUpdate[] = []; const skipped: SkippedCatalogPin[] = []; + const baseVersions = + input.baseDockerfile === undefined ? undefined : slimVersions(input.baseDockerfile); + const changed = (alias: string, version: string | undefined): boolean => + baseVersions === undefined || baseVersions.get(alias) !== version; for (const from of parseDockerfileServiceImages(input.dockerfile)) { if (isOrioleImage(from.image)) { + if (!changed(from.alias, undefined)) continue; skipped.push({ alias: from.alias, reason: `${from.alias} ${from.image} has no slim image.`, + blocking: false, }); continue; } const pin = slimCatalogPin(from.alias, from.image); - if (pin === undefined) continue; + if (pin === undefined || !changed(from.alias, pin.version)) continue; if (!VERSION_PATTERN.test(pin.version)) { throw new InvalidPayloadError(`invalid version for ${from.alias}: '${pin.version}'`); } @@ -220,17 +246,38 @@ export function planArtifactCatalogUpdate(input: { const entry = selectEntry(source, pin.service, pin.version); const reason = skipReason(from.alias, pin.version, entry); if (reason !== undefined) { - skipped.push({ alias: from.alias, reason }); + skipped.push({ alias: from.alias, reason, blocking: true }); continue; } if (entry.kind !== "default" && entry.kind !== "additional") continue; - const digest = imageHasDigest(entry.image) - ? input.digestFor(pin.service, pin.version) - : undefined; - if (imageHasDigest(entry.image) && digest === undefined) { + if (isOlderRelease(pin.version, entry.version)) { + skipped.push({ + alias: from.alias, + reason: `${pin.service} ${pin.version} is older than the catalog pin ${entry.version}.`, + blocking: true, + }); + continue; + } + if (entry.version === pin.version && !imageHasDigest(entry.image)) continue; + + const release = await input.publication(pin.service, pin.version); + if (release.status !== "published") { + skipped.push({ + alias: from.alias, + reason: + release.status === "missing" + ? `${pin.service}:${pin.version} has no published slim image and native release.` + : `${pin.service}:${pin.version} publication check failed.`, + blocking: true, + }); + continue; + } + const digest = imageHasDigest(entry.image) ? release.digest : undefined; + if (imageHasDigest(entry.image) && (digest === undefined || !DIGEST_PATTERN.test(digest))) { skipped.push({ alias: from.alias, reason: `${pin.service}:${pin.version} has no published slim manifest.`, + blocking: true, }); continue; } @@ -248,8 +295,9 @@ export function planArtifactCatalogUpdate(input: { return { source, updates, skipped }; } -async function publishedDigest(service: string, version: string): Promise { - const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; +type Probe = "published" | "missing" | "lookup-failed"; + +async function probeManifest(reference: string): Promise<{ probe: Probe; digest?: string }> { const proc = Bun.spawn(["regctl", "manifest", "head", reference], { stdout: "pipe", stderr: "pipe", @@ -259,39 +307,78 @@ async function publishedDigest(service: string, version: string): Promise { + const tag = `${service}-${version}`; + const headers: Record = { + Accept: "application/vnd.github+json", + "User-Agent": "supabase-cli-catalog-sync", + }; + const token = process.env.GITHUB_TOKEN; + if (token !== undefined && token !== "") headers.Authorization = `Bearer ${token}`; + try { + const response = await fetch(`${NATIVE_RELEASES}/${encodeURIComponent(tag)}`, { headers }); + if (response.status === 200) return "published"; + if (response.status === 404) return "missing"; + console.log(`::warning ::${tag} native release check returned HTTP ${response.status}.`); + return "lookup-failed"; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.log(`::warning ::${tag} native release check failed: ${message}`); + return "lookup-failed"; } - return live; +} + +async function lookupPublication(service: string, version: string): Promise { + const reference = `${SLIM_IMAGE_PREFIX}${service}:${version}`; + const [manifest, native] = await Promise.all([ + probeManifest(reference), + probeNativeRelease(service, version), + ]); + if (manifest.probe === "lookup-failed" || native === "lookup-failed") { + return { status: "lookup-failed" }; + } + if (manifest.probe === "missing" || native === "missing") return { status: "missing" }; + return { status: "published", digest: manifest.digest }; } async function main(argv: ReadonlyArray): Promise { - const [dockerfilePath = DOCKERFILE_PATH, catalogPath = CATALOG_PATH] = argv; + const [dockerfilePath = DOCKERFILE_PATH, catalogPath = CATALOG_PATH, baseDockerfilePath] = argv; const dockerfile = await Bun.file(dockerfilePath).text(); const catalog = await Bun.file(catalogPath).text(); - const digests = new Map(); - for (const pin of catalogDigestPins(dockerfile, catalog)) { - const key = `${pin.service}:${pin.version}`; - if (!digests.has(key)) { - const digest = await publishedDigest(pin.service, pin.version); - if (digest !== undefined) digests.set(key, digest); - } - } - - const plan = planArtifactCatalogUpdate({ + const baseDockerfile = + baseDockerfilePath === undefined ? undefined : await Bun.file(baseDockerfilePath).text(); + const plan = await planArtifactCatalogUpdate({ dockerfile, + baseDockerfile, catalog, - digestFor: (service, version) => digests.get(`${service}:${version}`), + publication: lookupPublication, }); - await Bun.write(catalogPath, plan.source); for (const skip of plan.skipped) { console.log(`::warning ::Left ${skip.alias} unchanged: ${skip.reason}`); } + if (plan.skipped.some((skip) => skip.blocking)) { + console.log("::error ::Refusing to commit a partial catalog update."); + process.exit(1); + } + await Bun.write(catalogPath, plan.source); if (plan.updates.length === 0) { console.log("Workload catalog already matches the Dockerfile."); return; diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index b9a8ec8d3a..c2f12fa74b 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -10,6 +10,8 @@ on: paths: - ".github/scripts/**" - "apps/cli/src/shared/services/Dockerfile" + - "apps/cli/src/shared/services/parse-dockerfile-service-images.ts" + - "apps/cli/src/shared/services/slim-images.ts" - "packages/stack/src/Artifacts.ts" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" diff --git a/.github/workflows/sync-artifacts-catalog.yml b/.github/workflows/sync-artifacts-catalog.yml index 68d68db111..d57b5d7b28 100644 --- a/.github/workflows/sync-artifacts-catalog.yml +++ b/.github/workflows/sync-artifacts-catalog.yml @@ -1,17 +1,10 @@ name: Sync Artifacts catalog -# Dependabot bumps apps/cli/src/shared/services/Dockerfile, the service-image -# manifest. This commits the matching slim pins into packages/stack/src/Artifacts.ts -# on that same branch. OrioleDB tags and any pin with no published slim image -# stay unchanged and do not block the other pins. Native URLs are derived from -# service + version; a digest already stored on the entry is replaced with the -# published slim manifest digest. -# -# The rewrite runs from the base revision. The pull request only supplies the -# Dockerfile and catalog text, and the job runs for Dependabot so a pull -# request cannot edit this workflow and then execute it with a write token. -# Dependabot's GITHUB_TOKEN is read-only unless this job raises contents. - +# The script is the base revision. The pull request only supplies the Dockerfile +# and catalog text, and the job runs for Dependabot so that pull request cannot +# edit this workflow and then execute it with a write token. The catalog commit +# is pushed with the app token so the new head triggers CI. Dependabot's +# GITHUB_TOKEN would not. on: pull_request: paths: @@ -31,7 +24,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 permissions: - contents: write + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -39,8 +32,8 @@ jobs: persist-credentials: true fetch-depth: 1 - # Bun only. The script imports no workspace packages, and Dependabot - # runs cannot read Actions secrets the full setup action would use. + # Bun only. The script imports no workspace packages. The firewall token + # is empty on a Dependabot run; setup is not required to format one file. - name: Install toolchains uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 with: @@ -57,11 +50,20 @@ jobs: echo "${RUNNER_TEMP}/regctl-bin" >> "$GITHUB_PATH" "${RUNNER_TEMP}/regctl-bin/regctl" version + - name: Generate token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.GH_APP_CLIENT_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + permission-contents: write + - name: Pin the catalog to the Dockerfile env: HEAD_SHA: ${{ github.event.pull_request.head.sha }} HEAD_REF: ${{ github.event.pull_request.head.ref }} PR_NUMBER: ${{ github.event.pull_request.number }} + APP_TOKEN: ${{ steps.app-token.outputs.token }} run: | set -euo pipefail git fetch --no-tags origin "pull/${PR_NUMBER}/head" @@ -76,15 +78,21 @@ jobs: git show "$HEAD_SHA:apps/cli/src/shared/services/Dockerfile" > "$dockerfile" git show "$HEAD_SHA:packages/stack/src/Artifacts.ts" > "$catalog" cp "$catalog" "$original" - bun .github/scripts/sync-artifacts-catalog.ts "$dockerfile" "$catalog" + bun .github/scripts/sync-artifacts-catalog.ts "$dockerfile" "$catalog" apps/cli/src/shared/services/Dockerfile if cmp -s "$original" "$catalog"; then echo "Workload catalog already matches the Dockerfile." exit 0 fi git checkout --detach "$HEAD_SHA" cp "$catalog" packages/stack/src/Artifacts.ts + bun x oxfmt@0.65.0 --config .oxfmtrc.json packages/stack/src/Artifacts.ts git add -- packages/stack/src/Artifacts.ts + if git diff --cached --quiet -- packages/stack/src/Artifacts.ts; then + echo "Workload catalog already matches the Dockerfile." + exit 0 + fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git commit -m "chore(stack): pin the workload catalog to the Dockerfile image tags" + git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" git push origin "HEAD:refs/heads/${HEAD_REF}" diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 57c658325a..af0635fc11 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -67,9 +67,8 @@ export interface SlimCatalogPin { /** OrioleDB tags are docker.io-only; slim-services does not publish them. */ export function isOrioleImage(image: string): boolean { - const tagSeparator = image.lastIndexOf(":"); - const tag = tagSeparator === -1 ? image : image.slice(tagSeparator + 1); - return tag.toLowerCase().includes("orioledb"); + const tag = imageTag(image); + return tag !== undefined && tag.toLowerCase().includes("orioledb"); } /** @@ -85,12 +84,11 @@ export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | u return undefined; } - const tagSeparator = image.lastIndexOf(":"); - if (tagSeparator === -1) { + const rawTag = imageTag(image); + if (rawTag === undefined) { return undefined; } - const rawTag = image.slice(tagSeparator + 1); const tag = alias === "vector" ? rawTag.replace(/-alpine$/, "") : rawTag; return { service, version: slimTagForService(service, tag) }; } diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 9b02b0dc2a..004e40294e 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -13,7 +13,7 @@ Some consumers cannot reach GitHub release assets or a single registry blob CDN. Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), digest-preserving, via `mirror-slim-image.yml` and `.github/scripts/mirror-slim-image.ts`. Publish each slim **native** archive as an OCI artifact on the same two repositories under `:version-native-` (not `:version-linux-*`, which are image platform tags). GitHub Releases remain the human HTTPS copy and `--clobber` on force. -`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`. A Dependabot Dockerfile bump commits that catalog pin onto the same pull request, so the docker.io tag, the slim image, and the native archive stay on one version. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that pull request merges. Natives follow the moved tag immediately. +`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`. A Dependabot Dockerfile bump commits that catalog pin onto the same pull request, so the docker.io tag, the slim image, and the native archive stay on one version. A PostgreSQL release line that is not the Dockerfile `pg` tag, and a same-version digest republish, stay put until that Dockerfile line changes. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that pull request merges. Natives follow the moved tag immediately. ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. Do not add a public S3 bucket or a second cloud vendor for this cut. From 355699d1a0a7f87778d14868dcf969fe5e6771a8 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 23 Sep 2026 17:31:13 +0200 Subject: [PATCH 6/6] ci(repo): keep the mirror payload check private The old catalog script was the only external caller, so the export failed knip. --- .github/scripts/slim-mirror-payload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/slim-mirror-payload.ts b/.github/scripts/slim-mirror-payload.ts index fe516d4377..74191dfebf 100644 --- a/.github/scripts/slim-mirror-payload.ts +++ b/.github/scripts/slim-mirror-payload.ts @@ -31,7 +31,7 @@ export const nativeTagPattern = (version: string): RegExp => new RegExp(`^${escapeRegExp(version)}-native-(${NATIVE_TARGETS.join("|")})$`); /** `service` / `version` / `digest` are the image fields. Extra keys such as `natives[]` are ignored. */ -export const validatePayload = (input: { +const validatePayload = (input: { readonly service: string; readonly version: string; readonly digest: string;