From c4fe894e560d55c015ee93d8051c73a944a36151 Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:18:22 -0300 Subject: [PATCH 1/4] fix(build): retry cosign signing when on_existing_tag skips the build When on_existing_tag: skip skips build/push because the tag already exists, the cosign steps were also skipped since they depended on steps.build-push.outputs.digest and should_build != 'false'. A transient cosign/Rekor failure on an already-pushed tag then had no retry path short of a full rebuild or a brand new release. Add a 'Resolve digest for existing tag' step that resolves the digest from the existing registry tag via docker buildx imagetools inspect when the build was skipped, and gate the cosign steps on enable_cosign_sign alone, feeding steps.build-push.outputs.digest || steps.existing-digest.outputs.digest. Closes #559 --- .github/workflows/build.yml | 32 +++++++++++++++++++++++++++----- docs/build-workflow.md | 2 +- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d6d486f7..8e37d16f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -471,11 +471,33 @@ jobs: github_token=${{ secrets.MANAGE_TOKEN }} # ----------------- Cosign Image Signing ----------------- + # Lets `on_existing_tag: skip` retry just the signing step for an already-pushed + # tag (e.g. after a transient cosign/Rekor failure) without a full rebuild. + - name: Resolve digest for existing tag + if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build == 'false' + id: existing-digest + env: + APP_NAME: ${{ matrix.app.name }} + VERSION: ${{ steps.version.outputs.version }} + ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} + DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} + GHCR_ORG: ${{ steps.image-names.outputs.ghcr_org }} + run: | + set -euo pipefail + TAG="${VERSION#v}" + if [ "$ENABLE_DOCKERHUB" == "true" ]; then + REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" + else + REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" + fi + DIGEST=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' | jq -r '.digest') + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + - name: Build cosign image references - if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' + if: inputs.enable_cosign_sign id: cosign-refs env: - DIGEST: ${{ steps.build-push.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} @@ -500,7 +522,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Sign container images with cosign - if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' + if: inputs.enable_cosign_sign id: cosign-sign continue-on-error: ${{ inputs.continue_gitops_on_signing_failure || false }} uses: LerianStudio/github-actions-shared-workflows/src/security/cosign-sign@v1 @@ -511,10 +533,10 @@ jobs: max-delay: ${{ inputs.cosign_max_delay }} - name: Report cosign signing status - if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' && steps.cosign-sign.outcome == 'failure' + if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' env: REFS: ${{ steps.cosign-refs.outputs.refs }} - DIGEST: ${{ steps.build-push.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} APP_NAME: ${{ matrix.app.name }} run: | echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." diff --git a/docs/build-workflow.md b/docs/build-workflow.md index deebe56c..1fbd8de5 100644 --- a/docs/build-workflow.md +++ b/docs/build-workflow.md @@ -132,7 +132,7 @@ Uses `secrets: inherit` pattern. Required secrets: Before building, the workflow checks whether the target image tag already exists in each enabled registry (via `docker manifest inspect`, reusing the registry logins). This avoids a full rebuild that would only fail at push time on registries with tag immutability enabled. Behaviour is controlled by `on_existing_tag`: - **`fail`** (default): abort early with a clear error instead of rebuilding then failing at push. -- **`skip`**: skip the build/push but still emit the GitOps tag artifacts (from the version), so a re-run remains idempotent for the downstream GitOps update. +- **`skip`**: skip the build/push but still emit the GitOps tag artifacts (from the version), so a re-run remains idempotent for the downstream GitOps update. Cosign signing (if enabled) also retries in this mode: the digest is resolved from the existing registry tag via `docker buildx imagetools inspect` instead of the build/push step, so a transient signing failure on an already-pushed tag can be recovered with a plain re-run instead of cutting a new release. - **`warn`**: emit a warning and build anyway (push may still fail on immutable registries). A non-existent tag (or a check that errors out, e.g. transient registry issues) is treated as "not present" so the check never blocks a legitimate build. From e158f99f160a7559aa6180a4c0779241455264dc Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:31:04 -0300 Subject: [PATCH 2/4] fix(build): resolve cosign digest per registry independently Address CodeRabbit finding on PR #603: when both DockerHub and GHCR are enabled, the existing-tag digest resolution always picked DockerHub (or GHCR alone) and reused that single digest for both registries' refs. If a tag existed in only one registry, this could construct a nonexistent ref for the other registry using the wrong digest. Resolve each enabled registry's digest independently and only emit a ref for a registry when its own digest was actually found. --- .github/workflows/build.yml | 46 +++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e37d16f..7337e36b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -480,24 +480,42 @@ jobs: APP_NAME: ${{ matrix.app.name }} VERSION: ${{ steps.version.outputs.version }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} + ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} GHCR_ORG: ${{ steps.image-names.outputs.ghcr_org }} run: | - set -euo pipefail + set -uo pipefail TAG="${VERSION#v}" + + # Each registry is inspected independently: a tag can be immutable-blocked + # in one registry but absent from another (e.g. a prior partial push), so + # reusing one registry's digest for the other's ref would sign a + # nonexistent (or wrong) reference. if [ "$ENABLE_DOCKERHUB" == "true" ]; then REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" - else + DIGEST="" + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + fi + echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" + fi + + if [ "$ENABLE_GHCR" == "true" ]; then REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" + DIGEST="" + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + fi + echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi - DIGEST=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' | jq -r '.digest') - echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" - name: Build cosign image references if: inputs.enable_cosign_sign id: cosign-refs env: - DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest }} + DOCKERHUB_DIGEST: ${{ steps.existing-digest.outputs.dockerhub_digest }} + GHCR_DIGEST: ${{ steps.existing-digest.outputs.ghcr_digest }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} @@ -506,13 +524,21 @@ jobs: run: | REFS="" - if [ "$ENABLE_DOCKERHUB" == "true" ]; then - REFS="docker.io/${DOCKERHUB_ORG}/${APP_NAME}@${DIGEST}" + # A single build-push pushes the identical content to every enabled + # registry, so DIGEST (from build-push) applies to both. When the build + # was skipped (existing tag), fall back to each registry's own resolved + # digest instead, and only emit a ref when that registry's digest was + # actually found. + DOCKERHUB_FINAL="${DIGEST:-$DOCKERHUB_DIGEST}" + GHCR_FINAL="${DIGEST:-$GHCR_DIGEST}" + + if [ "$ENABLE_DOCKERHUB" == "true" ] && [ -n "$DOCKERHUB_FINAL" ]; then + REFS="docker.io/${DOCKERHUB_ORG}/${APP_NAME}@${DOCKERHUB_FINAL}" fi - if [ "$ENABLE_GHCR" == "true" ]; then + if [ "$ENABLE_GHCR" == "true" ] && [ -n "$GHCR_FINAL" ]; then [ -n "$REFS" ] && REFS="${REFS}"$'\n' - REFS="${REFS}ghcr.io/${GHCR_ORG}/${APP_NAME}@${DIGEST}" + REFS="${REFS}ghcr.io/${GHCR_ORG}/${APP_NAME}@${GHCR_FINAL}" fi { @@ -536,7 +562,7 @@ jobs: if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' env: REFS: ${{ steps.cosign-refs.outputs.refs }} - DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.dockerhub_digest || steps.existing-digest.outputs.ghcr_digest }} APP_NAME: ${{ matrix.app.name }} run: | echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." From 9d71835da44a30e7e9d97ae97366cc5b9975291c Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:47:26 -0300 Subject: [PATCH 3/4] fix(build): surface digest inspection failures, stop collapsing report digests Address CodeRabbit findings on PR #603: - Resolve digest for existing tag: distinguish a genuine 'tag not found' from an inspect/lookup failure (transient network error, auth issue) by emitting a ::warning:: with the actual error in the latter case, instead of silently treating both as 'no digest'. Kept fail-open (skip that registry, don't abort the job) to match this file's existing convention for the same tradeoff in the pre-flight tag-existence check just above. - Report cosign signing status: stop collapsing to a single DIGEST value that discarded one registry's digest when both were resolved; REFS already lists each registry's fully-qualified ref with its own digest, so the redundant/lossy single-digest line was removed instead of duplicating that same registry-qualified data. --- .github/workflows/build.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7337e36b..92eda9da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,8 +494,11 @@ jobs: if [ "$ENABLE_DOCKERHUB" == "true" ]; then REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" DIGEST="" - if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." + else + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" fi echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -503,8 +506,11 @@ jobs: if [ "$ENABLE_GHCR" == "true" ]; then REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" DIGEST="" - if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." + else + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" fi echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -562,7 +568,6 @@ jobs: if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' env: REFS: ${{ steps.cosign-refs.outputs.refs }} - DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.dockerhub_digest || steps.existing-digest.outputs.ghcr_digest }} APP_NAME: ${{ matrix.app.name }} run: | echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." @@ -570,14 +575,13 @@ jobs: echo "### ⚠️ Unsigned image — manual action required" echo "" echo "- **App:** \`${APP_NAME}\`" - echo "- **Digest:** \`${DIGEST}\`" - echo "- **Refs:**" + echo "- **Refs (each already includes its own digest):**" echo '```' echo "${REFS}" echo '```' echo "" echo "GitOps artifact upload was allowed to continue (\`continue_gitops_on_signing_failure: true\`)." - echo "The image is immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against the digest above before promoting to production." + echo "The image(s) above are immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against each ref above before promoting to production." } >> "$GITHUB_STEP_SUMMARY" # ----------------- GitOps Artifacts ----------------- From 71fe9a402e5dbf6ebb72c1a8527228321a512955 Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:59:53 -0300 Subject: [PATCH 4/4] fix(build): escape workflow-command output, soften partial-failure wording Address CodeRabbit findings on PR #603: - Resolve digest for existing tag: percent-encode the captured 'docker buildx imagetools inspect' output (%, CR, LF) before embedding it in a ::warning:: line, so it can't corrupt or spoof a workflow command annotation. - Report cosign signing status: cosign-sign signs refs one by one, so a failure partway through the list doesn't mean every ref is unsigned. Reworded the summary to reflect that signing didn't complete for every ref instead of asserting all refs are unsigned. --- .github/workflows/build.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92eda9da..1e13c033 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -487,6 +487,19 @@ jobs: set -uo pipefail TAG="${VERSION#v}" + # GitHub Actions parses every line of step output for workflow commands + # (::warning::, ::error::, etc.). Registry inspect output is not + # attacker-controlled here, but it could still contain newlines or a + # stray "::" that would corrupt or spoof an annotation, so it's + # percent-encoded into a single line before being embedded in one. + escape_for_warning() { + local s="$1" + s="${s//%/%25}" + s="${s//$'\r'/%0D}" + s="${s//$'\n'/%0A}" + printf '%s' "$s" + } + # Each registry is inspected independently: a tag can be immutable-blocked # in one registry but absent from another (e.g. a prior partial push), so # reusing one registry's digest for the other's ref would sign a @@ -498,7 +511,7 @@ jobs: DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." else - echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")" fi echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -510,7 +523,7 @@ jobs: DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." else - echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")" fi echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -570,18 +583,18 @@ jobs: REFS: ${{ steps.cosign-refs.outputs.refs }} APP_NAME: ${{ matrix.app.name }} run: | - echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." + echo "::warning::Cosign signing did not complete for every ref for ${APP_NAME} — one or more of the refs below may still be unsigned." { - echo "### ⚠️ Unsigned image — manual action required" + echo "### ⚠️ Signing did not complete for every ref — manual action required" echo "" echo "- **App:** \`${APP_NAME}\`" - echo "- **Refs (each already includes its own digest):**" + echo "- **Requested refs (each already includes its own digest):**" echo '```' echo "${REFS}" echo '```' echo "" echo "GitOps artifact upload was allowed to continue (\`continue_gitops_on_signing_failure: true\`)." - echo "The image(s) above are immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against each ref above before promoting to production." + echo "The image(s) above are immutable and already present in the registry, but signing failed after all retries for at least one ref — some refs above may already be signed, others may not be. Verify with \`cosign verify\` and run \`cosign sign\` manually against any ref that isn't before promoting to production." } >> "$GITHUB_STEP_SUMMARY" # ----------------- GitOps Artifacts -----------------