fix(build): retry cosign signing when on_existing_tag skips the build

.github/workflows/build.yml

@@ -471,11 +471,70 @@ jobs:
             github_token=${{ secrets.MANAGE_TOKEN }}
 
       # ----------------- Cosign Image Signing -----------------
+      # Lets `on_existing_tag: skip` retry just the signing step for an already-pushed
+      # tag (e.g. after a transient cosign/Rekor failure) without a full rebuild.
+      - name: Resolve digest for existing tag
+        if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build == 'false'
+        id: existing-digest
+        env:
+          APP_NAME: ${{ matrix.app.name }}
+          VERSION: ${{ steps.version.outputs.version }}
+          ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }}
+          ENABLE_GHCR: ${{ inputs.enable_ghcr }}
+          DOCKERHUB_ORG: ${{ inputs.dockerhub_org }}
+          GHCR_ORG: ${{ steps.image-names.outputs.ghcr_org }}
+        run: |
+          set -uo pipefail
+          TAG="${VERSION#v}"
+
+          # GitHub Actions parses every line of step output for workflow commands
+          # (::warning::, ::error::, etc.). Registry inspect output is not
+          # attacker-controlled here, but it could still contain newlines or a
+          # stray "::" that would corrupt or spoof an annotation, so it's
+          # percent-encoded into a single line before being embedded in one.
+          escape_for_warning() {
+            local s="$1"
+            s="${s//%/%25}"
+            s="${s//$'\r'/%0D}"
+            s="${s//$'\n'/%0A}"
+            printf '%s' "$s"
+          }
+
+          # Each registry is inspected independently: a tag can be immutable-blocked
+          # in one registry but absent from another (e.g. a prior partial push), so
+          # reusing one registry's digest for the other's ref would sign a
+          # nonexistent (or wrong) reference.
+          if [ "$ENABLE_DOCKERHUB" == "true" ]; then
+            REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}"
+            DIGEST=""
+            if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then
+              DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty')
+              [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry."
+            else
+              echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")"
+            fi
+            echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT"
+          fi
+
+          if [ "$ENABLE_GHCR" == "true" ]; then
+            REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}"
+            DIGEST=""
+            if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then
+              DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty')
+              [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry."
+            else
+              echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")"
+            fi
+            echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Build cosign image references
-        if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false'
+        if: inputs.enable_cosign_sign
         id: cosign-refs
         env:
           DIGEST: ${{ steps.build-push.outputs.digest }}
+          DOCKERHUB_DIGEST: ${{ steps.existing-digest.outputs.dockerhub_digest }}
+          GHCR_DIGEST: ${{ steps.existing-digest.outputs.ghcr_digest }}
           ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }}
           ENABLE_GHCR: ${{ inputs.enable_ghcr }}
           DOCKERHUB_ORG: ${{ inputs.dockerhub_org }}
@@ -484,13 +543,21 @@ jobs:
         run: |
           REFS=""
 
-          if [ "$ENABLE_DOCKERHUB" == "true" ]; then
-            REFS="docker.io/${DOCKERHUB_ORG}/${APP_NAME}@${DIGEST}"
+          # A single build-push pushes the identical content to every enabled
+          # registry, so DIGEST (from build-push) applies to both. When the build
+          # was skipped (existing tag), fall back to each registry's own resolved
+          # digest instead, and only emit a ref when that registry's digest was
+          # actually found.
+          DOCKERHUB_FINAL="${DIGEST:-$DOCKERHUB_DIGEST}"
+          GHCR_FINAL="${DIGEST:-$GHCR_DIGEST}"
+
+          if [ "$ENABLE_DOCKERHUB" == "true" ] && [ -n "$DOCKERHUB_FINAL" ]; then
+            REFS="docker.io/${DOCKERHUB_ORG}/${APP_NAME}@${DOCKERHUB_FINAL}"
           fi
 
-          if [ "$ENABLE_GHCR" == "true" ]; then
+          if [ "$ENABLE_GHCR" == "true" ] && [ -n "$GHCR_FINAL" ]; then
             [ -n "$REFS" ] && REFS="${REFS}"$'\n'
-            REFS="${REFS}ghcr.io/${GHCR_ORG}/${APP_NAME}@${DIGEST}"
+            REFS="${REFS}ghcr.io/${GHCR_ORG}/${APP_NAME}@${GHCR_FINAL}"
           fi
 
           {
@@ -500,7 +567,7 @@ jobs:
           } >> "$GITHUB_OUTPUT"
 
       - name: Sign container images with cosign
-        if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false'
+        if: inputs.enable_cosign_sign
         id: cosign-sign
         continue-on-error: ${{ inputs.continue_gitops_on_signing_failure || false }}
         uses: LerianStudio/github-actions-shared-workflows/src/security/cosign-sign@v1
@@ -511,25 +578,23 @@ jobs:
           max-delay: ${{ inputs.cosign_max_delay }}
 
       - name: Report cosign signing status
-        if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' && steps.cosign-sign.outcome == 'failure'
+        if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure'
         env:
           REFS: ${{ steps.cosign-refs.outputs.refs }}
-          DIGEST: ${{ steps.build-push.outputs.digest }}
           APP_NAME: ${{ matrix.app.name }}
         run: |
-          echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry."
+          echo "::warning::Cosign signing did not complete for every ref for ${APP_NAME} — one or more of the refs below may still be unsigned."
           {
-            echo "### ⚠️ Unsigned image — manual action required"
+            echo "### ⚠️ Signing did not complete for every ref — manual action required"
             echo ""
             echo "- **App:** \`${APP_NAME}\`"
-            echo "- **Digest:** \`${DIGEST}\`"
-            echo "- **Refs:**"
+            echo "- **Requested refs (each already includes its own digest):**"
             echo '```'
             echo "${REFS}"
             echo '```'
             echo ""
             echo "GitOps artifact upload was allowed to continue (\`continue_gitops_on_signing_failure: true\`)."
-            echo "The image is immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against the digest above before promoting to production."
+            echo "The image(s) above are immutable and already present in the registry, but signing failed after all retries for at least one ref — some refs above may already be signed, others may not be. Verify with \`cosign verify\` and run \`cosign sign\` manually against any ref that isn't before promoting to production."
           } >> "$GITHUB_STEP_SUMMARY"
 
       # ----------------- GitOps Artifacts -----------------

docs/build-workflow.md

@@ -132,7 +132,7 @@ Uses `secrets: inherit` pattern. Required secrets:
 Before building, the workflow checks whether the target image tag already exists in each enabled registry (via `docker manifest inspect`, reusing the registry logins). This avoids a full rebuild that would only fail at push time on registries with tag immutability enabled. Behaviour is controlled by `on_existing_tag`:
 
 - **`fail`** (default): abort early with a clear error instead of rebuilding then failing at push.
-- **`skip`**: skip the build/push but still emit the GitOps tag artifacts (from the version), so a re-run remains idempotent for the downstream GitOps update.
+- **`skip`**: skip the build/push but still emit the GitOps tag artifacts (from the version), so a re-run remains idempotent for the downstream GitOps update. Cosign signing (if enabled) also retries in this mode: the digest is resolved from the existing registry tag via `docker buildx imagetools inspect` instead of the build/push step, so a transient signing failure on an already-pushed tag can be recovered with a plain re-run instead of cutting a new release.
 - **`warn`**: emit a warning and build anyway (push may still fail on immutable registries).
 
 A non-existent tag (or a check that errors out, e.g. transient registry issues) is treated as "not present" so the check never blocks a legitimate build.