fix(build): add jitter and delay cap to cosign retry backoff

.github/workflows/build.yml

@@ -156,13 +156,17 @@ on:
         type: boolean
         default: true
       cosign_max_attempts:
-        description: 'Maximum cosign signing attempts per image reference. Increase to absorb transient OIDC/Fulcio rate limits.'
+        description: 'Maximum cosign signing attempts per image reference. Increase to absorb transient OIDC/Fulcio/Rekor rate limits.'
         type: string
-        default: '3'
+        default: '5'
       cosign_initial_delay:
-        description: 'Initial delay (seconds) between cosign retries. Grows exponentially (×3) after each failed attempt.'
+        description: 'Initial delay (seconds) between cosign retries. Grows exponentially (×3) after each failed attempt, capped at cosign_max_delay, then randomized (equal jitter).'
         type: string
         default: '5'
+      cosign_max_delay:
+        description: 'Maximum delay (seconds) between cosign retries. Caps the exponential backoff before jitter is applied.'
+        type: string
+        default: '60'
       continue_gitops_on_signing_failure:
         description: 'When true, continue to GitOps artifact upload even if cosign signing fails after all retries. Image stays in registry unsigned and a warning is emitted; manual signing is required afterwards.'
         type: boolean
@@ -504,6 +508,7 @@ jobs:
           image-refs: ${{ steps.cosign-refs.outputs.refs }}
           max-attempts: ${{ inputs.cosign_max_attempts }}
           initial-delay: ${{ inputs.cosign_initial_delay }}
+          max-delay: ${{ inputs.cosign_max_delay }}
 
       - name: Report cosign signing status
         if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' && steps.cosign-sign.outcome == 'failure'

docs/build-workflow.md

@@ -111,8 +111,9 @@ jobs:
 | `force_full_matrix` | boolean | `false` | When `true`, build all `filter_paths` components on every run regardless of what changed. Use for tightly-coupled services that must always share the same image tag (e.g., auth + identity always released together) |
 | `force_multiplatform` | boolean | `false` | Force multi-platform build (amd64+arm64) even for beta/rc tags |
 | `enable_cosign_sign` | boolean | `true` | Sign images with cosign keyless (OIDC) signing. Requires `id-token: write` in caller |
-| `cosign_max_attempts` | string | `3` | Max cosign signing attempts per image. Increase to absorb transient OIDC/Fulcio rate limits |
-| `cosign_initial_delay` | string | `5` | Initial delay (seconds) between cosign retries. Grows ×3 each failed attempt |
+| `cosign_max_attempts` | string | `5` | Max cosign signing attempts per image. Increase to absorb transient OIDC/Fulcio/Rekor rate limits |
+| `cosign_initial_delay` | string | `5` | Initial delay (seconds) between cosign retries. Grows ×3 each failed attempt, capped at `cosign_max_delay`, then randomized (equal jitter) |
+| `cosign_max_delay` | string | `60` | Maximum delay (seconds) between cosign retries. Caps the exponential backoff before jitter is applied |
 | `continue_gitops_on_signing_failure` | boolean | `false` | Allow GitOps artifact upload to continue when cosign signing fails after all retries. Image stays unsigned in registry; manual `cosign sign` required |
 
 ## Secrets
@@ -281,7 +282,7 @@ jobs:
 
 Cosign signing depends on the Sigstore OIDC/Fulcio infrastructure, which can hit transient rate limits or 5xx responses. Two layers protect releases:
 
-1. **Retry with exponential backoff** — controlled by `cosign_max_attempts` (default `3`) and `cosign_initial_delay` (default `5`s, grows ×3 between attempts). Increase both for releases that consistently brush against rate limits.
+1. **Retry with exponential backoff and jitter** — controlled by `cosign_max_attempts` (default `5`) and `cosign_initial_delay` (default `5`s, grows ×3 between attempts, capped at `cosign_max_delay` — default `60`s — then randomized with equal jitter to avoid multiple jobs retrying against Rekor at the same instant). Increase these for releases that consistently brush against rate limits or multi-minute Rekor outages.
 
 2. **Optional GitOps continuation** — when `continue_gitops_on_signing_failure: true`, signing failure does not block the GitOps artifact upload. The image is already pushed and immutable in the registry; this prevents a transient signing failure from leaving the release in a broken half-state where the image exists but no GitOps PR was opened.
 
@@ -293,6 +294,7 @@ jobs:
       enable_cosign_sign: true
       cosign_max_attempts: 5
       cosign_initial_delay: 15
+      cosign_max_delay: 60
       continue_gitops_on_signing_failure: true
     secrets: inherit
 ```

src/security/cosign-sign/README.md

@@ -14,8 +14,9 @@ Composite action that signs container images using [Sigstore cosign](https://git
 | `image-refs` | Newline-separated fully qualified image references to sign (e.g., `docker.io/org/app@sha256:abc...`) | Yes | — |
 | `cosign-version` | Cosign version to install | No | `v2.5.0` |
 | `dry-run` | Log what would be signed without actually signing | No | `false` |
-| `max-attempts` | Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio failures) | No | `3` |
-| `initial-delay` | Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt. | No | `5` |
+| `max-attempts` | Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio/Rekor failures) | No | `5` |
+| `initial-delay` | Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt, capped at `max-delay`, then randomized (equal jitter) to avoid thundering-herd retries. | No | `5` |
+| `max-delay` | Maximum delay in seconds between retry attempts. Caps the exponential backoff before jitter is applied. | No | `60` |
 
 ## Outputs
 
@@ -83,9 +84,9 @@ permissions:
 
 ## Retry behavior
 
-The signing step retries automatically on transient failures (e.g., malformed OIDC responses from Fulcio, token endpoint flakiness). By default it attempts up to **3 times** with exponential backoff starting at **5s** and growing ×3 per retry (5s → 15s → 45s).
+The signing step retries automatically on transient failures (e.g., malformed OIDC responses from Fulcio, token endpoint flakiness, Rekor log entry lookups). By default it attempts up to **5 times** with exponential backoff starting at **5s**, growing ×3 per retry, capped at **60s**, and randomized (equal jitter) to avoid multiple concurrent jobs retrying against Rekor at the same instant.
 
-Tune via `max-attempts` and `initial-delay` if your environment needs a different policy:
+Tune via `max-attempts`, `initial-delay`, and `max-delay` if your environment needs a different policy:
 
 ```yaml
       - name: Sign container image
@@ -94,4 +95,5 @@ Tune via `max-attempts` and `initial-delay` if your environment needs a differen
           image-refs: docker.io/myorg/myapp@${{ steps.build-push.outputs.digest }}
           max-attempts: "5"
           initial-delay: "10"
+          max-delay: "60"
 ```

src/security/cosign-sign/action.yml

@@ -14,13 +14,17 @@ inputs:
     required: false
     default: "false"
   max-attempts:
-    description: Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio failures)
+    description: Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio/Rekor failures)
     required: false
-    default: "3"
+    default: "5"
   initial-delay:
-    description: Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt.
+    description: Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt, capped at max-delay, then randomized (equal jitter) to avoid thundering-herd retries.
     required: false
     default: "5"
+  max-delay:
+    description: Maximum delay in seconds between retry attempts. Caps the exponential backoff before jitter is applied.
+    required: false
+    default: "60"
 
 outputs:
   signed-refs:
@@ -95,6 +99,7 @@ runs:
         VALID_COUNT: ${{ steps.validate.outputs.count }}
         MAX_ATTEMPTS: ${{ inputs.max-attempts }}
         INITIAL_DELAY: ${{ inputs.initial-delay }}
+        MAX_DELAY: ${{ inputs.max-delay }}
       run: |
         sign_with_retry() {
           local ref="$1"
@@ -107,9 +112,18 @@ runs:
               return 0
             fi
             if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then
-              echo "::warning::cosign sign failed (attempt $attempt/$MAX_ATTEMPTS) — retrying in ${delay}s…"
-              sleep "$delay"
+              local capped_delay="$delay"
+              if [ "$capped_delay" -gt "$MAX_DELAY" ]; then
+                capped_delay="$MAX_DELAY"
+              fi
+              local half=$((capped_delay / 2))
+              local sleep_time=$((half + RANDOM % (half + 1)))
+              echo "::warning::cosign sign failed (attempt $attempt/$MAX_ATTEMPTS) — retrying in ${sleep_time}s (base ${capped_delay}s, jittered)…"
+              sleep "$sleep_time"
               delay=$((delay * 3))
+              if [ "$delay" -gt "$MAX_DELAY" ]; then
+                delay="$MAX_DELAY"
+              fi
             fi
           done