diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b646466..71efa74e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -156,13 +156,17 @@ on: type: boolean default: true cosign_max_attempts: - description: 'Maximum cosign signing attempts per image reference. Increase to absorb transient OIDC/Fulcio rate limits.' + description: 'Maximum cosign signing attempts per image reference. Increase to absorb transient OIDC/Fulcio/Rekor rate limits.' type: string - default: '3' + default: '5' cosign_initial_delay: - description: 'Initial delay (seconds) between cosign retries. Grows exponentially (×3) after each failed attempt.' + description: 'Initial delay (seconds) between cosign retries. Grows exponentially (×3) after each failed attempt, capped at cosign_max_delay, then randomized (equal jitter).' type: string default: '5' + cosign_max_delay: + description: 'Maximum delay (seconds) between cosign retries. Caps the exponential backoff before jitter is applied.' + type: string + default: '60' continue_gitops_on_signing_failure: description: 'When true, continue to GitOps artifact upload even if cosign signing fails after all retries. Image stays in registry unsigned and a warning is emitted; manual signing is required afterwards.' type: boolean @@ -504,6 +508,7 @@ jobs: image-refs: ${{ steps.cosign-refs.outputs.refs }} max-attempts: ${{ inputs.cosign_max_attempts }} initial-delay: ${{ inputs.cosign_initial_delay }} + max-delay: ${{ inputs.cosign_max_delay }} - name: Report cosign signing status if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' && steps.cosign-sign.outcome == 'failure' diff --git a/docs/build-workflow.md b/docs/build-workflow.md index ed972a01..deebe56c 100644 --- a/docs/build-workflow.md +++ b/docs/build-workflow.md @@ -111,8 +111,9 @@ jobs: | `force_full_matrix` | boolean | `false` | When `true`, build all `filter_paths` components on every run regardless of what changed. Use for tightly-coupled services that must always share the same image tag (e.g., auth + identity always released together) | | `force_multiplatform` | boolean | `false` | Force multi-platform build (amd64+arm64) even for beta/rc tags | | `enable_cosign_sign` | boolean | `true` | Sign images with cosign keyless (OIDC) signing. Requires `id-token: write` in caller | -| `cosign_max_attempts` | string | `3` | Max cosign signing attempts per image. Increase to absorb transient OIDC/Fulcio rate limits | -| `cosign_initial_delay` | string | `5` | Initial delay (seconds) between cosign retries. Grows ×3 each failed attempt | +| `cosign_max_attempts` | string | `5` | Max cosign signing attempts per image. Increase to absorb transient OIDC/Fulcio/Rekor rate limits | +| `cosign_initial_delay` | string | `5` | Initial delay (seconds) between cosign retries. Grows ×3 each failed attempt, capped at `cosign_max_delay`, then randomized (equal jitter) | +| `cosign_max_delay` | string | `60` | Maximum delay (seconds) between cosign retries. Caps the exponential backoff before jitter is applied | | `continue_gitops_on_signing_failure` | boolean | `false` | Allow GitOps artifact upload to continue when cosign signing fails after all retries. Image stays unsigned in registry; manual `cosign sign` required | ## Secrets @@ -281,7 +282,7 @@ jobs: Cosign signing depends on the Sigstore OIDC/Fulcio infrastructure, which can hit transient rate limits or 5xx responses. Two layers protect releases: -1. **Retry with exponential backoff** — controlled by `cosign_max_attempts` (default `3`) and `cosign_initial_delay` (default `5`s, grows ×3 between attempts). Increase both for releases that consistently brush against rate limits. +1. **Retry with exponential backoff and jitter** — controlled by `cosign_max_attempts` (default `5`) and `cosign_initial_delay` (default `5`s, grows ×3 between attempts, capped at `cosign_max_delay` — default `60`s — then randomized with equal jitter to avoid multiple jobs retrying against Rekor at the same instant). Increase these for releases that consistently brush against rate limits or multi-minute Rekor outages. 2. **Optional GitOps continuation** — when `continue_gitops_on_signing_failure: true`, signing failure does not block the GitOps artifact upload. The image is already pushed and immutable in the registry; this prevents a transient signing failure from leaving the release in a broken half-state where the image exists but no GitOps PR was opened. @@ -293,6 +294,7 @@ jobs: enable_cosign_sign: true cosign_max_attempts: 5 cosign_initial_delay: 15 + cosign_max_delay: 60 continue_gitops_on_signing_failure: true secrets: inherit ``` diff --git a/src/security/cosign-sign/README.md b/src/security/cosign-sign/README.md index e6fbacef..04eca0d7 100644 --- a/src/security/cosign-sign/README.md +++ b/src/security/cosign-sign/README.md @@ -14,8 +14,9 @@ Composite action that signs container images using [Sigstore cosign](https://git | `image-refs` | Newline-separated fully qualified image references to sign (e.g., `docker.io/org/app@sha256:abc...`) | Yes | — | | `cosign-version` | Cosign version to install | No | `v2.5.0` | | `dry-run` | Log what would be signed without actually signing | No | `false` | -| `max-attempts` | Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio failures) | No | `3` | -| `initial-delay` | Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt. | No | `5` | +| `max-attempts` | Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio/Rekor failures) | No | `5` | +| `initial-delay` | Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt, capped at `max-delay`, then randomized (equal jitter) to avoid thundering-herd retries. | No | `5` | +| `max-delay` | Maximum delay in seconds between retry attempts. Caps the exponential backoff before jitter is applied. | No | `60` | ## Outputs @@ -83,9 +84,9 @@ permissions: ## Retry behavior -The signing step retries automatically on transient failures (e.g., malformed OIDC responses from Fulcio, token endpoint flakiness). By default it attempts up to **3 times** with exponential backoff starting at **5s** and growing ×3 per retry (5s → 15s → 45s). +The signing step retries automatically on transient failures (e.g., malformed OIDC responses from Fulcio, token endpoint flakiness, Rekor log entry lookups). By default it attempts up to **5 times** with exponential backoff starting at **5s**, growing ×3 per retry, capped at **60s**, and randomized (equal jitter) to avoid multiple concurrent jobs retrying against Rekor at the same instant. -Tune via `max-attempts` and `initial-delay` if your environment needs a different policy: +Tune via `max-attempts`, `initial-delay`, and `max-delay` if your environment needs a different policy: ```yaml - name: Sign container image @@ -94,4 +95,5 @@ Tune via `max-attempts` and `initial-delay` if your environment needs a differen image-refs: docker.io/myorg/myapp@${{ steps.build-push.outputs.digest }} max-attempts: "5" initial-delay: "10" + max-delay: "60" ``` diff --git a/src/security/cosign-sign/action.yml b/src/security/cosign-sign/action.yml index 314c2c32..b74bf4b7 100644 --- a/src/security/cosign-sign/action.yml +++ b/src/security/cosign-sign/action.yml @@ -14,13 +14,17 @@ inputs: required: false default: "false" max-attempts: - description: Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio failures) + description: Maximum number of signing attempts per image reference (retry on transient OIDC/Fulcio/Rekor failures) required: false - default: "3" + default: "5" initial-delay: - description: Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt. + description: Initial delay in seconds between retry attempts. Delay grows exponentially (×3) after each failed attempt, capped at max-delay, then randomized (equal jitter) to avoid thundering-herd retries. required: false default: "5" + max-delay: + description: Maximum delay in seconds between retry attempts. Caps the exponential backoff before jitter is applied. + required: false + default: "60" outputs: signed-refs: @@ -95,6 +99,7 @@ runs: VALID_COUNT: ${{ steps.validate.outputs.count }} MAX_ATTEMPTS: ${{ inputs.max-attempts }} INITIAL_DELAY: ${{ inputs.initial-delay }} + MAX_DELAY: ${{ inputs.max-delay }} run: | sign_with_retry() { local ref="$1" @@ -107,9 +112,18 @@ runs: return 0 fi if [ "$attempt" -lt "$MAX_ATTEMPTS" ]; then - echo "::warning::cosign sign failed (attempt $attempt/$MAX_ATTEMPTS) — retrying in ${delay}s…" - sleep "$delay" + local capped_delay="$delay" + if [ "$capped_delay" -gt "$MAX_DELAY" ]; then + capped_delay="$MAX_DELAY" + fi + local half=$((capped_delay / 2)) + local sleep_time=$((half + RANDOM % (half + 1))) + echo "::warning::cosign sign failed (attempt $attempt/$MAX_ATTEMPTS) — retrying in ${sleep_time}s (base ${capped_delay}s, jittered)…" + sleep "$sleep_time" delay=$((delay * 3)) + if [ "$delay" -gt "$MAX_DELAY" ]; then + delay="$MAX_DELAY" + fi fi done