From ffe046f5b331a52805fb983a25623b9fa0d3efa4 Mon Sep 17 00:00:00 2001 From: Sonic Shih Date: Fri, 16 Oct 2026 01:50:07 +0800 Subject: [PATCH] fix(ci): retry transient tested-image reads --- .github/scripts/read-tested-image.sh | 38 ++++++++++++++++++++++---- .github/scripts/test-tested-image.sh | 40 ++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/scripts/read-tested-image.sh b/.github/scripts/read-tested-image.sh index 5faac6734..8c11c693f 100644 --- a/.github/scripts/read-tested-image.sh +++ b/.github/scripts/read-tested-image.sh @@ -11,8 +11,34 @@ esac [[ $source_sha =~ ^[0-9a-f]{40}$ && ${GITHUB_REPOSITORY:-} == proerror77/monday ]] || exit 2 [[ $mode == required || $mode == optional ]] || exit 2 mkdir -p "$destination" +retry_work=$(mktemp -d "$destination/.github-read.XXXXXX") +trap 'rm -rf "$retry_work"' EXIT +# Retry only transient read failures. Commit no partial JSON or download bytes. +read_github() { + local stage=$1 output=$2 attempt status + shift 2 + for attempt in 1 2 3; do + if "$@" >"$retry_work/response" 2>"$retry_work/error"; then + mv "$retry_work/response" "$output" + return 0 + else status=$?; fi + printf 'GitHub read %s failed (attempt %s/3, exit %s).\n' "$stage" "$attempt" "$status" >&2 + cat "$retry_work/error" >&2 + if (( attempt == 3 )) || ! grep -Eq 'HTTP (500|502|503|504)([^0-9]|$)' "$retry_work/error"; then + # Only complete, authenticated absence may return the optional rebuild code. + return 1 + fi + sleep "$attempt" + done +} +download_bundle() { + local partial + partial=$(mktemp -d "$retry_work/bundle.XXXXXX") + gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name "$name" --dir "$partial" || return $? + mv "$partial" "$destination/bundle" +} missing() { echo 'No successful exact-source tested image artifact is available.' >&2; [[ $mode != optional ]] || exit 75; exit 1; } -gh api "repos/$GITHUB_REPOSITORY/actions/workflows/$workflow/runs?head_sha=$source_sha&event=push&per_page=100" >"$destination/runs.json" +read_github workflow-runs "$destination/runs.json" gh api "repos/$GITHUB_REPOSITORY/actions/workflows/$workflow/runs?head_sha=$source_sha&event=push&per_page=100" run=$(jq -c --arg sha "$source_sha" --arg repo "$GITHUB_REPOSITORY" --arg path ".github/workflows/$workflow" ' [.workflow_runs[] | select(.head_sha==$sha and .head_branch=="main" and .event=="push" and .path==$path and .head_repository.full_name==$repo)] | max_by(.id) @@ -21,12 +47,13 @@ run=$(jq -c --arg sha "$source_sha" --arg repo "$GITHUB_REPOSITORY" --arg path " jq -e '.status=="completed" and .conclusion=="success" and (.id|type=="number" and .>0 and floor==.) and (.run_attempt|type=="number" and .>0 and floor==.)' <<<"$run" >/dev/null || { echo 'Latest image-producing workflow did not succeed' >&2; exit 1; } run_id=$(jq -r .id <<<"$run") attempt=$(jq -r .run_attempt <<<"$run") -gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/attempts/$attempt/jobs?per_page=100" >"$destination/jobs.json" +printf 'Read tested %s image for source %s from run %s attempt %s.\n' "$product" "$source_sha" "$run_id" "$attempt" +read_github producer-jobs "$destination/jobs.json" gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/attempts/$attempt/jobs?per_page=100" job=$(jq -er --arg name "$job_name" --argjson run "$run_id" --argjson attempt "$attempt" ' [.[].jobs[]?|select(.name==$name and .run_id==$run and .run_attempt==$attempt and .status=="completed" and .conclusion=="success")] | if length==1 then .[0].id else error("missing/ambiguous successful image producer") end' "$destination/jobs.json") name="tested-image-$product-$source_sha-$attempt" -gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" >"$destination/artifacts.json" +read_github artifact-list "$destination/artifacts.json" gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/artifacts?per_page=100" # A partial listing must not authorize the optional publisher to rebuild. # Count every page before deciding whether the retained image is absent. jq -e ' @@ -42,7 +69,8 @@ jq -e ' count=$(jq --arg name "$name" '[.[].artifacts[]|select(.name==$name and .expired==false)]|length' "$destination/artifacts.json") [[ $count != 0 ]] || missing [[ $count == 1 ]] || { echo 'Ambiguous retained tested image artifact' >&2; exit 1; } -gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name "$name" --dir "$destination/bundle" +[[ ! -e $destination/bundle && ! -L $destination/bundle ]] || exit 1 +read_github artifact-download "$retry_work/download-output" download_bundle manifest="$destination/bundle/image.json" [[ -f $manifest && ! -L $manifest && -f $destination/bundle/image.tar && ! -L $destination/bundle/image.tar ]] || exit 1 jq -e --arg sha "$source_sha" --arg run "$run_id" --arg attempt "$attempt" --arg image "$product" --argjson job "$job" --arg name "$job_name" ' @@ -59,7 +87,7 @@ docker load --input "$destination/bundle/image.tar" >/dev/null [[ $(docker image inspect --format '{{.Id}}' "$image_id") == "$image_id" ]] [[ $(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image_id") == "$source_sha" ]] [[ $(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image_id") == linux/amd64 ]] -gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" >"$destination/final-run.json" +read_github final-run "$destination/final-run.json" gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id" jq -e --arg sha "$source_sha" --arg repo "$GITHUB_REPOSITORY" --argjson run "$run_id" --argjson attempt "$attempt" ' .id==$run and .run_attempt==$attempt and .head_sha==$sha and .head_branch=="main" and .head_repository.full_name==$repo and .status=="completed" and .conclusion=="success"' "$destination/final-run.json" >/dev/null diff --git a/.github/scripts/test-tested-image.sh b/.github/scripts/test-tested-image.sh index c0f8c17c1..207d3104a 100644 --- a/.github/scripts/test-tested-image.sh +++ b/.github/scripts/test-tested-image.sh @@ -16,6 +16,27 @@ printf '%s\n' '[{"jobs":[{"id":7,"name":"Container Security - Image Scan (hft-co cat >"$scratch/bin/gh" <<'MOCK' #!/usr/bin/env bash set -euo pipefail +stage=unknown +case "$*" in + api*security-enabled.yml/runs*|api*ci.yml/runs*) stage=workflow-runs ;; + api*42/attempts/2/jobs*) stage=producer-jobs ;; + api*actions/runs/42) stage=final-run ;; + api*42/artifacts*) stage=artifact-list ;; + 'run download 42 '*) stage=artifact-download ;; +esac +if [[ ${TEST_READ_STAGE:-} == "$stage" ]]; then + count=0 + [[ ! -f $FIXTURE_ROOT/retry-count ]] || count=$(cat "$FIXTURE_ROOT/retry-count") + count=$((count+1)); printf '%s\n' "$count" >"$FIXTURE_ROOT/retry-count" + if (( count <= ${TEST_FAIL_COUNT:-1} )); then + if [[ $stage == artifact-download ]]; then + mkdir -p "${!#}"; printf partial >"${!#}/partial-only" + fi + printf '{"partial":' + printf 'gh: simulated error (HTTP %s)\n' "${TEST_HTTP_STATUS:-502}" >&2 + exit "${TEST_EXIT_STATUS:-1}" + fi +fi case "$*" in api*security-enabled.yml/runs*|api*ci.yml/runs*) cat "$FIXTURE_ROOT/runs.json" ;; api*42/attempts/2/jobs*) cat "$FIXTURE_ROOT/jobs.json" ;; @@ -41,10 +62,29 @@ case "$*" in *) exit 92 ;; esac MOCK +printf '#!/usr/bin/env bash\nexit 0\n' >"$scratch/bin/sleep" chmod +x "$scratch/bin/"* export PATH="$scratch/bin:$PATH" bash "$root/read-tested-image.sh" "$TEST_SOURCE" "$scratch/good" [[ $(cat "$scratch/good/image-id.txt") == "$TEST_IMAGE" ]] +# Transient failures at every read boundary restart the read with clean bytes. +for stage in workflow-runs producer-jobs artifact-list artifact-download final-run; do + rm -f "$FIXTURE_ROOT/retry-count" + TEST_READ_STAGE=$stage TEST_FAIL_COUNT=2 bash "$root/read-tested-image.sh" "$TEST_SOURCE" "$scratch/retry-$stage" >"$scratch/retry-$stage.log" 2>&1 + [[ $(cat "$FIXTURE_ROOT/retry-count") == 3 ]] + [[ $(cat "$scratch/retry-$stage/image-id.txt") == "$TEST_IMAGE" ]] + grep -Fq "GitHub read $stage failed (attempt 2/3" "$scratch/retry-$stage.log" +done +for code in 500 502 503 504 401 403 404; do + rm -f "$FIXTURE_ROOT/retry-count" + status=0 + TEST_READ_STAGE=artifact-list TEST_HTTP_STATUS=$code TEST_FAIL_COUNT=9 TEST_EXIT_STATUS=75 \ + bash "$root/read-tested-image.sh" "$TEST_SOURCE" "$scratch/exhaust-$code" optional >"$scratch/exhaust-$code.log" 2>&1 || status=$? + [[ $status == 1 && ! -e $scratch/exhaust-$code/image-id.txt && ! -e $scratch/exhaust-$code/artifacts.json ]] + expected=1; [[ $code != 5* ]] || expected=3 + [[ $(cat "$FIXTURE_ROOT/retry-count") == "$expected" ]] +done +printf 'PASS: bounded transient GET/download retries discard partial bytes; auth, absence and exhausted transport failures cannot authorize rebuild\n' # A retained exact-source image after page 1 must be reused. Missing evidence # alone permits the explicit manual path to build; incomplete evidence does not. cp "$FIXTURE_ROOT/artifacts.json" "$scratch/valid-artifacts.json"