From b400f6bbfad016d4c80682f1d9ac1382101dad86 Mon Sep 17 00:00:00 2001 From: Linwei Shang Date: Sat, 4 Jul 2026 22:10:17 -0400 Subject: [PATCH 1/2] ci: make crates.io publish idempotent and retry index lag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The publish job ran six `cargo publish` commands unguarded under `set -e`. Two failure modes broke it: 1. Partial release: if a later crate fails, the already-uploaded crates can't be re-published — a re-dispatch aborts immediately on the first `cargo publish` ("crate version already uploaded"). 2. Index lag: a dependent crate (e.g. icx -> ic-utils) can fail to resolve a just-uploaded dependency until the crates.io index catches up, which can take a minute or two. Both hit the 0.48.0 release: ic-transport-types, ic-agent, ic-identity-hsm, and ic-utils published, then `icx` failed to resolve `ic-utils = ^0.48.0` because the index hadn't surfaced it yet. Wrap each publish in a helper that treats "already uploaded/exists" as success (idempotent re-runs) and retries other failures up to 6x with a 30s backoff (rides out index lag). Re-dispatching now finishes icx and icx-cert without re-publishing the four crates already live. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish.yml | 37 +++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cf64b086..821e6dbf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,11 +11,36 @@ jobs: - uses: rust-lang/crates-io-auth-action@b7e9a28eded4986ec6b1fa40eeee8f8f165559ec # v1 id: auth - run: | - cargo publish -p ic-transport-types - cargo publish -p ic-agent - cargo publish -p ic-identity-hsm - cargo publish -p ic-utils - cargo publish -p icx - cargo publish -p icx-cert + set -euo pipefail + + # Publish one crate, tolerating the two things that make a naive + # `cargo publish` loop brittle: + # 1. Re-running after a partial release: a crate already uploaded at + # this version is a success, not a failure (lets a re-dispatch + # finish the crates that didn't make it the first time). + # 2. crates.io index lag: a dependent crate can fail to resolve a + # just-uploaded dependency for up to a minute or two. Retry. + publish() { + local crate="$1" attempt + for attempt in $(seq 1 6); do + if cargo publish -p "$crate" 2>&1 | tee /tmp/publish.log; then + echo "== $crate published" + return 0 + fi + if grep -qiE "already (uploaded|exists)" /tmp/publish.log; then + echo "== $crate already published at this version; skipping" + return 0 + fi + echo "== $crate publish attempt $attempt failed (likely dependency index lag); retrying in 30s" + sleep 30 + done + echo "== ERROR: $crate failed to publish after retries" >&2 + return 1 + } + + # Order matters: dependencies before dependents. + for crate in ic-transport-types ic-agent ic-identity-hsm ic-utils icx icx-cert; do + publish "$crate" + done env: CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }} From 72f77109e9d909649ad98af595d4d2c32182ea2d Mon Sep 17 00:00:00 2001 From: Linwei Shang Date: Sat, 4 Jul 2026 22:20:23 -0400 Subject: [PATCH 2/2] ci: skip final-attempt sleep in publish retry loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't sleep 30s after the last attempt fails — it only delayed the job's failure. Gate the backoff on there being another attempt to come, and include attempt count in the messages. Addresses Copilot review feedback on #735. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 821e6dbf..66d32a7f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,8 +21,8 @@ jobs: # 2. crates.io index lag: a dependent crate can fail to resolve a # just-uploaded dependency for up to a minute or two. Retry. publish() { - local crate="$1" attempt - for attempt in $(seq 1 6); do + local crate="$1" attempt max=6 + for attempt in $(seq 1 "$max"); do if cargo publish -p "$crate" 2>&1 | tee /tmp/publish.log; then echo "== $crate published" return 0 @@ -31,10 +31,12 @@ jobs: echo "== $crate already published at this version; skipping" return 0 fi - echo "== $crate publish attempt $attempt failed (likely dependency index lag); retrying in 30s" - sleep 30 + if [ "$attempt" -lt "$max" ]; then + echo "== $crate publish attempt $attempt/$max failed (likely dependency index lag); retrying in 30s" + sleep 30 + fi done - echo "== ERROR: $crate failed to publish after retries" >&2 + echo "== ERROR: $crate failed to publish after $max attempts" >&2 return 1 }