diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 36c12353..e062d8c9 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -2520,14 +2520,46 @@ cmd_rollback() { local target target="$(_gh_tag_commit "$host" "$agent/$to")" [ -z "$target" ] && { echo "::error::release tag $agent/$to not found on $host" >&2; return 1; } - echo "rolling back $agent/$ring -> $to (${target:0:12}) on $host" + # Resolve the RING tag with fallback (major-scoped-channels epic #657 F4): PREFER the + # v-scoped `/v-` within the agent's current major line when it exists, + # else the legacy bare `/`. Moving the bare tag on a v-scoped fleet relocated + # a tag no caller pins and still exited 0 — a silent no-op on the emergency lever (#1103). + # First check if the v-scoped tier exists; if not, fall back to bare. + # (The `--to` release tag above is always the bare immutable `/vX.Y.Z`, so it is + # correctly resolved without a major.) + local ring_tag ring_commit + IFS=$'\t' read -r ring_tag ring_commit < <(_resolved_channel "$agent" "$ring") + # If the resolved tag (v-scoped or bare from _resolved_channel's logic) doesn't exist, + # and we got v-scoped, try the bare tag as a fallback. The agent may have released v.*.* + # but not yet seeded channel tier tags, so bare still matters as an intermediate step. + if [ -z "$ring_commit" ] && [[ "$ring_tag" == *"/v"[0-9]* ]]; then + # Constructed ring_tag was v-scoped and doesn't exist; try bare. + local bare_commit bare_tag + bare_tag="$agent/$ring" + bare_commit="$(_channel_tag_commit "$agent" "$ring")" + if [ -n "$bare_commit" ]; then + ring_tag="$bare_tag" + ring_commit="$bare_commit" + else + # Both v-scoped and bare are absent; report the bare tag in the error message. + ring_tag="$bare_tag" + fi + fi + # Fail loudly rather than no-op (#1103): if BOTH v-scoped and bare (if tried) don't exist, + # refuse to create-and-move a channel tag nobody pins. The worst failure mode for rollback + # is a silent no-op that exits 0, so make it loud. + if [ -z "$ring_commit" ]; then + echo "::error::rollback target ring tag $ring_tag does not exist on $host — refusing to create-and-move a channel tag nobody pins (that would be a silent no-op). Check the ring name and the agent's major line." >&2 + return 1 + fi + echo "rolling back $ring_tag -> $to (${target:0:12}) on $host" if [ "$dry" = true ]; then - echo "[DRY-RUN] would: gh api PATCH repos/$host/git/refs/tags/$agent/$ring sha=$target (force)" + echo "[DRY-RUN] would: gh api PATCH repos/$host/git/refs/tags/$ring_tag sha=$target (force)" return 0 fi - _gh_move_tag "$host" "$agent/$ring" "$target" \ - || { echo "::error::failed to move $agent/$ring -> $to on $host" >&2; return 1; } - echo "rolled back $agent/$ring -> $to" + _gh_move_tag "$host" "$ring_tag" "$target" \ + || { echo "::error::failed to move $ring_tag -> $to on $host" >&2; return 1; } + echo "rolled back $ring_tag -> $to" } # ── blocker-issue + fleet-status automation (auto-triage of held promotions) ──── diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index 7b095050..0a0e2687 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1205,6 +1205,92 @@ GITEOF ! grep -Ev '^[[:space:]]*#' "$ORCH" | grep -Eq '\bgit[[:space:]]+(tag|push)\b' } +# ── orchestrator: rollback resolves the ring tag v-scope-aware, like promote (#1103) ── +# cmd_rollback used to move the BARE / tag regardless of the agent's major +# line. On a v-scoped fleet (all stubs pin /v-) that moved a tag nobody +# reads and still exited 0 — a silent no-op on the emergency lever. It must resolve the +# ring tag the SAME way cmd_promote/_resolved_channel do: prefer /v-. + +@test "orchestrator: rollback targets the v-scoped ring tag on a v-scoped agent, not the bare tier (#1103)" { + # The real shape from the issue: dev-lead on major 139, the v139-* family present. + # `rollback dev-lead stable --to v139.8.0` MUST target dev-lead/v139-stable — the tag the + # fleet stubs actually pin — never the bare dev-lead/stable that nobody reads. + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v139-next\nrefs/tags/dev-lead/v139.8.0\n' ;; # channel anchor + latest release + *"git/ref/tags/dev-lead/v139-next"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # channel anchor exists → major=139 + *"git/ref/tags/dev-lead/v139-stable"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # the pinned channel tag + *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # the release --to target + *"git/ref/tags/dev-lead/stable"*) echo "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef commit" ;; # bare tier — must NOT be targeted + *) echo "" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + # GITHUB_REPOSITORY=.github forces THIS_REPO=.github → dev-lead (host=.github-private) is cross-repo. + run env GITHUB_REPOSITORY="combsmith/.github" CANARY_RINGS="$RINGS" \ + bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"rolling back dev-lead/v139-stable"* ]] + [[ "$output" == *"refs/tags/dev-lead/v139-stable"* ]] + # The exact defect: the bare ring tag must never be the PATCH target. + [[ "$output" != *"refs/tags/dev-lead/stable"* ]] +} + +@test "orchestrator: rollback fails loudly when the resolved ring tag does not exist (no silent no-op) (#1103)" { + # dev-lead on major 139, release v139.8.0 present, but NEITHER the v-scoped channel tag + # v139-stable NOR the bare stable exists. Rollback must NOT create-and-move a tag nobody + # pins and exit 0 — it must fail with a clear error. + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v139-next\nrefs/tags/dev-lead/v139.8.0\n' ;; # channel anchor + release + *"git/ref/tags/dev-lead/v139-next"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # channel anchor exists → major=139 + *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # release --to resolves + *) echo "" ;; # every channel tag is absent +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env GITHUB_REPOSITORY="combsmith/.github" CANARY_RINGS="$RINGS" \ + bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run + [ "$status" -eq 1 ] + [[ "$output" == *"::error::"* ]] + # The resolution falls back to the bare tier (v-scoped absent), so the loud error names the + # resolved-but-absent tag and refuses — never a "[DRY-RUN] would … (force)" success line. + [[ "$output" == *"dev-lead/stable does not exist"* ]] + [[ "$output" != *"DRY-RUN"* ]] +} + +@test "orchestrator: rollback falls back to the bare ring tag when no v-scoped channel tag exists (#1103)" { + # Pre-F4 shape: releases exist (so a current major is derived) but the CHANNEL tags are + # still bare — v139-stable is absent, dev-lead/stable is what the fleet pins. Rollback must + # then move the bare tag, byte-identical to _resolved_channel's fall-back (matches promote). + _make_stub_bin + cat > "$STUB_BIN/gh" <<'GHEOF' +#!/usr/bin/env bash +case "$*" in + *"matching-refs/tags/dev-lead/v"*) printf 'refs/tags/dev-lead/v139-next\nrefs/tags/dev-lead/v139.8.0\n' ;; # channel anchor + release + *"git/ref/tags/dev-lead/v139-next"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # channel anchor exists → major=139 + *"git/ref/tags/dev-lead/v139.8.0"*) echo "90dcc01ca854d7acd65044bb8defcc32789a8335 commit" ;; # release --to target + *"git/ref/tags/dev-lead/v139-stable"*) echo "" ;; # v-scoped channel tag ABSENT + *"git/ref/tags/dev-lead/stable"*) echo "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef commit" ;; # bare channel tag present + *) echo "" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + + run env GITHUB_REPOSITORY="combsmith/.github" CANARY_RINGS="$RINGS" \ + bash "$ORCH" rollback dev-lead stable --to v139.8.0 --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"rolling back dev-lead/stable"* ]] + [[ "$output" == *"refs/tags/dev-lead/stable"* ]] + [[ "$output" != *"v139-stable"* ]] +} + # ── orchestrator: full graduated verdicts (cut date + gh run data → gate state) ─ # Lay out next = candidate (cccc); ring0/ring1/stable = prior (bbbb): frontier = ring0, # transition next->ring0, source = next. The release tag cccc is dated `cut_days` ago,