diff --git a/.changes/work-items/fix-work-items-release-outranked-hand-back-dac20e43.md b/.changes/work-items/fix-work-items-release-outranked-hand-back-dac20e43.md new file mode 100644 index 00000000000..145d302cc41 --- /dev/null +++ b/.changes/work-items/fix-work-items-release-outranked-hand-back-dac20e43.md @@ -0,0 +1,8 @@ +--- +bump: patch +--- + +### Fixed + +- The GitHub adapter's `release` of a live lease that an earlier live lease outranks backs off as `renew-lease` does before it exits `7`: it tries to supersede its own lease twice and hands the assignee back once one try succeeds, else keeps the assignee and says so. Before, it wrote nothing, so the lease and its login stayed until the TTL ran out. +- The assignee hand-back judges each lease list attempt at a clock read taken just before it. Before, it read the clock after the list, so a same-login lease renewed meanwhile could look expired and its holder was unassigned. diff --git a/plugins/work-items/tools/work-item-tracker/CONTRACT.md b/plugins/work-items/tools/work-item-tracker/CONTRACT.md index 05cb510d4da..6a389ef0f95 100644 --- a/plugins/work-items/tools/work-item-tracker/CONTRACT.md +++ b/plugins/work-items/tools/work-item-tracker/CONTRACT.md @@ -749,7 +749,11 @@ Claim sequence (race-safe, same-identity aware): comment, the foreign lease wins → supersede own comment, exit `7`. On that loss, on the step 2 back-off, and on any failure from step 1 until the emit, the GitHub adapter runs the **hand-back** after superseding its own lease. Each of its lease lists retries once before it - counts as failed. It lists the leases (a failed list counts as empty) and stops if one shows a + counts as failed, and each attempt counts a lease live when it is live at a clock read taken + just before the list or at one taken after it: a renew only moves an expiry later, so a + renewing holder never looks expired at the earlier read, a holder whose clock runs up to the + skew bound ahead and renews during the list is inside that bound at the later read, and a + lease that expires during the list still counts as live, so its login stays for reclaim. It lists the leases (a failed list counts as empty) and stops if one shows a live lease of the login other than its own; else it reads the assignees and stops only if that read succeeds without the login; else it removes the login, re-lists the leases (stopping on a failed list), and assigns the login again if a live lease of it other than its own has @@ -779,10 +783,13 @@ that is all of: on the addressed item, held by the authenticated login (or the ` which, like `claim`'s, replaces the `gh api user` lookup and is asserted by the caller; the comment edit still needs the token's write access), and, for a live lease, not outranked by an earlier live lease under claim step 4 (the completed-run rule included); a newer live lease is a claimant -that has not backed off yet and does not stop the release. Anything else is refused with exit `7` -and no lease is written: a cross-item handle, another login's lease, a live handle an earlier -live lease outranks, or an expired handle under a newer non-superseded lease. Only that last -refusal edits anything: it runs the hand-back below, since its lease no longer holds the item. The login check +that has not backed off yet and does not stop the release. Anything else is refused with exit `7`: +a cross-item handle, another login's lease, a live handle an earlier live lease outranks, or an +expired handle under a newer non-superseded lease. The first two write nothing. An outranked live +handle backs off as `renew-lease`'s back-off does: it tries the supersede twice and runs the +hand-back below only after one succeeded, since a lease it could not supersede may still be live, +so the holder stays assigned and the message says so. An expired handle under a newer lease runs +the hand-back alone, since its lease no longer holds the item. The login check exists because lease handles are public (a GitHub comment id), and within one login the handle tells sessions apart, as for `renew-lease`. An already-superseded or expired lease is an idempotent no-op (`released: false`, exit `0`), because it already blocks nobody; an diff --git a/plugins/work-items/tools/work-item-tracker/adapters/github/common.sh b/plugins/work-items/tools/work-item-tracker/adapters/github/common.sh index bcae2da78ba..2c107a2d9e1 100755 --- a/plugins/work-items/tools/work-item-tracker/adapters/github/common.sh +++ b/plugins/work-items/tools/work-item-tracker/adapters/github/common.sh @@ -504,35 +504,40 @@ wit_lease_released_by_run() { # leave a holder unassigned (CONTRACT.md "Lease protocol"). A failed assignee read # still removes, since an assignee that no lease holds keeps the item off the frontier for good. wit_hand_back_assignee() { - local owner="$1" repo="$2" number="$3" login="$4" own="$5" leases - leases="$(wit_list_lease_comments "$owner" "$repo" "$number")" || - leases="$(wit_list_lease_comments "$owner" "$repo" "$number")" || leases='[]' - _wit_login_holds_other "$leases" "$login" "$own" && return 0 + local owner="$1" repo="$2" number="$3" login="$4" own="$5" leases now + _wit_list_at_clock "$owner" "$repo" "$number" || _wit_list_at_clock "$owner" "$repo" "$number" || leases='[]' + _wit_login_holds_other "$leases" "$login" "$own" "$now" && return 0 if wit_try_read_assignees "$owner" "$repo" "$number"; then jq -e --arg l "$login" 'any(.[]; . == $l)' <<<"$WIT_GH_OUT" >/dev/null || return 0 fi wit_try_remove_assignee "$owner" "$repo" "$number" "$login" - leases="$(wit_list_lease_comments "$owner" "$repo" "$number")" || - leases="$(wit_list_lease_comments "$owner" "$repo" "$number")" || return 0 - _wit_login_holds_other "$leases" "$login" "$own" || return 0 + _wit_list_at_clock "$owner" "$repo" "$number" || _wit_list_at_clock "$owner" "$repo" "$number" || return 0 + _wit_login_holds_other "$leases" "$login" "$own" "$now" || return 0 wit_try_add_assignee "$owner" "$repo" "$number" "$login" # The lease that earned the re-add may have ended meanwhile (a renew that revived it for a # moment backs off and finds the login already gone), so the re-add must not outlive it. - leases="$(wit_list_lease_comments "$owner" "$repo" "$number")" || - leases="$(wit_list_lease_comments "$owner" "$repo" "$number")" || return 0 - _wit_login_holds_other "$leases" "$login" "$own" || wit_try_remove_assignee "$owner" "$repo" "$number" "$login" + _wit_list_at_clock "$owner" "$repo" "$number" || _wit_list_at_clock "$owner" "$repo" "$number" || return 0 + _wit_login_holds_other "$leases" "$login" "$own" "$now" || wit_try_remove_assignee "$owner" "$repo" "$number" "$login" } -# _wit_login_holds_other : 0 when a live lease other than -# is held by . -_wit_login_holds_other() { - local row cand now +# _wit_list_at_clock : set the caller's now to a clock read, then its +# leases to a lease list taken after it. A renew only moves an expiry later, so judging the +# listed bodies at that earlier read never makes a renewing holder look expired; a lease that +# expires during the list still counts as live, and its login then stays for reclaim. +_wit_list_at_clock() { now="$(date -u +%s)" + leases="$(wit_list_lease_comments "$1" "$2" "$3")" +} + +# _wit_login_holds_other : 0 when a live lease +# other than is held by at or at the current clock. +_wit_login_holds_other() { + local row cand now="$4" while IFS= read -r row; do [[ -n "$row" && "$(jq -r '.id' <<<"$row")" != "$3" ]] || continue cand="$(wit_lease_json "$(jq -r '.body' <<<"$row")")" [[ -n "$cand" && "$(jq -r '.holder // empty' <<<"$cand")" == "$2" ]] || continue - wit_lease_is_live "$cand" "$now" && return 0 + { wit_lease_is_live "$cand" "$now" || wit_lease_is_live "$cand" "$(date -u +%s)"; } && return 0 done < <(jq -c '.[]' <<<"$1") return 1 } diff --git a/plugins/work-items/tools/work-item-tracker/adapters/github/lease-coordination.test.sh b/plugins/work-items/tools/work-item-tracker/adapters/github/lease-coordination.test.sh index b6bbc0f0ea2..a7cd760ee82 100755 --- a/plugins/work-items/tools/work-item-tracker/adapters/github/lease-coordination.test.sh +++ b/plugins/work-items/tools/work-item-tracker/adapters/github/lease-coordination.test.sh @@ -414,8 +414,8 @@ assert_not_contains "release does not supersede a foreign lease" "$(calls)" "PAT cleanup_scenario # [outranked] two live leases: the earlier one holds the item (claim step 4) and the -# newer one is a claimant that has not backed off. The newer handle is refused; the -# earlier one is released. +# newer one is a claimant that has not backed off. The newer handle is refused and +# superseded; the earlier one is released. two_live_leases() { jq -cn --arg o "$(marker alice "$FUTURE" 24)" --arg n "$(marker alice "$FUTURE" 24)" \ '[{id:100, node_id:"MDEx", body:$o, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}, @@ -428,7 +428,8 @@ two_live_leases rc=0 bash "$RELEASE" "$ID" --lease-comment-id 200 >/dev/null 2>&1 || rc=$? assert_eq "release refuses (7) a handle outranked by an earlier live lease" "7" "$rc" -assert_not_contains "release writes nothing for an outranked handle" "$(calls)" "PATCH" +assert_contains "release supersedes an outranked handle" "$(calls)" "PATCH 200" +assert_not_contains "release leaves the earlier live lease alone" "$(calls)" "PATCH 100" cleanup_scenario new_scenario printf 'alice\n' >"$GH_STUB_DIR/login" @@ -1117,15 +1118,48 @@ refused_release_case() { refused_release_case "a refused release under a lane lease unassigns alice" "$LANE" '[]' refused_release_case "a refused release under alice's own newer lease keeps her" alice '["alice"]' -# [release clock] the lease expires between two clock reads; release decides on one read, -# so it releases the lease that was live at that read. +# [outranked live release] alice's live lease 2 is outranked by an earlier live lease 1: the +# release is refused (7), supersedes lease 2 and hands back alice unless lease 1 is alice's own. +outranked_release_case() { + new_scenario + : >"$GH_STUB_DIR/stateful" + printf 'alice\n' >"$GH_STUB_DIR/login" + printf '%s\n' "$3" >"$GH_STUB_DIR/assignees" + jq -cn --arg a "$(marker "$2" "$FUTURE" 24)" --arg b "$(marker alice "$FUTURE" 24)" \ + '[{id:1, node_id:"MDEx", body:$a, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}, + {id:2, node_id:"MDEx", body:$b, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}]' >"$GH_STUB_DIR/lease-comments" + rc=0 + bash "$RELEASE" "$ID" --lease-comment-id 2 >/dev/null 2>&1 || rc=$? + assert_eq "$1: exit" "7" "$rc" + assert_eq "$1: own lease 2 superseded" "true" "$(superseded 2)" + assert_eq "$1: earlier lease 1 left live" "false" "$(superseded 1)" + assert_eq "$1: assignees" "$4" "$(assignees_now)" + cleanup_scenario +} +outranked_release_case "an outranked release under bob's earlier lease unassigns alice" bob '["bob","alice"]' '["bob"]' +outranked_release_case "an outranked release under alice's own earlier lease keeps her" alice '["alice"]' '["alice"]' + +# [outranked release, supersede fails twice] lease 2 may still be live, so alice stays. new_scenario : >"$GH_STUB_DIR/stateful" printf 'alice\n' >"$GH_STUB_DIR/login" -printf '["alice"]\n' >"$GH_STUB_DIR/assignees" +printf '["bob","alice"]\n' >"$GH_STUB_DIR/assignees" +jq -cn --arg a "$(marker bob "$FUTURE" 24)" --arg b "$(marker alice "$FUTURE" 24)" \ + '[{id:1, node_id:"MDEx", body:$a, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}, + {id:2, node_id:"MDEx", body:$b, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}]' >"$GH_STUB_DIR/lease-comments" +: >"$GH_STUB_DIR/patch-lost-1" +: >"$GH_STUB_DIR/patch-lost-2" +rc=0 +err="$(bash "$RELEASE" "$ID" --lease-comment-id 2 2>&1 >/dev/null)" || rc=$? +assert_eq "an outranked release whose supersede fails twice exits conflict (7)" "7" "$rc" +assert_eq "that release tries the supersede twice" "2" "$(grep -c '^PATCH 2' "$GH_STUB_DIR/calls.log")" +assert_eq "lease 2 is not superseded" "false" "$(superseded 2)" +assert_not_contains "that release never removes alice" "$(calls)" "REMOVE_ASSIGNEE alice" +assert_eq "alice stays assigned for lease 2" '["bob","alice"]' "$(assignees_now)" +assert_contains "the refusal says the assignee is kept" "$err" "could not supersede the lease, assignee kept" +cleanup_scenario + T0=1783828800 -lease_array 1 "$(marker alice "2026-07-12T04:00:00Z" 1)" >"$GH_STUB_DIR/lease-comments" -printf '%s\n' "$((T0 + 3599))" "$((T0 + 3600))" >"$GH_STUB_DIR/clock" # date: `date -u +%s` answers the next line of $GH_STUB_DIR/clock, repeating the last. # shellcheck disable=SC2329 # exported to the release.sh child date() { @@ -1138,6 +1172,89 @@ date() { fi } export -f date + +# [hand-back clock] releasing lease 1, the hand-back lists alice's lease 2 (old expiry +# T0+3600); a same-login renew then extends it and the clock passes that old expiry before +# the judgment. Lease 2 was live at a clock read before the list, so alice is never removed. +new_scenario +: >"$GH_STUB_DIR/stateful" +printf 'alice\n' >"$GH_STUB_DIR/login" +printf '["alice"]\n' >"$GH_STUB_DIR/assignees" +jq -cn --arg a "$(marker alice "2026-07-12T04:00:00Z" 24)" --arg b "$(marker alice "2026-07-12T04:00:00Z" 1)" \ + '[{id:1, node_id:"MDEx", body:$a, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}, + {id:2, node_id:"MDEx", body:$b, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}]' >"$GH_STUB_DIR/lease-comments" +printf '%s\n' "$((T0 + 10))" >"$GH_STUB_DIR/clock" +h="$(next_read list 2)-after" +# shellcheck disable=SC2016 # expanded by the hook's bash +hook "$h" bash -c 'put_lease 2 "$1" && printf "%s\n" "$2" >"$GH_STUB_DIR/clock"' _ \ + "$(marker alice "2026-07-12T05:00:00Z" 1)" "$((T0 + 3600))" +rc=0 +bash "$RELEASE" "$ID" --lease-comment-id 1 >/dev/null 2>&1 || rc=$? +assert_eq "the release of lease 1 succeeds (0)" "0" "$rc" +assert_eq "the renew landed after the hand-back's list" "0" "$(hook_rc "$h")" +assert_not_contains "the hand-back never removes alice for renewed lease 2" "$(calls)" "REMOVE_ASSIGNEE alice" +assert_eq "alice stays assigned" '["alice"]' "$(assignees_now)" +cleanup_scenario + +# [hand-back clock, later lists] releasing lease 1, the hand-back's first list shows no +# lease 2, so it removes alice; lease 2 (old expiry T0+3600) is in every later list. On the +# named list a renew extends lease 2 and the clock passes its old expiry before the judgment; +# judged at a clock read before that list, alice ends assigned. +handback_clock_case() { + new_scenario + : >"$GH_STUB_DIR/stateful" + printf 'alice\n' >"$GH_STUB_DIR/login" + printf '["alice"]\n' >"$GH_STUB_DIR/assignees" + jq -cn --arg a "$(marker alice "2026-07-12T04:00:00Z" 24)" --arg b "$(marker alice "2026-07-12T04:00:00Z" 1)" \ + '[{id:1, node_id:"MDEx", body:$a, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}, + {id:2, node_id:"MDEx", body:$b, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}]' >"$GH_STUB_DIR/lease-comments" + lease_array 1 "$(superseded_marker alice)" >"$GH_STUB_DIR/lease-comments-2" + printf '%s\n' "$((T0 + 10))" >"$GH_STUB_DIR/clock" + # shellcheck disable=SC2016 # expanded by the hook's bash + hook "list-$2-after" bash -c 'put_lease 2 "$1" && printf "%s\n" "$2" >"$GH_STUB_DIR/clock"' _ \ + "$(marker alice "2026-07-12T05:00:00Z" 1)" "$((T0 + 3600))" + rc=0 + bash "$RELEASE" "$ID" --lease-comment-id 1 >/dev/null 2>&1 || rc=$? + assert_eq "$1: release exit" "0" "$rc" + assert_eq "$1: the renew landed after that list" "0" "$(hook_rc "list-$2-after")" + assert_eq "$1: alice ends assigned" '["alice"]' "$(assignees_now)" + cleanup_scenario +} +handback_clock_case "renew during the hand-back's re-list" 3 +handback_clock_case "renew during the hand-back's confirm list" 4 + +# [hand-back clock, fast holder] releasing lease 1, the hand-back reads its clock (T0+10), +# then a same-login holder whose clock runs 300 s fast renews lease 2 two seconds into the +# list (renewed_at T0+312). That is past the skew bound at the earlier read and inside it at +# a read after the list, so alice is never removed. +new_scenario +: >"$GH_STUB_DIR/stateful" +printf 'alice\n' >"$GH_STUB_DIR/login" +printf '["alice"]\n' >"$GH_STUB_DIR/assignees" +jq -cn --arg a "$(marker alice "2026-07-12T04:00:00Z" 24)" --arg b "$(marker alice "2026-07-12T04:00:00Z" 1)" \ + '[{id:1, node_id:"MDEx", body:$a, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}, + {id:2, node_id:"MDEx", body:$b, created_at:"2020-01-01T00:00:00Z", author_association:"MEMBER"}]' >"$GH_STUB_DIR/lease-comments" +printf '%s\n' "$((T0 + 10))" >"$GH_STUB_DIR/clock" +h="$(next_read list 2)" +# shellcheck disable=SC2016 # expanded by the hook's bash +hook "$h" bash -c 'put_lease 2 "$1" && printf "%s\n" "$2" >"$GH_STUB_DIR/clock"' _ \ + "$(marker alice "2026-07-12T04:05:12Z" 1)" "$((T0 + 12))" +rc=0 +bash "$RELEASE" "$ID" --lease-comment-id 1 >/dev/null 2>&1 || rc=$? +assert_eq "the release of lease 1 succeeds (0) beside a fast-clock holder" "0" "$rc" +assert_eq "the fast-clock renew landed during the hand-back's list" "0" "$(hook_rc "$h")" +assert_not_contains "the hand-back never removes alice for the fast-clock lease 2" "$(calls)" "REMOVE_ASSIGNEE alice" +assert_eq "alice stays assigned beside the fast-clock holder" '["alice"]' "$(assignees_now)" +cleanup_scenario + +# [release clock] the lease expires between two clock reads; release decides on one read, +# so it releases the lease that was live at that read. +new_scenario +: >"$GH_STUB_DIR/stateful" +printf 'alice\n' >"$GH_STUB_DIR/login" +printf '["alice"]\n' >"$GH_STUB_DIR/assignees" +lease_array 1 "$(marker alice "2026-07-12T04:00:00Z" 1)" >"$GH_STUB_DIR/lease-comments" +printf '%s\n' "$((T0 + 3599))" "$((T0 + 3600))" >"$GH_STUB_DIR/clock" rc=0 bash "$RELEASE" "$ID" --lease-comment-id 1 >/dev/null 2>&1 || rc=$? assert_eq "release of a lease live at its clock read succeeds (0)" "0" "$rc" diff --git a/plugins/work-items/tools/work-item-tracker/adapters/github/release.sh b/plugins/work-items/tools/work-item-tracker/adapters/github/release.sh index 5cbb0828467..7ccc61097a8 100755 --- a/plugins/work-items/tools/work-item-tracker/adapters/github/release.sh +++ b/plugins/work-items/tools/work-item-tracker/adapters/github/release.sh @@ -112,8 +112,15 @@ fi # that has not backed off yet and does not stop the holder's release. wit_lease_winner "$leases" "$now_epoch" "$lease_comment_id" wit_lease_released_by_run if [[ "$WIT_WINNER_ID" != "$lease_comment_id" ]]; then - printf 'release: comment %s is outranked by the earlier live lease, comment %s\n' \ - "$lease_comment_id" "${WIT_WINNER_ID:-none}" >&2 + outcome="lease superseded" + if wit_try_supersede_lease_comment "$owner" "$repo" "$lease_comment_id" "$lease_json" || + wit_try_supersede_lease_comment "$owner" "$repo" "$lease_comment_id" "$lease_json"; then + wit_hand_back_assignee "$owner" "$repo" "$number" "$login" "$lease_comment_id" + else + outcome="could not supersede the lease, assignee kept" + fi + printf 'release: comment %s is outranked by the earlier live lease, comment %s; %s\n' \ + "$lease_comment_id" "${WIT_WINNER_ID:-none}" "$outcome" >&2 exit "$EX_CONFLICT" fi