diff --git a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs index 886139a905..ce4e77b31a 100755 --- a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs +++ b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs @@ -176,11 +176,14 @@ // flake-only sets. Nothing else re-runs a lane run, while fix-ci does not wait // for a flake re-run. A lane run's listing does not name its PR, so lane runs // go oldest first among those whose creation matches an open head's newest -// checks completion (laneOrder), then the rest oldest first: a run for a -// closed PR, or a second run for one head, takes no place before a live one, -// and the newer runs each kill-switch trip adds no longer push the oldest -// live ones back pass after pass. One pass uploads at most MAX_IN_FLIGHT -// records, so what would pass that waits too. A pass that defers anything but +// checks completion (laneOrder), then the rest oldest first: a second run for +// one head takes no place before a live one, and the newer runs each +// kill-switch trip adds no longer push the oldest live ones back pass after +// pass. A lost lane run that follows no open head's newest checks +// completion is not re-run (laneOrder): it was created outside every open head's +// window, so a re-run would judge nothing live, and after a brake the backlog +// is mostly such runs. One pass uploads at most MAX_IN_FLIGHT records, so what +// would pass that waits too. A pass that defers anything but // a flake-only set writes no `clean`, so the pacer runs the next tick. // Writes are at least WRITE_GAP_MS apart. The first rate-limit error stops the // run. @@ -358,6 +361,12 @@ export const RECORDED_LANES = new Map([ [".github/workflows/pr-verify.yml", JUDGE_STEP], [".github/workflows/pr-review-main.yml", REVIEW_STEP], ]); +// The checks-run conclusions that start each recorded lane's work: the +// conclusions its jobs' `github.event.workflow_run.conclusion` test passes. +export const LANE_TRIGGERS = new Map([ + [".github/workflows/pr-verify.yml", new Set(["success", "failure"])], + [".github/workflows/pr-review-main.yml", new Set(["success"])], +]); // The recorded lanes whose workflow runs the upstream-status gate; pr-verify.yml // runs none, so no outage holds its re-run. export const GATED_LANES = new Set([".github/workflows/pr-review-main.yml"]); @@ -718,14 +727,34 @@ const byAge = (a, b) => Date.parse(a.created_at) - Date.parse(b.created_at) || a // log, and compare the run's created_at with that checks run's updated_at. const MATCH_BEFORE_MS = 60_000; const MATCH_AFTER_MS = 15 * 60_000; +// How long after an open head's newest checks completion a lost lane run may +// still be that head's and so be kept: eight times MATCH_AFTER_MS, a judgment +// margin for a queue backed up far past the measured pair. Without it, one +// open head left untouched for hours keeps every stale run queued. +const OWNED_AFTER_MS = 2 * 3_600_000; // When a head's newest completed pull_request checks run finished, the one // whose completion started the head's live lane runs. const checksDone = (own) => { - const done = own - .filter((r) => RECORDED_UPSTREAMS.has(r.path) && r.event === "pull_request" && r.status === "completed" && isTime(r.updated_at)) - .map((r) => Date.parse(r.updated_at)); - return done.length > 0 ? Math.max(...done) : undefined; + const done = own.filter((r) => RECORDED_UPSTREAMS.has(r.path) && r.event === "pull_request" && r.status === "completed" && isTime(r.updated_at)); + return done.length > 0 ? Math.max(...done.map((r) => Date.parse(r.updated_at))) : undefined; +}; + +// Per recorded lane, when the head's newest checks run that starts it +// (LANE_TRIGGERS) finished: a newer run that does not start the lane replaces +// none of its lost runs. +const laneStarts = (own) => { + const starts = new Map(); + for (const r of own) { + if (RECORDED_UPSTREAMS.has(r.path) && r.event === "pull_request" && r.status === "completed" && isTime(r.updated_at)) { + for (const [lanePath, conclusions] of LANE_TRIGGERS) { + if (conclusions.has(r.conclusion)) { + starts.set(lanePath, Math.max(starts.get(lanePath) ?? 0, Date.parse(r.updated_at))); + } + } + } + } + return starts; }; // The lane items in place order: those matched to an open head, oldest first, @@ -735,7 +764,16 @@ const checksDone = (own) => { // taken once, so a duplicate run for a head and a run whose PR is gone sort // last. The match sets the order only; check-trusted-trigger and resolve-pr // still bind the re-run to its PR. -function laneOrder(lanes, openChecks) { +// A lost lane run is `superseded` and left out when, for every open head, it +// was created more than MATCH_BEFORE_MS before the head's newest checks +// completion that starts its lane (laneStarts) or more than OWNED_AFTER_MS +// after it: a lane run starts after the checks completion it follows, so it +// follows no open head's current checks: its PR closed, its head moved, or its +// head's checks completed again and started a newer run of its lane. Nothing +// is left out unless `drop` (every open head's runs were read) holds and some +// open head has a completion that starts the lane; a fault-held run is never +// left out. +function laneOrder(lanes, openChecks, drop) { const pairs = []; for (const lane of lanes) { for (const { head, done } of openChecks) { @@ -754,7 +792,15 @@ function laneOrder(lanes, openChecks) { taken.add(key); } } - return [...lanes.filter((l) => matched.has(l)).sort(byAge), ...lanes.filter((l) => !matched.has(l)).sort(byAge)]; + const superseded = drop + ? lanes.filter((l) => { + const starts = openChecks.map((c) => c.starts.get(l.path)).filter((t) => t !== undefined); + const created = Date.parse(l.created_at); + return l.kind === "lane" && starts.length > 0 && !starts.some((t) => created >= t - MATCH_BEFORE_MS && created <= t + OWNED_AFTER_MS); + }) + : []; + const rest = lanes.filter((l) => !matched.has(l) && !superseded.includes(l)); + return { order: [...lanes.filter((l) => matched.has(l)).sort(byAge), ...rest.sort(byAge)], superseded }; } // A job whose steps did not list in full may have lost its runner, which cuts @@ -1030,6 +1076,8 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp // Each open head's newest checks completion, {head, done}, for laneOrder; // taken before any skip, so a busy head still matches its lane runs. const openChecks = []; + // False once a head's run listing failed before its completion was taken. + let everyHeadRead = true; // Heads whose checks passed, each with its earliest checks run's creation. const passed = []; // Heads with a pass whose spec may have changed since (`prefilterSpec`). @@ -1083,7 +1131,7 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp const own = listing.workflow_runs.filter((r) => r.head_repository?.full_name === repo); const done = checksDone(own); if (done !== undefined) { - openChecks.push({ head: pull.head.sha, done }); + openChecks.push({ head: pull.head.sha, done, starts: laneStarts(own) }); } const { rerun, again, lane, inFlight: busy } = classify(own, since); inFlight += busy ? 1 : 0; @@ -1267,6 +1315,7 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp ready.push(set.map(withRecord)); } } catch (error) { + everyHeadRead &&= openChecks.at(-1)?.head === pull.head.sha; noteFailure(`#${pull.number}`, error); } } @@ -1630,7 +1679,11 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp for (const set of ready.filter((s) => !flakeOnly(s))) { place(`#${set[0].pr}`, set); } - for (const lane of laneOrder(lanes, openChecks)) { + const { order, superseded } = laneOrder(lanes, openChecks, everyHeadRead); + for (const lane of superseded) { + say(`lane run ${lane.id} (${lane.path}) ended ${lane.conclusion}; it follows no open pull request's newest checks run, so it judges no open head (its PR closed, its head moved, or its head's checks completed again since); not re-run`); + } + for (const lane of order) { place(`lane run ${lane.id} (${lane.path})`, [lane]); } for (const set of ready.filter(flakeOnly)) { diff --git a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs index a51848809b..ee2dd8cadf 100644 --- a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs +++ b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs @@ -19,7 +19,7 @@ import { test } from "node:test"; import { createGitHub, GitHubError } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs"; import { createRateLimitLatch, RateLimitStop } from "../../actions/log-api-budget/rate-guard.mjs"; -import { CI_WORKFLOWS, FLOOR_NOTE, floorNoted, GITHUB_STATUS_URL, JUDGE_PREDECESSOR, JUDGE_STEP, KILL_SWITCH_NOTE, main, NPM_STATUS_URL, MISSING_REVIEW_NOTE, PENDING_BACKOFF_MINUTES, PENDING_REVIEW_NOTE, prefilterSpec, REVIEW_STEP, REVIEW_STOP_NOTE, REVIEW_STOP_STEP, REVIEW_SWITCHES, SETTLED_VERSION, TOO_OLD_DAYS, UPSTREAM_STOP_STEP, VERDICT_STOP_STEP, WRITE_STEP } from "./reconcile.mjs"; +import { CI_WORKFLOWS, FLOOR_NOTE, floorNoted, GITHUB_STATUS_URL, JUDGE_PREDECESSOR, JUDGE_STEP, KILL_SWITCH_NOTE, LANE_TRIGGERS, main, NPM_STATUS_URL, MISSING_REVIEW_NOTE, PENDING_BACKOFF_MINUTES, PENDING_REVIEW_NOTE, prefilterSpec, REVIEW_STEP, REVIEW_STOP_NOTE, REVIEW_STOP_STEP, REVIEW_SWITCHES, SETTLED_VERSION, TOO_OLD_DAYS, UPSTREAM_STOP_STEP, VERDICT_STOP_STEP, WRITE_STEP } from "./reconcile.mjs"; import { switchHold } from "../pr-merge/merge-holds.mjs"; import { tempDir } from "../../../lib/test-tmp.mjs"; @@ -1185,7 +1185,9 @@ test("a main-run review re-run shares the cap of 3 with pr-verify re-runs, oldes const doneChecks = (id, n, done) => run(id, sha(n), { conclusion: "success", created: "2026-10-19T16:50:00Z", updated: done }); const deferredLanes = (log) => log.mock.calls.map((c) => c.arguments[0]).filter((line) => / deferred, /.test(line)).map((line) => Number(/lane run ([0-9]+) /.exec(line)?.[1])); -test("lane runs of open heads go first, oldest first, ahead of an older run whose PR is gone and of newer runs", async (t) => { +const supersededLanes = (log) => log.mock.calls.map((c) => c.arguments[0]).filter((line) => /judges no open head/.test(line)).map((line) => Number(/lane run ([0-9]+) /.exec(line)?.[1])); + +test("lane runs of open heads go first, oldest first, ahead of newer runs, and a run older than every open head's checks is not re-run", async (t) => { const log = t.mock.method(console, "log", () => {}); const { code, reruns } = await reconcile({ pulls: [pr(1), pr(2)], @@ -1201,8 +1203,9 @@ test("lane runs of open heads go first, oldest first, ahead of an older run whos ], }); assert.equal(code, 0); - assert.deepEqual(reruns, [211, 212, 210], "the two matched runs, then the oldest unmatched one"); - assert.deepEqual(deferredLanes(log), [221, 222, 223], "every lane run not re-run is named as deferred"); + assert.deepEqual(reruns, [211, 212, 221], "the two matched runs, then the oldest unmatched one"); + assert.deepEqual(deferredLanes(log), [222, 223], "every live lane run not re-run is named as deferred"); + assert.deepEqual(supersededLanes(log), [210], "the run created an hour before the oldest open head's checks completed is named, not deferred"); }); test("a second lane run for one head goes behind every matched run, while each lane matches the head once", async (t) => { @@ -1225,9 +1228,10 @@ test("a second lane run for one head goes behind every matched run, while each l test("a lane run created more than 60 s before or 15 minutes after a head's checks completion is not matched", async () => { const { code, reruns } = await reconcile({ - pulls: [pr(1)], + pulls: [pr(1), pr(2)], runs: [ doneChecks(101, 1, "2026-10-19T17:00:00Z"), + doneChecks(102, 2, "2026-10-19T15:00:00Z"), verifyRun(240, { created: "2026-10-19T16:00:00Z" }), verifyRun(241, { created: "2026-10-19T16:58:59Z" }), verifyRun(242, { created: "2026-10-19T17:15:01Z" }), @@ -1237,7 +1241,118 @@ test("a lane run created more than 60 s before or 15 minutes after a head's chec assert.equal(code, 0); // 241 is 61 s before the completion and 242 is 15 min 1 s after; 244, at // 14 min 59 s after, is the one match, and the rest go oldest first. - assert.deepEqual(reruns, [244, 240, 241]); + assert.deepEqual(reruns, [244, 240, 241], "head 2's checks done at 15:00 match no run, and keep every run created after them"); +}); + +test("a lost lane run created before every open head's checks completed is not re-run, while the head's newest run is, and the pass writes clean", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { code, reruns, outputs } = await reconcile({ + pulls: [pr(1)], + runs: [ + doneChecks(101, 1, "2026-10-19T19:00:00Z"), + verifyRun(250, { created: "2026-10-19T17:00:04Z" }), + verifyRun(251, { created: "2026-10-19T16:30:00Z" }), + verifyRun(252, { created: "2026-10-19T19:00:04Z" }), + heldRun(253, { created: "2026-10-19T16:00:00Z" }), + ], + }); + assert.equal(code, 0); + assert.deepEqual(reruns, [252, 253], "the head's newest run, and the held run, which is never left out"); + assert.deepEqual(supersededLanes(log).sort(), [250, 251], "250 followed head 1's checks before they completed again at 19:00, and 251 follows no open head"); + assert.deepEqual(deferredLanes(log), []); + assert.equal(outputs.clean, "true", "a left-out run defers nothing"); +}); + +test("a lost lane run created from 60 s before to 2 hours after an open head's checks completed is re-run, outside the 15-minute match window too", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { reruns } = await reconcile({ + pulls: [pr(1)], + runs: [ + doneChecks(101, 1, "2026-10-19T17:00:00Z"), + verifyRun(260, { created: "2026-10-19T17:40:00Z" }), + verifyRun(261, { created: "2026-10-19T16:59:30Z" }), + verifyRun(262, { created: "2026-10-19T18:59:00Z" }), + verifyRun(263, { created: "2026-10-19T16:58:59Z" }), + verifyRun(264, { created: "2026-10-19T19:00:01Z" }), + ], + }); + assert.deepEqual(reruns, [261, 260, 262], "30 s before, 40 minutes after and 1 minute short of 2 hours after are kept"); + assert.deepEqual(supersededLanes(log).sort(), [263, 264], "61 s before and 2 hours 1 s after are left out"); +}); + +test("an open head untouched for hours keeps only the lost runs near its own checks, not every later stale run", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { reruns } = await reconcile({ + pulls: [pr(1), pr(2)], + runs: [ + doneChecks(101, 1, "2026-10-19T08:00:00Z"), + doneChecks(102, 2, "2026-10-19T19:00:00Z"), + verifyRun(265, { created: "2026-10-19T09:00:00Z" }), + verifyRun(266, { created: "2026-10-19T15:00:00Z" }), + ], + }); + assert.deepEqual(reruns, [265], "an hour after head 1's checks, so possibly head 1's"); + assert.deepEqual(supersededLanes(log), [266], "seven hours after head 1's checks and four before head 2's"); +}); + +test("no lost lane run is left out when an open head's runs could not be listed", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { reruns } = await reconcile({ + pulls: [pr(1), pr(2)], + runs: [doneChecks(101, 1, "2026-10-19T17:00:00Z"), verifyRun(270, { created: "2026-10-19T16:00:00Z" })], + fail: { [`GET /repos/${REPO}/actions/runs?head_sha=${sha(2)}&per_page=100`]: new GitHubError(500, "boom") }, + }); + assert.deepEqual(reruns, [270], "head 2's checks may have completed before 16:00"); + assert.deepEqual(supersededLanes(log), []); +}); + +test("a newer failed checks run leaves out a lost verify run, which it starts again, but keeps a lost main-run review, which only a passed one starts", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { reruns, outputs } = await reconcile({ + pulls: [pr(1)], + runs: [ + run(101, sha(1), { conclusion: "success", created: "2026-10-19T16:50:00Z", updated: "2026-10-19T17:00:00Z" }), + failed(102, sha(1), { jobs: MIXED_JOBS, updated: "2026-10-19T19:00:00Z" }), + mainReviewRun(330, { created: "2026-10-19T17:00:04Z" }), + verifyRun(331, { created: "2026-10-19T17:00:30Z" }), + ], + }); + assert.deepEqual(reruns, [330], "pr-review-main runs only on a passed checks run, so nothing replaces the 17:00:04 review"); + assert.deepEqual(supersededLanes(log), [331], "pr-verify runs on a failed checks run too, so the 19:00 run started a newer one"); + assert.deepEqual(deferredLanes(log), []); + assert.equal(outputs.clean, "true", "the review takes a place this pass, so no recovery work waits"); +}); + +test("pr-verify and pr-review-main still start on the checks conclusions LANE_TRIGGERS names", () => { + for (const [lanePath, conclusions] of LANE_TRIGGERS) { + const workflow = readFileSync(new URL(`../../../${lanePath}`, import.meta.url), "utf8"); + const tests = [...workflow.matchAll(/github\.event\.workflow_run\.conclusion == '([a-z_]+)'/g)].map((m) => m[1]); + assert.ok(tests.length > 0, `${lanePath} tests the upstream conclusion`); + assert.deepEqual([...new Set(tests)].sort(), [...conclusions].sort(), lanePath); + } +}); + +test("left-out lane runs take no place: four live runs fill the 3 places and one waits, deferred", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { reruns, outputs } = await reconcile({ + pulls: [pr(1), pr(2), pr(3), pr(4)], + runs: [ + doneChecks(101, 1, "2026-10-19T17:00:00Z"), + doneChecks(102, 2, "2026-10-19T17:10:00Z"), + doneChecks(103, 3, "2026-10-19T17:20:00Z"), + doneChecks(104, 4, "2026-10-19T17:30:00Z"), + verifyRun(279, { created: "2026-10-19T15:00:00Z" }), + verifyRun(280, { created: "2026-10-19T16:00:00Z" }), + verifyRun(281, { created: "2026-10-19T17:00:04Z" }), + verifyRun(282, { created: "2026-10-19T17:10:04Z" }), + verifyRun(283, { created: "2026-10-19T17:20:04Z" }), + verifyRun(284, { created: "2026-10-19T17:30:04Z" }), + ], + }); + assert.deepEqual(reruns, [281, 282, 283]); + assert.deepEqual(deferredLanes(log), [284]); + assert.deepEqual(supersededLanes(log), [279, 280]); + assert.equal(outputs.clean, "", "a deferred live run keeps the pacer running the next tick"); }); test("a non-operational Actions status records nothing", async () => { @@ -3114,7 +3229,7 @@ test("a CI Status that failed alone at the rate-limit floor has its failed jobs }); test("a Mark Lanes refusal at the floor is re-run before lane runs; a real Mark Lanes failure waits behind them as a flake", async () => { - const lanes = [201, 202, 203].map((id, i) => verifyRun(id, { created: `2026-10-19T19:1${i}:00Z` })); + const lanes = [201, 202, 203].map((id, i) => verifyRun(id, { created: `2026-10-19T19:3${i}:00Z` })); const floor = await reconcile({ pulls: [pr(1)], runs: [failed(101, sha(1), { jobs: markLanes() }), ...lanes] }); assert.deepEqual(floor.failedJobReruns, [101]); assert.deepEqual(floor.reruns, [201, 202]);