Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 66 additions & 13 deletions .github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -176,11 +176,14 @@
// flake-only sets. Nothing else re-runs a lane run, while fix-ci does not wait
// for a flake re-run. A lane run's listing does not name its PR, so lane runs
// go oldest first among those whose creation matches an open head's newest
// checks completion (laneOrder), then the rest oldest first: a run for a
// closed PR, or a second run for one head, takes no place before a live one,
// and the newer runs each kill-switch trip adds no longer push the oldest
// live ones back pass after pass. One pass uploads at most MAX_IN_FLIGHT
// records, so what would pass that waits too. A pass that defers anything but
// checks completion (laneOrder), then the rest oldest first: a second run for
// one head takes no place before a live one, and the newer runs each
// kill-switch trip adds no longer push the oldest live ones back pass after
// pass. A lost lane run that follows no open head's newest checks
// completion is not re-run (laneOrder): it was created outside every open head's
// window, so a re-run would judge nothing live, and after a brake the backlog
// is mostly such runs. One pass uploads at most MAX_IN_FLIGHT records, so what
// would pass that waits too. A pass that defers anything but
// a flake-only set writes no `clean`, so the pacer runs the next tick.
// Writes are at least WRITE_GAP_MS apart. The first rate-limit error stops the
// run.
Expand Down Expand Up @@ -358,6 +361,12 @@ export const RECORDED_LANES = new Map([
[".github/workflows/pr-verify.yml", JUDGE_STEP],
[".github/workflows/pr-review-main.yml", REVIEW_STEP],
]);
// The checks-run conclusions that start each recorded lane's work: the
// conclusions its jobs' `github.event.workflow_run.conclusion` test passes.
export const LANE_TRIGGERS = new Map([
[".github/workflows/pr-verify.yml", new Set(["success", "failure"])],
[".github/workflows/pr-review-main.yml", new Set(["success"])],
]);
// The recorded lanes whose workflow runs the upstream-status gate; pr-verify.yml
// runs none, so no outage holds its re-run.
export const GATED_LANES = new Set([".github/workflows/pr-review-main.yml"]);
Expand Down Expand Up @@ -718,14 +727,34 @@ const byAge = (a, b) => Date.parse(a.created_at) - Date.parse(b.created_at) || a
// log, and compare the run's created_at with that checks run's updated_at.
const MATCH_BEFORE_MS = 60_000;
const MATCH_AFTER_MS = 15 * 60_000;
// How long after an open head's newest checks completion a lost lane run may
// still be that head's and so be kept: eight times MATCH_AFTER_MS, a judgment
// margin for a queue backed up far past the measured pair. Without it, one
// open head left untouched for hours keeps every stale run queued.
const OWNED_AFTER_MS = 2 * 3_600_000;

// When a head's newest completed pull_request checks run finished, the one
// whose completion started the head's live lane runs.
const checksDone = (own) => {
const done = own
.filter((r) => RECORDED_UPSTREAMS.has(r.path) && r.event === "pull_request" && r.status === "completed" && isTime(r.updated_at))
.map((r) => Date.parse(r.updated_at));
return done.length > 0 ? Math.max(...done) : undefined;
const done = own.filter((r) => RECORDED_UPSTREAMS.has(r.path) && r.event === "pull_request" && r.status === "completed" && isTime(r.updated_at));
return done.length > 0 ? Math.max(...done.map((r) => Date.parse(r.updated_at))) : undefined;
};

// Per recorded lane, when the head's newest checks run that starts it
// (LANE_TRIGGERS) finished: a newer run that does not start the lane replaces
// none of its lost runs.
const laneStarts = (own) => {
const starts = new Map();
for (const r of own) {
if (RECORDED_UPSTREAMS.has(r.path) && r.event === "pull_request" && r.status === "completed" && isTime(r.updated_at)) {
for (const [lanePath, conclusions] of LANE_TRIGGERS) {
if (conclusions.has(r.conclusion)) {
starts.set(lanePath, Math.max(starts.get(lanePath) ?? 0, Date.parse(r.updated_at)));
}
}
}
}
return starts;
};

// The lane items in place order: those matched to an open head, oldest first,
Expand All @@ -735,7 +764,16 @@ const checksDone = (own) => {
// taken once, so a duplicate run for a head and a run whose PR is gone sort
// last. The match sets the order only; check-trusted-trigger and resolve-pr
// still bind the re-run to its PR.
function laneOrder(lanes, openChecks) {
// A lost lane run is `superseded` and left out when, for every open head, it
// was created more than MATCH_BEFORE_MS before the head's newest checks
// completion that starts its lane (laneStarts) or more than OWNED_AFTER_MS
// after it: a lane run starts after the checks completion it follows, so it
// follows no open head's current checks: its PR closed, its head moved, or its
// head's checks completed again and started a newer run of its lane. Nothing
// is left out unless `drop` (every open head's runs were read) holds and some
// open head has a completion that starts the lane; a fault-held run is never
// left out.
function laneOrder(lanes, openChecks, drop) {
const pairs = [];
for (const lane of lanes) {
for (const { head, done } of openChecks) {
Expand All @@ -754,7 +792,15 @@ function laneOrder(lanes, openChecks) {
taken.add(key);
}
}
return [...lanes.filter((l) => matched.has(l)).sort(byAge), ...lanes.filter((l) => !matched.has(l)).sort(byAge)];
const superseded = drop
? lanes.filter((l) => {
const starts = openChecks.map((c) => c.starts.get(l.path)).filter((t) => t !== undefined);
const created = Date.parse(l.created_at);
return l.kind === "lane" && starts.length > 0 && !starts.some((t) => created >= t - MATCH_BEFORE_MS && created <= t + OWNED_AFTER_MS);
})
: [];
const rest = lanes.filter((l) => !matched.has(l) && !superseded.includes(l));
return { order: [...lanes.filter((l) => matched.has(l)).sort(byAge), ...rest.sort(byAge)], superseded };
}

// A job whose steps did not list in full may have lost its runner, which cuts
Expand Down Expand Up @@ -1030,6 +1076,8 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
// Each open head's newest checks completion, {head, done}, for laneOrder;
// taken before any skip, so a busy head still matches its lane runs.
const openChecks = [];
// False once a head's run listing failed before its completion was taken.
let everyHeadRead = true;
// Heads whose checks passed, each with its earliest checks run's creation.
const passed = [];
// Heads with a pass whose spec may have changed since (`prefilterSpec`).
Expand Down Expand Up @@ -1083,7 +1131,7 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
const own = listing.workflow_runs.filter((r) => r.head_repository?.full_name === repo);
const done = checksDone(own);
if (done !== undefined) {
openChecks.push({ head: pull.head.sha, done });
openChecks.push({ head: pull.head.sha, done, starts: laneStarts(own) });
}
const { rerun, again, lane, inFlight: busy } = classify(own, since);
inFlight += busy ? 1 : 0;
Expand Down Expand Up @@ -1267,6 +1315,7 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
ready.push(set.map(withRecord));
}
} catch (error) {
everyHeadRead &&= openChecks.at(-1)?.head === pull.head.sha;
noteFailure(`#${pull.number}`, error);
}
}
Expand Down Expand Up @@ -1630,7 +1679,11 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneAp
for (const set of ready.filter((s) => !flakeOnly(s))) {
place(`#${set[0].pr}`, set);
}
for (const lane of laneOrder(lanes, openChecks)) {
const { order, superseded } = laneOrder(lanes, openChecks, everyHeadRead);
for (const lane of superseded) {
say(`lane run ${lane.id} (${lane.path}) ended ${lane.conclusion}; it follows no open pull request's newest checks run, so it judges no open head (its PR closed, its head moved, or its head's checks completed again since); not re-run`);
}
for (const lane of order) {
place(`lane run ${lane.id} (${lane.path})`, [lane]);
}
for (const set of ready.filter(flakeOnly)) {
Expand Down
129 changes: 122 additions & 7 deletions .github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { test } from "node:test";

import { createGitHub, GitHubError } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs";
import { createRateLimitLatch, RateLimitStop } from "../../actions/log-api-budget/rate-guard.mjs";
import { CI_WORKFLOWS, FLOOR_NOTE, floorNoted, GITHUB_STATUS_URL, JUDGE_PREDECESSOR, JUDGE_STEP, KILL_SWITCH_NOTE, main, NPM_STATUS_URL, MISSING_REVIEW_NOTE, PENDING_BACKOFF_MINUTES, PENDING_REVIEW_NOTE, prefilterSpec, REVIEW_STEP, REVIEW_STOP_NOTE, REVIEW_STOP_STEP, REVIEW_SWITCHES, SETTLED_VERSION, TOO_OLD_DAYS, UPSTREAM_STOP_STEP, VERDICT_STOP_STEP, WRITE_STEP } from "./reconcile.mjs";
import { CI_WORKFLOWS, FLOOR_NOTE, floorNoted, GITHUB_STATUS_URL, JUDGE_PREDECESSOR, JUDGE_STEP, KILL_SWITCH_NOTE, LANE_TRIGGERS, main, NPM_STATUS_URL, MISSING_REVIEW_NOTE, PENDING_BACKOFF_MINUTES, PENDING_REVIEW_NOTE, prefilterSpec, REVIEW_STEP, REVIEW_STOP_NOTE, REVIEW_STOP_STEP, REVIEW_SWITCHES, SETTLED_VERSION, TOO_OLD_DAYS, UPSTREAM_STOP_STEP, VERDICT_STOP_STEP, WRITE_STEP } from "./reconcile.mjs";
import { switchHold } from "../pr-merge/merge-holds.mjs";
import { tempDir } from "../../../lib/test-tmp.mjs";

Expand Down Expand Up @@ -1185,7 +1185,9 @@ test("a main-run review re-run shares the cap of 3 with pr-verify re-runs, oldes
const doneChecks = (id, n, done) => run(id, sha(n), { conclusion: "success", created: "2026-10-19T16:50:00Z", updated: done });
const deferredLanes = (log) => log.mock.calls.map((c) => c.arguments[0]).filter((line) => / deferred, /.test(line)).map((line) => Number(/lane run ([0-9]+) /.exec(line)?.[1]));

test("lane runs of open heads go first, oldest first, ahead of an older run whose PR is gone and of newer runs", async (t) => {
const supersededLanes = (log) => log.mock.calls.map((c) => c.arguments[0]).filter((line) => /judges no open head/.test(line)).map((line) => Number(/lane run ([0-9]+) /.exec(line)?.[1]));

test("lane runs of open heads go first, oldest first, ahead of newer runs, and a run older than every open head's checks is not re-run", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { code, reruns } = await reconcile({
pulls: [pr(1), pr(2)],
Expand All @@ -1201,8 +1203,9 @@ test("lane runs of open heads go first, oldest first, ahead of an older run whos
],
});
assert.equal(code, 0);
assert.deepEqual(reruns, [211, 212, 210], "the two matched runs, then the oldest unmatched one");
assert.deepEqual(deferredLanes(log), [221, 222, 223], "every lane run not re-run is named as deferred");
assert.deepEqual(reruns, [211, 212, 221], "the two matched runs, then the oldest unmatched one");
assert.deepEqual(deferredLanes(log), [222, 223], "every live lane run not re-run is named as deferred");
assert.deepEqual(supersededLanes(log), [210], "the run created an hour before the oldest open head's checks completed is named, not deferred");
});

test("a second lane run for one head goes behind every matched run, while each lane matches the head once", async (t) => {
Expand All @@ -1225,9 +1228,10 @@ test("a second lane run for one head goes behind every matched run, while each l

test("a lane run created more than 60 s before or 15 minutes after a head's checks completion is not matched", async () => {
const { code, reruns } = await reconcile({
pulls: [pr(1)],
pulls: [pr(1), pr(2)],
runs: [
doneChecks(101, 1, "2026-10-19T17:00:00Z"),
doneChecks(102, 2, "2026-10-19T15:00:00Z"),
verifyRun(240, { created: "2026-10-19T16:00:00Z" }),
verifyRun(241, { created: "2026-10-19T16:58:59Z" }),
verifyRun(242, { created: "2026-10-19T17:15:01Z" }),
Expand All @@ -1237,7 +1241,118 @@ test("a lane run created more than 60 s before or 15 minutes after a head's chec
assert.equal(code, 0);
// 241 is 61 s before the completion and 242 is 15 min 1 s after; 244, at
// 14 min 59 s after, is the one match, and the rest go oldest first.
assert.deepEqual(reruns, [244, 240, 241]);
assert.deepEqual(reruns, [244, 240, 241], "head 2's checks done at 15:00 match no run, and keep every run created after them");
});

test("a lost lane run created before every open head's checks completed is not re-run, while the head's newest run is, and the pass writes clean", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { code, reruns, outputs } = await reconcile({
pulls: [pr(1)],
runs: [
doneChecks(101, 1, "2026-10-19T19:00:00Z"),
verifyRun(250, { created: "2026-10-19T17:00:04Z" }),
verifyRun(251, { created: "2026-10-19T16:30:00Z" }),
verifyRun(252, { created: "2026-10-19T19:00:04Z" }),
heldRun(253, { created: "2026-10-19T16:00:00Z" }),
],
});
assert.equal(code, 0);
assert.deepEqual(reruns, [252, 253], "the head's newest run, and the held run, which is never left out");
assert.deepEqual(supersededLanes(log).sort(), [250, 251], "250 followed head 1's checks before they completed again at 19:00, and 251 follows no open head");
assert.deepEqual(deferredLanes(log), []);
assert.equal(outputs.clean, "true", "a left-out run defers nothing");
});

test("a lost lane run created from 60 s before to 2 hours after an open head's checks completed is re-run, outside the 15-minute match window too", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { reruns } = await reconcile({
pulls: [pr(1)],
runs: [
doneChecks(101, 1, "2026-10-19T17:00:00Z"),
verifyRun(260, { created: "2026-10-19T17:40:00Z" }),
verifyRun(261, { created: "2026-10-19T16:59:30Z" }),
verifyRun(262, { created: "2026-10-19T18:59:00Z" }),
verifyRun(263, { created: "2026-10-19T16:58:59Z" }),
verifyRun(264, { created: "2026-10-19T19:00:01Z" }),
],
});
assert.deepEqual(reruns, [261, 260, 262], "30 s before, 40 minutes after and 1 minute short of 2 hours after are kept");
assert.deepEqual(supersededLanes(log).sort(), [263, 264], "61 s before and 2 hours 1 s after are left out");
});

test("an open head untouched for hours keeps only the lost runs near its own checks, not every later stale run", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { reruns } = await reconcile({
pulls: [pr(1), pr(2)],
runs: [
doneChecks(101, 1, "2026-10-19T08:00:00Z"),
doneChecks(102, 2, "2026-10-19T19:00:00Z"),
verifyRun(265, { created: "2026-10-19T09:00:00Z" }),
verifyRun(266, { created: "2026-10-19T15:00:00Z" }),
],
});
assert.deepEqual(reruns, [265], "an hour after head 1's checks, so possibly head 1's");
assert.deepEqual(supersededLanes(log), [266], "seven hours after head 1's checks and four before head 2's");
});

test("no lost lane run is left out when an open head's runs could not be listed", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { reruns } = await reconcile({
pulls: [pr(1), pr(2)],
runs: [doneChecks(101, 1, "2026-10-19T17:00:00Z"), verifyRun(270, { created: "2026-10-19T16:00:00Z" })],
fail: { [`GET /repos/${REPO}/actions/runs?head_sha=${sha(2)}&per_page=100`]: new GitHubError(500, "boom") },
});
assert.deepEqual(reruns, [270], "head 2's checks may have completed before 16:00");
assert.deepEqual(supersededLanes(log), []);
});

test("a newer failed checks run leaves out a lost verify run, which it starts again, but keeps a lost main-run review, which only a passed one starts", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { reruns, outputs } = await reconcile({
pulls: [pr(1)],
runs: [
run(101, sha(1), { conclusion: "success", created: "2026-10-19T16:50:00Z", updated: "2026-10-19T17:00:00Z" }),
failed(102, sha(1), { jobs: MIXED_JOBS, updated: "2026-10-19T19:00:00Z" }),
mainReviewRun(330, { created: "2026-10-19T17:00:04Z" }),
verifyRun(331, { created: "2026-10-19T17:00:30Z" }),
],
});
assert.deepEqual(reruns, [330], "pr-review-main runs only on a passed checks run, so nothing replaces the 17:00:04 review");
assert.deepEqual(supersededLanes(log), [331], "pr-verify runs on a failed checks run too, so the 19:00 run started a newer one");
assert.deepEqual(deferredLanes(log), []);
assert.equal(outputs.clean, "true", "the review takes a place this pass, so no recovery work waits");
});

test("pr-verify and pr-review-main still start on the checks conclusions LANE_TRIGGERS names", () => {
for (const [lanePath, conclusions] of LANE_TRIGGERS) {
const workflow = readFileSync(new URL(`../../../${lanePath}`, import.meta.url), "utf8");
const tests = [...workflow.matchAll(/github\.event\.workflow_run\.conclusion == '([a-z_]+)'/g)].map((m) => m[1]);
assert.ok(tests.length > 0, `${lanePath} tests the upstream conclusion`);
assert.deepEqual([...new Set(tests)].sort(), [...conclusions].sort(), lanePath);
}
});

test("left-out lane runs take no place: four live runs fill the 3 places and one waits, deferred", async (t) => {
const log = t.mock.method(console, "log", () => {});
const { reruns, outputs } = await reconcile({
pulls: [pr(1), pr(2), pr(3), pr(4)],
runs: [
doneChecks(101, 1, "2026-10-19T17:00:00Z"),
doneChecks(102, 2, "2026-10-19T17:10:00Z"),
doneChecks(103, 3, "2026-10-19T17:20:00Z"),
doneChecks(104, 4, "2026-10-19T17:30:00Z"),
verifyRun(279, { created: "2026-10-19T15:00:00Z" }),
verifyRun(280, { created: "2026-10-19T16:00:00Z" }),
verifyRun(281, { created: "2026-10-19T17:00:04Z" }),
verifyRun(282, { created: "2026-10-19T17:10:04Z" }),
verifyRun(283, { created: "2026-10-19T17:20:04Z" }),
verifyRun(284, { created: "2026-10-19T17:30:04Z" }),
],
});
assert.deepEqual(reruns, [281, 282, 283]);
assert.deepEqual(deferredLanes(log), [284]);
assert.deepEqual(supersededLanes(log), [279, 280]);
assert.equal(outputs.clean, "", "a deferred live run keeps the pacer running the next tick");
});

test("a non-operational Actions status records nothing", async () => {
Expand Down Expand Up @@ -3114,7 +3229,7 @@ test("a CI Status that failed alone at the rate-limit floor has its failed jobs
});

test("a Mark Lanes refusal at the floor is re-run before lane runs; a real Mark Lanes failure waits behind them as a flake", async () => {
const lanes = [201, 202, 203].map((id, i) => verifyRun(id, { created: `2026-10-19T19:1${i}:00Z` }));
const lanes = [201, 202, 203].map((id, i) => verifyRun(id, { created: `2026-10-19T19:3${i}:00Z` }));
const floor = await reconcile({ pulls: [pr(1)], runs: [failed(101, sha(1), { jobs: markLanes() }), ...lanes] });
assert.deepEqual(floor.failedJobReruns, [101]);
assert.deepEqual(floor.reruns, [201, 202]);
Expand Down
Loading