From b9d51baa6972487e0e769688ece6d4078ab8d678 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 09:05:14 +0000 Subject: [PATCH 1/2] CI: run the Firefox/Safari/phone matrix alongside Chromium, split desktop Safari and Firefox four ways The release browser matrix waited for all three Production UI Chromium shards before starting, though it only reads the shared Playwright build. Main run 38352161182 had the build ready at minute 4.1 and the matrix idle until 15.2, ending at 34.1. On dispatch runs a single red Chromium shard skipped the whole matrix, so cross-browser failures surfaced only one full round later. The matrix now starts with Production UI. When UI changed it keeps its mockups-only Chromium path, since Production UI in the same run (or the merged PR's identical tree) proves production Chromium and still fails the run if red. Desktop WebKit (17.2/13.0/14.2 min) and Firefox (13.4/9.0/8.7) become the slowest legs once parallel, so both split four ways. No test is skipped. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- .github/workflows/ci.yml | 45 ++++++++++++++++-------- tests/ci-browser-matrix-coverage.test.ts | 10 +++--- tests/ci-main-tree-proof.test.ts | 14 +++++--- tests/playwright-release-shards.test.ts | 6 ++-- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1ca6a4b8b..38ca5bfd96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1929,9 +1929,15 @@ jobs: needs.changes.result == 'success' && needs.static-pr.result == 'success' && (needs.build.result == 'success' || needs.build.result == 'skipped') && - (needs.ui-playwright-build.result == 'success' || needs.ui-playwright-build.result == 'skipped') && - (needs.ui-critical.result == 'success' || needs.ui-critical.result == 'skipped') - needs: [changes, static-pr, build, ui-playwright-build, ui-critical] + (needs.ui-playwright-build.result == 'success' || needs.ui-playwright-build.result == 'skipped') + # Starts as soon as the shared Playwright build is ready, ALONGSIDE Production UI rather than + # after it. Waiting for Production UI bought nothing: these legs only read the shared build, + # and the run still fails on a red Production UI (it is required in its own right). Measured + # 2026-10-13: main run 38352161182 had the build ready at minute 4.1, but every leg waited until + # minute 15.2 for the three Chromium shards and the run ended at 34.1; dispatch runs + # 38369056741 and 38370421833 had one red Chromium shard, so the whole Firefox/WebKit matrix + # was skipped and its verdict deferred to another full round after the fix. + needs: [changes, static-pr, build, ui-playwright-build] runs-on: ubuntu-24.04 # Per ENGINE, not for the whole matrix. Before 2026-09-07 one job ran # chromium-mockups + firefox + webkit sequentially under `workers: 1` / @@ -1992,14 +1998,21 @@ jobs: # merge (main runs took 36-40 min). Tablet now splits four ways (~12 min # each by its new timing table) and webkit three ways (19.5 -> 13 min by # its existing table), so every tablet test runs and main finishes sooner. + # + # 2026-10-13: with the matrix no longer queued behind Production UI, desktop WebKit and + # Firefox were the slowest legs (main run 38352161182: webkit 17.2/13.0/14.2 min of tests, + # firefox 13.4/9.0/8.7). Both now split four ways, which brings them under the iPhone legs + # (their slowest single file, ui-smoke, is ~11 min and sets the floor). include: - { engine: chromium, project: chromium-mockups, shard: 1, total: 1 } - - { engine: firefox, project: firefox, shard: 1, total: 3 } - - { engine: firefox, project: firefox, shard: 2, total: 3 } - - { engine: firefox, project: firefox, shard: 3, total: 3 } - - { engine: webkit, project: webkit, shard: 1, total: 3 } - - { engine: webkit, project: webkit, shard: 2, total: 3 } - - { engine: webkit, project: webkit, shard: 3, total: 3 } + - { engine: firefox, project: firefox, shard: 1, total: 4 } + - { engine: firefox, project: firefox, shard: 2, total: 4 } + - { engine: firefox, project: firefox, shard: 3, total: 4 } + - { engine: firefox, project: firefox, shard: 4, total: 4 } + - { engine: webkit, project: webkit, shard: 1, total: 4 } + - { engine: webkit, project: webkit, shard: 2, total: 4 } + - { engine: webkit, project: webkit, shard: 3, total: 4 } + - { engine: webkit, project: webkit, shard: 4, total: 4 } - { engine: webkit, project: mobile-webkit, shard: 1, total: 5 } - { engine: webkit, project: mobile-webkit, shard: 2, total: 5 } - { engine: webkit, project: mobile-webkit, shard: 3, total: 5 } @@ -2104,9 +2117,11 @@ jobs: # includes mockups, whose public flag is baked in, so it builds its own. # If the producer was out of scope, the wrapper also builds normally. PLAYWRIGHT_REUSE_BUILD: ${{ matrix.engine != 'chromium' && needs.ui-playwright-build.result == 'success' }} - # "true" only when this main push skipped Production UI because the identical tree already - # passed it on the merged PR (Main tree proof) — the same proof the primary path relies on. - UI_PROVEN_BY_PR: ${{ needs.changes.outputs.tree_proven_ui }} + # When UI changed, production Chromium is proven either by Production UI in this same run + # (it runs whenever UI changed and no proof exists, and the run fails if it fails) or, on a + # main push, by the merged PR's identical tree (Main tree proof). Either way this matrix + # need not repeat it. + UI_CHANGED: ${{ needs.changes.outputs.ui_changed }} run: | # Both paths below are exhaustive across playwright.config.ts projects # for their own path, and disjoint between engines. The fail-safe union @@ -2114,9 +2129,9 @@ jobs: # vanish from CI silently, so # `tests/ci-browser-matrix-coverage.test.ts` re-derives both unions from # this file plus playwright.config.ts and fails closed on any drift. - if [ "${{ needs.changes.outputs.ui_changed }}" = "true" ] && { [ "${{ needs.ui-critical.result }}" = "success" ] || [ "$UI_PROVEN_BY_PR" = "true" ]; }; then - # Production Chromium already passed — in this run, or on the merged PR's - # byte-identical tree. Keep mockup Chromium plus the cross-browser + if [ "$UI_CHANGED" = "true" ]; then + # Production Chromium is proven by Production UI in this run, or on the merged + # PR's byte-identical tree. Keep mockup Chromium plus the cross-browser # backstop without repeating it. case "$ENGINE" in chromium) PROJECTS="--project=chromium-mockups" ;; diff --git a/tests/ci-browser-matrix-coverage.test.ts b/tests/ci-browser-matrix-coverage.test.ts index 374d3ac8ee..acf6c9dd86 100644 --- a/tests/ci-browser-matrix-coverage.test.ts +++ b/tests/ci-browser-matrix-coverage.test.ts @@ -132,8 +132,8 @@ describe("release-browser-matrix engine coverage", () => { .sort(), ); - // `chromium` (the production project) is the one the ui-critical job already - // proved in this run. Re-running it here is the waste the primary path exists + // `chromium` (the production project) is the one the ui-critical job proves in + // this same run. Re-running it here is the waste the primary path exists // to avoid — but only Chromium, never the cross-engine backstop. expect([...byEngine.values()].flat()).not.toContain("chromium"); expect([...byEngine.values()].flat()).toContain("firefox"); @@ -149,11 +149,11 @@ describe("release-browser-matrix engine coverage", () => { shard: number; total: number; }>; - expect(jobs).toHaveLength(21); + expect(jobs).toHaveLength(23); for (const [project, engine, total] of [ ["chromium-mockups", "chromium", 1], - ["firefox", "firefox", 3], - ["webkit", "webkit", 3], + ["firefox", "firefox", 4], + ["webkit", "webkit", 4], ["mobile-webkit", "webkit", 5], ["mobile-pwa-standalone", "webkit", 5], ["tablet", "webkit", 4], diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index 822a496b7c..a8c666471d 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -278,16 +278,20 @@ describe("main tree proof workflow wiring", () => { ]) { expect(ifOf(job)).not.toContain("tree_proven"); } - // The matrix tolerates the proof-skipped Production UI and still runs on main... - expect(ifOf("release-browser-matrix")).toContain("needs.ui-critical.result == 'skipped'"); + // The matrix does not wait on Production UI at all (it runs alongside it), so a proof-skipped + // Production UI cannot hold it back, and it still runs on main... + expect(ifOf("release-browser-matrix")).not.toContain("ui-critical"); + expect(jobs["release-browser-matrix"].needs).not.toContain("ui-critical"); expect(ifOf("release-browser-matrix")).toContain("github.ref == 'refs/heads/main'"); - // ...and takes its primary (mockups-only Chromium) path when the PR proved production Chromium, + // ...and takes its primary (mockups-only Chromium) path whenever UI changed, because production + // Chromium is then proven by Production UI in the same run or by the merged PR's identical tree, // instead of re-running production Chromium unsharded on the fail-safe path. const matrixStep = (jobs["release-browser-matrix"].steps as Step[]).find( (step) => step.name === "Full browser UI matrix", ); - expect(matrixStep?.env?.UI_PROVEN_BY_PR).toBe("${{ needs.changes.outputs.tree_proven_ui }}"); - expect(matrixStep?.run).toContain('[ "$UI_PROVEN_BY_PR" = "true" ]'); + expect(matrixStep?.env?.UI_CHANGED).toBe("${{ needs.changes.outputs.ui_changed }}"); + expect(matrixStep?.run).toContain('if [ "$UI_CHANGED" = "true" ]; then'); + expect(ifOf("ui-critical")).toContain("needs.changes.outputs.ui_changed == 'true'"); }); it("keeps the scheduled full run and the push-run concurrency unchanged", () => { diff --git a/tests/playwright-release-shards.test.ts b/tests/playwright-release-shards.test.ts index b993187059..f0ae42e290 100644 --- a/tests/playwright-release-shards.test.ts +++ b/tests/playwright-release-shards.test.ts @@ -111,10 +111,10 @@ describe("release-browser-matrix duration shards", () => { }); it("keeps the committed leg counts within the measured balance", () => { - // The matrix keeps firefox 3, webkit 3, five legs per iPhone project and tablet 4. + // The matrix keeps firefox 4, webkit 4, five legs per iPhone project and tablet 4. const expectedMaxMinutes = { - firefox: [3, 9], - webkit: [3, 14], + firefox: [4, 7], + webkit: [4, 11], "mobile-webkit": [5, 12], "mobile-pwa-standalone": [5, 10], tablet: [4, 13], From 7af1daabc322141bc420f9c7c4e670e1f78223f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 10:30:33 +0000 Subject: [PATCH 2/2] CI: let main skip the release browser matrix when a full draft run already passed the identical tree The Firefox/WebKit/iPhone matrix never runs on pull_request events, so main always re-ran it after every merge (about 20 to 30 minutes holding Railway's Wait for CI), even when the draft's full-suite dispatch run had just passed the byte-identical tree. The main tree proof now also looks for the newest green workflow_dispatch run of ci.yml on the merged PR head, recorded by its own Change scope as testing this exact tree within 24 hours, with every release-browser-matrix leg green. Only then does main skip the matrix. The schedule, release branches and dispatch runs never take this path. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- .github/workflows/ci.yml | 4 + docs/ci-operations.md | 14 +-- scripts/ci-main-tree-proof.mjs | 153 ++++++++++++++++++++++++++----- tests/ci-main-tree-proof.test.ts | 36 ++++---- 4 files changed, 161 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38ca5bfd96..41a04fda70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,9 @@ jobs: tree_proven_coverage: ${{ steps.tree-proof.outputs.coverage }} tree_proven_ui: ${{ steps.tree-proof.outputs.ui }} tree_proven_lighthouse: ${{ steps.tree-proof.outputs.lighthouse }} + # True only when a green full-suite dispatch run already passed every release-matrix leg on + # this exact tree (the merged PR's head). Proved separately because PR runs never run it. + tree_proven_matrix: ${{ steps.tree-proof.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1919,6 +1922,7 @@ jobs: startsWith(github.ref, 'refs/heads/release/') || ( github.ref == 'refs/heads/main' && + needs.changes.outputs.tree_proven_matrix != 'true' && ( needs.changes.outputs.ui_changed == 'true' || needs.changes.outputs.perf_changed == 'true' || diff --git a/docs/ci-operations.md b/docs/ci-operations.md index 2178f7ee0d..6159273c71 100644 --- a/docs/ci-operations.md +++ b/docs/ci-operations.md @@ -8,13 +8,13 @@ These changes came out of the 6 October 2026 CI-efficiency review and its handov which runs repeat work. They don't change tests, assertions, thresholds, budgets, browser projects, shard counts, required check names, security gates or `main` concurrency. -| Change | PR | What it does | What still always runs | -| ------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the Firefox/WebKit `release-browser-matrix`, failure routing; the weekly schedule and every dispatch run in full. | -| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | -| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | -| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | -| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | +| Change | PR | What it does | What still always runs | +| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. Also skips the Firefox/WebKit `release-browser-matrix` when a green full-suite `workflow_dispatch` run on the same PR head recorded testing that tree. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the release matrix without its own dispatch-run proof, failure routing; the weekly schedule and every dispatch run in full. | +| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | +| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | +| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | +| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | ### How a skip is proven (`scripts/ci-main-tree-proof.mjs`) diff --git a/scripts/ci-main-tree-proof.mjs b/scripts/ci-main-tree-proof.mjs index cf872490d4..397c8958e8 100644 --- a/scripts/ci-main-tree-proof.mjs +++ b/scripts/ci-main-tree-proof.mjs @@ -34,10 +34,17 @@ * Only jobs whose result depends solely on the tree are eligible: unit coverage, the Chromium * production UI suite and the Lighthouse budget. Everything with main-only behaviour keeps * running on every push: static checks, safety, Build (its deployment boot smoke uses - * production secrets and runs only on main), the Playwright build, visual baselines, the - * Firefox/WebKit release-browser-matrix (it never runs on a PR, so nothing proved it) and + * production secrets and runs only on main), the Playwright build, visual baselines and * main-failure routing. The weekly scheduled run and every dispatch never take this path. * + * The Firefox/WebKit release-browser-matrix never runs on a pull_request event, so a PR run + * cannot prove it. A full-suite `workflow_dispatch` run on the PR head can (the draft "hold" + * rule runs one before a PR is marked ready). The `matrix` group therefore takes its proof from + * the newest successful dispatch run of THIS workflow for the same head H, under the same checks + * 4-8 above: same repository, within MAX_PROOF_AGE_HOURS, its Change scope recorded + * `event=workflow_dispatch` with a tree equal to tree(HEAD), and every matrix leg concluded + * success. Absent such a run, main runs the matrix exactly as before. + * * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse ("true"/"false"), proof_run_url, reason. * Uses only the job's GITHUB_TOKEN (actions: read, pull-requests: read) — about four REST * calls per main push. Every proved job checks out the run's GITHUB_SHA (no `ref:` override, @@ -104,7 +111,19 @@ export const PROOF_GROUPS = { lighthouse: [/^Lighthouse budget$/], }; -const NONE = Object.freeze({ coverage: false, ui: false, lighthouse: false }); +/** The release matrix legs a full-suite dispatch run must have run green; see the docblock. */ +export const MATRIX_PROOF_PATTERNS = Object.freeze([/^release-browser-matrix \(.+\)$/]); + +const NONE = Object.freeze({ coverage: false, ui: false, lighthouse: false, matrix: false }); + +/** True only when every matrix-leg pattern matches at least one job and all of them succeeded. Pure. */ +export function classifyMatrixProof(jobs) { + const list = Array.isArray(jobs) ? jobs : []; + return MATRIX_PROOF_PATTERNS.every((pattern) => { + const legs = list.filter((job) => pattern.test(String(job?.name ?? ""))); + return legs.length > 0 && legs.every((job) => job?.status === "completed" && job?.conclusion === "success"); + }); +} /** Which proof groups a PR run's job list establishes. Pure. */ export function classifyProof(jobs) { @@ -142,13 +161,16 @@ export function pickMergedPull(pulls, sha) { return matches.length === 1 ? matches[0] : null; } -/** The newest successful, fresh pull_request run of ci.yml for `headSha`, or null. Pure. */ -export function selectProvingRun(runs, { headSha, now = new Date(), maxAgeHours = DEFAULT_MAX_PROOF_AGE_HOURS }) { +/** The newest successful, fresh `event` run (pull_request by default) of ci.yml for `headSha`, or null. Pure. */ +export function selectProvingRun( + runs, + { headSha, now = new Date(), maxAgeHours = DEFAULT_MAX_PROOF_AGE_HOURS, event = "pull_request" }, +) { const maxAgeMs = maxAgeHours * 3_600_000; const candidates = (Array.isArray(runs) ? runs : []) .filter( (run) => - run?.event === "pull_request" && + run?.event === event && run?.head_sha === headSha && run?.path === CI_WORKFLOW_PATH && run?.status === "completed" && @@ -235,34 +257,55 @@ export async function evaluate({ const maxAgeHours = Number(env.MAX_PROOF_AGE_HOURS || DEFAULT_MAX_PROOF_AGE_HOURS); if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0) return { proof: NONE, reason: "invalid MAX_PROOF_AGE_HOURS" }; + const prOutcome = await provenRun({ request, repo, head, mainTree, now, maxAgeHours, event: "pull_request" }); + const matrixOutcome = await provenRun({ + request, + repo, + head, + mainTree, + now, + maxAgeHours, + event: "workflow_dispatch", + }); + const proof = { + ...(prOutcome.jobs ? classifyProof(prOutcome.jobs) : { coverage: false, ui: false, lighthouse: false }), + matrix: matrixOutcome.jobs ? classifyMatrixProof(matrixOutcome.jobs) : false, + }; + return { + proof, + reason: `PR #${pull.number}: ${prOutcome.reason}; release matrix: ${matrixOutcome.reason}`, + runUrl: prOutcome.runUrl || matrixOutcome.runUrl || "", + }; +} + +/** + * Checks 5-6 for one event kind: the newest fresh green `event` run of ci.yml for `head` whose own + * Change scope recorded testing exactly `mainTree`. Returns its job list, or `jobs: null` and why. + */ +async function provenRun({ request, repo, head, mainTree, now, maxAgeHours, event }) { const runs = await request( - `/repos/${repo}/actions/workflows/ci.yml/runs?event=pull_request&head_sha=${head}&status=completed&per_page=30`, + `/repos/${repo}/actions/workflows/ci.yml/runs?event=${event}&head_sha=${head}&status=completed&per_page=30`, ); - const run = selectProvingRun(runs?.workflow_runs, { headSha: head, now, maxAgeHours }); - if (!run) { - return { - proof: NONE, - reason: `no successful pull_request CI run for PR #${pull.number} head within ${maxAgeHours}h`, - }; - } + const run = selectProvingRun(runs?.workflow_runs, { headSha: head, now, maxAgeHours, event }); + if (!run) return { jobs: null, reason: `no successful ${event} CI run for the head within ${maxAgeHours}h` }; const jobs = await request(`/repos/${repo}/actions/runs/${run.id}/jobs?filter=latest&per_page=100`); if (!Array.isArray(jobs?.jobs) || jobs.total_count > jobs.jobs.length) { - return { proof: NONE, reason: "could not read the proving run's complete job list" }; + return { jobs: null, reason: `could not read run ${run.id}'s complete job list` }; } const scopeJobs = jobs.jobs.filter((job) => job?.name === "Change scope"); if (scopeJobs.length !== 1 || scopeJobs[0].conclusion !== "success") { - return { proof: NONE, reason: "the proving run has no single successful Change scope job" }; + return { jobs: null, reason: `run ${run.id} has no single successful Change scope job` }; } const tested = parseTestedTree(await request(`/repos/${repo}/check-runs/${scopeJobs[0].id}/annotations?per_page=50`)); - if (!tested || tested.event !== "pull_request") { - return { proof: NONE, reason: `run ${run.id} did not record the tree it tested` }; + if (!tested || tested.event !== event) { + return { jobs: null, reason: `run ${run.id} did not record the tree it tested` }; } if (tested.tree !== mainTree) { - return { proof: NONE, reason: `run ${run.id} tested tree ${tested.tree.slice(0, 12)}, not this main tree` }; + return { jobs: null, reason: `run ${run.id} tested tree ${tested.tree.slice(0, 12)}, not this main tree` }; } return { - proof: classifyProof(jobs.jobs), - reason: `run ${run.id} (PR #${pull.number}) recorded testing ${tested.sha.slice(0, 12)} whose tree equals this main tree`, + jobs: jobs.jobs, + reason: `run ${run.id} recorded testing ${tested.sha.slice(0, 12)} whose tree equals this main tree`, runUrl: run.html_url ?? "", }; } @@ -351,7 +394,7 @@ export async function evaluatePullRequestUpdate({ `/repos/${repo}/actions/workflows/ci.yml/runs?event=pull_request&branch=${encodeURIComponent(headRef)}&status=success&per_page=30`, ); const candidates = selectPullRequestCandidates(runs?.workflow_runs, { prNumber, now, maxAgeHours }); - const proof = { coverage: false, ui: false, lighthouse: false }; + const proof = { coverage: false, ui: false, lighthouse: false, matrix: false }; const notes = []; let runUrl = ""; for (const run of candidates) { @@ -395,7 +438,7 @@ export async function evaluatePullRequestUpdate({ } const changed = runGit(["diff", "--name-only", "--no-renames", oldTree, newTree]).split("\n").filter(Boolean); if (changed.length === 0) { - for (const group of Object.keys(proof)) proof[group] ||= legs[group]; + for (const group of Object.keys(legs)) proof[group] ||= legs[group]; notes.push(`run ${run.id}: identical tested tree`); } else if (changed.every(isUnrelatedInput)) { proof.ui ||= legs.ui; @@ -422,6 +465,7 @@ function writeOutputs({ proof, reason, runUrl = "" }) { `coverage=${proof.coverage === true}`, `ui=${proof.ui === true}`, `lighthouse=${proof.lighthouse === true}`, + `matrix=${proof.matrix === true}`, `proof_run_url=${runUrl}`, `reason=${String(reason).replace(/[\r\n]+/g, " ")}`, ]; @@ -634,6 +678,69 @@ async function selfTest() { result = await evaluate({ ...base, request: request([{ ...run, conclusion: "cancelled" }]) }); assert(!result.proof.ui, "a cancelled run proves nothing"); assert(parseTestedTree(testedRecord)?.tree === tree, "a well-formed record parses"); + assert(result.proof.matrix === false, "a pull_request run never proves the release matrix"); + + // The release matrix: proved only by a green full-suite dispatch run on the same head and tree. + const matrixLegs = [ + { ...ok("Change scope"), id: 601 }, + ok("release-browser-matrix (chromium, chromium-mockups, 1, 1)"), + ok("release-browser-matrix (webkit, mobile-webkit, 1, 5)"), + ]; + const dispatchRun = { ...run, id: 6, event: "workflow_dispatch" }; + const dispatchRecord = record(`event=workflow_dispatch sha=${pr.head.sha} tree=${tree}`); + const matrixRequest = + ({ legs = matrixLegs, annotations = dispatchRecord, dispatchRuns = [dispatchRun] } = {}) => + async (path) => { + if (path.includes("/workflows/") && path.includes("event=workflow_dispatch")) + return { workflow_runs: dispatchRuns }; + if (path.includes("/runs/6/jobs")) return { total_count: legs.length, jobs: legs }; + if (path.includes("/check-runs/601/annotations")) return annotations; + return request()(path); + }; + result = await evaluate({ ...base, request: matrixRequest() }); + assert(result.proof.matrix && result.proof.ui, `a green dispatch run proves the matrix (${result.reason})`); + assert(classifyMatrixProof(matrixLegs), "all-green legs prove the matrix"); + assert(!classifyMatrixProof([ok("release-browser-matrix")]), "a skipped (unexpanded) matrix proves nothing"); + result = await evaluate({ + ...base, + request: matrixRequest({ + legs: matrixLegs.map((job, index) => (index === 2 ? { ...job, conclusion: "failure" } : job)), + }), + }); + assert(!result.proof.matrix, "one red leg un-proves the matrix"); + result = await evaluate({ ...base, request: matrixRequest({ legs: matrixLegs.slice(0, 1) }) }); + assert(!result.proof.matrix, "a dispatch run without matrix legs proves nothing"); + result = await evaluate({ + ...base, + request: matrixRequest({ + annotations: record(`event=workflow_dispatch sha=${pr.head.sha} tree=${"3".repeat(40)}`), + }), + }); + assert(!result.proof.matrix, "a dispatch run that tested another tree proves nothing"); + result = await evaluate({ ...base, request: matrixRequest({ annotations: testedRecord }) }); + assert(!result.proof.matrix, "a dispatch run must record a dispatch event"); + result = await evaluate({ + ...base, + request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, conclusion: "failure" }] }), + }); + assert(!result.proof.matrix, "a red dispatch run proves nothing"); + result = await evaluate({ + ...base, + request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, updated_at: "2026-10-05T10:00:00Z" }] }), + }); + assert(!result.proof.matrix, "a dispatch run older than 24h is stale"); + result = await evaluate({ + ...base, + runGit: gitMap({ "rev-parse HEAD^{tree}": "2".repeat(40) }), + request: matrixRequest(), + }); + assert(!result.proof.matrix, "a tree mismatch never proves the matrix"); + result = await evaluate({ + ...base, + env: { ...env, GITHUB_EVENT_NAME: "workflow_dispatch" }, + request: matrixRequest(), + }); + assert(!result.proof.matrix, "a dispatch run never skips its own matrix"); const recorded = []; const originalLog = console.log; console.log = (line) => recorded.push(line); diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index a8c666471d..8c82ebe49a 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -22,7 +22,8 @@ import { * Main tree proof: a push to `main` skips only the jobs the merged PR's own CI run already passed * on a byte-identical tree. Everything here pins the fail-closed shape — an absent or partial proof * must leave the push doing the full job set it did before, and the jobs with main-only behaviour - * (Build + deployment boot smoke, the Firefox/WebKit release matrix) must never become skippable. + * (Build + deployment boot smoke) must never become skippable. The Firefox/WebKit release matrix + * is skippable only on its own proof: a green full-suite dispatch run on the identical tree. */ const workflowText = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"); const workflow = createRequire(import.meta.url)("js-yaml").load(workflowText); @@ -156,13 +157,13 @@ describe("main tree proof script", () => { }; const result = await evaluate({ env: env as unknown as NodeJS.ProcessEnv, now, request, runGit, runGitOk }); expect(fetched).toEqual(["+refs/pull/42/head:refs/remotes/tree-proof/42"]); - expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true }); + expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true, matrix: false }); expect(result.runUrl).toBe("https://example.test/run/9"); // Same evidence, one byte of tree drift: nothing is proven. gitTable["rev-parse HEAD^{tree}"] = "e".repeat(40); const drifted = await evaluate({ env: env as unknown as NodeJS.ProcessEnv, now, request, runGit, runGitOk }); - expect(drifted.proof).toEqual({ coverage: false, ui: false, lighthouse: false }); + expect(drifted.proof).toEqual({ coverage: false, ui: false, lighthouse: false, matrix: false }); }); it("propagates API errors so the step's catch turns them into a full run", async () => { @@ -203,6 +204,7 @@ describe("main tree proof script", () => { expect(result.stdout).toContain("coverage=false"); expect(result.stdout).toContain("ui=false"); expect(result.stdout).toContain("lighthouse=false"); + expect(result.stdout).toContain("matrix=false"); }); }); @@ -223,6 +225,7 @@ describe("main tree proof workflow wiring", () => { expect(jobs.changes.outputs.tree_proven_coverage).toBe("${{ steps.tree-proof.outputs.coverage }}"); expect(jobs.changes.outputs.tree_proven_ui).toBe("${{ steps.tree-proof.outputs.ui }}"); expect(jobs.changes.outputs.tree_proven_lighthouse).toBe("${{ steps.tree-proof.outputs.lighthouse }}"); + expect(jobs.changes.outputs.tree_proven_matrix).toBe("${{ steps.tree-proof.outputs.matrix }}"); // The changes job checks out full history, which HEAD^1 and the ancestry check rely on. const checkout = steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@")); expect(checkout?.with?.["fetch-depth"]).toBe(0); @@ -261,21 +264,22 @@ describe("main tree proof workflow wiring", () => { "coverage", "coverage-shards", "lighthouse-budget", + "release-browser-matrix", "ui-critical", "ui-critical-fast", ]); + // The matrix is gated on its own dispatch-run proof, and only inside its main-push clause, so the + // weekly schedule, release branches and every dispatch still run it in full. + const matrixIf = ifOf("release-browser-matrix"); + expect(matrixIf.match(/tree_proven_\w+/g)).toEqual(["tree_proven_matrix"]); + expect(matrixIf).toMatch( + /github\.ref == 'refs\/heads\/main' &&\s+needs\.changes\.outputs\.tree_proven_matrix != 'true' &&/, + ); expect(ifOf("ui-critical-fast")).toContain("needs.changes.outputs.tree_proven_ui != 'true'"); }); - it("keeps Build (deployment boot smoke) and the release browser matrix running on proven pushes", () => { - for (const job of [ - "build", - "release-browser-matrix", - "static-pr", - "safety", - "ui-playwright-build", - "visual-baseline", - ]) { + it("keeps Build (deployment boot smoke) running on proven pushes, and the matrix on a PR-only proof", () => { + for (const job of ["build", "static-pr", "safety", "ui-playwright-build", "visual-baseline"]) { expect(ifOf(job)).not.toContain("tree_proven"); } // The matrix does not wait on Production UI at all (it runs alongside it), so a proof-skipped @@ -386,7 +390,7 @@ describe("PR branch update reuse", () => { it("carries every group when the newly tested tree is identical to a recorded green one", async () => { const result = await evaluatePullRequestUpdate(setup()); - expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true }); + expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true, matrix: false }); expect(result.runUrl).toBe("https://example.test/run/5"); }); @@ -394,7 +398,7 @@ describe("PR branch update reuse", () => { const result = await evaluatePullRequestUpdate( setup({ changed: ["README.md", "worker/requirements.txt", "eval/docling/pyproject.toml"] }), ); - expect(result.proof).toEqual({ coverage: false, ui: true, lighthouse: true }); + expect(result.proof).toEqual({ coverage: false, ui: true, lighthouse: true, matrix: false }); }); it("runs everything when any changed path could be an input", async () => { @@ -410,12 +414,12 @@ describe("PR branch update reuse", () => { "scripts/ci-main-tree-proof.mjs", ]) { const result = await evaluatePullRequestUpdate(setup({ changed: ["README.md", input] })); - expect(result.proof, input).toEqual({ coverage: false, ui: false, lighthouse: false }); + expect(result.proof, input).toEqual({ coverage: false, ui: false, lighthouse: false, matrix: false }); } }); it("fails open to the full run on every doubt", async () => { - const none = { coverage: false, ui: false, lighthouse: false }; + const none = { coverage: false, ui: false, lighthouse: false, matrix: false }; expect((await evaluatePullRequestUpdate({ ...setup(), env: { ...env, PR_ACTION: "opened" } })).proof).toEqual(none); expect((await evaluatePullRequestUpdate({ ...setup(), env: { ...env, PR_HEAD_REPO: "fork/r" } })).proof).toEqual( none,