From cee89fd0f99a407d09198792a9c41e2439cfbc65 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Tue, 13 Oct 2026 19:47:02 +0800 Subject: [PATCH 1/3] CI: always run the cross-browser release matrix on main again --- .github/workflows/ci.yml | 4 ---- docs/ci-operations.md | 14 +++++++------- tests/ci-main-tree-proof.test.ts | 13 ++++--------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41a04fda70..38ca5bfd96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,9 +107,6 @@ jobs: tree_proven_coverage: ${{ steps.tree-proof.outputs.coverage }} tree_proven_ui: ${{ steps.tree-proof.outputs.ui }} tree_proven_lighthouse: ${{ steps.tree-proof.outputs.lighthouse }} - # True only when a green full-suite dispatch run already passed every release-matrix leg on - # this exact tree (the merged PR's head). Proved separately because PR runs never run it. - tree_proven_matrix: ${{ steps.tree-proof.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1922,7 +1919,6 @@ jobs: startsWith(github.ref, 'refs/heads/release/') || ( github.ref == 'refs/heads/main' && - needs.changes.outputs.tree_proven_matrix != 'true' && ( needs.changes.outputs.ui_changed == 'true' || needs.changes.outputs.perf_changed == 'true' || diff --git a/docs/ci-operations.md b/docs/ci-operations.md index 6159273c71..5a2e6e7de9 100644 --- a/docs/ci-operations.md +++ b/docs/ci-operations.md @@ -8,13 +8,13 @@ These changes came out of the 6 October 2026 CI-efficiency review and its handov which runs repeat work. They don't change tests, assertions, thresholds, budgets, browser projects, shard counts, required check names, security gates or `main` concurrency. -| Change | PR | What it does | What still always runs | -| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. Also skips the Firefox/WebKit `release-browser-matrix` when a green full-suite `workflow_dispatch` run on the same PR head recorded testing that tree. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the release matrix without its own dispatch-run proof, failure routing; the weekly schedule and every dispatch run in full. | -| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | -| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | -| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | -| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | +| Change | PR | What it does | What still always runs | +| ------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. Never skips the Firefox/WebKit `release-browser-matrix`. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the Firefox/WebKit release matrix, failure routing; the weekly schedule and every dispatch run in full. | +| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | +| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | +| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | +| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | ### How a skip is proven (`scripts/ci-main-tree-proof.mjs`) diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index 8c82ebe49a..bb0a773b12 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -225,7 +225,8 @@ describe("main tree proof workflow wiring", () => { expect(jobs.changes.outputs.tree_proven_coverage).toBe("${{ steps.tree-proof.outputs.coverage }}"); expect(jobs.changes.outputs.tree_proven_ui).toBe("${{ steps.tree-proof.outputs.ui }}"); expect(jobs.changes.outputs.tree_proven_lighthouse).toBe("${{ steps.tree-proof.outputs.lighthouse }}"); - expect(jobs.changes.outputs.tree_proven_matrix).toBe("${{ steps.tree-proof.outputs.matrix }}"); + // The release matrix has no tree proof: main always runs it when UI, perf or the lockfile changed. + expect(jobs.changes.outputs.tree_proven_matrix).toBeUndefined(); // The changes job checks out full history, which HEAD^1 and the ancestry check rely on. const checkout = steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@")); expect(checkout?.with?.["fetch-depth"]).toBe(0); @@ -264,17 +265,11 @@ describe("main tree proof workflow wiring", () => { "coverage", "coverage-shards", "lighthouse-budget", - "release-browser-matrix", "ui-critical", "ui-critical-fast", ]); - // The matrix is gated on its own dispatch-run proof, and only inside its main-push clause, so the - // weekly schedule, release branches and every dispatch still run it in full. - const matrixIf = ifOf("release-browser-matrix"); - expect(matrixIf.match(/tree_proven_\w+/g)).toEqual(["tree_proven_matrix"]); - expect(matrixIf).toMatch( - /github\.ref == 'refs\/heads\/main' &&\s+needs\.changes\.outputs\.tree_proven_matrix != 'true' &&/, - ); + // The Firefox/WebKit/phone matrix never runs on a PR, so no proof can skip it on main. + expect(ifOf("release-browser-matrix")).not.toMatch(/tree_proven_/); expect(ifOf("ui-critical-fast")).toContain("needs.changes.outputs.tree_proven_ui != 'true'"); }); From a260e13dcad1cb569e9343ff1642381b3bb17762 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Tue, 13 Oct 2026 20:30:42 +0800 Subject: [PATCH 2/3] CI: stop reporting the release matrix as skipped by the tree proof --- scripts/ci-main-tree-proof.mjs | 9 ++++----- tests/ci-main-tree-proof.test.ts | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/ci-main-tree-proof.mjs b/scripts/ci-main-tree-proof.mjs index 397c8958e8..aecd0e684a 100644 --- a/scripts/ci-main-tree-proof.mjs +++ b/scripts/ci-main-tree-proof.mjs @@ -43,7 +43,8 @@ * the newest successful dispatch run of THIS workflow for the same head H, under the same checks * 4-8 above: same repository, within MAX_PROOF_AGE_HOURS, its Change scope recorded * `event=workflow_dispatch` with a tree equal to tree(HEAD), and every matrix leg concluded - * success. Absent such a run, main runs the matrix exactly as before. + * success. That value is no longer an output: ci.yml always runs the matrix on main, so the + * proof is informational only and never reported as a skip. * * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse ("true"/"false"), proof_run_url, reason. * Uses only the job's GITHUB_TOKEN (actions: read, pull-requests: read) — about four REST @@ -465,16 +466,14 @@ function writeOutputs({ proof, reason, runUrl = "" }) { `coverage=${proof.coverage === true}`, `ui=${proof.ui === true}`, `lighthouse=${proof.lighthouse === true}`, - `matrix=${proof.matrix === true}`, `proof_run_url=${runUrl}`, `reason=${String(reason).replace(/[\r\n]+/g, " ")}`, ]; console.log(lines.join("\n")); if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); if (process.env.GITHUB_STEP_SUMMARY) { - const skipped = Object.entries(proof) - .filter(([, value]) => value === true) - .map(([group]) => group); + // Only these groups gate a job; `matrix` is still computed but main always runs the release matrix. + const skipped = ["coverage", "ui", "lighthouse"].filter((group) => proof[group] === true); appendFileSync( process.env.GITHUB_STEP_SUMMARY, [ diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index bb0a773b12..5cc98f1769 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -204,7 +204,7 @@ describe("main tree proof script", () => { expect(result.stdout).toContain("coverage=false"); expect(result.stdout).toContain("ui=false"); expect(result.stdout).toContain("lighthouse=false"); - expect(result.stdout).toContain("matrix=false"); + expect(result.stdout).not.toContain("matrix="); }); }); From e7a6a28429a980304bdc24786f7fe7f62428d160 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Tue, 13 Oct 2026 20:38:07 +0800 Subject: [PATCH 3/3] CI: drop the unused dispatch-run matrix lookup from the tree proof; test the summary --- scripts/ci-main-tree-proof.mjs | 108 ++++++++----------------------- tests/ci-main-tree-proof.test.ts | 12 +++- 2 files changed, 39 insertions(+), 81 deletions(-) diff --git a/scripts/ci-main-tree-proof.mjs b/scripts/ci-main-tree-proof.mjs index aecd0e684a..b1960e3233 100644 --- a/scripts/ci-main-tree-proof.mjs +++ b/scripts/ci-main-tree-proof.mjs @@ -37,14 +37,9 @@ * production secrets and runs only on main), the Playwright build, visual baselines and * main-failure routing. The weekly scheduled run and every dispatch never take this path. * - * The Firefox/WebKit release-browser-matrix never runs on a pull_request event, so a PR run - * cannot prove it. A full-suite `workflow_dispatch` run on the PR head can (the draft "hold" - * rule runs one before a PR is marked ready). The `matrix` group therefore takes its proof from - * the newest successful dispatch run of THIS workflow for the same head H, under the same checks - * 4-8 above: same repository, within MAX_PROOF_AGE_HOURS, its Change scope recorded - * `event=workflow_dispatch` with a tree equal to tree(HEAD), and every matrix leg concluded - * success. That value is no longer an output: ci.yml always runs the matrix on main, so the - * proof is informational only and never reported as a skip. + * The Firefox/WebKit release-browser-matrix never runs on a pull_request event, so nothing can + * prove it: every main push that changes UI, perf inputs or the lockfile runs it, whatever this + * script finds, and no dispatch-run lookup is made for it. * * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse ("true"/"false"), proof_run_url, reason. * Uses only the job's GITHUB_TOKEN (actions: read, pull-requests: read) — about four REST @@ -259,23 +254,15 @@ export async function evaluate({ const maxAgeHours = Number(env.MAX_PROOF_AGE_HOURS || DEFAULT_MAX_PROOF_AGE_HOURS); if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0) return { proof: NONE, reason: "invalid MAX_PROOF_AGE_HOURS" }; const prOutcome = await provenRun({ request, repo, head, mainTree, now, maxAgeHours, event: "pull_request" }); - const matrixOutcome = await provenRun({ - request, - repo, - head, - mainTree, - now, - maxAgeHours, - event: "workflow_dispatch", - }); const proof = { ...(prOutcome.jobs ? classifyProof(prOutcome.jobs) : { coverage: false, ui: false, lighthouse: false }), - matrix: matrixOutcome.jobs ? classifyMatrixProof(matrixOutcome.jobs) : false, + // ci.yml always runs the release matrix on qualifying main pushes, so it is never looked up or proved. + matrix: false, }; return { proof, - reason: `PR #${pull.number}: ${prOutcome.reason}; release matrix: ${matrixOutcome.reason}`, - runUrl: prOutcome.runUrl || matrixOutcome.runUrl || "", + reason: `PR #${pull.number}: ${prOutcome.reason}`, + runUrl: prOutcome.runUrl || "", }; } @@ -461,6 +448,11 @@ export async function evaluatePullRequestUpdate({ }; } +/** Groups this push actually skips. The release matrix is never one: ci.yml has no gate for it. Pure. */ +export function skippedGroups(proof) { + return ["coverage", "ui", "lighthouse"].filter((group) => proof?.[group] === true); +} + function writeOutputs({ proof, reason, runUrl = "" }) { const lines = [ `coverage=${proof.coverage === true}`, @@ -472,8 +464,7 @@ function writeOutputs({ proof, reason, runUrl = "" }) { console.log(lines.join("\n")); if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); if (process.env.GITHUB_STEP_SUMMARY) { - // Only these groups gate a job; `matrix` is still computed but main always runs the release matrix. - const skipped = ["coverage", "ui", "lighthouse"].filter((group) => proof[group] === true); + const skipped = skippedGroups(proof); appendFileSync( process.env.GITHUB_STEP_SUMMARY, [ @@ -679,67 +670,24 @@ async function selfTest() { assert(parseTestedTree(testedRecord)?.tree === tree, "a well-formed record parses"); assert(result.proof.matrix === false, "a pull_request run never proves the release matrix"); - // The release matrix: proved only by a green full-suite dispatch run on the same head and tree. - const matrixLegs = [ - { ...ok("Change scope"), id: 601 }, - ok("release-browser-matrix (chromium, chromium-mockups, 1, 1)"), - ok("release-browser-matrix (webkit, mobile-webkit, 1, 5)"), - ]; - const dispatchRun = { ...run, id: 6, event: "workflow_dispatch" }; - const dispatchRecord = record(`event=workflow_dispatch sha=${pr.head.sha} tree=${tree}`); - const matrixRequest = - ({ legs = matrixLegs, annotations = dispatchRecord, dispatchRuns = [dispatchRun] } = {}) => - async (path) => { - if (path.includes("/workflows/") && path.includes("event=workflow_dispatch")) - return { workflow_runs: dispatchRuns }; - if (path.includes("/runs/6/jobs")) return { total_count: legs.length, jobs: legs }; - if (path.includes("/check-runs/601/annotations")) return annotations; - return request()(path); - }; - result = await evaluate({ ...base, request: matrixRequest() }); - assert(result.proof.matrix && result.proof.ui, `a green dispatch run proves the matrix (${result.reason})`); - assert(classifyMatrixProof(matrixLegs), "all-green legs prove the matrix"); - assert(!classifyMatrixProof([ok("release-browser-matrix")]), "a skipped (unexpanded) matrix proves nothing"); - result = await evaluate({ - ...base, - request: matrixRequest({ - legs: matrixLegs.map((job, index) => (index === 2 ? { ...job, conclusion: "failure" } : job)), - }), - }); - assert(!result.proof.matrix, "one red leg un-proves the matrix"); - result = await evaluate({ ...base, request: matrixRequest({ legs: matrixLegs.slice(0, 1) }) }); - assert(!result.proof.matrix, "a dispatch run without matrix legs proves nothing"); - result = await evaluate({ - ...base, - request: matrixRequest({ - annotations: record(`event=workflow_dispatch sha=${pr.head.sha} tree=${"3".repeat(40)}`), - }), - }); - assert(!result.proof.matrix, "a dispatch run that tested another tree proves nothing"); - result = await evaluate({ ...base, request: matrixRequest({ annotations: testedRecord }) }); - assert(!result.proof.matrix, "a dispatch run must record a dispatch event"); - result = await evaluate({ - ...base, - request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, conclusion: "failure" }] }), - }); - assert(!result.proof.matrix, "a red dispatch run proves nothing"); - result = await evaluate({ - ...base, - request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, updated_at: "2026-10-05T10:00:00Z" }] }), - }); - assert(!result.proof.matrix, "a dispatch run older than 24h is stale"); - result = await evaluate({ - ...base, - runGit: gitMap({ "rev-parse HEAD^{tree}": "2".repeat(40) }), - request: matrixRequest(), - }); - assert(!result.proof.matrix, "a tree mismatch never proves the matrix"); + // The release matrix is never proved or reported as skipped. + assert(classifyMatrixProof([ok("release-browser-matrix (webkit, webkit, 1, 4)")]), "legs still classify"); + assert( + skippedGroups({ coverage: true, ui: false, lighthouse: true, matrix: true }).join(",") === "coverage,lighthouse", + "the summary never lists the release matrix as skipped", + ); + const paths = []; result = await evaluate({ ...base, - env: { ...env, GITHUB_EVENT_NAME: "workflow_dispatch" }, - request: matrixRequest(), + request: async (path) => { + paths.push(path); + return request()(path); + }, }); - assert(!result.proof.matrix, "a dispatch run never skips its own matrix"); + assert( + result.proof.ui && !paths.some((path) => path.includes("event=workflow_dispatch")), + "no dispatch-run lookup is made, so a failing one cannot clear the PR proof", + ); const recorded = []; const originalLog = console.log; console.log = (line) => recorded.push(line); diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index 5cc98f1769..402c308134 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -15,6 +15,7 @@ import { PROOF_GROUPS, reconstructTestedTree, selectPullRequestCandidates, + skippedGroups, UNRELATED_INPUT_RULES, } from "../scripts/ci-main-tree-proof.mjs"; @@ -48,6 +49,15 @@ describe("main tree proof script", () => { expect(result.stdout).toContain("ci-main-tree-proof self-test passed."); }); + it("never lists the release matrix as skipped in the step summary", () => { + expect(skippedGroups({ coverage: true, ui: true, lighthouse: true, matrix: true })).toEqual([ + "coverage", + "ui", + "lighthouse", + ]); + expect(skippedGroups({ coverage: false, ui: false, lighthouse: false, matrix: true })).toEqual([]); + }); + it("names exactly the PR-run jobs each group skips, matching ci.yml job names", () => { const names = Object.values(jobs).map((job) => String((job as { name?: string }).name ?? "")); for (const name of [ @@ -225,7 +235,7 @@ describe("main tree proof workflow wiring", () => { expect(jobs.changes.outputs.tree_proven_coverage).toBe("${{ steps.tree-proof.outputs.coverage }}"); expect(jobs.changes.outputs.tree_proven_ui).toBe("${{ steps.tree-proof.outputs.ui }}"); expect(jobs.changes.outputs.tree_proven_lighthouse).toBe("${{ steps.tree-proof.outputs.lighthouse }}"); - // The release matrix has no tree proof: main always runs it when UI, perf or the lockfile changed. + // The release matrix has no tree proof: every main push that changes UI, perf inputs or the lockfile runs it. expect(jobs.changes.outputs.tree_proven_matrix).toBeUndefined(); // The changes job checks out full history, which HEAD^1 and the ancestry check rely on. const checkout = steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@"));