diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88e0f398e..68924a2271 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,9 +107,6 @@ jobs: tree_proven_coverage: ${{ steps.tree-proof.outputs.coverage }} tree_proven_ui: ${{ steps.tree-proof.outputs.ui }} tree_proven_lighthouse: ${{ steps.tree-proof.outputs.lighthouse }} - # True only when a green full-suite dispatch run already passed every release-matrix leg on - # this exact tree (the merged PR's head). Proved separately because PR runs never run it. - tree_proven_matrix: ${{ steps.tree-proof.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1922,7 +1919,6 @@ jobs: startsWith(github.ref, 'refs/heads/release/') || ( github.ref == 'refs/heads/main' && - needs.changes.outputs.tree_proven_matrix != 'true' && ( needs.changes.outputs.ui_changed == 'true' || needs.changes.outputs.perf_changed == 'true' || diff --git a/docs/ci-operations.md b/docs/ci-operations.md index 6159273c71..5a2e6e7de9 100644 --- a/docs/ci-operations.md +++ b/docs/ci-operations.md @@ -8,13 +8,13 @@ These changes came out of the 6 October 2026 CI-efficiency review and its handov which runs repeat work. They don't change tests, assertions, thresholds, budgets, browser projects, shard counts, required check names, security gates or `main` concurrency. -| Change | PR | What it does | What still always runs | -| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. Also skips the Firefox/WebKit `release-browser-matrix` when a green full-suite `workflow_dispatch` run on the same PR head recorded testing that tree. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the release matrix without its own dispatch-run proof, failure routing; the weekly schedule and every dispatch run in full. | -| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | -| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | -| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | -| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | +| Change | PR | What it does | What still always runs | +| ------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. Never skips the Firefox/WebKit `release-browser-matrix`. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the Firefox/WebKit release matrix, failure routing; the weekly schedule and every dispatch run in full. | +| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | +| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | +| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | +| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | ### How a skip is proven (`scripts/ci-main-tree-proof.mjs`) diff --git a/scripts/ci-main-tree-proof.mjs b/scripts/ci-main-tree-proof.mjs index e6533d8011..908dcffce2 100644 --- a/scripts/ci-main-tree-proof.mjs +++ b/scripts/ci-main-tree-proof.mjs @@ -37,13 +37,9 @@ * production secrets and runs only on main), the Playwright build, visual baselines and * main-failure routing. The weekly scheduled run and every dispatch never take this path. * - * The Firefox/WebKit release-browser-matrix never runs on a pull_request event, so a PR run - * cannot prove it. A full-suite `workflow_dispatch` run on the PR head can (the draft "hold" - * rule runs one before a PR is marked ready). The `matrix` group therefore takes its proof from - * the newest successful dispatch run of THIS workflow for the same head H, under the same checks - * 4-8 above: same repository, within MAX_PROOF_AGE_HOURS, its Change scope recorded - * `event=workflow_dispatch` with a tree equal to tree(HEAD), and every matrix leg concluded - * success. Absent such a run, main runs the matrix exactly as before. + * The Firefox/WebKit release-browser-matrix never runs on a pull_request event, so nothing can + * prove it: every main push that changes UI, perf inputs or the lockfile runs it, whatever this + * script finds, and no dispatch-run lookup is made for it. * * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse ("true"/"false"), proof_run_url, reason. * Uses only the job's GITHUB_TOKEN (actions: read, pull-requests: read) — about four REST @@ -258,23 +254,15 @@ export async function evaluate({ const maxAgeHours = Number(env.MAX_PROOF_AGE_HOURS || DEFAULT_MAX_PROOF_AGE_HOURS); if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0) return { proof: NONE, reason: "invalid MAX_PROOF_AGE_HOURS" }; const prOutcome = await provenRun({ request, repo, head, mainTree, now, maxAgeHours, event: "pull_request" }); - const matrixOutcome = await provenRun({ - request, - repo, - head, - mainTree, - now, - maxAgeHours, - event: "workflow_dispatch", - }); const proof = { ...(prOutcome.jobs ? classifyProof(prOutcome.jobs) : { coverage: false, ui: false, lighthouse: false }), - matrix: matrixOutcome.jobs ? classifyMatrixProof(matrixOutcome.jobs) : false, + // ci.yml always runs the release matrix on qualifying main pushes, so it is never looked up or proved. + matrix: false, }; return { proof, - reason: `PR #${pull.number}: ${prOutcome.reason}; release matrix: ${matrixOutcome.reason}`, - runUrl: prOutcome.runUrl || matrixOutcome.runUrl || "", + reason: `PR #${pull.number}: ${prOutcome.reason}`, + runUrl: prOutcome.runUrl || "", }; } @@ -460,21 +448,23 @@ export async function evaluatePullRequestUpdate({ }; } +/** Groups this push actually skips. The release matrix is never one: ci.yml has no gate for it. Pure. */ +export function skippedGroups(proof) { + return ["coverage", "ui", "lighthouse"].filter((group) => proof?.[group] === true); +} + function writeOutputs({ proof, reason, runUrl = "" }) { const lines = [ `coverage=${proof.coverage === true}`, `ui=${proof.ui === true}`, `lighthouse=${proof.lighthouse === true}`, - `matrix=${proof.matrix === true}`, `proof_run_url=${runUrl}`, `reason=${String(reason).replace(/[\r\n]+/g, " ")}`, ]; console.log(lines.join("\n")); if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); if (process.env.GITHUB_STEP_SUMMARY) { - const skipped = Object.entries(proof) - .filter(([, value]) => value === true) - .map(([group]) => group); + const skipped = skippedGroups(proof); appendFileSync( process.env.GITHUB_STEP_SUMMARY, [ @@ -681,67 +671,24 @@ async function selfTest() { assert(parseTestedTree(testedRecord)?.tree === tree, "a well-formed record parses"); assert(result.proof.matrix === false, "a pull_request run never proves the release matrix"); - // The release matrix: proved only by a green full-suite dispatch run on the same head and tree. - const matrixLegs = [ - { ...ok("Change scope"), id: 601 }, - ok("release-browser-matrix (chromium, chromium-mockups, 1, 1)"), - ok("release-browser-matrix (webkit, mobile-webkit, 1, 5)"), - ]; - const dispatchRun = { ...run, id: 6, event: "workflow_dispatch" }; - const dispatchRecord = record(`event=workflow_dispatch sha=${pr.head.sha} tree=${tree}`); - const matrixRequest = - ({ legs = matrixLegs, annotations = dispatchRecord, dispatchRuns = [dispatchRun] } = {}) => - async (path) => { - if (path.includes("/workflows/") && path.includes("event=workflow_dispatch")) - return { workflow_runs: dispatchRuns }; - if (path.includes("/runs/6/jobs")) return { total_count: legs.length, jobs: legs }; - if (path.includes("/check-runs/601/annotations")) return annotations; - return request()(path); - }; - result = await evaluate({ ...base, request: matrixRequest() }); - assert(result.proof.matrix && result.proof.ui, `a green dispatch run proves the matrix (${result.reason})`); - assert(classifyMatrixProof(matrixLegs), "all-green legs prove the matrix"); - assert(!classifyMatrixProof([ok("release-browser-matrix")]), "a skipped (unexpanded) matrix proves nothing"); - result = await evaluate({ - ...base, - request: matrixRequest({ - legs: matrixLegs.map((job, index) => (index === 2 ? { ...job, conclusion: "failure" } : job)), - }), - }); - assert(!result.proof.matrix, "one red leg un-proves the matrix"); - result = await evaluate({ ...base, request: matrixRequest({ legs: matrixLegs.slice(0, 1) }) }); - assert(!result.proof.matrix, "a dispatch run without matrix legs proves nothing"); - result = await evaluate({ - ...base, - request: matrixRequest({ - annotations: record(`event=workflow_dispatch sha=${pr.head.sha} tree=${"3".repeat(40)}`), - }), - }); - assert(!result.proof.matrix, "a dispatch run that tested another tree proves nothing"); - result = await evaluate({ ...base, request: matrixRequest({ annotations: testedRecord }) }); - assert(!result.proof.matrix, "a dispatch run must record a dispatch event"); - result = await evaluate({ - ...base, - request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, conclusion: "failure" }] }), - }); - assert(!result.proof.matrix, "a red dispatch run proves nothing"); - result = await evaluate({ - ...base, - request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, updated_at: "2026-10-05T10:00:00Z" }] }), - }); - assert(!result.proof.matrix, "a dispatch run older than 24h is stale"); - result = await evaluate({ - ...base, - runGit: gitMap({ "rev-parse HEAD^{tree}": "2".repeat(40) }), - request: matrixRequest(), - }); - assert(!result.proof.matrix, "a tree mismatch never proves the matrix"); + // The release matrix is never proved or reported as skipped. + assert(classifyMatrixProof([ok("release-browser-matrix (webkit, webkit, 1, 4)")]), "legs still classify"); + assert( + skippedGroups({ coverage: true, ui: false, lighthouse: true, matrix: true }).join(",") === "coverage,lighthouse", + "the summary never lists the release matrix as skipped", + ); + const paths = []; result = await evaluate({ ...base, - env: { ...env, GITHUB_EVENT_NAME: "workflow_dispatch" }, - request: matrixRequest(), + request: async (path) => { + paths.push(path); + return request()(path); + }, }); - assert(!result.proof.matrix, "a dispatch run never skips its own matrix"); + assert( + result.proof.ui && !paths.some((path) => path.includes("event=workflow_dispatch")), + "no dispatch-run lookup is made, so a failing one cannot clear the PR proof", + ); const recorded = []; const originalLog = console.log; console.log = (line) => recorded.push(line); diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index 9554e74b5b..7576251fa3 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -15,6 +15,7 @@ import { PROOF_GROUPS, reconstructTestedTree, selectPullRequestCandidates, + skippedGroups, UNRELATED_INPUT_RULES, } from "../scripts/ci-main-tree-proof.mjs"; @@ -48,6 +49,15 @@ describe("main tree proof script", () => { expect(result.stdout).toContain("ci-main-tree-proof self-test passed."); }); + it("never lists the release matrix as skipped in the step summary", () => { + expect(skippedGroups({ coverage: true, ui: true, lighthouse: true, matrix: true })).toEqual([ + "coverage", + "ui", + "lighthouse", + ]); + expect(skippedGroups({ coverage: false, ui: false, lighthouse: false, matrix: true })).toEqual([]); + }); + it("exits non-zero when a self-test assertion fails, writing no GitHub outputs", () => { const outDir = mkdtempSync(path.join(tmpdir(), "tree-proof-self-test-")); const outputFile = path.join(outDir, "output"); @@ -232,7 +242,7 @@ describe("main tree proof script", () => { expect(result.stdout).toContain("coverage=false"); expect(result.stdout).toContain("ui=false"); expect(result.stdout).toContain("lighthouse=false"); - expect(result.stdout).toContain("matrix=false"); + expect(result.stdout).not.toContain("matrix="); }); }); @@ -253,7 +263,8 @@ describe("main tree proof workflow wiring", () => { expect(jobs.changes.outputs.tree_proven_coverage).toBe("${{ steps.tree-proof.outputs.coverage }}"); expect(jobs.changes.outputs.tree_proven_ui).toBe("${{ steps.tree-proof.outputs.ui }}"); expect(jobs.changes.outputs.tree_proven_lighthouse).toBe("${{ steps.tree-proof.outputs.lighthouse }}"); - expect(jobs.changes.outputs.tree_proven_matrix).toBe("${{ steps.tree-proof.outputs.matrix }}"); + // The release matrix has no tree proof: every main push that changes UI, perf inputs or the lockfile runs it. + expect(jobs.changes.outputs.tree_proven_matrix).toBeUndefined(); // The changes job checks out full history, which HEAD^1 and the ancestry check rely on. const checkout = steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@")); expect(checkout?.with?.["fetch-depth"]).toBe(0); @@ -292,17 +303,11 @@ describe("main tree proof workflow wiring", () => { "coverage", "coverage-shards", "lighthouse-budget", - "release-browser-matrix", "ui-critical", "ui-critical-fast", ]); - // The matrix is gated on its own dispatch-run proof, and only inside its main-push clause, so the - // weekly schedule, release branches and every dispatch still run it in full. - const matrixIf = ifOf("release-browser-matrix"); - expect(matrixIf.match(/tree_proven_\w+/g)).toEqual(["tree_proven_matrix"]); - expect(matrixIf).toMatch( - /github\.ref == 'refs\/heads\/main' &&\s+needs\.changes\.outputs\.tree_proven_matrix != 'true' &&/, - ); + // The Firefox/WebKit/phone matrix never runs on a PR, so no proof can skip it on main. + expect(ifOf("release-browser-matrix")).not.toMatch(/tree_proven_/); expect(ifOf("ui-critical-fast")).toContain("needs.changes.outputs.tree_proven_ui != 'true'"); });