diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1ca6a4b8b..41a04fda70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,6 +107,9 @@ jobs: tree_proven_coverage: ${{ steps.tree-proof.outputs.coverage }} tree_proven_ui: ${{ steps.tree-proof.outputs.ui }} tree_proven_lighthouse: ${{ steps.tree-proof.outputs.lighthouse }} + # True only when a green full-suite dispatch run already passed every release-matrix leg on + # this exact tree (the merged PR's head). Proved separately because PR runs never run it. + tree_proven_matrix: ${{ steps.tree-proof.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -1919,6 +1922,7 @@ jobs: startsWith(github.ref, 'refs/heads/release/') || ( github.ref == 'refs/heads/main' && + needs.changes.outputs.tree_proven_matrix != 'true' && ( needs.changes.outputs.ui_changed == 'true' || needs.changes.outputs.perf_changed == 'true' || @@ -1929,9 +1933,15 @@ jobs: needs.changes.result == 'success' && needs.static-pr.result == 'success' && (needs.build.result == 'success' || needs.build.result == 'skipped') && - (needs.ui-playwright-build.result == 'success' || needs.ui-playwright-build.result == 'skipped') && - (needs.ui-critical.result == 'success' || needs.ui-critical.result == 'skipped') - needs: [changes, static-pr, build, ui-playwright-build, ui-critical] + (needs.ui-playwright-build.result == 'success' || needs.ui-playwright-build.result == 'skipped') + # Starts as soon as the shared Playwright build is ready, ALONGSIDE Production UI rather than + # after it. Waiting for Production UI bought nothing: these legs only read the shared build, + # and the run still fails on a red Production UI (it is required in its own right). Measured + # 2026-10-13: main run 38352161182 had the build ready at minute 4.1, but every leg waited until + # minute 15.2 for the three Chromium shards and the run ended at 34.1; dispatch runs + # 38369056741 and 38370421833 had one red Chromium shard, so the whole Firefox/WebKit matrix + # was skipped and its verdict deferred to another full round after the fix. + needs: [changes, static-pr, build, ui-playwright-build] runs-on: ubuntu-24.04 # Per ENGINE, not for the whole matrix. Before 2026-09-07 one job ran # chromium-mockups + firefox + webkit sequentially under `workers: 1` / @@ -1992,14 +2002,21 @@ jobs: # merge (main runs took 36-40 min). Tablet now splits four ways (~12 min # each by its new timing table) and webkit three ways (19.5 -> 13 min by # its existing table), so every tablet test runs and main finishes sooner. + # + # 2026-10-13: with the matrix no longer queued behind Production UI, desktop WebKit and + # Firefox were the slowest legs (main run 38352161182: webkit 17.2/13.0/14.2 min of tests, + # firefox 13.4/9.0/8.7). Both now split four ways, which brings them under the iPhone legs + # (their slowest single file, ui-smoke, is ~11 min and sets the floor). include: - { engine: chromium, project: chromium-mockups, shard: 1, total: 1 } - - { engine: firefox, project: firefox, shard: 1, total: 3 } - - { engine: firefox, project: firefox, shard: 2, total: 3 } - - { engine: firefox, project: firefox, shard: 3, total: 3 } - - { engine: webkit, project: webkit, shard: 1, total: 3 } - - { engine: webkit, project: webkit, shard: 2, total: 3 } - - { engine: webkit, project: webkit, shard: 3, total: 3 } + - { engine: firefox, project: firefox, shard: 1, total: 4 } + - { engine: firefox, project: firefox, shard: 2, total: 4 } + - { engine: firefox, project: firefox, shard: 3, total: 4 } + - { engine: firefox, project: firefox, shard: 4, total: 4 } + - { engine: webkit, project: webkit, shard: 1, total: 4 } + - { engine: webkit, project: webkit, shard: 2, total: 4 } + - { engine: webkit, project: webkit, shard: 3, total: 4 } + - { engine: webkit, project: webkit, shard: 4, total: 4 } - { engine: webkit, project: mobile-webkit, shard: 1, total: 5 } - { engine: webkit, project: mobile-webkit, shard: 2, total: 5 } - { engine: webkit, project: mobile-webkit, shard: 3, total: 5 } @@ -2104,9 +2121,11 @@ jobs: # includes mockups, whose public flag is baked in, so it builds its own. # If the producer was out of scope, the wrapper also builds normally. PLAYWRIGHT_REUSE_BUILD: ${{ matrix.engine != 'chromium' && needs.ui-playwright-build.result == 'success' }} - # "true" only when this main push skipped Production UI because the identical tree already - # passed it on the merged PR (Main tree proof) — the same proof the primary path relies on. - UI_PROVEN_BY_PR: ${{ needs.changes.outputs.tree_proven_ui }} + # When UI changed, production Chromium is proven either by Production UI in this same run + # (it runs whenever UI changed and no proof exists, and the run fails if it fails) or, on a + # main push, by the merged PR's identical tree (Main tree proof). Either way this matrix + # need not repeat it. + UI_CHANGED: ${{ needs.changes.outputs.ui_changed }} run: | # Both paths below are exhaustive across playwright.config.ts projects # for their own path, and disjoint between engines. The fail-safe union @@ -2114,9 +2133,9 @@ jobs: # vanish from CI silently, so # `tests/ci-browser-matrix-coverage.test.ts` re-derives both unions from # this file plus playwright.config.ts and fails closed on any drift. - if [ "${{ needs.changes.outputs.ui_changed }}" = "true" ] && { [ "${{ needs.ui-critical.result }}" = "success" ] || [ "$UI_PROVEN_BY_PR" = "true" ]; }; then - # Production Chromium already passed — in this run, or on the merged PR's - # byte-identical tree. Keep mockup Chromium plus the cross-browser + if [ "$UI_CHANGED" = "true" ]; then + # Production Chromium is proven by Production UI in this run, or on the merged + # PR's byte-identical tree. Keep mockup Chromium plus the cross-browser # backstop without repeating it. case "$ENGINE" in chromium) PROJECTS="--project=chromium-mockups" ;; diff --git a/docs/ci-operations.md b/docs/ci-operations.md index 2178f7ee0d..6159273c71 100644 --- a/docs/ci-operations.md +++ b/docs/ci-operations.md @@ -8,13 +8,13 @@ These changes came out of the 6 October 2026 CI-efficiency review and its handov which runs repeat work. They don't change tests, assertions, thresholds, budgets, browser projects, shard counts, required check names, security gates or `main` concurrency. -| Change | PR | What it does | What still always runs | -| ------------------------------------------- | ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the Firefox/WebKit `release-browser-matrix`, failure routing; the weekly schedule and every dispatch run in full. | -| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | -| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | -| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | -| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | +| Change | PR | What it does | What still always runs | +| ------------------------------------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Proven-tree skip on `main` | #3313 | On a push to `main` whose tree equals the tree the merged PR's own `pull_request` run **recorded testing**, skips unit coverage, Production UI and Lighthouse if every leg of that job passed there. Also skips the Firefox/WebKit `release-browser-matrix` when a green full-suite `workflow_dispatch` run on the same PR head recorded testing that tree. | Static checks, safety, Build + deployment boot smoke, Playwright build, visual baselines, the release matrix without its own dispatch-run proof, failure routing; the weekly schedule and every dispatch run in full. | +| Lighthouse requirement matches its producer | `ci/lighthouse-required-condition` | `PR required` demands a `Lighthouse budget` success on every event where that job is expected to run (schedule, ordinary dispatch and lockfile-only pushes included). A skip is accepted only for the skip label, baseline-refresh dispatch, drafts/out-of-scope changes, or a proven skip. | The producer, budget, baseline and thresholds are unchanged. | +| PR branch-update reuse | `ci/pr-update-proof` | On `synchronize` of a same-repo PR, reuses an earlier green run of the **same PR**. If the newly tested tree is identical, coverage, UI and Lighthouse carry. If the delta lies entirely outside the build/browser/Lighthouse inputs, UI and Lighthouse carry. | Lint, typecheck, unit coverage (unless the tree is identical), safety, secret scanning, SAST, Build. | +| Scheduled npm cache keys | #3318 | Six scheduled workflows hash `package-lock.json` + `.nvmrc`, the key CI's `npm ci` jobs populate, instead of the lockfile-only key a bare `setup-node` can fill empty. | Every `npm ci`. | +| Merge-queue Gitleaks | #3322 | `merge_group` scans the full immutable `base_sha..head_sha` range with the pinned binary and fails closed on a missing, zero, absent or non-ancestor base. | Push/PR/schedule/dispatch scans, `contents: read`, no persisted credentials. | ### How a skip is proven (`scripts/ci-main-tree-proof.mjs`) diff --git a/scripts/ci-main-tree-proof.mjs b/scripts/ci-main-tree-proof.mjs index cf872490d4..397c8958e8 100644 --- a/scripts/ci-main-tree-proof.mjs +++ b/scripts/ci-main-tree-proof.mjs @@ -34,10 +34,17 @@ * Only jobs whose result depends solely on the tree are eligible: unit coverage, the Chromium * production UI suite and the Lighthouse budget. Everything with main-only behaviour keeps * running on every push: static checks, safety, Build (its deployment boot smoke uses - * production secrets and runs only on main), the Playwright build, visual baselines, the - * Firefox/WebKit release-browser-matrix (it never runs on a PR, so nothing proved it) and + * production secrets and runs only on main), the Playwright build, visual baselines and * main-failure routing. The weekly scheduled run and every dispatch never take this path. * + * The Firefox/WebKit release-browser-matrix never runs on a pull_request event, so a PR run + * cannot prove it. A full-suite `workflow_dispatch` run on the PR head can (the draft "hold" + * rule runs one before a PR is marked ready). The `matrix` group therefore takes its proof from + * the newest successful dispatch run of THIS workflow for the same head H, under the same checks + * 4-8 above: same repository, within MAX_PROOF_AGE_HOURS, its Change scope recorded + * `event=workflow_dispatch` with a tree equal to tree(HEAD), and every matrix leg concluded + * success. Absent such a run, main runs the matrix exactly as before. + * * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse ("true"/"false"), proof_run_url, reason. * Uses only the job's GITHUB_TOKEN (actions: read, pull-requests: read) — about four REST * calls per main push. Every proved job checks out the run's GITHUB_SHA (no `ref:` override, @@ -104,7 +111,19 @@ export const PROOF_GROUPS = { lighthouse: [/^Lighthouse budget$/], }; -const NONE = Object.freeze({ coverage: false, ui: false, lighthouse: false }); +/** The release matrix legs a full-suite dispatch run must have run green; see the docblock. */ +export const MATRIX_PROOF_PATTERNS = Object.freeze([/^release-browser-matrix \(.+\)$/]); + +const NONE = Object.freeze({ coverage: false, ui: false, lighthouse: false, matrix: false }); + +/** True only when every matrix-leg pattern matches at least one job and all of them succeeded. Pure. */ +export function classifyMatrixProof(jobs) { + const list = Array.isArray(jobs) ? jobs : []; + return MATRIX_PROOF_PATTERNS.every((pattern) => { + const legs = list.filter((job) => pattern.test(String(job?.name ?? ""))); + return legs.length > 0 && legs.every((job) => job?.status === "completed" && job?.conclusion === "success"); + }); +} /** Which proof groups a PR run's job list establishes. Pure. */ export function classifyProof(jobs) { @@ -142,13 +161,16 @@ export function pickMergedPull(pulls, sha) { return matches.length === 1 ? matches[0] : null; } -/** The newest successful, fresh pull_request run of ci.yml for `headSha`, or null. Pure. */ -export function selectProvingRun(runs, { headSha, now = new Date(), maxAgeHours = DEFAULT_MAX_PROOF_AGE_HOURS }) { +/** The newest successful, fresh `event` run (pull_request by default) of ci.yml for `headSha`, or null. Pure. */ +export function selectProvingRun( + runs, + { headSha, now = new Date(), maxAgeHours = DEFAULT_MAX_PROOF_AGE_HOURS, event = "pull_request" }, +) { const maxAgeMs = maxAgeHours * 3_600_000; const candidates = (Array.isArray(runs) ? runs : []) .filter( (run) => - run?.event === "pull_request" && + run?.event === event && run?.head_sha === headSha && run?.path === CI_WORKFLOW_PATH && run?.status === "completed" && @@ -235,34 +257,55 @@ export async function evaluate({ const maxAgeHours = Number(env.MAX_PROOF_AGE_HOURS || DEFAULT_MAX_PROOF_AGE_HOURS); if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0) return { proof: NONE, reason: "invalid MAX_PROOF_AGE_HOURS" }; + const prOutcome = await provenRun({ request, repo, head, mainTree, now, maxAgeHours, event: "pull_request" }); + const matrixOutcome = await provenRun({ + request, + repo, + head, + mainTree, + now, + maxAgeHours, + event: "workflow_dispatch", + }); + const proof = { + ...(prOutcome.jobs ? classifyProof(prOutcome.jobs) : { coverage: false, ui: false, lighthouse: false }), + matrix: matrixOutcome.jobs ? classifyMatrixProof(matrixOutcome.jobs) : false, + }; + return { + proof, + reason: `PR #${pull.number}: ${prOutcome.reason}; release matrix: ${matrixOutcome.reason}`, + runUrl: prOutcome.runUrl || matrixOutcome.runUrl || "", + }; +} + +/** + * Checks 5-6 for one event kind: the newest fresh green `event` run of ci.yml for `head` whose own + * Change scope recorded testing exactly `mainTree`. Returns its job list, or `jobs: null` and why. + */ +async function provenRun({ request, repo, head, mainTree, now, maxAgeHours, event }) { const runs = await request( - `/repos/${repo}/actions/workflows/ci.yml/runs?event=pull_request&head_sha=${head}&status=completed&per_page=30`, + `/repos/${repo}/actions/workflows/ci.yml/runs?event=${event}&head_sha=${head}&status=completed&per_page=30`, ); - const run = selectProvingRun(runs?.workflow_runs, { headSha: head, now, maxAgeHours }); - if (!run) { - return { - proof: NONE, - reason: `no successful pull_request CI run for PR #${pull.number} head within ${maxAgeHours}h`, - }; - } + const run = selectProvingRun(runs?.workflow_runs, { headSha: head, now, maxAgeHours, event }); + if (!run) return { jobs: null, reason: `no successful ${event} CI run for the head within ${maxAgeHours}h` }; const jobs = await request(`/repos/${repo}/actions/runs/${run.id}/jobs?filter=latest&per_page=100`); if (!Array.isArray(jobs?.jobs) || jobs.total_count > jobs.jobs.length) { - return { proof: NONE, reason: "could not read the proving run's complete job list" }; + return { jobs: null, reason: `could not read run ${run.id}'s complete job list` }; } const scopeJobs = jobs.jobs.filter((job) => job?.name === "Change scope"); if (scopeJobs.length !== 1 || scopeJobs[0].conclusion !== "success") { - return { proof: NONE, reason: "the proving run has no single successful Change scope job" }; + return { jobs: null, reason: `run ${run.id} has no single successful Change scope job` }; } const tested = parseTestedTree(await request(`/repos/${repo}/check-runs/${scopeJobs[0].id}/annotations?per_page=50`)); - if (!tested || tested.event !== "pull_request") { - return { proof: NONE, reason: `run ${run.id} did not record the tree it tested` }; + if (!tested || tested.event !== event) { + return { jobs: null, reason: `run ${run.id} did not record the tree it tested` }; } if (tested.tree !== mainTree) { - return { proof: NONE, reason: `run ${run.id} tested tree ${tested.tree.slice(0, 12)}, not this main tree` }; + return { jobs: null, reason: `run ${run.id} tested tree ${tested.tree.slice(0, 12)}, not this main tree` }; } return { - proof: classifyProof(jobs.jobs), - reason: `run ${run.id} (PR #${pull.number}) recorded testing ${tested.sha.slice(0, 12)} whose tree equals this main tree`, + jobs: jobs.jobs, + reason: `run ${run.id} recorded testing ${tested.sha.slice(0, 12)} whose tree equals this main tree`, runUrl: run.html_url ?? "", }; } @@ -351,7 +394,7 @@ export async function evaluatePullRequestUpdate({ `/repos/${repo}/actions/workflows/ci.yml/runs?event=pull_request&branch=${encodeURIComponent(headRef)}&status=success&per_page=30`, ); const candidates = selectPullRequestCandidates(runs?.workflow_runs, { prNumber, now, maxAgeHours }); - const proof = { coverage: false, ui: false, lighthouse: false }; + const proof = { coverage: false, ui: false, lighthouse: false, matrix: false }; const notes = []; let runUrl = ""; for (const run of candidates) { @@ -395,7 +438,7 @@ export async function evaluatePullRequestUpdate({ } const changed = runGit(["diff", "--name-only", "--no-renames", oldTree, newTree]).split("\n").filter(Boolean); if (changed.length === 0) { - for (const group of Object.keys(proof)) proof[group] ||= legs[group]; + for (const group of Object.keys(legs)) proof[group] ||= legs[group]; notes.push(`run ${run.id}: identical tested tree`); } else if (changed.every(isUnrelatedInput)) { proof.ui ||= legs.ui; @@ -422,6 +465,7 @@ function writeOutputs({ proof, reason, runUrl = "" }) { `coverage=${proof.coverage === true}`, `ui=${proof.ui === true}`, `lighthouse=${proof.lighthouse === true}`, + `matrix=${proof.matrix === true}`, `proof_run_url=${runUrl}`, `reason=${String(reason).replace(/[\r\n]+/g, " ")}`, ]; @@ -634,6 +678,69 @@ async function selfTest() { result = await evaluate({ ...base, request: request([{ ...run, conclusion: "cancelled" }]) }); assert(!result.proof.ui, "a cancelled run proves nothing"); assert(parseTestedTree(testedRecord)?.tree === tree, "a well-formed record parses"); + assert(result.proof.matrix === false, "a pull_request run never proves the release matrix"); + + // The release matrix: proved only by a green full-suite dispatch run on the same head and tree. + const matrixLegs = [ + { ...ok("Change scope"), id: 601 }, + ok("release-browser-matrix (chromium, chromium-mockups, 1, 1)"), + ok("release-browser-matrix (webkit, mobile-webkit, 1, 5)"), + ]; + const dispatchRun = { ...run, id: 6, event: "workflow_dispatch" }; + const dispatchRecord = record(`event=workflow_dispatch sha=${pr.head.sha} tree=${tree}`); + const matrixRequest = + ({ legs = matrixLegs, annotations = dispatchRecord, dispatchRuns = [dispatchRun] } = {}) => + async (path) => { + if (path.includes("/workflows/") && path.includes("event=workflow_dispatch")) + return { workflow_runs: dispatchRuns }; + if (path.includes("/runs/6/jobs")) return { total_count: legs.length, jobs: legs }; + if (path.includes("/check-runs/601/annotations")) return annotations; + return request()(path); + }; + result = await evaluate({ ...base, request: matrixRequest() }); + assert(result.proof.matrix && result.proof.ui, `a green dispatch run proves the matrix (${result.reason})`); + assert(classifyMatrixProof(matrixLegs), "all-green legs prove the matrix"); + assert(!classifyMatrixProof([ok("release-browser-matrix")]), "a skipped (unexpanded) matrix proves nothing"); + result = await evaluate({ + ...base, + request: matrixRequest({ + legs: matrixLegs.map((job, index) => (index === 2 ? { ...job, conclusion: "failure" } : job)), + }), + }); + assert(!result.proof.matrix, "one red leg un-proves the matrix"); + result = await evaluate({ ...base, request: matrixRequest({ legs: matrixLegs.slice(0, 1) }) }); + assert(!result.proof.matrix, "a dispatch run without matrix legs proves nothing"); + result = await evaluate({ + ...base, + request: matrixRequest({ + annotations: record(`event=workflow_dispatch sha=${pr.head.sha} tree=${"3".repeat(40)}`), + }), + }); + assert(!result.proof.matrix, "a dispatch run that tested another tree proves nothing"); + result = await evaluate({ ...base, request: matrixRequest({ annotations: testedRecord }) }); + assert(!result.proof.matrix, "a dispatch run must record a dispatch event"); + result = await evaluate({ + ...base, + request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, conclusion: "failure" }] }), + }); + assert(!result.proof.matrix, "a red dispatch run proves nothing"); + result = await evaluate({ + ...base, + request: matrixRequest({ dispatchRuns: [{ ...dispatchRun, updated_at: "2026-10-05T10:00:00Z" }] }), + }); + assert(!result.proof.matrix, "a dispatch run older than 24h is stale"); + result = await evaluate({ + ...base, + runGit: gitMap({ "rev-parse HEAD^{tree}": "2".repeat(40) }), + request: matrixRequest(), + }); + assert(!result.proof.matrix, "a tree mismatch never proves the matrix"); + result = await evaluate({ + ...base, + env: { ...env, GITHUB_EVENT_NAME: "workflow_dispatch" }, + request: matrixRequest(), + }); + assert(!result.proof.matrix, "a dispatch run never skips its own matrix"); const recorded = []; const originalLog = console.log; console.log = (line) => recorded.push(line); diff --git a/tests/ci-browser-matrix-coverage.test.ts b/tests/ci-browser-matrix-coverage.test.ts index 374d3ac8ee..acf6c9dd86 100644 --- a/tests/ci-browser-matrix-coverage.test.ts +++ b/tests/ci-browser-matrix-coverage.test.ts @@ -132,8 +132,8 @@ describe("release-browser-matrix engine coverage", () => { .sort(), ); - // `chromium` (the production project) is the one the ui-critical job already - // proved in this run. Re-running it here is the waste the primary path exists + // `chromium` (the production project) is the one the ui-critical job proves in + // this same run. Re-running it here is the waste the primary path exists // to avoid — but only Chromium, never the cross-engine backstop. expect([...byEngine.values()].flat()).not.toContain("chromium"); expect([...byEngine.values()].flat()).toContain("firefox"); @@ -149,11 +149,11 @@ describe("release-browser-matrix engine coverage", () => { shard: number; total: number; }>; - expect(jobs).toHaveLength(21); + expect(jobs).toHaveLength(23); for (const [project, engine, total] of [ ["chromium-mockups", "chromium", 1], - ["firefox", "firefox", 3], - ["webkit", "webkit", 3], + ["firefox", "firefox", 4], + ["webkit", "webkit", 4], ["mobile-webkit", "webkit", 5], ["mobile-pwa-standalone", "webkit", 5], ["tablet", "webkit", 4], diff --git a/tests/ci-main-tree-proof.test.ts b/tests/ci-main-tree-proof.test.ts index 822a496b7c..8c82ebe49a 100644 --- a/tests/ci-main-tree-proof.test.ts +++ b/tests/ci-main-tree-proof.test.ts @@ -22,7 +22,8 @@ import { * Main tree proof: a push to `main` skips only the jobs the merged PR's own CI run already passed * on a byte-identical tree. Everything here pins the fail-closed shape — an absent or partial proof * must leave the push doing the full job set it did before, and the jobs with main-only behaviour - * (Build + deployment boot smoke, the Firefox/WebKit release matrix) must never become skippable. + * (Build + deployment boot smoke) must never become skippable. The Firefox/WebKit release matrix + * is skippable only on its own proof: a green full-suite dispatch run on the identical tree. */ const workflowText = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"); const workflow = createRequire(import.meta.url)("js-yaml").load(workflowText); @@ -156,13 +157,13 @@ describe("main tree proof script", () => { }; const result = await evaluate({ env: env as unknown as NodeJS.ProcessEnv, now, request, runGit, runGitOk }); expect(fetched).toEqual(["+refs/pull/42/head:refs/remotes/tree-proof/42"]); - expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true }); + expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true, matrix: false }); expect(result.runUrl).toBe("https://example.test/run/9"); // Same evidence, one byte of tree drift: nothing is proven. gitTable["rev-parse HEAD^{tree}"] = "e".repeat(40); const drifted = await evaluate({ env: env as unknown as NodeJS.ProcessEnv, now, request, runGit, runGitOk }); - expect(drifted.proof).toEqual({ coverage: false, ui: false, lighthouse: false }); + expect(drifted.proof).toEqual({ coverage: false, ui: false, lighthouse: false, matrix: false }); }); it("propagates API errors so the step's catch turns them into a full run", async () => { @@ -203,6 +204,7 @@ describe("main tree proof script", () => { expect(result.stdout).toContain("coverage=false"); expect(result.stdout).toContain("ui=false"); expect(result.stdout).toContain("lighthouse=false"); + expect(result.stdout).toContain("matrix=false"); }); }); @@ -223,6 +225,7 @@ describe("main tree proof workflow wiring", () => { expect(jobs.changes.outputs.tree_proven_coverage).toBe("${{ steps.tree-proof.outputs.coverage }}"); expect(jobs.changes.outputs.tree_proven_ui).toBe("${{ steps.tree-proof.outputs.ui }}"); expect(jobs.changes.outputs.tree_proven_lighthouse).toBe("${{ steps.tree-proof.outputs.lighthouse }}"); + expect(jobs.changes.outputs.tree_proven_matrix).toBe("${{ steps.tree-proof.outputs.matrix }}"); // The changes job checks out full history, which HEAD^1 and the ancestry check rely on. const checkout = steps.find((step) => String(step.uses ?? "").startsWith("actions/checkout@")); expect(checkout?.with?.["fetch-depth"]).toBe(0); @@ -261,33 +264,38 @@ describe("main tree proof workflow wiring", () => { "coverage", "coverage-shards", "lighthouse-budget", + "release-browser-matrix", "ui-critical", "ui-critical-fast", ]); + // The matrix is gated on its own dispatch-run proof, and only inside its main-push clause, so the + // weekly schedule, release branches and every dispatch still run it in full. + const matrixIf = ifOf("release-browser-matrix"); + expect(matrixIf.match(/tree_proven_\w+/g)).toEqual(["tree_proven_matrix"]); + expect(matrixIf).toMatch( + /github\.ref == 'refs\/heads\/main' &&\s+needs\.changes\.outputs\.tree_proven_matrix != 'true' &&/, + ); expect(ifOf("ui-critical-fast")).toContain("needs.changes.outputs.tree_proven_ui != 'true'"); }); - it("keeps Build (deployment boot smoke) and the release browser matrix running on proven pushes", () => { - for (const job of [ - "build", - "release-browser-matrix", - "static-pr", - "safety", - "ui-playwright-build", - "visual-baseline", - ]) { + it("keeps Build (deployment boot smoke) running on proven pushes, and the matrix on a PR-only proof", () => { + for (const job of ["build", "static-pr", "safety", "ui-playwright-build", "visual-baseline"]) { expect(ifOf(job)).not.toContain("tree_proven"); } - // The matrix tolerates the proof-skipped Production UI and still runs on main... - expect(ifOf("release-browser-matrix")).toContain("needs.ui-critical.result == 'skipped'"); + // The matrix does not wait on Production UI at all (it runs alongside it), so a proof-skipped + // Production UI cannot hold it back, and it still runs on main... + expect(ifOf("release-browser-matrix")).not.toContain("ui-critical"); + expect(jobs["release-browser-matrix"].needs).not.toContain("ui-critical"); expect(ifOf("release-browser-matrix")).toContain("github.ref == 'refs/heads/main'"); - // ...and takes its primary (mockups-only Chromium) path when the PR proved production Chromium, + // ...and takes its primary (mockups-only Chromium) path whenever UI changed, because production + // Chromium is then proven by Production UI in the same run or by the merged PR's identical tree, // instead of re-running production Chromium unsharded on the fail-safe path. const matrixStep = (jobs["release-browser-matrix"].steps as Step[]).find( (step) => step.name === "Full browser UI matrix", ); - expect(matrixStep?.env?.UI_PROVEN_BY_PR).toBe("${{ needs.changes.outputs.tree_proven_ui }}"); - expect(matrixStep?.run).toContain('[ "$UI_PROVEN_BY_PR" = "true" ]'); + expect(matrixStep?.env?.UI_CHANGED).toBe("${{ needs.changes.outputs.ui_changed }}"); + expect(matrixStep?.run).toContain('if [ "$UI_CHANGED" = "true" ]; then'); + expect(ifOf("ui-critical")).toContain("needs.changes.outputs.ui_changed == 'true'"); }); it("keeps the scheduled full run and the push-run concurrency unchanged", () => { @@ -382,7 +390,7 @@ describe("PR branch update reuse", () => { it("carries every group when the newly tested tree is identical to a recorded green one", async () => { const result = await evaluatePullRequestUpdate(setup()); - expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true }); + expect(result.proof).toEqual({ coverage: true, ui: true, lighthouse: true, matrix: false }); expect(result.runUrl).toBe("https://example.test/run/5"); }); @@ -390,7 +398,7 @@ describe("PR branch update reuse", () => { const result = await evaluatePullRequestUpdate( setup({ changed: ["README.md", "worker/requirements.txt", "eval/docling/pyproject.toml"] }), ); - expect(result.proof).toEqual({ coverage: false, ui: true, lighthouse: true }); + expect(result.proof).toEqual({ coverage: false, ui: true, lighthouse: true, matrix: false }); }); it("runs everything when any changed path could be an input", async () => { @@ -406,12 +414,12 @@ describe("PR branch update reuse", () => { "scripts/ci-main-tree-proof.mjs", ]) { const result = await evaluatePullRequestUpdate(setup({ changed: ["README.md", input] })); - expect(result.proof, input).toEqual({ coverage: false, ui: false, lighthouse: false }); + expect(result.proof, input).toEqual({ coverage: false, ui: false, lighthouse: false, matrix: false }); } }); it("fails open to the full run on every doubt", async () => { - const none = { coverage: false, ui: false, lighthouse: false }; + const none = { coverage: false, ui: false, lighthouse: false, matrix: false }; expect((await evaluatePullRequestUpdate({ ...setup(), env: { ...env, PR_ACTION: "opened" } })).proof).toEqual(none); expect((await evaluatePullRequestUpdate({ ...setup(), env: { ...env, PR_HEAD_REPO: "fork/r" } })).proof).toEqual( none, diff --git a/tests/playwright-release-shards.test.ts b/tests/playwright-release-shards.test.ts index b993187059..f0ae42e290 100644 --- a/tests/playwright-release-shards.test.ts +++ b/tests/playwright-release-shards.test.ts @@ -111,10 +111,10 @@ describe("release-browser-matrix duration shards", () => { }); it("keeps the committed leg counts within the measured balance", () => { - // The matrix keeps firefox 3, webkit 3, five legs per iPhone project and tablet 4. + // The matrix keeps firefox 4, webkit 4, five legs per iPhone project and tablet 4. const expectedMaxMinutes = { - firefox: [3, 9], - webkit: [3, 14], + firefox: [4, 7], + webkit: [4, 11], "mobile-webkit": [5, 12], "mobile-pwa-standalone": [5, 10], tablet: [4, 13],