From 342815cc0b6dc56d643dcaed807c801b358cde50 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 12:03:27 +0000 Subject: [PATCH 1/6] CI follow-ups: keep PR proof when the dispatch lookup fails, link the matrix proving run, steady two browser clicks A failed workflow_dispatch lookup no longer throws away the PR run's proof: only the release matrix proof is withheld, so main still skips coverage, Production UI and Lighthouse it already proved. The dispatch run that proves the matrix gets its own matrix_proof_run_url output and summary line, and PR required lists the matrix among jobs proven on the identical tree. Firefox dropped a My Day calendar click on three main runs in a row (the chosen day once, Today once), and iPad WebKit dropped the DSM Compare remove click once. Both now repeat a click that left no trace while still asserting the full outcome. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- .github/workflows/ci.yml | 2 ++ scripts/ci-main-tree-proof.mjs | 30 +++++++++++++++++++++++++----- tests/ui-my-day-calendar.spec.ts | 22 ++++++++++++++++++---- tests/ui-route-coverage.spec.ts | 9 +++++++-- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41a04fda70..229660c593 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1662,6 +1662,7 @@ jobs: TREE_PROVEN_COVERAGE: ${{ needs.changes.outputs.tree_proven_coverage }} TREE_PROVEN_UI: ${{ needs.changes.outputs.tree_proven_ui }} TREE_PROVEN_LIGHTHOUSE: ${{ needs.changes.outputs.tree_proven_lighthouse }} + TREE_PROVEN_MATRIX: ${{ needs.changes.outputs.tree_proven_matrix }} run: | set -euo pipefail @@ -1902,6 +1903,7 @@ jobs: if [ "$TREE_PROVEN_COVERAGE" = "true" ]; then proven+=("coverage"); fi if [ "$TREE_PROVEN_UI" = "true" ]; then proven+=("production-ui"); fi if [ "$TREE_PROVEN_LIGHTHOUSE" = "true" ]; then proven+=("lighthouse-budget"); fi + if [ "$TREE_PROVEN_MATRIX" = "true" ]; then proven+=("release-browser-matrix"); fi if [ ${#proven[@]} -gt 0 ]; then echo "::notice title=Proven on the identical tree::Not re-run because an earlier green run" \ "already passed them on the tree it recorded testing, and nothing they read has changed" \ diff --git a/scripts/ci-main-tree-proof.mjs b/scripts/ci-main-tree-proof.mjs index 397c8958e8..f003120a9d 100644 --- a/scripts/ci-main-tree-proof.mjs +++ b/scripts/ci-main-tree-proof.mjs @@ -45,7 +45,8 @@ * `event=workflow_dispatch` with a tree equal to tree(HEAD), and every matrix leg concluded * success. Absent such a run, main runs the matrix exactly as before. * - * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse ("true"/"false"), proof_run_url, reason. + * Outputs (GITHUB_OUTPUT): coverage, ui, lighthouse, matrix ("true"/"false"), proof_run_url, + * matrix_proof_run_url, reason. A failed dispatch lookup only withholds the matrix proof. * Uses only the job's GITHUB_TOKEN (actions: read, pull-requests: read) — about four REST * calls per main push. Every proved job checks out the run's GITHUB_SHA (no `ref:` override, * pinned by tests/ci-main-tree-proof.test.ts), so the Change scope record covers every leg. @@ -258,6 +259,7 @@ export async function evaluate({ const maxAgeHours = Number(env.MAX_PROOF_AGE_HOURS || DEFAULT_MAX_PROOF_AGE_HOURS); if (!Number.isFinite(maxAgeHours) || maxAgeHours <= 0) return { proof: NONE, reason: "invalid MAX_PROOF_AGE_HOURS" }; const prOutcome = await provenRun({ request, repo, head, mainTree, now, maxAgeHours, event: "pull_request" }); + // The matrix lookup is optional: if it fails, the matrix runs and the PR proof above still stands. const matrixOutcome = await provenRun({ request, repo, @@ -266,7 +268,7 @@ export async function evaluate({ now, maxAgeHours, event: "workflow_dispatch", - }); + }).catch((error) => ({ jobs: null, reason: `dispatch lookup failed (${error?.message ?? error})` })); const proof = { ...(prOutcome.jobs ? classifyProof(prOutcome.jobs) : { coverage: false, ui: false, lighthouse: false }), matrix: matrixOutcome.jobs ? classifyMatrixProof(matrixOutcome.jobs) : false, @@ -274,7 +276,8 @@ export async function evaluate({ return { proof, reason: `PR #${pull.number}: ${prOutcome.reason}; release matrix: ${matrixOutcome.reason}`, - runUrl: prOutcome.runUrl || matrixOutcome.runUrl || "", + runUrl: prOutcome.jobs ? (prOutcome.runUrl ?? "") : "", + matrixRunUrl: proof.matrix ? (matrixOutcome.runUrl ?? "") : "", }; } @@ -460,13 +463,14 @@ export async function evaluatePullRequestUpdate({ }; } -function writeOutputs({ proof, reason, runUrl = "" }) { +function writeOutputs({ proof, reason, runUrl = "", matrixRunUrl = "" }) { const lines = [ `coverage=${proof.coverage === true}`, `ui=${proof.ui === true}`, `lighthouse=${proof.lighthouse === true}`, `matrix=${proof.matrix === true}`, `proof_run_url=${runUrl}`, + `matrix_proof_run_url=${matrixRunUrl}`, `reason=${String(reason).replace(/[\r\n]+/g, " ")}`, ]; console.log(lines.join("\n")); @@ -482,6 +486,7 @@ function writeOutputs({ proof, reason, runUrl = "" }) { "", `- Result: ${reason}`, runUrl ? `- Proving PR run: ${runUrl}` : "- Proving PR run: none", + ...(matrixRunUrl ? [`- Proving full-suite dispatch run (release matrix): ${matrixRunUrl}`] : []), `- Skipped on this push (already green on the identical tree): ${skipped.length ? skipped.join(", ") : "nothing — full run"}`, "", ].join("\n"), @@ -686,7 +691,7 @@ async function selfTest() { ok("release-browser-matrix (chromium, chromium-mockups, 1, 1)"), ok("release-browser-matrix (webkit, mobile-webkit, 1, 5)"), ]; - const dispatchRun = { ...run, id: 6, event: "workflow_dispatch" }; + const dispatchRun = { ...run, id: 6, event: "workflow_dispatch", html_url: "https://example.test/runs/6" }; const dispatchRecord = record(`event=workflow_dispatch sha=${pr.head.sha} tree=${tree}`); const matrixRequest = ({ legs = matrixLegs, annotations = dispatchRecord, dispatchRuns = [dispatchRun] } = {}) => @@ -699,6 +704,21 @@ async function selfTest() { }; result = await evaluate({ ...base, request: matrixRequest() }); assert(result.proof.matrix && result.proof.ui, `a green dispatch run proves the matrix (${result.reason})`); + assert( + result.matrixRunUrl === dispatchRun.html_url && result.runUrl !== dispatchRun.html_url, + "the summary links the PR run and the dispatch run separately", + ); + result = await evaluate({ + ...base, + request: async (path) => { + if (path.includes("event=workflow_dispatch")) throw new Error("GitHub API 502"); + return request()(path); + }, + }); + assert( + result.proof.ui && result.proof.coverage && !result.proof.matrix && result.matrixRunUrl === "", + `a failed dispatch lookup keeps the PR proof and runs the matrix (${result.reason})`, + ); assert(classifyMatrixProof(matrixLegs), "all-green legs prove the matrix"); assert(!classifyMatrixProof([ok("release-browser-matrix")]), "a skipped (unexpanded) matrix proves nothing"); result = await evaluate({ diff --git a/tests/ui-my-day-calendar.spec.ts b/tests/ui-my-day-calendar.spec.ts index bc48b31903..d4349ab77c 100644 --- a/tests/ui-my-day-calendar.spec.ts +++ b/tests/ui-my-day-calendar.spec.ts @@ -65,6 +65,19 @@ async function expectMonth(calendar: Locator, title: string) { await expect(calendar.getByRole("heading", { level: 2, name: title })).toBeVisible(); } +/** + * Clicks a calendar control and waits for what it does, clicking again if the first click left no + * trace. Firefox on main dropped one of these clicks in three runs in a row on 13 Oct 2026 (once the + * chosen day, once Today), each time after the calendar had settled. The outcome is still asserted + * in full; a repeat click is harmless because both actions are idempotent. + */ +async function clickUntil(control: Locator, outcome: () => Promise) { + await expect(async () => { + await control.click({ timeout: 5_000 }); + await outcome(); + }).toPass({ timeout: 30_000, intervals: [1_000, 2_000, 4_000] }); +} + async function expectNoSidewaysScroll(page: Page, label: string) { const { scrollWidth, clientWidth } = await page.evaluate(() => ({ scrollWidth: document.documentElement.scrollWidth, @@ -102,13 +115,14 @@ test("moves between months, shows your older shifts, and returns to today", asyn await expect(page.getByTestId("my-day-calendar-history-notice")).toHaveCount(0); expect(datedReads.some((query) => query.includes("from=2026-07-31"))).toBe(true); - await calendar.getByTestId("my-day-calendar-day-2026-08-12").click(); - await expect(calendar.getByTestId("my-day-calendar-day-2026-08-12")).toHaveAttribute("aria-pressed", "true"); + const chosenDay = calendar.getByTestId("my-day-calendar-day-2026-08-12"); + await clickUntil(chosenDay, () => expect(chosenDay).toHaveAttribute("aria-pressed", "true", { timeout: 5_000 })); const day = calendar.getByTestId("my-day-calendar-day"); await expect(day.getByTestId(`my-day-calendar-item-shift:${older.id}`)).toBeVisible(); - await calendar.getByTestId("my-day-calendar-today").click(); - await expectMonth(calendar, "October 2026"); + await clickUntil(calendar.getByTestId("my-day-calendar-today"), () => + expect(calendar.getByRole("heading", { level: 2, name: "October 2026" })).toBeVisible({ timeout: 5_000 }), + ); await expect(calendar.getByTestId("my-day-calendar-today")).toHaveCount(0); await expect(today).toHaveAttribute("aria-pressed", "true"); }); diff --git a/tests/ui-route-coverage.spec.ts b/tests/ui-route-coverage.spec.ts index 5b277969bc..ba9f7ca396 100644 --- a/tests/ui-route-coverage.spec.ts +++ b/tests/ui-route-coverage.spec.ts @@ -487,8 +487,13 @@ test.describe("previously uncovered production routes", () => { // the URL: a same-route `?ids=` soft navigation fires no document load, which // `waitForURL` waits for, and inside `Promise.all` its timeout also hid whether the // click itself ever landed (WebKit, 12 Oct 2026). - await remove.click(); - await expect(currentPage).toHaveURL(/\/dsm\/compare\?ids=bipolar-ii-disorder$/, { timeout: 30_000 }); + // iPad WebKit on main dropped this click once on 13 Oct 2026 (URL unchanged for 30 s), so a + // click that leaves no trace is repeated while the button is still there. Removal must still + // land on exactly the remaining id. + await expect(async () => { + if (await remove.isVisible()) await remove.click({ timeout: 5_000 }); + await expect(currentPage).toHaveURL(/\/dsm\/compare\?ids=bipolar-ii-disorder$/, { timeout: 10_000 }); + }).toPass({ timeout: 40_000, intervals: [1_000, 2_000, 4_000] }); // Empty dashed panel is suppressed in favour of the compact slot rail // and inline starter chips when fewer than two diagnoses remain. await expect(currentPage.getByTestId("compare-slot-tile-compact").first()).toBeVisible(); From d31629c549137a351d1652a35ab3fe87aa3a70e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 12:17:17 +0000 Subject: [PATCH 2/6] Fix PR required aggregate tests for the matrix proof notice The aggregate test harness runs under set -u, so it needs the new TREE_PROVEN_MATRIX binding, and the routing contract forbids naming the matrix job inside pr-required, so the notice uses a plain label. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- .github/workflows/ci.yml | 2 +- tests/ci-cache-safety.test.ts | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 229660c593..a8e470ec2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1903,7 +1903,7 @@ jobs: if [ "$TREE_PROVEN_COVERAGE" = "true" ]; then proven+=("coverage"); fi if [ "$TREE_PROVEN_UI" = "true" ]; then proven+=("production-ui"); fi if [ "$TREE_PROVEN_LIGHTHOUSE" = "true" ]; then proven+=("lighthouse-budget"); fi - if [ "$TREE_PROVEN_MATRIX" = "true" ]; then proven+=("release-browser-matrix"); fi + if [ "$TREE_PROVEN_MATRIX" = "true" ]; then proven+=("Firefox/WebKit release matrix"); fi if [ ${#proven[@]} -gt 0 ]; then echo "::notice title=Proven on the identical tree::Not re-run because an earlier green run" \ "already passed them on the tree it recorded testing, and nothing they read has changed" \ diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index 3fdf36593d..9eb8aded0c 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -713,6 +713,7 @@ describe.skipIf(process.platform === "win32")("PR required aggregate — cancell TREE_PROVEN_COVERAGE: "", TREE_PROVEN_UI: "", TREE_PROVEN_LIGHTHOUSE: "", + TREE_PROVEN_MATRIX: "", }; function runAggregate(overrides: Record = {}) { @@ -783,6 +784,15 @@ describe.skipIf(process.platform === "win32")("PR required aggregate — cancell expect(proven.status).toBe(0); expect(proven.output).toContain("Proven on the identical tree"); expect(proven.output).toContain("coverage production-ui lighthouse-budget"); + expect( + runAggregate({ + ...heavyMainPush, + TREE_PROVEN_COVERAGE: "true", + TREE_PROVEN_UI: "true", + TREE_PROVEN_LIGHTHOUSE: "true", + TREE_PROVEN_MATRIX: "true", + }).output, + ).toContain("lighthouse-budget Firefox/WebKit release matrix"); // Without proof the same skips stay red - one group at a time, so no proof leaks across jobs. expect(runAggregate(heavyMainPush).status).not.toBe(0); From 836101a46081a559674ad0ab68ba340e607df31e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 12:19:40 +0000 Subject: [PATCH 3/6] Assert the matrix-proof aggregate exits cleanly Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- tests/ci-cache-safety.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index 9eb8aded0c..46792fc24d 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -784,15 +784,15 @@ describe.skipIf(process.platform === "win32")("PR required aggregate — cancell expect(proven.status).toBe(0); expect(proven.output).toContain("Proven on the identical tree"); expect(proven.output).toContain("coverage production-ui lighthouse-budget"); - expect( - runAggregate({ - ...heavyMainPush, - TREE_PROVEN_COVERAGE: "true", - TREE_PROVEN_UI: "true", - TREE_PROVEN_LIGHTHOUSE: "true", - TREE_PROVEN_MATRIX: "true", - }).output, - ).toContain("lighthouse-budget Firefox/WebKit release matrix"); + const matrixProven = runAggregate({ + ...heavyMainPush, + TREE_PROVEN_COVERAGE: "true", + TREE_PROVEN_UI: "true", + TREE_PROVEN_LIGHTHOUSE: "true", + TREE_PROVEN_MATRIX: "true", + }); + expect(matrixProven.status).toBe(0); + expect(matrixProven.output).toContain("lighthouse-budget Firefox/WebKit release matrix"); // Without proof the same skips stay red - one group at a time, so no proof leaks across jobs. expect(runAggregate(heavyMainPush).status).not.toBe(0); From fadfd489bca109675b84176537d32aa8c3156f02 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 13:41:17 +0000 Subject: [PATCH 4/6] Keep calendar click retries inside the test timeout Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- tests/ui-my-day-calendar.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ui-my-day-calendar.spec.ts b/tests/ui-my-day-calendar.spec.ts index d4349ab77c..77e585b33c 100644 --- a/tests/ui-my-day-calendar.spec.ts +++ b/tests/ui-my-day-calendar.spec.ts @@ -75,7 +75,7 @@ async function clickUntil(control: Locator, outcome: () => Promise) { await expect(async () => { await control.click({ timeout: 5_000 }); await outcome(); - }).toPass({ timeout: 30_000, intervals: [1_000, 2_000, 4_000] }); + }).toPass({ timeout: 20_000, intervals: [1_000, 2_000, 4_000] }); } async function expectNoSidewaysScroll(page: Page, label: string) { From 2707729371a5325a966f532ebb1e0ab9eada4734 Mon Sep 17 00:00:00 2001 From: BigSimmo Date: Tue, 13 Oct 2026 21:56:27 +0800 Subject: [PATCH 5/6] Test: cap the calendar click retry at 20s so two calls fit the 60s test timeout --- tests/ui-my-day-calendar.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ui-my-day-calendar.spec.ts b/tests/ui-my-day-calendar.spec.ts index d4349ab77c..77e585b33c 100644 --- a/tests/ui-my-day-calendar.spec.ts +++ b/tests/ui-my-day-calendar.spec.ts @@ -75,7 +75,7 @@ async function clickUntil(control: Locator, outcome: () => Promise) { await expect(async () => { await control.click({ timeout: 5_000 }); await outcome(); - }).toPass({ timeout: 30_000, intervals: [1_000, 2_000, 4_000] }); + }).toPass({ timeout: 20_000, intervals: [1_000, 2_000, 4_000] }); } async function expectNoSidewaysScroll(page: Page, label: string) { From 91f5d7dff8a0b962499f5d9dba1e1065558ee00b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 14:40:48 +0000 Subject: [PATCH 6/6] Test: retry the calendar click only while the control is still on screen Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_014qMDuRLipTaGrY6EyC89hm --- tests/ui-my-day-calendar.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/ui-my-day-calendar.spec.ts b/tests/ui-my-day-calendar.spec.ts index 77e585b33c..5a84eee63e 100644 --- a/tests/ui-my-day-calendar.spec.ts +++ b/tests/ui-my-day-calendar.spec.ts @@ -73,7 +73,8 @@ async function expectMonth(calendar: Locator, title: string) { */ async function clickUntil(control: Locator, outcome: () => Promise) { await expect(async () => { - await control.click({ timeout: 5_000 }); + // Today disappears once October shows, so a click that landed but rendered slowly is not repeated. + if (await control.isVisible()) await control.click({ timeout: 5_000 }); await outcome(); }).toPass({ timeout: 20_000, intervals: [1_000, 2_000, 4_000] }); }