From acb3d82ebd0d1123a624051a5b562ffc7dad5aab Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 09:29:13 +0000 Subject: [PATCH 1/5] ci: fail browser install when system libraries are missing On a browser-cache hit the release matrix ran install-deps and then the browser download in a shell without set -e, so a failed install-deps was masked by the successful download. The step reported success with apt libraries missing, and every WebKit test died at launch with libevent-2.1.so.7 not found (main runs 38224928367 and 38228339791). - set -e in the release matrix and both Chromium setup actions - launch-probe the installed engine inside each install attempt so a missing library fails and retries in the install step - finish an interrupted dpkg before retrying after a timeout Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCghuougzFFXRF3dQQEcYr --- .github/actions/setup-lighthouse-chromium/action.yml | 3 +++ .github/actions/setup-ui-e2e/action.yml | 3 +++ .github/workflows/ci.yml | 10 ++++++++++ tests/ci-browser-matrix-coverage.test.ts | 12 ++++++++++++ 4 files changed, 28 insertions(+) diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index 5c92958293..c4cb2bdfaf 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -55,6 +55,9 @@ runs: EOF fi install_chromium() { + # Fail the attempt if install-deps fails, rather than letting the browser + # download that follows mask it (see release-browser-matrix in ci.yml). + set -e if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps chromium npx playwright install chromium diff --git a/.github/actions/setup-ui-e2e/action.yml b/.github/actions/setup-ui-e2e/action.yml index 94c785a613..cab24abf03 100644 --- a/.github/actions/setup-ui-e2e/action.yml +++ b/.github/actions/setup-ui-e2e/action.yml @@ -42,6 +42,9 @@ runs: EOF fi install_chromium() { + # Fail the attempt if install-deps fails, rather than letting the browser + # download that follows mask it (see release-browser-matrix in ci.yml). + set -e if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps chromium npx playwright install chromium diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d9c8b79d..d8950d3ce7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2048,19 +2048,29 @@ jobs: Acquire::https::Timeout "30"; EOF fi + # `set -e`: without it a failed `install-deps` on a cache hit was masked by the + # browser download that followed, so the attempt "passed" with apt libraries + # missing and every WebKit test died at launch (libevent-2.1.so.7, main run + # 38228339791). The launch probe proves the engine actually starts on this + # runner, so a missing library fails here and retries instead of in each test. install_browsers() { + set -e if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps "$BROWSER_ENGINE" npx playwright install "$BROWSER_ENGINE" else npx playwright install --with-deps "$BROWSER_ENGINE" fi + node -e 'require("playwright")[process.env.BROWSER_ENGINE].launch().then((browser) => browser.close())' } for attempt in 1 2 3; do if timeout 180 bash -c "$(declare -f install_browsers); install_browsers"; then exit 0 fi echo "Playwright browsers install attempt $attempt timed out or failed, retrying..." >&2 + # A timeout can kill apt mid-install; finish any half-configured packages so + # the next attempt is not refused by an interrupted dpkg. + if command -v sudo >/dev/null 2>&1; then sudo dpkg --configure -a || true; fi sleep 5 done exit 1 diff --git a/tests/ci-browser-matrix-coverage.test.ts b/tests/ci-browser-matrix-coverage.test.ts index 4bc109f0dd..31c439ccf1 100644 --- a/tests/ci-browser-matrix-coverage.test.ts +++ b/tests/ci-browser-matrix-coverage.test.ts @@ -191,6 +191,18 @@ describe("release-browser-matrix engine coverage", () => { expect(releaseJob).toContain('npx playwright install --with-deps "$BROWSER_ENGINE"'); }); + it("fails the install attempt when system libraries are missing instead of failing every test at launch", () => { + const steps = parsedWorkflow.jobs["release-browser-matrix"].steps as WorkflowStep[]; + const install = steps.find((step) => step.name === "Install Playwright browsers"); + const script = String(install?.run ?? ""); + // Main run 38228339791: install-deps failed on a cache hit, the browser download after + // it succeeded, the attempt reported success, and every WebKit test died at launch. + expect(script).toMatch(/install_browsers\(\) \{\n\s+set -e\n/); + expect(script).toContain('require("playwright")[process.env.BROWSER_ENGINE].launch()'); + expect(script.indexOf("launch()")).toBeGreaterThan(script.indexOf('install --with-deps "$BROWSER_ENGINE"')); + expect(script).toContain("sudo dpkg --configure -a || true"); + }); + it("reuses only the same run's matching production build, preserving the mockup build and no-producer fallback", () => { const job = parsedWorkflow.jobs["release-browser-matrix"]; const steps = job.steps as WorkflowStep[]; From 7e2d71267a579a83a263faf76dc6d1f1e7d3b40c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 10:27:44 +0000 Subject: [PATCH 2/5] ci: point apt's mirror list off the stalling azure mirror and free the apt lock on retry The dispatched browser matrix (run 38234519037) now failed at install, as intended, and showed the real cause: the runner's apt sources resolve through /etc/apt/apt-mirrors.txt, which the existing sed did not rewrite, so apt-get update waited on azure.archive.ubuntu.com until the 180 s attempt timeout. timeout killed the attempt's shell but not the root apt-get, which kept the apt lock and failed both retries. - rewrite apt-mirrors.txt alongside the sources files (all three installers) - pkill the orphaned apt-get before each retry Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCghuougzFFXRF3dQQEcYr --- .../actions/setup-lighthouse-chromium/action.yml | 4 +++- .github/actions/setup-ui-e2e/action.yml | 4 +++- .github/workflows/ci.yml | 16 ++++++++++++---- tests/ci-browser-matrix-coverage.test.ts | 4 ++++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index c4cb2bdfaf..b0c181b5fc 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -46,7 +46,7 @@ runs: shell: bash run: | if command -v sudo >/dev/null 2>&1; then - sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true sudo mkdir -p /etc/apt/apt.conf.d sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' Acquire::Retries "3"; @@ -70,6 +70,8 @@ runs: exit 0 fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 + # A timed-out attempt can leave its root apt-get holding the apt lock. + if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true; fi sleep 5 done exit 1 diff --git a/.github/actions/setup-ui-e2e/action.yml b/.github/actions/setup-ui-e2e/action.yml index cab24abf03..4eb2e58c2c 100644 --- a/.github/actions/setup-ui-e2e/action.yml +++ b/.github/actions/setup-ui-e2e/action.yml @@ -33,7 +33,7 @@ runs: shell: bash run: | if command -v sudo >/dev/null 2>&1; then - sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true sudo mkdir -p /etc/apt/apt.conf.d sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' Acquire::Retries "3"; @@ -57,6 +57,8 @@ runs: exit 0 fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 + # A timed-out attempt can leave its root apt-get holding the apt lock. + if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true; fi sleep 5 done exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8950d3ce7..81353d70cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2040,7 +2040,7 @@ jobs: BROWSER_ENGINE: ${{ matrix.engine }} run: | if command -v sudo >/dev/null 2>&1; then - sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true sudo mkdir -p /etc/apt/apt.conf.d sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' Acquire::Retries "3"; @@ -2048,6 +2048,9 @@ jobs: Acquire::https::Timeout "30"; EOF fi + # The runner's apt sources resolve through the mirror list in apt-mirrors.txt, so + # rewriting only the sources files left apt waiting on the azure mirror until the + # attempt timed out. # `set -e`: without it a failed `install-deps` on a cache hit was masked by the # browser download that followed, so the attempt "passed" with apt libraries # missing and every WebKit test died at launch (libevent-2.1.so.7, main run @@ -2068,9 +2071,14 @@ jobs: exit 0 fi echo "Playwright browsers install attempt $attempt timed out or failed, retrying..." >&2 - # A timeout can kill apt mid-install; finish any half-configured packages so - # the next attempt is not refused by an interrupted dpkg. - if command -v sudo >/dev/null 2>&1; then sudo dpkg --configure -a || true; fi + # `timeout` kills the attempt's shell but not the root apt-get it started, which + # then holds the apt lock and fails every retry (dispatch run 38234519037). Stop + # it, then finish any half-configured packages before the next attempt. + if command -v sudo >/dev/null 2>&1; then + sudo pkill -x apt-get || true + sleep 2 + sudo dpkg --configure -a || true + fi sleep 5 done exit 1 diff --git a/tests/ci-browser-matrix-coverage.test.ts b/tests/ci-browser-matrix-coverage.test.ts index 31c439ccf1..b291d811b3 100644 --- a/tests/ci-browser-matrix-coverage.test.ts +++ b/tests/ci-browser-matrix-coverage.test.ts @@ -201,6 +201,10 @@ describe("release-browser-matrix engine coverage", () => { expect(script).toContain('require("playwright")[process.env.BROWSER_ENGINE].launch()'); expect(script.indexOf("launch()")).toBeGreaterThan(script.indexOf('install --with-deps "$BROWSER_ENGINE"')); expect(script).toContain("sudo dpkg --configure -a || true"); + // Dispatch run 38234519037: the azure mirror in apt-mirrors.txt stalled apt past the + // attempt timeout, and the orphaned apt-get then held the lock for every retry. + expect(script).toContain("/etc/apt/apt-mirrors.txt"); + expect(script).toContain("sudo pkill -x apt-get || true"); }); it("reuses only the same run's matching production build, preserving the mockup build and no-producer fallback", () => { From e829771dc751ddf654224df5b605413e7c819d80 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 10:30:01 +0000 Subject: [PATCH 3/5] ci: finish an interrupted dpkg after killing apt-get in the Chromium setup actions Matches the release matrix recovery so a timed-out attempt that leaves dpkg half-configured does not fail every retry in the required UI and Lighthouse gates. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCghuougzFFXRF3dQQEcYr --- .github/actions/setup-lighthouse-chromium/action.yml | 9 +++++++-- .github/actions/setup-ui-e2e/action.yml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index b0c181b5fc..0de96c5425 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -70,8 +70,13 @@ runs: exit 0 fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 - # A timed-out attempt can leave its root apt-get holding the apt lock. - if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true; fi + # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg + # half-configured; clear both before the next attempt. + if command -v sudo >/dev/null 2>&1; then + sudo pkill -x apt-get || true + sleep 2 + sudo dpkg --configure -a || true + fi sleep 5 done exit 1 diff --git a/.github/actions/setup-ui-e2e/action.yml b/.github/actions/setup-ui-e2e/action.yml index 4eb2e58c2c..e41a99e37e 100644 --- a/.github/actions/setup-ui-e2e/action.yml +++ b/.github/actions/setup-ui-e2e/action.yml @@ -57,8 +57,13 @@ runs: exit 0 fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 - # A timed-out attempt can leave its root apt-get holding the apt lock. - if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true; fi + # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg + # half-configured; clear both before the next attempt. + if command -v sudo >/dev/null 2>&1; then + sudo pkill -x apt-get || true + sleep 2 + sudo dpkg --configure -a || true + fi sleep 5 done exit 1 From c7c557e7afceef390eb51662d2685449ac97eb18 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 11:13:14 +0000 Subject: [PATCH 4/5] docs(ledger): queue the Safari test failures revealed by the WebKit launch fix Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCghuougzFFXRF3dQQEcYr --- .../693e8f34-1b14-4e61-a87c-430fff130dad.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json diff --git a/docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json b/docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json new file mode 100644 index 0000000000..9999a6709f --- /dev/null +++ b/docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "693e8f34-1b14-4e61-a87c-430fff130dad", + "createdOn": "2026-10-12", + "action": "add", + "payload": { + "pri": "P2", + "type": "issue", + "summary": "Safari (WebKit) browser tests: 26 genuine failures now visible after the launch fix in PR #3487", + "detail": "Before PR #3487 every WebKit leg of the release browser matrix died at browser launch (missing libevent), hiding real results. Dispatch run 38236585559 (head 7e2d712) launched every leg; these tests then failed. tablet 2/4 (16): ui-accessibility 176/377/597, ui-overlap 239/437/549, ui-teaching 83, ui-tools 587/672/1084/2183/2295/2490/2674/2946, ui-universal-search 360. tablet 4/4 (4): ui-my-day-calendar 116, ui-roster-team 100/174, ui-stress 346. tablet 3/4: ui-cme-phone 351. mobile-webkit 2/5: ui-accessibility 295. mobile-webkit 5/5 and mobile-pwa-standalone 4/5: ui-on-call-boards 558 (declares no anchor the page does not render). mobile-pwa-standalone 5/5: ui-phone-scroll-assessments 19, ui-phone-scroll-page-owned 454. tablet 1/4 was still running at triage. Many tablet failures are 1.0m timeouts, so some may share one cause (tablet viewport or WebKit timing). Next action: rerun the matrix on main after #3487 merges, group failures by cause, fix app or test per group. Never skip or quarantine a test to get green.", + "source": "PR #3487 browser-matrix dispatch 2026-10-12", + "issueUlid": "01M4QX1X5FCJENZWYHCG1971RC" + } +} From 54f252492b415118ae6396f8041ed18eec1b800c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 11:24:18 +0000 Subject: [PATCH 5/5] ci: bound apt recovery between browser-install retries and stop orphaned dpkg Review follow-up: the recovery ran outside the 180 s attempt timeout, so a stalled dpkg --configure -a could hold the loop to the job limit, and only apt-get was stopped, so a dpkg child could keep the lock. Stop dpkg too and give recovery its own 120 s bound. The contract test now also pins the launch probe after the cache-hit/cold-cache branch closes. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01DCghuougzFFXRF3dQQEcYr --- .github/actions/setup-lighthouse-chromium/action.yml | 5 +++-- .github/actions/setup-ui-e2e/action.yml | 5 +++-- .github/workflows/ci.yml | 6 ++++-- tests/ci-browser-matrix-coverage.test.ts | 6 ++++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index 0de96c5425..0957cfcfce 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -71,11 +71,12 @@ runs: fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg - # half-configured; clear both before the next attempt. + # half-configured; clear both (recovery bounded) before the next attempt. if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true + sudo pkill -x dpkg || true sleep 2 - sudo dpkg --configure -a || true + sudo timeout 120 dpkg --configure -a || true fi sleep 5 done diff --git a/.github/actions/setup-ui-e2e/action.yml b/.github/actions/setup-ui-e2e/action.yml index e41a99e37e..00cfb7a9ca 100644 --- a/.github/actions/setup-ui-e2e/action.yml +++ b/.github/actions/setup-ui-e2e/action.yml @@ -58,11 +58,12 @@ runs: fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg - # half-configured; clear both before the next attempt. + # half-configured; clear both (recovery bounded) before the next attempt. if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true + sudo pkill -x dpkg || true sleep 2 - sudo dpkg --configure -a || true + sudo timeout 120 dpkg --configure -a || true fi sleep 5 done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81353d70cd..c1ca6a4b8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2073,11 +2073,13 @@ jobs: echo "Playwright browsers install attempt $attempt timed out or failed, retrying..." >&2 # `timeout` kills the attempt's shell but not the root apt-get it started, which # then holds the apt lock and fails every retry (dispatch run 38234519037). Stop - # it, then finish any half-configured packages before the next attempt. + # it (and any dpkg it started), then finish half-configured packages, bounded so a stalled + # recovery cannot hold the loop, before the next attempt. if command -v sudo >/dev/null 2>&1; then sudo pkill -x apt-get || true + sudo pkill -x dpkg || true sleep 2 - sudo dpkg --configure -a || true + sudo timeout 120 dpkg --configure -a || true fi sleep 5 done diff --git a/tests/ci-browser-matrix-coverage.test.ts b/tests/ci-browser-matrix-coverage.test.ts index b291d811b3..374d3ac8ee 100644 --- a/tests/ci-browser-matrix-coverage.test.ts +++ b/tests/ci-browser-matrix-coverage.test.ts @@ -199,12 +199,14 @@ describe("release-browser-matrix engine coverage", () => { // it succeeded, the attempt reported success, and every WebKit test died at launch. expect(script).toMatch(/install_browsers\(\) \{\n\s+set -e\n/); expect(script).toContain('require("playwright")[process.env.BROWSER_ENGINE].launch()'); - expect(script.indexOf("launch()")).toBeGreaterThan(script.indexOf('install --with-deps "$BROWSER_ENGINE"')); - expect(script).toContain("sudo dpkg --configure -a || true"); + // The probe sits after the cache-hit/cold-cache branch closes, so both paths are verified. + expect(script).toMatch(/\n\s+fi\n\s+node -e 'require\("playwright"\)\[process\.env\.BROWSER_ENGINE\]\.launch\(\)/); + expect(script).toContain("sudo timeout 120 dpkg --configure -a || true"); // Dispatch run 38234519037: the azure mirror in apt-mirrors.txt stalled apt past the // attempt timeout, and the orphaned apt-get then held the lock for every retry. expect(script).toContain("/etc/apt/apt-mirrors.txt"); expect(script).toContain("sudo pkill -x apt-get || true"); + expect(script).toContain("sudo pkill -x dpkg || true"); }); it("reuses only the same run's matching production build, preserving the mockup build and no-producer fallback", () => {