diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index 5c92958293..0957cfcfce 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -46,7 +46,7 @@ runs: shell: bash run: | if command -v sudo >/dev/null 2>&1; then - sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true sudo mkdir -p /etc/apt/apt.conf.d sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' Acquire::Retries "3"; @@ -55,6 +55,9 @@ runs: EOF fi install_chromium() { + # Fail the attempt if install-deps fails, rather than letting the browser + # download that follows mask it (see release-browser-matrix in ci.yml). + set -e if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps chromium npx playwright install chromium @@ -67,6 +70,14 @@ runs: exit 0 fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 + # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg + # half-configured; clear both (recovery bounded) before the next attempt. + if command -v sudo >/dev/null 2>&1; then + sudo pkill -x apt-get || true + sudo pkill -x dpkg || true + sleep 2 + sudo timeout 120 dpkg --configure -a || true + fi sleep 5 done exit 1 diff --git a/.github/actions/setup-ui-e2e/action.yml b/.github/actions/setup-ui-e2e/action.yml index 94c785a613..00cfb7a9ca 100644 --- a/.github/actions/setup-ui-e2e/action.yml +++ b/.github/actions/setup-ui-e2e/action.yml @@ -33,7 +33,7 @@ runs: shell: bash run: | if command -v sudo >/dev/null 2>&1; then - sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true sudo mkdir -p /etc/apt/apt.conf.d sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' Acquire::Retries "3"; @@ -42,6 +42,9 @@ runs: EOF fi install_chromium() { + # Fail the attempt if install-deps fails, rather than letting the browser + # download that follows mask it (see release-browser-matrix in ci.yml). + set -e if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps chromium npx playwright install chromium @@ -54,6 +57,14 @@ runs: exit 0 fi echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 + # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg + # half-configured; clear both (recovery bounded) before the next attempt. + if command -v sudo >/dev/null 2>&1; then + sudo pkill -x apt-get || true + sudo pkill -x dpkg || true + sleep 2 + sudo timeout 120 dpkg --configure -a || true + fi sleep 5 done exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d9c8b79d..c1ca6a4b8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2040,7 +2040,7 @@ jobs: BROWSER_ENGINE: ${{ matrix.engine }} run: | if command -v sudo >/dev/null 2>&1; then - sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true sudo mkdir -p /etc/apt/apt.conf.d sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' Acquire::Retries "3"; @@ -2048,19 +2048,39 @@ jobs: Acquire::https::Timeout "30"; EOF fi + # The runner's apt sources resolve through the mirror list in apt-mirrors.txt, so + # rewriting only the sources files left apt waiting on the azure mirror until the + # attempt timed out. + # `set -e`: without it a failed `install-deps` on a cache hit was masked by the + # browser download that followed, so the attempt "passed" with apt libraries + # missing and every WebKit test died at launch (libevent-2.1.so.7, main run + # 38228339791). The launch probe proves the engine actually starts on this + # runner, so a missing library fails here and retries instead of in each test. install_browsers() { + set -e if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps "$BROWSER_ENGINE" npx playwright install "$BROWSER_ENGINE" else npx playwright install --with-deps "$BROWSER_ENGINE" fi + node -e 'require("playwright")[process.env.BROWSER_ENGINE].launch().then((browser) => browser.close())' } for attempt in 1 2 3; do if timeout 180 bash -c "$(declare -f install_browsers); install_browsers"; then exit 0 fi echo "Playwright browsers install attempt $attempt timed out or failed, retrying..." >&2 + # `timeout` kills the attempt's shell but not the root apt-get it started, which + # then holds the apt lock and fails every retry (dispatch run 38234519037). Stop + # it (and any dpkg it started), then finish half-configured packages, bounded so a stalled + # recovery cannot hold the loop, before the next attempt. + if command -v sudo >/dev/null 2>&1; then + sudo pkill -x apt-get || true + sudo pkill -x dpkg || true + sleep 2 + sudo timeout 120 dpkg --configure -a || true + fi sleep 5 done exit 1 diff --git a/docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json b/docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json new file mode 100644 index 0000000000..9999a6709f --- /dev/null +++ b/docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "693e8f34-1b14-4e61-a87c-430fff130dad", + "createdOn": "2026-10-12", + "action": "add", + "payload": { + "pri": "P2", + "type": "issue", + "summary": "Safari (WebKit) browser tests: 26 genuine failures now visible after the launch fix in PR #3487", + "detail": "Before PR #3487 every WebKit leg of the release browser matrix died at browser launch (missing libevent), hiding real results. Dispatch run 38236585559 (head 7e2d712) launched every leg; these tests then failed. tablet 2/4 (16): ui-accessibility 176/377/597, ui-overlap 239/437/549, ui-teaching 83, ui-tools 587/672/1084/2183/2295/2490/2674/2946, ui-universal-search 360. tablet 4/4 (4): ui-my-day-calendar 116, ui-roster-team 100/174, ui-stress 346. tablet 3/4: ui-cme-phone 351. mobile-webkit 2/5: ui-accessibility 295. mobile-webkit 5/5 and mobile-pwa-standalone 4/5: ui-on-call-boards 558 (declares no anchor the page does not render). mobile-pwa-standalone 5/5: ui-phone-scroll-assessments 19, ui-phone-scroll-page-owned 454. tablet 1/4 was still running at triage. Many tablet failures are 1.0m timeouts, so some may share one cause (tablet viewport or WebKit timing). Next action: rerun the matrix on main after #3487 merges, group failures by cause, fix app or test per group. Never skip or quarantine a test to get green.", + "source": "PR #3487 browser-matrix dispatch 2026-10-12", + "issueUlid": "01M4QX1X5FCJENZWYHCG1971RC" + } +} diff --git a/tests/ci-browser-matrix-coverage.test.ts b/tests/ci-browser-matrix-coverage.test.ts index 4bc109f0dd..374d3ac8ee 100644 --- a/tests/ci-browser-matrix-coverage.test.ts +++ b/tests/ci-browser-matrix-coverage.test.ts @@ -191,6 +191,24 @@ describe("release-browser-matrix engine coverage", () => { expect(releaseJob).toContain('npx playwright install --with-deps "$BROWSER_ENGINE"'); }); + it("fails the install attempt when system libraries are missing instead of failing every test at launch", () => { + const steps = parsedWorkflow.jobs["release-browser-matrix"].steps as WorkflowStep[]; + const install = steps.find((step) => step.name === "Install Playwright browsers"); + const script = String(install?.run ?? ""); + // Main run 38228339791: install-deps failed on a cache hit, the browser download after + // it succeeded, the attempt reported success, and every WebKit test died at launch. + expect(script).toMatch(/install_browsers\(\) \{\n\s+set -e\n/); + expect(script).toContain('require("playwright")[process.env.BROWSER_ENGINE].launch()'); + // The probe sits after the cache-hit/cold-cache branch closes, so both paths are verified. + expect(script).toMatch(/\n\s+fi\n\s+node -e 'require\("playwright"\)\[process\.env\.BROWSER_ENGINE\]\.launch\(\)/); + expect(script).toContain("sudo timeout 120 dpkg --configure -a || true"); + // Dispatch run 38234519037: the azure mirror in apt-mirrors.txt stalled apt past the + // attempt timeout, and the orphaned apt-get then held the lock for every retry. + expect(script).toContain("/etc/apt/apt-mirrors.txt"); + expect(script).toContain("sudo pkill -x apt-get || true"); + expect(script).toContain("sudo pkill -x dpkg || true"); + }); + it("reuses only the same run's matching production build, preserving the mockup build and no-producer fallback", () => { const job = parsedWorkflow.jobs["release-browser-matrix"]; const steps = job.steps as WorkflowStep[];