ci: fail browser install when system libraries are missing (WebKit could not launch)

.github/actions/setup-lighthouse-chromium/action.yml

@@ -46,7 +46,7 @@ runs:
       shell: bash
       run: |
         if command -v sudo >/dev/null 2>&1; then
-          sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true
+          sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true
           sudo mkdir -p /etc/apt/apt.conf.d
           sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF'
         Acquire::Retries "3";
@@ -55,6 +55,9 @@ runs:
         EOF
         fi
         install_chromium() {
+          # Fail the attempt if install-deps fails, rather than letting the browser
+          # download that follows mask it (see release-browser-matrix in ci.yml).
+          set -e
           if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then
             npx playwright install-deps chromium
             npx playwright install chromium
@@ -67,6 +70,14 @@ runs:
             exit 0
           fi
           echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2
+          # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg
+          # half-configured; clear both (recovery bounded) before the next attempt.
+          if command -v sudo >/dev/null 2>&1; then
+            sudo pkill -x apt-get || true
+            sudo pkill -x dpkg || true
+            sleep 2
+            sudo timeout 120 dpkg --configure -a || true
+          fi
           sleep 5
         done
         exit 1

.github/actions/setup-ui-e2e/action.yml

@@ -33,7 +33,7 @@ runs:
       shell: bash
       run: |
         if command -v sudo >/dev/null 2>&1; then
-          sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true
+          sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true
           sudo mkdir -p /etc/apt/apt.conf.d
           sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF'
         Acquire::Retries "3";
@@ -42,6 +42,9 @@ runs:
         EOF
         fi
         install_chromium() {
+          # Fail the attempt if install-deps fails, rather than letting the browser
+          # download that follows mask it (see release-browser-matrix in ci.yml).
+          set -e
           if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then
             npx playwright install-deps chromium
             npx playwright install chromium
@@ -54,6 +57,14 @@ runs:
             exit 0
           fi
           echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2
+          # A timed-out attempt can leave its root apt-get holding the apt lock, or dpkg
+          # half-configured; clear both (recovery bounded) before the next attempt.
+          if command -v sudo >/dev/null 2>&1; then
+            sudo pkill -x apt-get || true
+            sudo pkill -x dpkg || true
+            sleep 2
+            sudo timeout 120 dpkg --configure -a || true
+          fi
           sleep 5
         done
         exit 1

.github/workflows/ci.yml

@@ -2040,27 +2040,47 @@ jobs:
           BROWSER_ENGINE: ${{ matrix.engine }}
         run: |
           if command -v sudo >/dev/null 2>&1; then
-            sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true
+            sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list /etc/apt/apt-mirrors.txt 2>/dev/null || true
             sudo mkdir -p /etc/apt/apt.conf.d
             sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF'
           Acquire::Retries "3";
           Acquire::http::Timeout "30";
           Acquire::https::Timeout "30";
           EOF
           fi
+          # The runner's apt sources resolve through the mirror list in apt-mirrors.txt, so
+          # rewriting only the sources files left apt waiting on the azure mirror until the
+          # attempt timed out.
+          # `set -e`: without it a failed `install-deps` on a cache hit was masked by the
+          # browser download that followed, so the attempt "passed" with apt libraries
+          # missing and every WebKit test died at launch (libevent-2.1.so.7, main run
+          # 38228339791). The launch probe proves the engine actually starts on this
+          # runner, so a missing library fails here and retries instead of in each test.
           install_browsers() {
+            set -e
             if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then
               npx playwright install-deps "$BROWSER_ENGINE"
               npx playwright install "$BROWSER_ENGINE"
             else
               npx playwright install --with-deps "$BROWSER_ENGINE"
             fi
+            node -e 'require("playwright")[process.env.BROWSER_ENGINE].launch().then((browser) => browser.close())'
           }
           for attempt in 1 2 3; do
             if timeout 180 bash -c "$(declare -f install_browsers); install_browsers"; then
               exit 0
             fi
             echo "Playwright browsers install attempt $attempt timed out or failed, retrying..." >&2
+            # `timeout` kills the attempt's shell but not the root apt-get it started, which
+            # then holds the apt lock and fails every retry (dispatch run 38234519037). Stop
+            # it (and any dpkg it started), then finish half-configured packages, bounded so a stalled
+            # recovery cannot hold the loop, before the next attempt.
+            if command -v sudo >/dev/null 2>&1; then
+              sudo pkill -x apt-get || true
+              sudo pkill -x dpkg || true
+              sleep 2
+              sudo timeout 120 dpkg --configure -a || true
+            fi
             sleep 5
           done
           exit 1

docs/outstanding-issues-inbox/693e8f34-1b14-4e61-a87c-430fff130dad.json

@@ -0,0 +1,14 @@
+{
+  "version": 2,
+  "id": "693e8f34-1b14-4e61-a87c-430fff130dad",
+  "createdOn": "2026-10-12",
+  "action": "add",
+  "payload": {
+    "pri": "P2",
+    "type": "issue",
+    "summary": "Safari (WebKit) browser tests: 26 genuine failures now visible after the launch fix in PR #3487",
+    "detail": "Before PR #3487 every WebKit leg of the release browser matrix died at browser launch (missing libevent), hiding real results. Dispatch run 38236585559 (head 7e2d712) launched every leg; these tests then failed. tablet 2/4 (16): ui-accessibility 176/377/597, ui-overlap 239/437/549, ui-teaching 83, ui-tools 587/672/1084/2183/2295/2490/2674/2946, ui-universal-search 360. tablet 4/4 (4): ui-my-day-calendar 116, ui-roster-team 100/174, ui-stress 346. tablet 3/4: ui-cme-phone 351. mobile-webkit 2/5: ui-accessibility 295. mobile-webkit 5/5 and mobile-pwa-standalone 4/5: ui-on-call-boards 558 (declares no anchor the page does not render). mobile-pwa-standalone 5/5: ui-phone-scroll-assessments 19, ui-phone-scroll-page-owned 454. tablet 1/4 was still running at triage. Many tablet failures are 1.0m timeouts, so some may share one cause (tablet viewport or WebKit timing). Next action: rerun the matrix on main after #3487 merges, group failures by cause, fix app or test per group. Never skip or quarantine a test to get green.",
+    "source": "PR #3487 browser-matrix dispatch 2026-10-12",
+    "issueUlid": "01M4QX1X5FCJENZWYHCG1971RC"
+  }
+}

tests/ci-browser-matrix-coverage.test.ts

@@ -191,6 +191,24 @@ describe("release-browser-matrix engine coverage", () => {
     expect(releaseJob).toContain('npx playwright install --with-deps "$BROWSER_ENGINE"');
   });
 
+  it("fails the install attempt when system libraries are missing instead of failing every test at launch", () => {
+    const steps = parsedWorkflow.jobs["release-browser-matrix"].steps as WorkflowStep[];
+    const install = steps.find((step) => step.name === "Install Playwright browsers");
+    const script = String(install?.run ?? "");
+    // Main run 38228339791: install-deps failed on a cache hit, the browser download after
+    // it succeeded, the attempt reported success, and every WebKit test died at launch.
+    expect(script).toMatch(/install_browsers\(\) \{\n\s+set -e\n/);
+    expect(script).toContain('require("playwright")[process.env.BROWSER_ENGINE].launch()');
+    // The probe sits after the cache-hit/cold-cache branch closes, so both paths are verified.
+    expect(script).toMatch(/\n\s+fi\n\s+node -e 'require\("playwright"\)\[process\.env\.BROWSER_ENGINE\]\.launch\(\)/);
+    expect(script).toContain("sudo timeout 120 dpkg --configure -a || true");
+    // Dispatch run 38234519037: the azure mirror in apt-mirrors.txt stalled apt past the
+    // attempt timeout, and the orphaned apt-get then held the lock for every retry.
+    expect(script).toContain("/etc/apt/apt-mirrors.txt");
+    expect(script).toContain("sudo pkill -x apt-get || true");
+    expect(script).toContain("sudo pkill -x dpkg || true");
+  });
+
   it("reuses only the same run's matching production build, preserving the mockup build and no-producer fallback", () => {
     const job = parsedWorkflow.jobs["release-browser-matrix"];
     const steps = job.steps as WorkflowStep[];