From f31e1cf6e420f48ad5b74352ae639a38fd59d367 Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:45:29 -0400 Subject: [PATCH 1/4] fix(web): let the production build sign in on the local network adapter-node 6 takes every request to be https unless PROTOCOL_HEADER names a header that says otherwise, so SvelteKit's CSRF check refused the sign-in form posted from http://127.0.0.1. dev-network.ts --production now sets PROTOCOL_HEADER=x-forwarded-proto. The OAuth client also refused the network's http PDS outside dev. It now allows http when ALLOW_PRIVATE_NETWORK is set, which is already what lets the production build reach a loopback PDS. --- apps/web/scripts/dev-network.ts | 10 ++++++- apps/web/src/lib/server/atproto/oauth.test.ts | 30 +++++++++++++++++++ apps/web/src/lib/server/atproto/oauth.ts | 4 ++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/apps/web/scripts/dev-network.ts b/apps/web/scripts/dev-network.ts index 485a8e5f2..2b75bb92d 100644 --- a/apps/web/scripts/dev-network.ts +++ b/apps/web/scripts/dev-network.ts @@ -381,7 +381,15 @@ async function main() { ); // The production build only reaches this network's loopback PDS and PLC with the SSRF guard off. - const production = { NODE_ENV: 'production', HOST: '127.0.0.1', ALLOW_PRIVATE_NETWORK: 'true' }; + // adapter-node takes a request to be https unless PROTOCOL_HEADER's header says otherwise, so a + // form posted from http://127.0.0.1 (signing in) fails SvelteKit's CSRF check without + // `X-Forwarded-Proto: http`, which scripts/lighthouse.ts sends while it signs in. + const production = { + NODE_ENV: 'production', + HOST: '127.0.0.1', + PROTOCOL_HEADER: 'x-forwarded-proto', + ALLOW_PRIVATE_NETWORK: 'true' + }; const app = START_APP ? spawn(PRODUCTION ? process.execPath : 'pnpm', PRODUCTION ? ['build'] : ['exec', 'vite', 'dev'], { diff --git a/apps/web/src/lib/server/atproto/oauth.test.ts b/apps/web/src/lib/server/atproto/oauth.test.ts index 1d48d8141..d435f2412 100644 --- a/apps/web/src/lib/server/atproto/oauth.test.ts +++ b/apps/web/src/lib/server/atproto/oauth.test.ts @@ -239,6 +239,36 @@ describe('OAuth scopes', () => { }); }); +describe('OAuthService', () => { + /** Signing in at an http PDS from a production build with ALLOW_PRIVATE_NETWORK as given. */ + const signInAtHttpPds = async (allowPrivateNetwork: string) => { + const oauth = new OAuthService({ + config: loadConfig({ + NODE_ENV: 'production', + ALLOW_PRIVATE_NETWORK: allowPrivateNetwork, + ROOT_IDENTIFIER: 'did:plc:root', + ROOT_PASSWORD: 'x', + PUBLIC_URL: 'https://localnetwork.test', + HAPPYVIEW_URL: 'http://127.0.0.1:8000' + }), + db: await openDatabase(':memory:'), + logger: silentLogger + }); + + // Port 9 is one fetch refuses to connect to, so nothing leaves the machine either way. + return oauth.client.authorize('http://127.0.0.1:9/'); + }; + + it('signs in at an http PDS in a production build only on a private network it may reach', async () => { + const refusedHttp = expect.stringMatching(/Unsecure resource metadata URL \(http:\)/); + + await expect(signInAtHttpPds('false')).rejects.toMatchObject({ cause: { message: refusedHttp } }); + await expect(signInAtHttpPds('true')).rejects.toMatchObject({ + cause: { message: expect.not.stringMatching(/Unsecure/) } + }); + }); +}); + describe('OAuthService.authorize', () => { const user = 'did:plc:user'; const page = new URL('https://pds.test/oauth/authorize'); diff --git a/apps/web/src/lib/server/atproto/oauth.ts b/apps/web/src/lib/server/atproto/oauth.ts index 7a499f365..487ee4edd 100644 --- a/apps/web/src/lib/server/atproto/oauth.ts +++ b/apps/web/src/lib/server/atproto/oauth.ts @@ -229,7 +229,9 @@ export class OAuthService { constructor(private readonly deps: { config: Config; db: Db; logger: Logger }) { this.client = new NodeOAuthClient({ clientMetadata: clientMetadata(deps.config, DECLARED_SCOPES), - allowHttp: deps.config.dev, + // An http PDS only lives on a private network, which ALLOW_PRIVATE_NETWORK already lets the + // production build reach (scripts/dev-network.ts --production, for Lighthouse's signed-in pages). + allowHttp: deps.config.dev || deps.config.allowPrivateNetwork, plcDirectoryUrl: deps.config.plcUrl, handleResolver: deps.config.handleResolverUrl, requestLock: requestLocalLock, From 588395774b499617b9eada41440b978b68d93ad9 Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:45:47 -0400 Subject: [PATCH 2/4] test(web): measure every page with Lighthouse against its history lighthouse/pages.json lists the pages to measure, each signed out or signed in as a seeded account, and lighthouserc.cjs takes its pull request pages from it. `pnpm test:lighthouse:full` starts the network once, signs in with Playwright, and runs lhci collect on every page on a phone and a desktop against the existing budgets. scripts/lighthouse-history.ts then holds each page's medians against the last five recorded runs and fails on a regression past a metric's tolerance or on a bad result (a failed run, a redirect, an error status, a failed budget). The e2e sign-in helper moves to tests/sign-in.ts so both use it. --- .github/labeler.yml | 1 + .github/workflows/web-ci.yml | 5 +- apps/web/.gitignore | 2 + apps/web/README.md | 60 ++-- apps/web/lighthouse/pages.json | 46 +++ apps/web/lighthouse/pages.schema.json | 35 ++ apps/web/lighthouserc.cjs | 6 +- apps/web/package.json | 1 + apps/web/scripts/lib/lighthouse.test.ts | 255 ++++++++++++++ apps/web/scripts/lib/lighthouse.ts | 444 ++++++++++++++++++++++++ apps/web/scripts/lighthouse-history.ts | 62 ++++ apps/web/scripts/lighthouse.ts | 256 ++++++++++++++ apps/web/tests/e2e/app.e2e.ts | 15 +- apps/web/tests/sign-in.ts | 19 + 14 files changed, 1167 insertions(+), 40 deletions(-) create mode 100644 apps/web/lighthouse/pages.json create mode 100644 apps/web/lighthouse/pages.schema.json create mode 100644 apps/web/scripts/lib/lighthouse.test.ts create mode 100644 apps/web/scripts/lib/lighthouse.ts create mode 100644 apps/web/scripts/lighthouse-history.ts create mode 100644 apps/web/scripts/lighthouse.ts create mode 100644 apps/web/tests/sign-in.ts diff --git a/.github/labeler.yml b/.github/labeler.yml index b6fa3d055..a5c7e2123 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -272,6 +272,7 @@ tests: - "**/{vitest,vitest.stryker,playwright}.config.ts" - "**/stryker*.{mjs,ts}" - "apps/web/src/lib/server/testing/**" + - "apps/web/{lighthouse/**,scripts/lighthouse*.ts,scripts/lib/lighthouse.ts}" - "apps/locallibrary/bombadil/**" - "fuzz/**" - ".config/nextest.toml" diff --git a/.github/workflows/web-ci.yml b/.github/workflows/web-ci.yml index 7d4837a65..28f098948 100644 --- a/.github/workflows/web-ci.yml +++ b/.github/workflows/web-ci.yml @@ -89,8 +89,8 @@ jobs: # What the pages Lighthouse measures are built and served from: the app's code, static # files and Lua (HappyView answers the directory's queries), the dev network and seed # data it measures them on, its build and Lighthouse config, the dependencies, and the - # README and guides of the one tool page lighthouserc.cjs visits (add a tool here when - # its page goes in there). Left out: the app's tests and the helpers and fixture only + # README and guides of the one tool page lighthouse/pages.json marks `pr` (add a tool here + # when its page is marked). Left out: the app's tests and the helpers and fixture only # they import (src/lib/server/testing, the mix fixture), stories and Storybook docs, and # everything outside these paths (tests/, .storybook/, the lint and format configs, the # website's README, the Dockerfile). @@ -103,6 +103,7 @@ jobs: - 'apps/web/vite.config.ts' - 'apps/web/tsconfig.json' - 'apps/web/lighthouserc.cjs' + - 'apps/web/lighthouse/**' - 'apps/locallibrary/README.md' - 'apps/locallibrary/docs/*.md' - 'packages/tokens/**' diff --git a/apps/web/.gitignore b/apps/web/.gitignore index 0f3866750..6e237e3c5 100644 --- a/apps/web/.gitignore +++ b/apps/web/.gitignore @@ -17,6 +17,8 @@ test-results *.db-shm /.lighthouseci /lighthouse-report +/lighthouse-full +/lighthouse-history*.jsonl *storybook.log storybook-static diff --git a/apps/web/README.md b/apps/web/README.md index 96a6ba76e..6a7c614fb 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -215,32 +215,46 @@ The network is temporary; Ctrl-C stops it, and restarting resets everything (and ## Scripts -| Script | What it does | -| ----------------------------------- | ------------------------------------------------------------------------- | -| `pnpm dev` | Local atproto network, HappyView, and the dev server | -| `pnpm dev:network` / `pnpm dev:app` | The network and HappyView only / the dev server only (reads `.env.local`) | -| `pnpm build` / `pnpm start` | Production build (adapter-node) / run it | -| `pnpm check` | `svelte-check` with TypeScript 6 | -| `pnpm lint` | Oxlint with type-aware rules (tsgolint) and the anti-slop plugin | -| `pnpm format` / `pnpm format:check` | Oxfmt | -| `pnpm test` | Unit tests (Node) + component tests (Vitest browser mode, Chromium) | -| `pnpm test:integration` | Full flows against an in-process spaces PDS and a HappyView container | -| `pnpm test:coverage` | Unit + component + story + integration tests with `@vitest/coverage-v8` | -| `pnpm test:build` | Production build, checked for sourcemaps and the client JS budgets | -| `pnpm storybook` | The design system in Storybook, at localhost:6006 | -| `pnpm test:storybook` | Every story rendered in Chromium; an accessibility violation fails it | -| `pnpm build-storybook` | Static Storybook in `storybook-static/` | -| `pnpm test:e2e` | Playwright (desktop browsers and phones) on an isolated network, axe | -| `pnpm test:lighthouse` | Lighthouse CI on the production build (`lighthouserc.cjs`) | -| `pnpm knip` | Unused files, exports, and dependencies | -| `pnpm lexicons:publish` | Publish lexicons from the root account and check or set their DNS | -| `pnpm db:backup` | Back the database up to the bucket now ([Backups](#backups)) | -| `pnpm db:backups` | List the bucket's backups | -| `pnpm db:restore [path]` | Restore a backup (or `latest`) over the database, or to `path` | -| `pnpm verify` | CI's quick checks, in order | +| Script | What it does | +| ----------------------------------- | -------------------------------------------------------------------------------------------- | +| `pnpm dev` | Local atproto network, HappyView, and the dev server | +| `pnpm dev:network` / `pnpm dev:app` | The network and HappyView only / the dev server only (reads `.env.local`) | +| `pnpm build` / `pnpm start` | Production build (adapter-node) / run it | +| `pnpm check` | `svelte-check` with TypeScript 6 | +| `pnpm lint` | Oxlint with type-aware rules (tsgolint) and the anti-slop plugin | +| `pnpm format` / `pnpm format:check` | Oxfmt | +| `pnpm test` | Unit tests (Node) + component tests (Vitest browser mode, Chromium) | +| `pnpm test:integration` | Full flows against an in-process spaces PDS and a HappyView container | +| `pnpm test:coverage` | Unit + component + story + integration tests with `@vitest/coverage-v8` | +| `pnpm test:build` | Production build, checked for sourcemaps and the client JS budgets | +| `pnpm storybook` | The design system in Storybook, at localhost:6006 | +| `pnpm test:storybook` | Every story rendered in Chromium; an accessibility violation fails it | +| `pnpm build-storybook` | Static Storybook in `storybook-static/` | +| `pnpm test:e2e` | Playwright (desktop browsers and phones) on an isolated network, axe | +| `pnpm test:lighthouse` | Lighthouse CI on the production build (`lighthouserc.cjs`) | +| `pnpm test:lighthouse:full` | Lighthouse on every page, phone and desktop, against its history ([Lighthouse](#lighthouse)) | +| `pnpm knip` | Unused files, exports, and dependencies | +| `pnpm lexicons:publish` | Publish lexicons from the root account and check or set their DNS | +| `pnpm db:backup` | Back the database up to the bucket now ([Backups](#backups)) | +| `pnpm db:backups` | List the bucket's backups | +| `pnpm db:restore [path]` | Restore a backup (or `latest`) over the database, or to `path` | +| `pnpm verify` | CI's quick checks, in order | Pull requests run the e2e tests in Chromium only (`pnpm test:e2e --project chromium`); the other browsers and the phones run weekly. +### Lighthouse + +[`lighthouse/pages.json`](lighthouse/pages.json) lists the pages Lighthouse measures, each signed out or signed in as `neon.test` (`dj`) or `ada.test` (`admin`). Pull requests that change what the pages are built from measure the ones marked `pr` against the budgets in `lighthouserc.cjs`. `pnpm test:lighthouse:full` measures all of them on a phone and a desktop, three runs each, and keeps each page's medians in `lighthouse-history.jsonl`. Each run is held against the median of the last five recorded runs, and against the twenty before those to catch a slow creep. A regression past a metric's tolerance (in [`scripts/lib/lighthouse.ts`](scripts/lib/lighthouse.ts)) fails the run, as does a bad result: a failed run, a page that redirected or answered an error, or a failed budget (signed-in pages have no SEO budget, since search engines never see them). Runs that spread wide, and a new Lighthouse or a faster or slower machine, are noted too. + +To find the commit behind a regression, record a baseline of only the pages that moved on the last good commit (each history line names its commit), then let `git bisect` measure each commit against it, failing only on a regression (commits from before this suite landed lack its scripts): + +```bash +git checkout +pnpm build && pnpm exec tsx scripts/lighthouse.ts --page /charts && node scripts/lighthouse-history.ts --record --history lighthouse-history-bisect.jsonl +git bisect start +git bisect run sh -c 'pnpm build && pnpm exec tsx scripts/lighthouse.ts --page /charts && node scripts/lighthouse-history.ts --history lighthouse-history-bisect.jsonl --regressions-only' +``` + Git hooks (lefthook, from the root `lefthook.yml`) format and lint staged files on commit and run type checks and unit tests on push. ## Design system diff --git a/apps/web/lighthouse/pages.json b/apps/web/lighthouse/pages.json new file mode 100644 index 000000000..f910e7de5 --- /dev/null +++ b/apps/web/lighthouse/pages.json @@ -0,0 +1,46 @@ +{ + "$schema": "./pages.schema.json", + "pages": [ + { "path": "/", "pr": true }, + { "path": "/about" }, + { "path": "/for-djs" }, + { "path": "/privacy" }, + { "path": "/login" }, + { "path": "/projects" }, + { "path": "/projects?genres=house" }, + { "path": "/projects/halflight", "pr": true }, + { "path": "/projects/neon-tide" }, + { "path": "/projects/neon-tide/press-kits" }, + { "path": "/artists" }, + { "path": "/artists?q=okon" }, + { "path": "/charts" }, + { "path": "/shows" }, + { "path": "/match" }, + { "path": "/tools" }, + { "path": "/tools/locallibrary", "pr": true }, + { "path": "/tools/locallibrary/docs/install" }, + { "path": "/tools/localtoolbox" }, + { "path": "/dashboard", "account": "dj" }, + { "path": "/dashboard/activity", "account": "dj" }, + { "path": "/dashboard/bookings", "account": "dj" }, + { "path": "/dashboard/bookings/new?project=halflight", "account": "dj" }, + { "path": "/dashboard/inbox", "account": "dj" }, + { "path": "/dashboard/inbox/settings", "account": "dj" }, + { "path": "/dashboard/planner", "account": "dj" }, + { "path": "/dashboard/requests", "account": "dj" }, + { "path": "/report", "account": "dj" }, + { "path": "/dashboard/projects/new", "account": "dj" }, + { "path": "/dashboard/projects/neon-tide", "account": "dj" }, + { "path": "/dashboard/projects/neon-tide/profile", "account": "dj" }, + { "path": "/dashboard/projects/neon-tide/press-kits", "account": "dj" }, + { "path": "/admin", "account": "admin" }, + { "path": "/admin/audit", "account": "admin" }, + { "path": "/admin/projects", "account": "admin" }, + { "path": "/admin/projects/neon-tide", "account": "admin" }, + { "path": "/admin/requests", "account": "admin" }, + { "path": "/admin/spaces", "account": "admin" }, + { "path": "/admin/staff", "account": "admin" }, + { "path": "/admin/system", "account": "admin" }, + { "path": "/admin/verifications", "account": "admin" } + ] +} diff --git a/apps/web/lighthouse/pages.schema.json b/apps/web/lighthouse/pages.schema.json new file mode 100644 index 000000000..ba785783b --- /dev/null +++ b/apps/web/lighthouse/pages.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Lighthouse pages", + "description": "The pages Lighthouse measures (lighthouserc.cjs on pull requests, scripts/lighthouse.ts weekly), on the seeded network of scripts/dev-network.ts.", + "type": "object", + "required": ["pages"], + "additionalProperties": false, + "properties": { + "$schema": { "type": "string" }, + "pages": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { + "path": { + "description": "The page's path and query, on the seeded network.", + "type": "string", + "pattern": "^/" + }, + "account": { + "description": "The seeded account the page is measured signed in as: dj (neon.test) or admin (ada.test). Signed out without one.", + "enum": ["dj", "admin"] + }, + "pr": { + "description": "Also measured on pull requests that change what the pages are built from (web-ci.yml), against lighthouserc.cjs's budgets.", + "type": "boolean" + } + } + } + } + } +} diff --git a/apps/web/lighthouserc.cjs b/apps/web/lighthouserc.cjs index deba6e56e..0a79a6d13 100644 --- a/apps/web/lighthouserc.cjs +++ b/apps/web/lighthouserc.cjs @@ -1,8 +1,12 @@ // Lighthouse CI (`pnpm test:lighthouse`, which builds first). It measures the production build on the // same local network and seeded projects as the end-to-end tests (scripts/dev-network.ts), so the // pages have real data in them. Ports apart from the e2e run's, so the two can run side by side. +// The pages are the ones lighthouse/pages.json marks `pr`; scripts/lighthouse.ts measures all of them +// weekly, on phones and desktops, and tracks each against its history. const { chromium } = require('playwright'); +const { pages } = require('./lighthouse/pages.json'); + const PORT = 5198; const env = { @@ -24,7 +28,7 @@ module.exports = { // adapter-node prints this once the app has connected its root account and is serving. startServerReadyPattern: 'Listening on', startServerReadyTimeout: 180_000, - url: ['/', '/projects/halflight', '/tools/locallibrary'].map((path) => `http://127.0.0.1:${PORT}${path}`), + url: pages.filter((page) => page.pr).map(({ path }) => `http://127.0.0.1:${PORT}${path}`), numberOfRuns: 3, // Playwright's Chromium, which the e2e tests install anyway, rather than whichever Chrome the // machine has (or doesn't). diff --git a/apps/web/package.json b/apps/web/package.json index 2e9e4ae48..a7b0306db 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -32,6 +32,7 @@ "test:build": "vitest run --project build", "test:e2e": "playwright test", "test:lighthouse": "pnpm build && lhci autorun", + "test:lighthouse:full": "pnpm build && tsx scripts/lighthouse.ts && node scripts/lighthouse-history.ts --record", "lexicons:publish": "tsx scripts/publish-lexicons.ts", "fixtures:mix": "tsx scripts/mix-fixture.ts", "db:backup": "node scripts/db-backup.ts backup", diff --git a/apps/web/scripts/lib/lighthouse.test.ts b/apps/web/scripts/lib/lighthouse.test.ts new file mode 100644 index 000000000..300803770 --- /dev/null +++ b/apps/web/scripts/lib/lighthouse.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from 'vitest'; +import { + compare, + formatValue, + type HistoryEntry, + isFailed, + type Lhr, + METRICS, + median, + OLDER_RUNS, + RECENT_RUNS, + renderMarkdown, + sparkline, + summarizePage, + type Summary +} from './lighthouse.ts'; + +const metric = (id: string) => METRICS.find((candidate) => candidate.id === id)!; + +/** A run of `path` that scored `performance` with an LCP of `lcp` ms. */ +function lhr(path: string, { performance = 0.98, lcp = 1800, benchmarkIndex = 2000 } = {}): Lhr { + return { + lighthouseVersion: '12.6.1', + finalDisplayedUrl: `http://127.0.0.1:5198${path}`, + environment: { benchmarkIndex }, + categories: { performance: { score: performance }, accessibility: { score: 1 } }, + audits: { + 'largest-contentful-paint': { score: 0.9, numericValue: lcp }, + 'http-status-code': { score: 1 } + } + }; +} + +const summary = (pages: Summary['pages'], formFactor: Summary['formFactor'] = 'mobile'): Summary => ({ + formFactor, + lighthouseVersion: '12.6.1', + benchmarkIndex: 2000, + runs: 3, + pages +}); + +const page = (path: string, metrics: Record, problems: string[] = []) => ({ + path, + runs: 3, + problems, + metrics: Object.fromEntries( + Object.entries(metrics).map(([id, value]) => [id, { median: value, min: value, max: value }]) + ) +}); + +const entry = (results: HistoryEntry['results'], extra: Partial = {}): HistoryEntry => ({ + date: '2026-10-01T00:00:00.000Z', + commit: 'abc', + lighthouseVersion: '12.6.1', + benchmarkIndex: 2000, + results, + ...extra +}); + +const RUN = { date: '2026-10-31T00:00:00.000Z', commit: '0123456789abcdef' }; + +describe('median', () => { + it('takes the middle value, or the mean of the two middle ones', () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + }); +}); + +describe('summarizePage', () => { + it('keeps the median, lowest and highest of each metric over the runs', () => { + const result = summarizePage({ path: '/' }, [ + lhr('/', { lcp: 1700 }), + lhr('/', { lcp: 2100 }), + lhr('/', { lcp: 1800 }) + ]); + + expect(result.runs).toBe(3); + expect(result.problems).toEqual([]); + expect(result.metrics.lcp).toEqual({ median: 1800, min: 1700, max: 2100 }); + expect(result.metrics.accessibility).toEqual({ median: 1, min: 1, max: 1 }); + expect(result.metrics).not.toHaveProperty('tbt'); + }); + + it('reports failed runs and leaves them out of the numbers', () => { + const failed = { ...lhr('/', { lcp: 9000 }), runtimeError: { code: 'NO_FCP', message: 'The page did not paint' } }; + const result = summarizePage({ path: '/' }, [failed, lhr('/', { lcp: 1800 })]); + + expect(result.runs).toBe(1); + expect(result.problems).toEqual(['a run failed: NO_FCP (The page did not paint)']); + expect(result.metrics.lcp.median).toBe(1800); + }); + + it('reports a page that redirected, answered an error, or never ran', () => { + const redirected = { ...lhr('/login?next=%2Fdashboard'), audits: { 'http-status-code': { score: 0 } } }; + + expect(summarizePage({ path: '/dashboard', account: 'dj' }, [redirected]).problems).toEqual([ + 'ended up on /login?next=%2Fdashboard', + 'answered with an error status' + ]); + expect(summarizePage({ path: '/' }, []).problems).toEqual(['no run produced a result']); + }); + + it('adds lhci assertion failures to the problems', () => { + expect(summarizePage({ path: '/' }, [lhr('/')], ['failed largest-contentful-paint']).problems).toEqual([ + 'failed largest-contentful-paint' + ]); + }); +}); + +describe('compare', () => { + const history = Array.from({ length: RECENT_RUNS }, () => entry({ 'mobile /': { performance: 0.98, lcp: 1800 } })); + + it('flags a metric worse than the recent median by more than its tolerance', () => { + const result = compare([summary([page('/', { performance: 0.9, lcp: 2400 })])], history, RUN); + + expect(result.regressions.map(({ metric: { id }, baseline, current }) => [id, baseline, current])).toEqual([ + ['performance', 0.98, 0.9], + ['lcp', 1800, 2400] + ]); + expect(isFailed(result)).toBe(true); + }); + + it('leaves changes within tolerance alone, in both directions', () => { + // LCP's tolerance is the larger of 150 ms and 10%: 180 ms on 1.8 s. + const result = compare([summary([page('/', { performance: 0.96, lcp: 1970 })])], history, RUN); + + expect(result.regressions).toEqual([]); + expect(result.improvements).toEqual([]); + expect(isFailed(result)).toBe(false); + }); + + it('lists improvements past the tolerance', () => { + const result = compare([summary([page('/', { performance: 1, lcp: 1200 })])], history, RUN); + + expect(result.improvements.map(({ metric: { id } }) => id)).toEqual(['lcp']); + }); + + it('takes the median of the recent runs, so one outlier in them moves nothing', () => { + const withOutlier = [...history.slice(1), entry({ 'mobile /': { lcp: 6000 } })]; + + expect(compare([summary([page('/', { lcp: 1850 })])], withOutlier, RUN).regressions).toEqual([]); + }); + + it('flags a slow creep against the runs before the recent ones', () => { + const older = Array.from({ length: OLDER_RUNS }, () => entry({ 'mobile /': { lcp: 1500 } })); + const recent = [1600, 1700, 1750, 1800, 1850].map((lcp) => entry({ 'mobile /': { lcp } })); + const result = compare([summary([page('/', { lcp: 1800 })])], [...older, ...recent], RUN); + + expect(result.regressions).toEqual([]); + expect(result.drifts.map(({ metric: { id }, baseline }) => [id, baseline])).toEqual([['lcp', 1500]]); + expect(isFailed(result)).toBe(false); + }); + + it('keeps form factors apart', () => { + const result = compare([summary([page('/', { lcp: 2400 })], 'desktop')], history, RUN); + + expect(result.regressions).toEqual([]); + expect(result.added).toEqual(['desktop /']); + }); + + it('fails on a bad result even with nothing to compare against', () => { + const result = compare([summary([page('/dashboard', {}, ['ended up on /login'])])], [], RUN); + + expect(result.problems).toEqual([{ key: 'mobile /dashboard', problem: 'ended up on /login' }]); + expect(result.baselineRuns).toBe(0); + expect(isFailed(result)).toBe(true); + }); + + it('flags runs that spread wider than twice the tolerance', () => { + const spread = { ...page('/', {}), metrics: { lcp: { median: 1800, min: 1500, max: 2400 } } }; + + expect(compare([summary([spread])], history, RUN).unstable.map(({ key, metric: { id } }) => [key, id])).toEqual([ + ['mobile /', 'lcp'] + ]); + }); + + it('notes a new Lighthouse version and a machine of a different speed', () => { + const old = history.map((past) => ({ ...past, lighthouseVersion: '12.5.0', benchmarkIndex: 1000 })); + + expect(compare([summary([page('/', { lcp: 1800 })])], old, RUN).notes).toEqual([ + 'Lighthouse changed from 12.5.0 to 12.6.1 since the last run.', + "This machine benchmarked at 2000, against 1000 for the baseline's, so timings may have moved with it." + ]); + }); + + it('records each page’s medians under its form factor and path', () => { + const result = compare( + [summary([page('/', { lcp: 1800 })]), summary([page('/', { lcp: 900 })], 'desktop')], + [], + RUN + ); + + expect(result.entry).toEqual({ + ...RUN, + lighthouseVersion: '12.6.1', + benchmarkIndex: 2000, + results: { 'mobile /': { lcp: 1800 }, 'desktop /': { lcp: 900 } } + }); + }); +}); + +describe('formatValue', () => { + it('shows each unit the way Lighthouse does', () => { + expect(formatValue(metric('performance'), 0.973)).toBe('97'); + expect(formatValue(metric('lcp'), 1834)).toBe('1.83 s'); + expect(formatValue(metric('tbt'), 42.4)).toBe('42 ms'); + expect(formatValue(metric('bytes'), 204_800)).toBe('200 KiB'); + expect(formatValue(metric('cls'), 0.0123)).toBe('0.012'); + expect(formatValue(metric('dom'), 812)).toBe('812'); + }); +}); + +describe('sparkline', () => { + it('draws the lowest value lowest and the highest highest', () => { + expect(sparkline([1, 8, 4.5])).toBe('▁█▅'); + expect(sparkline([3, 3])).toBe('▁▁'); + }); +}); + +describe('renderMarkdown', () => { + it('puts regressions and bad results first, and every page in a table', () => { + const history = [entry({ 'mobile /': { performance: 0.98, lcp: 1800 } })]; + + const result = compare( + [summary([page('/', { performance: 0.98, lcp: 2400 }), page('/admin', {}, ['ended up on /login'])])], + history, + RUN + ); + + const markdown = renderMarkdown(result, history); + + expect(markdown).toContain('0123456789ab, Lighthouse 12.6.1, against the median of the last 1 recorded run.'); + expect(markdown).toContain( + '### Regressions (1)\n\n| Page | Metric | Baseline | Now |\n| --- | --- | ---: | ---: |\n| `mobile /` | LCP | 1.80 s | 2.40 s |' + ); + expect(markdown).toContain('### Bad results (1)\n\n- `mobile /admin`: ended up on /login'); + expect(markdown).toContain('New pages, with no history yet: `mobile /admin`'); + expect(markdown).toContain('| `mobile /` | 98 | | | | 2.40 s | | | | | ▁▁ | ▁█ |'); + }); + + it('shortens the commit but keeps a dirty marker', () => { + const dirty = { ...RUN, commit: `${RUN.commit}0123456789abcdef01234567-dirty` }; + + expect(renderMarkdown(compare([summary([page('/', { lcp: 1800 })])], [], dirty), [])).toContain( + '0123456789ab-dirty, Lighthouse' + ); + }); + + it('says when there is no history yet, without listing every page as new', () => { + const markdown = renderMarkdown(compare([summary([page('/', { lcp: 1800 })])], [], RUN), []); + + expect(markdown).toContain('No history yet: this run is the first baseline.'); + expect(markdown).not.toContain('New pages'); + }); +}); diff --git a/apps/web/scripts/lib/lighthouse.ts b/apps/web/scripts/lib/lighthouse.ts new file mode 100644 index 000000000..c2172b8a4 --- /dev/null +++ b/apps/web/scripts/lib/lighthouse.ts @@ -0,0 +1,444 @@ +// What scripts/lighthouse.ts keeps of each page's Lighthouse runs, and how scripts/lighthouse-history.ts +// holds that against the pages' history. No dependencies, so CI's history job runs it on plain Node. + +export type FormFactor = 'mobile' | 'desktop'; + +export type Account = 'dj' | 'admin'; + +/** A number tracked over time: a category's score, or an audit's numericValue. */ +export type Metric = { + id: string; + label: string; + category?: string; + audit?: string; + unit: 'score' | 'ms' | 'bytes' | 'count'; + higherIsBetter: boolean; + /** A change only counts past both: this much, and (when set) this share of the baseline. */ + tolerance: { absolute: number; relative?: number }; +}; + +// Tolerances sit above the run-to-run noise of Lighthouse's simulated throttling on a median of three: +// the category scores other than performance don't move between runs, so any drop counts. +export const METRICS: Metric[] = [ + { + id: 'performance', + label: 'Performance', + category: 'performance', + unit: 'score', + higherIsBetter: true, + tolerance: { absolute: 0.05 } + }, + { + id: 'accessibility', + label: 'Accessibility', + category: 'accessibility', + unit: 'score', + higherIsBetter: true, + tolerance: { absolute: 0.01 } + }, + { + id: 'best-practices', + label: 'Best practices', + category: 'best-practices', + unit: 'score', + higherIsBetter: true, + tolerance: { absolute: 0.01 } + }, + { id: 'seo', label: 'SEO', category: 'seo', unit: 'score', higherIsBetter: true, tolerance: { absolute: 0.01 } }, + { + id: 'fcp', + label: 'FCP', + audit: 'first-contentful-paint', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.1 } + }, + { + id: 'lcp', + label: 'LCP', + audit: 'largest-contentful-paint', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 150, relative: 0.1 } + }, + { + id: 'tbt', + label: 'TBT', + audit: 'total-blocking-time', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 50, relative: 0.25 } + }, + { + id: 'cls', + label: 'CLS', + audit: 'cumulative-layout-shift', + unit: 'count', + higherIsBetter: false, + tolerance: { absolute: 0.02 } + }, + { + id: 'si', + label: 'Speed Index', + audit: 'speed-index', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 200, relative: 0.15 } + }, + { + id: 'ttfb', + label: 'Server response', + audit: 'server-response-time', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.5 } + }, + { + id: 'js', + label: 'JS execution', + audit: 'bootup-time', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.25 } + }, + { + id: 'bytes', + label: 'Transfer size', + audit: 'total-byte-weight', + unit: 'bytes', + higherIsBetter: false, + tolerance: { absolute: 10_240, relative: 0.05 } + }, + { + id: 'dom', + label: 'DOM elements', + audit: 'dom-size', + unit: 'count', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.1 } + } +]; + +/** The parts of a Lighthouse result (LHR) read here. */ +export type Lhr = { + lighthouseVersion: string; + finalDisplayedUrl: string; + runtimeError?: { code: string; message: string }; + environment: { benchmarkIndex: number }; + categories: Record; + audits: Record; +}; + +export type Stat = { median: number; min: number; max: number }; + +export type PageSummary = { + path: string; + account?: Account; + /** Runs that produced a result. */ + runs: number; + /** Bad results: failed runs, a page that didn't load as itself, and lhci's failed assertions. */ + problems: string[]; + metrics: Record; +}; + +/** One form factor's pages from a run of scripts/lighthouse.ts (lighthouse-full/summary-
.json). */ +export type Summary = { + formFactor: FormFactor; + lighthouseVersion: string; + benchmarkIndex: number; + runs: number; + pages: PageSummary[]; +}; + +/** A line of the history file: each page's medians, keyed ` `. */ +export type HistoryEntry = { + date: string; + commit: string; + lighthouseVersion: string; + benchmarkIndex: number; + results: Record>; +}; + +export const median = (values: number[]) => { + const sorted = values.toSorted((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +const metricValue = (lhr: Lhr, metric: Metric) => + metric.category ? lhr.categories[metric.category]?.score : lhr.audits[metric.audit ?? '']?.numericValue; + +/** One page's runs, with what went wrong in them. */ +export function summarizePage( + page: { path: string; account?: Account }, + lhrs: Lhr[], + assertionFailures: string[] = [] +): PageSummary { + const problems: string[] = []; + const ok = lhrs.filter((lhr) => !lhr.runtimeError); + + for (const lhr of lhrs.filter((lhr) => lhr.runtimeError)) { + problems.push(`a run failed: ${lhr.runtimeError?.code} (${lhr.runtimeError?.message})`); + } + + if (!lhrs.length) problems.push('no run produced a result'); + + // A signed-in page measured signed out lands on /login, and a missing one on the error page. + const landed = new Set( + ok.map(({ finalDisplayedUrl }) => new URL(finalDisplayedUrl)).map((url) => url.pathname + url.search) + ); + + for (const path of landed) if (path !== page.path) problems.push(`ended up on ${path}`); + + if (ok.some((lhr) => lhr.audits['http-status-code']?.score === 0)) problems.push('answered with an error status'); + + problems.push(...assertionFailures); + + const metrics: Record = {}; + + for (const metric of METRICS) { + const values = ok.flatMap((lhr) => metricValue(lhr, metric) ?? []); + + if (values.length) + metrics[metric.id] = { median: median(values), min: Math.min(...values), max: Math.max(...values) }; + } + + return { ...page, runs: ok.length, problems, metrics }; +} + +export type Change = { key: string; metric: Metric; baseline: number; current: number }; + +export type Comparison = { + entry: HistoryEntry; + /** Worse than the median of the last few runs by more than the metric's tolerance. */ + regressions: Change[]; + /** Not a regression against the last few runs, but worse than the runs before them: a slow creep. */ + drifts: Change[]; + improvements: Change[]; + problems: { key: string; problem: string }[]; + /** Runs of one page that spread wider than twice the metric's tolerance, so its median is shaky. */ + unstable: { key: string; metric: Metric; stat: Stat }[]; + /** Pages with no history yet. */ + added: string[]; + /** Why this run's numbers may not line up with its baseline's. */ + notes: string[]; + baselineRuns: number; +}; + +export const RECENT_RUNS = 5; + +export const OLDER_RUNS = 20; + +const limit = (metric: Metric, baseline: number) => + Math.max(metric.tolerance.absolute, (metric.tolerance.relative ?? 0) * Math.abs(baseline)); + +/** How much worse `current` is than `baseline`, positive when worse. */ +const worsening = (metric: Metric, baseline: number, current: number) => + metric.higherIsBetter ? baseline - current : current - baseline; + +const baselineOf = (entries: HistoryEntry[], key: string, metric: Metric) => { + const values = entries.flatMap((entry) => entry.results[key]?.[metric.id] ?? []); + + return values.length ? median(values) : undefined; +}; + +/** This run's summaries against the history file's entries (oldest first), and the entry to record. */ +export function compare( + summaries: Summary[], + history: HistoryEntry[], + run: { date: string; commit: string } +): Comparison { + const keyed = summaries.flatMap((summary) => + summary.pages.map((page) => ({ key: `${summary.formFactor} ${page.path}`, page })) + ); + + const entry: HistoryEntry = { + ...run, + lighthouseVersion: summaries[0]?.lighthouseVersion ?? '', + benchmarkIndex: median(summaries.map((summary) => summary.benchmarkIndex)), + results: Object.fromEntries( + keyed.map(({ key, page }) => [ + key, + Object.fromEntries(Object.entries(page.metrics).map(([id, stat]) => [id, stat.median])) + ]) + ) + }; + + const recent = history.slice(-RECENT_RUNS); + const older = history.slice(-RECENT_RUNS - OLDER_RUNS, -RECENT_RUNS); + + const comparison: Comparison = { + entry, + regressions: [], + drifts: [], + improvements: [], + problems: keyed.flatMap(({ key, page }) => page.problems.map((problem) => ({ key, problem }))), + unstable: [], + added: [], + notes: [], + baselineRuns: recent.length + }; + + for (const { key, page } of keyed) { + if (!recent.some((past) => past.results[key])) comparison.added.push(key); + + for (const metric of METRICS) { + const stat = page.metrics[metric.id]; + + if (!stat) continue; + + if (stat.max - stat.min > 2 * limit(metric, stat.median)) comparison.unstable.push({ key, metric, stat }); + + const baseline = baselineOf(recent, key, metric); + + if (baseline === undefined) continue; + + const change = { key, metric, baseline, current: stat.median }; + const worse = worsening(metric, baseline, stat.median); + + if (worse > limit(metric, baseline)) { + comparison.regressions.push(change); + continue; + } + + if (-worse > limit(metric, baseline)) comparison.improvements.push(change); + + const longBaseline = baselineOf(older, key, metric); + + if (longBaseline !== undefined && worsening(metric, longBaseline, stat.median) > limit(metric, longBaseline)) { + comparison.drifts.push({ ...change, baseline: longBaseline }); + } + } + } + + const last = recent.at(-1); + + if (last && last.lighthouseVersion !== entry.lighthouseVersion) { + comparison.notes.push( + `Lighthouse changed from ${last.lighthouseVersion} to ${entry.lighthouseVersion} since the last run.` + ); + } + + // Lighthouse's estimate of the machine's CPU speed: a slower runner moves every timing. + const benchmark = recent.length ? median(recent.map((past) => past.benchmarkIndex)) : undefined; + + if (benchmark && Math.abs(entry.benchmarkIndex - benchmark) > 0.25 * benchmark) { + comparison.notes.push( + `This machine benchmarked at ${Math.round(entry.benchmarkIndex)}, against ${Math.round(benchmark)} for the baseline's, so timings may have moved with it.` + ); + } + + return comparison; +} + +export const isFailed = (comparison: Comparison) => comparison.regressions.length > 0 || comparison.problems.length > 0; + +export function formatValue(metric: Metric, value: number) { + if (metric.unit === 'score') return String(Math.round(value * 100)); + + if (metric.unit === 'bytes') return `${(value / 1024).toFixed(0)} KiB`; + + if (metric.unit === 'ms') return value >= 1000 ? `${(value / 1000).toFixed(2)} s` : `${Math.round(value)} ms`; + + return Number.isInteger(value) ? String(value) : value.toFixed(3); +} + +const SPARKS = '▁▂▃▄▅▆▇█'; + +/** A row of block characters for `values`, from the lowest (▁) to the highest (█). */ +export function sparkline(values: number[]) { + const low = Math.min(...values); + const range = Math.max(...values) - low; + + return values.map((value) => SPARKS[range ? Math.round(((value - low) / range) * (SPARKS.length - 1)) : 0]).join(''); +} + +const changeRows = (changes: Change[]) => [ + '| Page | Metric | Baseline | Now |', + '| --- | --- | ---: | ---: |', + ...changes.map( + ({ key, metric, baseline, current }) => + `| \`${key}\` | ${metric.label} | ${formatValue(metric, baseline)} | ${formatValue(metric, current)} |` + ) +]; + +const TABLE_METRICS = ['performance', 'accessibility', 'best-practices', 'seo', 'lcp', 'tbt', 'cls', 'ttfb', 'bytes']; + +/** The comparison as Markdown, for the run's summary or the terminal. `trend` is how many past runs the sparklines show. */ +export function renderMarkdown(comparison: Comparison, history: HistoryEntry[], trend = 12) { + const { entry, regressions, drifts, improvements, problems, unstable, added, notes, baselineRuns } = comparison; + + const lines = [ + '## Lighthouse history', + '', + `${entry.commit.replace(/^[0-9a-f]{12}[0-9a-f]*/, (sha) => sha.slice(0, 12))}, Lighthouse ${entry.lighthouseVersion}, against the median of the last ${baselineRuns} recorded run${baselineRuns === 1 ? '' : 's'}.`, + ...notes.map((note) => `\n> ${note}`), + '' + ]; + + if (!baselineRuns) lines.push('No history yet: this run is the first baseline.', ''); + + if (regressions.length) lines.push(`### Regressions (${regressions.length})`, '', ...changeRows(regressions), ''); + + if (problems.length) + lines.push( + `### Bad results (${problems.length})`, + '', + ...problems.map(({ key, problem }) => `- \`${key}\`: ${problem}`), + '' + ); + + if (drifts.length) { + lines.push( + `### Slow creep (${drifts.length})`, + '', + `Worse than the ${OLDER_RUNS} runs before the baseline's.`, + '', + ...changeRows(drifts), + '' + ); + } + + if (unstable.length) { + lines.push( + `### Unstable (${unstable.length})`, + '', + ...unstable.map( + ({ key, metric, stat }) => + `- \`${key}\`: ${metric.label} ranged ${formatValue(metric, stat.min)} to ${formatValue(metric, stat.max)}` + ), + '' + ); + } + + if (improvements.length) lines.push(`### Improvements (${improvements.length})`, '', ...changeRows(improvements), ''); + + if (baselineRuns && added.length) + lines.push(`New pages, with no history yet: ${added.map((key) => `\`${key}\``).join(', ')}`, ''); + + const metrics = METRICS.filter((metric) => TABLE_METRICS.includes(metric.id)); + + const past = history.slice(-trend); + + const trendOf = (key: string, id: string) => + sparkline([...past.flatMap((old) => old.results[key]?.[id] ?? []), entry.results[key][id]]); + + lines.push( + '
Every page', + '', + `| Page | ${metrics.map((metric) => metric.label).join(' | ')} | Performance trend | LCP trend |`, + `| --- | ${metrics.map(() => '---:').join(' | ')} | --- | --- |`, + ...Object.entries(entry.results).map( + ([key, values]) => + `| \`${key}\` | ${metrics.map((metric) => (metric.id in values ? formatValue(metric, values[metric.id]) : '')).join(' | ')} | ${ + 'performance' in values ? trendOf(key, 'performance') : '' + } | ${'lcp' in values ? trendOf(key, 'lcp') : ''} |` + ), + '', + '
', + '' + ); + + return lines.join('\n'); +} diff --git a/apps/web/scripts/lighthouse-history.ts b/apps/web/scripts/lighthouse-history.ts new file mode 100644 index 000000000..4f965e66e --- /dev/null +++ b/apps/web/scripts/lighthouse-history.ts @@ -0,0 +1,62 @@ +/** + * Holds a run of scripts/lighthouse.ts (lighthouse-full/summary-*.json) against the pages' history: a + * JSON Lines file, one line per recorded run. Prints what regressed, crept, broke or improved, and + * exits 1 on a regression or a bad result. Plain Node (no dependencies), so CI runs it uninstalled. + * + * node scripts/lighthouse-history.ts # compare only, say while bisecting + * node scripts/lighthouse-history.ts --record # compare, then add this run to the history + * + * --results the summaries' directory (lighthouse-full) + * --history the history (lighthouse-history.jsonl); CI keeps it as a workflow artifact + * --markdown append the report to a file (CI's $GITHUB_STEP_SUMMARY) instead of printing it + * --regressions-only exit 1 on a regression alone, so bad results every commit has don't fail a bisect + */ +import { execFileSync } from 'node:child_process'; +import { appendFileSync, existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { compare, type HistoryEntry, isFailed, renderMarkdown, type Summary } from './lib/lighthouse.ts'; + +const { values: args } = parseArgs({ + options: { + results: { type: 'string', default: 'lighthouse-full' }, + history: { type: 'string', default: 'lighthouse-history.jsonl' }, + markdown: { type: 'string' }, + record: { type: 'boolean', default: false }, + 'regressions-only': { type: 'boolean', default: false } + } +}); + +const summaries: Summary[] = readdirSync(args.results) + .filter((name) => /^summary-.+\.json$/.test(name)) + .map((name) => JSON.parse(readFileSync(join(args.results, name), 'utf8'))); + +if (!summaries.length) { + console.error(`No summaries in ${args.results}: run scripts/lighthouse.ts first.`); + process.exit(1); +} + +const history: HistoryEntry[] = existsSync(args.history) + ? readFileSync(args.history, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) + : []; + +// A run of uncommitted changes says so, so it isn't mistaken for the commit's own numbers. +const git = (...gitArgs: string[]) => execFileSync('git', gitArgs, { encoding: 'utf8' }).trim(); + +const commit = process.env.GITHUB_SHA ?? `${git('rev-parse', 'HEAD')}${git('status', '--porcelain') ? '-dirty' : ''}`; + +const comparison = compare(summaries, history, { date: new Date().toISOString(), commit }); + +const report = renderMarkdown(comparison, history); + +if (args.markdown) appendFileSync(args.markdown, `${report}\n`); +else console.log(report); + +if (args.record) appendFileSync(args.history, `${JSON.stringify(comparison.entry)}\n`); + +const isFailing = args['regressions-only'] ? comparison.regressions.length > 0 : isFailed(comparison); + +if (isFailing) process.exit(1); diff --git a/apps/web/scripts/lighthouse.ts b/apps/web/scripts/lighthouse.ts new file mode 100644 index 000000000..ac2d01abd --- /dev/null +++ b/apps/web/scripts/lighthouse.ts @@ -0,0 +1,256 @@ +/** + * Lighthouse CI on every page in lighthouse/pages.json, on a phone and a desktop, signed in where a page + * needs it, then each page's runs summed up for scripts/lighthouse-history.ts (`pnpm test:lighthouse:full` + * builds first, then records the run in the history). Uses lighthouserc.cjs's network, Chromium and + * budgets, and starts the network once for every page. + * + * tsx scripts/lighthouse.ts # every page, both form factors + * tsx scripts/lighthouse.ts --form-factor desktop # one form factor (CI runs one per job) + * tsx scripts/lighthouse.ts --page /charts --runs 5 # pages whose path contains /charts, five runs each + * + * Writes lighthouse-full/summary-.json, and each run's report under lighthouse-full//. + */ +import { spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { chromium } from 'playwright'; +import { signIn } from '../tests/sign-in.ts'; +import { type Account, type FormFactor, type Lhr, median, summarizePage, type Summary } from './lib/lighthouse.ts'; + +type Page = { path: string; account?: Account; pr?: boolean }; + +const require = createRequire(import.meta.url); + +const { ci } = require('../lighthouserc.cjs'); + +const { pages }: { pages: Page[] } = require('../lighthouse/pages.json'); + +// The seeded accounts of scripts/dev-network.ts. +const ACCOUNTS: Record = { + dj: { handle: 'neon.test', password: 'neon-pass' }, + admin: { handle: 'ada.test', password: 'ada-pass' } +}; + +const OUT = 'lighthouse-full'; + +const LHCI_DIR = '.lighthouseci'; + +const { values: args } = parseArgs({ + options: { + 'form-factor': { type: 'string', multiple: true, default: ['mobile', 'desktop'] }, + page: { type: 'string', multiple: true }, + runs: { type: 'string', default: String(ci.collect.numberOfRuns) } + } +}); + +const formFactors = args['form-factor'].filter( + (value): value is FormFactor => value === 'mobile' || value === 'desktop' +); + +const selected = pages.filter(({ path }) => !args.page || args.page.some((filter) => path.includes(filter))); + +const origin = new URL(ci.collect.url[0]).origin; + +if (!selected.length || formFactors.length !== args['form-factor'].length) { + console.error( + 'Nothing to measure: check --page (a part of a path in lighthouse/pages.json) and --form-factor (mobile or desktop).' + ); + process.exit(1); +} + +const log = (message: string) => console.log(`\x1b[35m▸\x1b[0m ${message}`); + +/** The network and the production build, as lhci's startServerCommand would start them; resolves once it serves. */ +async function startServer() { + const server = spawn(ci.collect.startServerCommand, { + shell: true, + detached: true, + stdio: ['ignore', 'pipe', 'inherit'] + }); + + const exited = once(server, 'exit').then(([code]) => { + throw new Error(`The network exited (${code}) before the app was serving.`); + }); + + const ready = new Promise((resolve) => { + let output = ''; + server.stdout.on('data', (chunk: Buffer) => { + process.stdout.write(chunk); + output += chunk.toString(); + + if (output.includes(ci.collect.startServerReadyPattern)) resolve(); + }); + }); + + const timeout = new Promise((_, reject) => + setTimeout( + () => reject(new Error('The network took too long to start.')), + ci.collect.startServerReadyTimeout + ).unref() + ); + + await Promise.race([ready, exited, timeout]); + + return async () => { + // The whole group, so the shell, tsx and the network all get it; the network then stops the + // app and removes HappyView's container (scripts/dev-network.ts). + if (server.exitCode !== null || !server.pid) return; + const exit = once(server, 'exit'); + process.kill(-server.pid, 'SIGTERM'); + await exit; + }; +} + +/** A Cookie header for each account the selected pages are measured as, from a real sign-in. */ +async function sessionCookies() { + const accounts = [...new Set(selected.map((page) => page.account).filter((account) => account !== undefined))]; + const browser = await chromium.launch(); + const cookies: Partial> = {}; + + try { + for (const account of accounts) { + // The production build's origin is https without this header (scripts/dev-network.ts). + const context = await browser.newContext({ baseURL: origin, extraHTTPHeaders: { 'X-Forwarded-Proto': 'http' } }); + await signIn(await context.newPage(), ACCOUNTS[account].handle, ACCOUNTS[account].password); + cookies[account] = (await context.cookies(origin)).map(({ name, value }) => `${name}=${value}`).join('; '); + await context.close(); + } + } finally { + await browser.close(); + } + + return cookies; +} + +const lhci = (command: string, config: string, ...flags: string[]) => + spawnSync('lhci', [command, `--config=${config}`, ...flags], { stdio: 'inherit' }); + +/** Each failed lhci assertion of the last `lhci assert`, by page path. */ +function assertionFailures() { + const failures = new Map(); + + let results: { + url: string; + level: string; + auditId: string; + auditProperty?: string; + actual: number; + operator: string; + expected: number; + }[]; + + try { + results = JSON.parse(readFileSync(join(LHCI_DIR, 'assertion-results.json'), 'utf8')); + } catch { + return failures; + } + + for (const result of results.filter(({ level }) => level === 'error')) { + const url = new URL(result.url); + const audit = result.auditProperty ? `${result.auditId}.${result.auditProperty}` : result.auditId; + const failure = `failed ${audit}: ${Math.round(result.actual * 100) / 100}, expected ${result.operator} ${result.expected}`; + failures.set(url.pathname + url.search, [...(failures.get(url.pathname + url.search) ?? []), failure]); + } + + return failures; +} + +/** The results of the last `lhci collect`, by the page path each run asked for. */ +function collectedLhrs() { + const lhrs = new Map(); + + for (const name of readdirSync(LHCI_DIR).filter((file) => file.startsWith('lhr-') && file.endsWith('.json'))) { + const lhr: Lhr & { requestedUrl: string } = JSON.parse(readFileSync(join(LHCI_DIR, name), 'utf8')); + const url = new URL(lhr.requestedUrl); + lhrs.set(url.pathname + url.search, [...(lhrs.get(url.pathname + url.search) ?? []), lhr]); + } + + return lhrs; +} + +// Whole, so a summary from an earlier run of another form factor isn't compared or recorded again. +rmSync(OUT, { recursive: true, force: true }); + +mkdirSync(join(OUT, 'config'), { recursive: true }); + +const stopServer = await startServer(); + +try { + const cookies = await sessionCookies(); + + for (const formFactor of formFactors) { + const summary: Summary = { + formFactor, + lighthouseVersion: '', + benchmarkIndex: 0, + runs: Number(args.runs), + pages: [] + }; + + const benchmarks: number[] = []; + + for (const account of [undefined, 'dj', 'admin'] satisfies (Account | undefined)[]) { + const group = selected.filter((page) => page.account === account); + + if (!group.length) continue; + + const name = `${formFactor}-${account ?? 'signed-out'}`; + const config = join(OUT, 'config', `${name}.json`); + const cookie = account && cookies[account]; + log( + `${group.length} page${group.length === 1 ? '' : 's'}, ${formFactor}, ${account ? `signed in as ${ACCOUNTS[account].handle}` : 'signed out'}` + ); + + writeFileSync( + config, + JSON.stringify({ + ci: { + collect: { + numberOfRuns: Number(args.runs), + chromePath: ci.collect.chromePath, + settings: { + ...(formFactor === 'desktop' && { preset: 'desktop' }), + ...(cookie && { extraHeaders: { Cookie: cookie } }) + } + }, + // Search engines never see a signed-in page, so its SEO score is tracked but not held to a budget. + assert: { + assertions: Object.fromEntries( + Object.entries(ci.assert.assertions).filter(([id]) => !account || id !== 'categories:seo') + ) + }, + upload: { target: 'filesystem', outputDir: join(OUT, formFactor, account ?? 'signed-out') } + } + }) + ); + + // One page per collect, adding to the last: lhci gives up on the pages after one that fails to + // load, and here that page just has no runs in its summary. + for (const [index, page] of group.entries()) { + lhci('collect', config, `--url=${origin}${page.path}`, ...(index ? ['--additive'] : [])); + } + + lhci('assert', config); + lhci('upload', config); + + const lhrs = collectedLhrs(); + const failures = assertionFailures(); + + for (const page of group) { + const runs = lhrs.get(page.path) ?? []; + summary.lighthouseVersion ||= runs[0]?.lighthouseVersion ?? ''; + benchmarks.push(...runs.map((lhr) => lhr.environment.benchmarkIndex)); + summary.pages.push(summarizePage({ path: page.path, account: page.account }, runs, failures.get(page.path))); + } + } + + summary.benchmarkIndex = benchmarks.length ? median(benchmarks) : 0; + writeFileSync(join(OUT, `summary-${formFactor}.json`), `${JSON.stringify(summary, null, '\t')}\n`); + log(`${OUT}/summary-${formFactor}.json`); + } +} finally { + await stopServer(); +} diff --git a/apps/web/tests/e2e/app.e2e.ts b/apps/web/tests/e2e/app.e2e.ts index 709444d09..70c6249e8 100644 --- a/apps/web/tests/e2e/app.e2e.ts +++ b/apps/web/tests/e2e/app.e2e.ts @@ -1,6 +1,7 @@ import { expect, type Page, test } from '@playwright/test'; import headline from '../../src/lib/headline.json' with { type: 'json' }; import { z } from 'zod'; +import { signIn } from '../sign-in.ts'; import { expectAccessible } from './axe.ts'; /** @@ -29,20 +30,6 @@ function keepPdsCookies(page: Page) { test.beforeEach(({ page }) => keepPdsCookies(page)); -/** Signs in through the local PDS's OAuth screens with a seeded test account. */ -async function signIn(page: Page, handle: string, password: string, next = '/dashboard') { - await page.goto(`/login?next=${encodeURIComponent(next)}`); - await page.waitForLoadState('networkidle'); - await page.getByPlaceholder('alice.bsky.social').fill(handle); - await page.getByRole('button', { name: 'Continue' }).click(); - await page.locator('input[type="password"]').fill(password); - await page.getByRole('button', { name: 'Sign in' }).click(); - await page.getByRole('button', { name: 'Authorize' }).click(); - const target = new URL(next, 'http://127.0.0.1:5199'); - await page.waitForURL((url) => url.port === '5199' && url.pathname === target.pathname); - await page.waitForLoadState('networkidle'); -} - /** * Signs out from the dashboard and waits for the redirect home: /logout records the audit entry and * revokes the OAuth session first, and a sign-in started before it finishes would find the old session. diff --git a/apps/web/tests/sign-in.ts b/apps/web/tests/sign-in.ts new file mode 100644 index 000000000..622ad1b2c --- /dev/null +++ b/apps/web/tests/sign-in.ts @@ -0,0 +1,19 @@ +import type { Page } from 'playwright'; + +/** + * Signs in through the local PDS's OAuth screens with a seeded test account (scripts/dev-network.ts), + * for the end-to-end tests and Lighthouse's signed-in pages. The page's context needs a baseURL. + */ +export async function signIn(page: Page, handle: string, password: string, next = '/dashboard') { + await page.goto(`/login?next=${encodeURIComponent(next)}`); + const app = new URL(page.url()); + await page.waitForLoadState('networkidle'); + await page.getByPlaceholder('alice.bsky.social').fill(handle); + await page.getByRole('button', { name: 'Continue' }).click(); + await page.locator('input[type="password"]').fill(password); + await page.getByRole('button', { name: 'Sign in' }).click(); + await page.getByRole('button', { name: 'Authorize' }).click(); + const target = new URL(next, app); + await page.waitForURL((url) => url.origin === app.origin && url.pathname === target.pathname); + await page.waitForLoadState('networkidle'); +} From 48247bc5f800545e403e071248d710e4ecb4b066 Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:45:54 -0400 Subject: [PATCH 3/4] ci(web): run the full Lighthouse suite weekly Web scheduled runs scripts/lighthouse.ts in one job per form factor, then compares their summaries with main's history in a job that installs nothing. The history is the lighthouse-history artifact, which each run on main downloads, adds to and uploads again. A regression or bad result fails the run, which opens the scheduled-failure issue. --- .github/NEW_PROJECT.md | 2 +- .github/workflows/web-scheduled.yml | 114 +++++++++++++++++++++++++++- README.md | 2 +- apps/web/README.md | 2 + 4 files changed, 114 insertions(+), 6 deletions(-) diff --git a/.github/NEW_PROJECT.md b/.github/NEW_PROJECT.md index 6ce17a64b..fb3e98453 100644 --- a/.github/NEW_PROJECT.md +++ b/.github/NEW_PROJECT.md @@ -53,7 +53,7 @@ Logging: the website logs with [pino](https://getpino.io) (`apps/web/src/lib/ser | Project | Tests | Coverage and other bars | | ------------ | ----------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| website | Vitest projects in `apps/web/vite.config.ts` (server, client in browser mode, integration, build, Storybook); Playwright e2e | Vitest thresholds in `vite.config.ts`; Lighthouse budgets in `lighthouserc.cjs`; Storybook stories fail on a11y violations | +| website | Vitest projects in `apps/web/vite.config.ts` (server, client in browser mode, integration, build, Storybook); Playwright e2e | Vitest thresholds in `vite.config.ts`; Lighthouse budgets in `lighthouserc.cjs`, and every page in `lighthouse/pages.json` weekly against its history; Storybook stories fail on a11y violations | | localtoolbox | Vitest with jsdom; its README is its spec, checked by `test/docs.test.ts` | per-file thresholds in `vitest.config.ts`; Stryker mutation score of at least 91 (`stryker.config.mjs`) | | Rust | cargo-nextest (`.config/nextest.toml`); unit tests in each module's `tests.rs`; Hegel property tests; fuzz targets in `fuzz/` | `cargo coverage` (`.cargo/config.toml`) fails under its thresholds; `tools/hegel-coverage.sh` holds the Hegel tests alone to them | diff --git a/.github/workflows/web-scheduled.yml b/.github/workflows/web-scheduled.yml index 01751b539..1f622b434 100644 --- a/.github/workflows/web-scheduled.yml +++ b/.github/workflows/web-scheduled.yml @@ -1,8 +1,10 @@ name: Web scheduled # The end-to-end tests in every Playwright project (playwright.config.ts): Chromium, Firefox and -# WebKit on the desktop, and a Pixel 7 and an iPhone 14. Weekly and on demand (Actions > Web -# scheduled > Run workflow, on any branch), so pull requests only wait for web-ci.yml's Chromium run. +# WebKit on the desktop, and a Pixel 7 and an iPhone 14. Then Lighthouse on every page in +# lighthouse/pages.json, on a phone and a desktop, held against each page's history. Weekly and on +# demand (Actions > Web scheduled > Run workflow, on any branch), so pull requests only wait for +# web-ci.yml's Chromium run and its Lighthouse run on three pages. on: schedule: - cron: "29 4 * * 1" @@ -47,10 +49,113 @@ jobs: name: playwright-report path: apps/web/playwright-report/ + # Every page in lighthouse/pages.json, signed in where it needs to be (scripts/lighthouse.ts), one + # job per form factor. Each job's summary is its lighthouse-summary- artifact, and every + # run's report its lighthouse- artifact. + lighthouse: + name: Lighthouse (${{ matrix.form-factor }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + form-factor: [mobile, desktop] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install --with-deps chromium + # As in web-ci.yml's Lighthouse job: lets Chromium's sandbox create its user namespaces. + - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Pull the HappyView image (scripts/lib/happyview.ts) + run: docker pull ghcr.io/gamesgamesgamesgamesgames/happyview:2.16.0 + - run: pnpm build + - run: pnpm exec tsx scripts/lighthouse.ts --form-factor "$FORM_FACTOR" + env: + FORM_FACTOR: ${{ matrix.form-factor }} + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: lighthouse-summary-${{ matrix.form-factor }} + path: apps/web/lighthouse-full/summary-*.json + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: lighthouse-${{ matrix.form-factor }} + path: apps/web/lighthouse-full/${{ matrix.form-factor }}/ + + # Each page against the median of its last recorded runs on main (scripts/lighthouse-history.ts), + # in the run's summary; a regression or a bad result fails the job. The history is the + # lighthouse-history artifact: each run on main downloads the newest one, adds itself and uploads + # it again, so it lasts as long as runs on main come less than 90 days apart. Plain Node with + # nothing installed, so no package from npm runs while the job can read the repository's artifacts. + lighthouse-history: + name: Lighthouse history + needs: lighthouse + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + # The history artifact from an earlier run. + actions: read + defaults: + run: + working-directory: apps/web + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: | + .node-version + apps/web/scripts/lighthouse-history.ts + apps/web/scripts/lib/lighthouse.ts + sparse-checkout-cone-mode: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .node-version + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: lighthouse-summary-* + path: apps/web/lighthouse-full + merge-multiple: true + - name: Download the history so far + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + run=$(gh api "repos/$GH_REPO/actions/artifacts?name=lighthouse-history&per_page=50" \ + --jq '[.artifacts[] | select((.expired | not) and .workflow_run.head_branch == "main")][0].workflow_run.id // empty') + if [ -n "$run" ]; then + gh run download "$run" --name lighthouse-history --dir . + else + echo "::notice::No Lighthouse history yet: this run starts it." + fi + # Runs on other branches compare without recording, so main's history stays main's. + - name: Compare with the history + env: + IS_MAIN: ${{ github.ref == 'refs/heads/main' }} + run: | + args=(--markdown "$GITHUB_STEP_SUMMARY") + if [ "$IS_MAIN" = true ]; then args+=(--record); fi + node scripts/lighthouse-history.ts "${args[@]}" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && hashFiles('apps/web/lighthouse-history.jsonl') != '' }} + with: + name: lighthouse-history + path: apps/web/lighthouse-history.jsonl + retention-days: 90 + # Weekly runs only: a failed job opens an issue, or comments on the open one, and the next run # that passes closes it (scheduled-issue.yml). report: - needs: [e2e] + needs: [e2e, lighthouse, lighthouse-history] if: ${{ !cancelled() && github.event_name == 'schedule' }} permissions: actions: read @@ -60,4 +165,5 @@ jobs: label: web details: >- The Playwright report, with each failed test's trace and screenshots, is the run's - playwright-report artifact. + playwright-report artifact. Lighthouse's regressions and bad results are in the run's + summary, and each page's reports in its lighthouse-mobile and lighthouse-desktop artifacts. diff --git a/README.md b/README.md index 3b1ba28ef..c2c60dacb 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ cargo build # locallibrary cargo test --workspace --locked ``` -The TypeScript apps share one lint config (`.oxlintrc.json`, which each app's own extends) and one format config (`.oxfmtrc.json`). CI runs each tool's checks only when its files (or the shared ones) change, and comments on a pull request with each tool's coverage and, for the website and the extension, its size against `main`. The slow suites (mutation testing, fuzzing, the website in every browser, the CLI's timings, a release build) run weekly, in the `*-scheduled.yml` workflows; a weekly run that fails opens an issue, which the next one that passes closes. +The TypeScript apps share one lint config (`.oxlintrc.json`, which each app's own extends) and one format config (`.oxfmtrc.json`). CI runs each tool's checks only when its files (or the shared ones) change, and comments on a pull request with each tool's coverage and, for the website and the extension, its size against `main`. The slow suites (mutation testing, fuzzing, the website in every browser, Lighthouse on every page of the website against its history, the CLI's timings, a release build) run weekly, in the `*-scheduled.yml` workflows; a weekly run that fails opens an issue, which the next one that passes closes. ## Contributing diff --git a/apps/web/README.md b/apps/web/README.md index 6a7c614fb..80918fe45 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -246,6 +246,8 @@ Pull requests run the e2e tests in Chromium only (`pnpm test:e2e --project chrom [`lighthouse/pages.json`](lighthouse/pages.json) lists the pages Lighthouse measures, each signed out or signed in as `neon.test` (`dj`) or `ada.test` (`admin`). Pull requests that change what the pages are built from measure the ones marked `pr` against the budgets in `lighthouserc.cjs`. `pnpm test:lighthouse:full` measures all of them on a phone and a desktop, three runs each, and keeps each page's medians in `lighthouse-history.jsonl`. Each run is held against the median of the last five recorded runs, and against the twenty before those to catch a slow creep. A regression past a metric's tolerance (in [`scripts/lib/lighthouse.ts`](scripts/lib/lighthouse.ts)) fails the run, as does a bad result: a failed run, a page that redirected or answered an error, or a failed budget (signed-in pages have no SEO budget, since search engines never see them). Runs that spread wide, and a new Lighthouse or a faster or slower machine, are noted too. +The Web scheduled workflow does the same weekly, one job per form factor, and keeps main's history as its `lighthouse-history` artifact. Its report is in the run's summary, with every run's report in the `lighthouse-mobile` and `lighthouse-desktop` artifacts. + To find the commit behind a regression, record a baseline of only the pages that moved on the last good commit (each history line names its commit), then let `git bisect` measure each commit against it, failing only on a regression (commits from before this suite landed lack its scripts): ```bash From ac1c97c7418415498a2fe806ffccc353ef6ddb07 Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:15:15 -0500 Subject: [PATCH 4/4] fix(ci): read Lighthouse history only from this workflow's own runs on main The history job took the newest lighthouse-history artifact from any run whose head branch was named main, which a pull request from a fork with a branch of that name also has, and unpacked it over apps/web, where it could replace the script the job runs next. It now only looks at this workflow's scheduled and hand-started runs in this repository, unpacks each into an empty directory, and copies out the history file alone. --- .github/workflows/web-scheduled.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/web-scheduled.yml b/.github/workflows/web-scheduled.yml index 1f622b434..f9bf6e697 100644 --- a/.github/workflows/web-scheduled.yml +++ b/.github/workflows/web-scheduled.yml @@ -125,15 +125,24 @@ jobs: pattern: lighthouse-summary-* path: apps/web/lighthouse-full merge-multiple: true + # Only from this workflow's own scheduled or hand-started runs on main in this repository, + # which a pull request (from a fork with a branch named main, say) can't start. Each is + # unpacked into an empty directory and only the history file is copied out, so an artifact + # can't put anything else in the checkout. - name: Download the history so far env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} + UNPACK: ${{ runner.temp }}/lighthouse-history run: | - run=$(gh api "repos/$GH_REPO/actions/artifacts?name=lighthouse-history&per_page=50" \ - --jq '[.artifacts[] | select((.expired | not) and .workflow_run.head_branch == "main")][0].workflow_run.id // empty') - if [ -n "$run" ]; then - gh run download "$run" --name lighthouse-history --dir . + runs=$(gh api "repos/$GH_REPO/actions/workflows/web-scheduled.yml/runs?branch=main&per_page=50" \ + --jq '.workflow_runs[] | select((.event == "schedule" or .event == "workflow_dispatch") and .head_repository.id == .repository.id) | .id') + for run in $runs; do + rm -rf "$UNPACK" + if gh run download "$run" --name lighthouse-history --dir "$UNPACK" 2>/dev/null; then break; fi + done + if [ -f "$UNPACK/lighthouse-history.jsonl" ] && [ ! -L "$UNPACK/lighthouse-history.jsonl" ]; then + cp "$UNPACK/lighthouse-history.jsonl" lighthouse-history.jsonl else echo "::notice::No Lighthouse history yet: this run starts it." fi