diff --git a/.github/NEW_PROJECT.md b/.github/NEW_PROJECT.md index aa3562e43..2b152c695 100644 --- a/.github/NEW_PROJECT.md +++ b/.github/NEW_PROJECT.md @@ -53,7 +53,7 @@ Logging: the website logs with [pino](https://getpino.io) (`apps/web/src/lib/ser | Project | Tests | Coverage and other bars | | ------------ | ----------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| website | Vitest projects in `apps/web/vite.config.ts` (server, client in browser mode, integration, build, Storybook); Playwright e2e | Vitest thresholds in `vite.config.ts`; Lighthouse budgets in `lighthouserc.cjs`; Storybook stories fail on a11y violations | +| website | Vitest projects in `apps/web/vite.config.ts` (server, client in browser mode, integration, build, Storybook); Playwright e2e | Vitest thresholds in `vite.config.ts`; Lighthouse budgets in `lighthouserc.cjs`, and every page in `lighthouse/pages.json` weekly against its history; Storybook stories fail on a11y violations | | localtoolbox | Vitest with jsdom; its README is its spec, checked by `test/docs.test.ts` | per-file thresholds in `vitest.config.ts`; Stryker mutation score of at least 91 (`stryker.config.mjs`) | | Rust | cargo-nextest (`.config/nextest.toml`); unit tests in each module's `tests.rs`; Hegel property tests; fuzz targets in `fuzz/` | `cargo coverage` (`.cargo/config.toml`) fails under its thresholds; `tools/hegel-coverage.sh` holds the Hegel tests alone to them | diff --git a/.github/labeler.yml b/.github/labeler.yml index b6fa3d055..a5c7e2123 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -272,6 +272,7 @@ tests: - "**/{vitest,vitest.stryker,playwright}.config.ts" - "**/stryker*.{mjs,ts}" - "apps/web/src/lib/server/testing/**" + - "apps/web/{lighthouse/**,scripts/lighthouse*.ts,scripts/lib/lighthouse.ts}" - "apps/locallibrary/bombadil/**" - "fuzz/**" - ".config/nextest.toml" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a60bd3370..cfaea9021 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -256,8 +256,8 @@ jobs: # What the pages Lighthouse measures are built and served from: the app's code, static # files and Lua (HappyView answers the directory's queries), the dev network and seed # data it measures them on, its build and Lighthouse config, the dependencies, and the - # README and guides of the one tool page lighthouserc.cjs visits (add a tool here when - # its page goes in there). Left out: the app's tests and the helpers and fixture only + # README and guides of the one tool page lighthouse/pages.json marks `pr` (add a tool here + # when its page is marked). Left out: the app's tests and the helpers and fixture only # they import (src/lib/server/testing, the mix fixture), stories and Storybook docs, and # everything outside these paths (tests/, .storybook/, the lint and format configs, the # website's README, the Dockerfile). @@ -270,6 +270,7 @@ jobs: - 'apps/web/vite.config.ts' - 'apps/web/tsconfig.json' - 'apps/web/lighthouserc.cjs' + - 'apps/web/lighthouse/**' - 'apps/locallibrary/README.md' - 'apps/locallibrary/docs/*.md' - 'packages/tokens/**' diff --git a/.github/workflows/web-scheduled.yml b/.github/workflows/web-scheduled.yml index 5c98a2e10..bc5fdf401 100644 --- a/.github/workflows/web-scheduled.yml +++ b/.github/workflows/web-scheduled.yml @@ -1,9 +1,11 @@ name: Web scheduled # The end-to-end tests in every Playwright project (playwright.config.ts): Chromium, Firefox and -# WebKit on the desktop, and a Pixel 7 and an iPhone 14. Weekly and on demand (Actions > Web -# scheduled > Run workflow, on any branch), so pull requests only wait for web-ci.yml's Chromium run. -# A week that changed none of what they test skips them (see the changes job). +# WebKit on the desktop, and a Pixel 7 and an iPhone 14. Then Lighthouse on every page in +# lighthouse/pages.json, on a phone and a desktop, held against each page's history. Weekly and on +# demand (Actions > Web scheduled > Run workflow, on any branch), so pull requests only wait for +# web-ci.yml's Chromium run and its Lighthouse run on three pages. A week that changed none of what +# they test skips them (see the changes job). on: schedule: - cron: "29 4 * * 1" @@ -23,7 +25,7 @@ defaults: # Pinned the same way as web-ci.yml. jobs: - # Whether anything the end-to-end tests read changed since the last weekly or hand-started run on + # Whether anything the end-to-end tests and Lighthouse read changed since the last weekly or hand-started run on # main that passed. A run that failed doesn't count, so they run again each week until one # passes. Started by hand, everything runs, as it does when none of the workflow's last 100 runs # passed and there's nothing to compare with. The paths are what the e2e tests run: the app, the @@ -92,10 +94,127 @@ jobs: name: playwright-report path: apps/web/playwright-report/ + # Every page in lighthouse/pages.json, signed in where it needs to be (scripts/lighthouse.ts), one + # job per form factor. Each job's summary is its lighthouse-summary-
artifact, and every + # run's report its lighthouse- artifact. + lighthouse: + name: Lighthouse (${{ matrix.form-factor }}) + needs: changes + if: needs.changes.outputs.run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + form-factor: [mobile, desktop] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .node-version + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm exec playwright install --with-deps chromium + # As in web-ci.yml's Lighthouse job: lets Chromium's sandbox create its user namespaces. + - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - name: Pull the HappyView image (scripts/lib/happyview.ts) + run: docker pull ghcr.io/gamesgamesgamesgamesgames/happyview:2.16.0 + - run: pnpm build + - run: pnpm exec tsx scripts/lighthouse.ts --form-factor "$FORM_FACTOR" + env: + FORM_FACTOR: ${{ matrix.form-factor }} + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: lighthouse-summary-${{ matrix.form-factor }} + path: apps/web/lighthouse-full/summary-*.json + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() }} + with: + name: lighthouse-${{ matrix.form-factor }} + path: apps/web/lighthouse-full/${{ matrix.form-factor }}/ + + # Each page against the median of its last recorded runs on main (scripts/lighthouse-history.ts), + # in the run's summary; a regression or a bad result fails the job. The history is the + # lighthouse-history artifact: each run on main downloads the newest one, adds itself and uploads + # it again, so it lasts as long as runs on main come less than 90 days apart. Plain Node with + # nothing installed, so no package from npm runs while the job can read the repository's artifacts. + # A week Lighthouse skips, it only uploads the history again, so the artifact doesn't expire. + lighthouse-history: + name: Lighthouse history + needs: [changes, lighthouse] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + # The history artifact from an earlier run. + actions: read + defaults: + run: + working-directory: apps/web + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: | + .node-version + apps/web/scripts/lighthouse-history.ts + apps/web/scripts/lib/lighthouse.ts + sparse-checkout-cone-mode: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .node-version + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: needs.lighthouse.result != 'skipped' + with: + pattern: lighthouse-summary-* + path: apps/web/lighthouse-full + merge-multiple: true + # Only from this workflow's own scheduled or hand-started runs on main in this repository, + # which a pull request (from a fork with a branch named main, say) can't start. Each is + # unpacked into an empty directory and only the history file is copied out, so an artifact + # can't put anything else in the checkout. + - name: Download the history so far + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + UNPACK: ${{ runner.temp }}/lighthouse-history + run: | + runs=$(gh api "repos/$GH_REPO/actions/workflows/web-scheduled.yml/runs?branch=main&per_page=50" \ + --jq '.workflow_runs[] | select((.event == "schedule" or .event == "workflow_dispatch") and .head_repository.id == .repository.id) | .id') + for run in $runs; do + rm -rf "$UNPACK" + if gh run download "$run" --name lighthouse-history --dir "$UNPACK" 2>/dev/null; then break; fi + done + if [ -f "$UNPACK/lighthouse-history.jsonl" ] && [ ! -L "$UNPACK/lighthouse-history.jsonl" ]; then + cp "$UNPACK/lighthouse-history.jsonl" lighthouse-history.jsonl + else + echo "::notice::No Lighthouse history yet: this run starts it." + fi + # Runs on other branches compare without recording, so main's history stays main's. + - name: Compare with the history + if: needs.lighthouse.result != 'skipped' + env: + IS_MAIN: ${{ github.ref == 'refs/heads/main' }} + run: | + args=(--markdown "$GITHUB_STEP_SUMMARY") + if [ "$IS_MAIN" = true ]; then args+=(--record); fi + node scripts/lighthouse-history.ts "${args[@]}" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && hashFiles('apps/web/lighthouse-history.jsonl') != '' }} + with: + name: lighthouse-history + path: apps/web/lighthouse-history.jsonl + retention-days: 90 + # Weekly runs only: a failed job opens an issue, or comments on the open one, and the next run # that passes closes it (scheduled-issue.yml). A skipped run counts as passed. report: - needs: [changes, e2e] + needs: [changes, e2e, lighthouse, lighthouse-history] # A week the suites skip follows one that passed, so there's no issue to close. if: ${{ !cancelled() && github.event_name == 'schedule' && needs.changes.outputs.run == 'true' }} permissions: @@ -106,4 +225,5 @@ jobs: label: web details: >- The Playwright report, with each failed test's trace and screenshots, is the run's - playwright-report artifact. + playwright-report artifact. Lighthouse's regressions and bad results are in the run's + summary, and each page's reports in its lighthouse-mobile and lighthouse-desktop artifacts. diff --git a/README.md b/README.md index 3b1ba28ef..c2c60dacb 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ cargo build # locallibrary cargo test --workspace --locked ``` -The TypeScript apps share one lint config (`.oxlintrc.json`, which each app's own extends) and one format config (`.oxfmtrc.json`). CI runs each tool's checks only when its files (or the shared ones) change, and comments on a pull request with each tool's coverage and, for the website and the extension, its size against `main`. The slow suites (mutation testing, fuzzing, the website in every browser, the CLI's timings, a release build) run weekly, in the `*-scheduled.yml` workflows; a weekly run that fails opens an issue, which the next one that passes closes. +The TypeScript apps share one lint config (`.oxlintrc.json`, which each app's own extends) and one format config (`.oxfmtrc.json`). CI runs each tool's checks only when its files (or the shared ones) change, and comments on a pull request with each tool's coverage and, for the website and the extension, its size against `main`. The slow suites (mutation testing, fuzzing, the website in every browser, Lighthouse on every page of the website against its history, the CLI's timings, a release build) run weekly, in the `*-scheduled.yml` workflows; a weekly run that fails opens an issue, which the next one that passes closes. ## Contributing diff --git a/apps/web/.gitignore b/apps/web/.gitignore index 0f3866750..6e237e3c5 100644 --- a/apps/web/.gitignore +++ b/apps/web/.gitignore @@ -17,6 +17,8 @@ test-results *.db-shm /.lighthouseci /lighthouse-report +/lighthouse-full +/lighthouse-history*.jsonl *storybook.log storybook-static diff --git a/apps/web/README.md b/apps/web/README.md index 5a430171a..e283816e1 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -215,32 +215,48 @@ The network is temporary; Ctrl-C stops it, and restarting resets everything (and ## Scripts -| Script | What it does | -| ----------------------------------- | ------------------------------------------------------------------------- | -| `pnpm dev` | Local atproto network, HappyView, and the dev server | -| `pnpm dev:network` / `pnpm dev:app` | The network and HappyView only / the dev server only (reads `.env.local`) | -| `pnpm build` / `pnpm start` | Production build (adapter-node) / run it | -| `pnpm check` | `svelte-check` with TypeScript 6 | -| `pnpm lint` | Oxlint with type-aware rules (tsgolint) and the anti-slop plugin | -| `pnpm format` / `pnpm format:check` | Oxfmt | -| `pnpm test` | Unit tests (Node) + component tests (Vitest browser mode, Chromium) | -| `pnpm test:integration` | Full flows against an in-process spaces PDS and a HappyView container | -| `pnpm test:coverage` | Unit + component + story + integration tests with `@vitest/coverage-v8` | -| `pnpm test:build` | Production build, checked for sourcemaps and the client JS budgets | -| `pnpm storybook` | The design system in Storybook, at localhost:6006 | -| `pnpm test:storybook` | Every story rendered in Chromium; an accessibility violation fails it | -| `pnpm build-storybook` | Static Storybook in `storybook-static/` | -| `pnpm test:e2e` | Playwright (desktop browsers and phones) on an isolated network, axe | -| `pnpm test:lighthouse` | Lighthouse CI on the production build (`lighthouserc.cjs`) | -| `pnpm knip` | Unused files, exports, and dependencies | -| `pnpm lexicons:publish` | Publish lexicons from the root account and check or set their DNS | -| `pnpm db:backup` | Back the database up to the bucket now ([Backups](#backups)) | -| `pnpm db:backups` | List the bucket's backups | -| `pnpm db:restore [path]` | Restore a backup (or `latest`) over the database, or to `path` | -| `pnpm verify` | CI's quick checks, in order | +| Script | What it does | +| ----------------------------------- | -------------------------------------------------------------------------------------------- | +| `pnpm dev` | Local atproto network, HappyView, and the dev server | +| `pnpm dev:network` / `pnpm dev:app` | The network and HappyView only / the dev server only (reads `.env.local`) | +| `pnpm build` / `pnpm start` | Production build (adapter-node) / run it | +| `pnpm check` | `svelte-check` with TypeScript 6 | +| `pnpm lint` | Oxlint with type-aware rules (tsgolint) and the anti-slop plugin | +| `pnpm format` / `pnpm format:check` | Oxfmt | +| `pnpm test` | Unit tests (Node) + component tests (Vitest browser mode, Chromium) | +| `pnpm test:integration` | Full flows against an in-process spaces PDS and a HappyView container | +| `pnpm test:coverage` | Unit + component + story + integration tests with `@vitest/coverage-v8` | +| `pnpm test:build` | Production build, checked for sourcemaps and the client JS budgets | +| `pnpm storybook` | The design system in Storybook, at localhost:6006 | +| `pnpm test:storybook` | Every story rendered in Chromium; an accessibility violation fails it | +| `pnpm build-storybook` | Static Storybook in `storybook-static/` | +| `pnpm test:e2e` | Playwright (desktop browsers and phones) on an isolated network, axe | +| `pnpm test:lighthouse` | Lighthouse CI on the production build (`lighthouserc.cjs`) | +| `pnpm test:lighthouse:full` | Lighthouse on every page, phone and desktop, against its history ([Lighthouse](#lighthouse)) | +| `pnpm knip` | Unused files, exports, and dependencies | +| `pnpm lexicons:publish` | Publish lexicons from the root account and check or set their DNS | +| `pnpm db:backup` | Back the database up to the bucket now ([Backups](#backups)) | +| `pnpm db:backups` | List the bucket's backups | +| `pnpm db:restore [path]` | Restore a backup (or `latest`) over the database, or to `path` | +| `pnpm verify` | CI's quick checks, in order | Pull requests run the e2e tests in Chromium only (`pnpm test:e2e --project chromium`); the other browsers and the phones run weekly. +### Lighthouse + +[`lighthouse/pages.json`](lighthouse/pages.json) lists the pages Lighthouse measures, each signed out or signed in as `neon.test` (`dj`) or `ada.test` (`admin`). Pull requests that change what the pages are built from measure the ones marked `pr` against the budgets in `lighthouserc.cjs`. `pnpm test:lighthouse:full` measures all of them on a phone and a desktop, three runs each, and keeps each page's medians in `lighthouse-history.jsonl`. Each run is held against the median of the last five recorded runs, and against the twenty before those to catch a slow creep. A regression past a metric's tolerance (in [`scripts/lib/lighthouse.ts`](scripts/lib/lighthouse.ts)) fails the run, as does a bad result: a failed run, a page that redirected or answered an error, or a failed budget (signed-in pages have no SEO budget, since search engines never see them). Runs that spread wide, and a new Lighthouse or a faster or slower machine, are noted too. + +The Web scheduled workflow does the same weekly, one job per form factor, and keeps main's history as its `lighthouse-history` artifact. A week that changed nothing the website is built from skips it, and only uploads the history again so it doesn't expire. Its report is in the run's summary, with every run's report in the `lighthouse-mobile` and `lighthouse-desktop` artifacts. + +To find the commit behind a regression, record a baseline of only the pages that moved on the last good commit (each history line names its commit), then let `git bisect` measure each commit against it, failing only on a regression (commits from before this suite landed lack its scripts): + +```bash +git checkout +pnpm build && pnpm exec tsx scripts/lighthouse.ts --page /charts && node scripts/lighthouse-history.ts --record --history lighthouse-history-bisect.jsonl +git bisect start +git bisect run sh -c 'pnpm build && pnpm exec tsx scripts/lighthouse.ts --page /charts && node scripts/lighthouse-history.ts --history lighthouse-history-bisect.jsonl --regressions-only' +``` + Git hooks (lefthook, from the root `lefthook.yml`) format and lint staged files on commit and run type checks and unit tests on push. ## Design system diff --git a/apps/web/lighthouse/pages.json b/apps/web/lighthouse/pages.json new file mode 100644 index 000000000..f910e7de5 --- /dev/null +++ b/apps/web/lighthouse/pages.json @@ -0,0 +1,46 @@ +{ + "$schema": "./pages.schema.json", + "pages": [ + { "path": "/", "pr": true }, + { "path": "/about" }, + { "path": "/for-djs" }, + { "path": "/privacy" }, + { "path": "/login" }, + { "path": "/projects" }, + { "path": "/projects?genres=house" }, + { "path": "/projects/halflight", "pr": true }, + { "path": "/projects/neon-tide" }, + { "path": "/projects/neon-tide/press-kits" }, + { "path": "/artists" }, + { "path": "/artists?q=okon" }, + { "path": "/charts" }, + { "path": "/shows" }, + { "path": "/match" }, + { "path": "/tools" }, + { "path": "/tools/locallibrary", "pr": true }, + { "path": "/tools/locallibrary/docs/install" }, + { "path": "/tools/localtoolbox" }, + { "path": "/dashboard", "account": "dj" }, + { "path": "/dashboard/activity", "account": "dj" }, + { "path": "/dashboard/bookings", "account": "dj" }, + { "path": "/dashboard/bookings/new?project=halflight", "account": "dj" }, + { "path": "/dashboard/inbox", "account": "dj" }, + { "path": "/dashboard/inbox/settings", "account": "dj" }, + { "path": "/dashboard/planner", "account": "dj" }, + { "path": "/dashboard/requests", "account": "dj" }, + { "path": "/report", "account": "dj" }, + { "path": "/dashboard/projects/new", "account": "dj" }, + { "path": "/dashboard/projects/neon-tide", "account": "dj" }, + { "path": "/dashboard/projects/neon-tide/profile", "account": "dj" }, + { "path": "/dashboard/projects/neon-tide/press-kits", "account": "dj" }, + { "path": "/admin", "account": "admin" }, + { "path": "/admin/audit", "account": "admin" }, + { "path": "/admin/projects", "account": "admin" }, + { "path": "/admin/projects/neon-tide", "account": "admin" }, + { "path": "/admin/requests", "account": "admin" }, + { "path": "/admin/spaces", "account": "admin" }, + { "path": "/admin/staff", "account": "admin" }, + { "path": "/admin/system", "account": "admin" }, + { "path": "/admin/verifications", "account": "admin" } + ] +} diff --git a/apps/web/lighthouse/pages.schema.json b/apps/web/lighthouse/pages.schema.json new file mode 100644 index 000000000..ba785783b --- /dev/null +++ b/apps/web/lighthouse/pages.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Lighthouse pages", + "description": "The pages Lighthouse measures (lighthouserc.cjs on pull requests, scripts/lighthouse.ts weekly), on the seeded network of scripts/dev-network.ts.", + "type": "object", + "required": ["pages"], + "additionalProperties": false, + "properties": { + "$schema": { "type": "string" }, + "pages": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { + "path": { + "description": "The page's path and query, on the seeded network.", + "type": "string", + "pattern": "^/" + }, + "account": { + "description": "The seeded account the page is measured signed in as: dj (neon.test) or admin (ada.test). Signed out without one.", + "enum": ["dj", "admin"] + }, + "pr": { + "description": "Also measured on pull requests that change what the pages are built from (web-ci.yml), against lighthouserc.cjs's budgets.", + "type": "boolean" + } + } + } + } + } +} diff --git a/apps/web/lighthouserc.cjs b/apps/web/lighthouserc.cjs index deba6e56e..0a79a6d13 100644 --- a/apps/web/lighthouserc.cjs +++ b/apps/web/lighthouserc.cjs @@ -1,8 +1,12 @@ // Lighthouse CI (`pnpm test:lighthouse`, which builds first). It measures the production build on the // same local network and seeded projects as the end-to-end tests (scripts/dev-network.ts), so the // pages have real data in them. Ports apart from the e2e run's, so the two can run side by side. +// The pages are the ones lighthouse/pages.json marks `pr`; scripts/lighthouse.ts measures all of them +// weekly, on phones and desktops, and tracks each against its history. const { chromium } = require('playwright'); +const { pages } = require('./lighthouse/pages.json'); + const PORT = 5198; const env = { @@ -24,7 +28,7 @@ module.exports = { // adapter-node prints this once the app has connected its root account and is serving. startServerReadyPattern: 'Listening on', startServerReadyTimeout: 180_000, - url: ['/', '/projects/halflight', '/tools/locallibrary'].map((path) => `http://127.0.0.1:${PORT}${path}`), + url: pages.filter((page) => page.pr).map(({ path }) => `http://127.0.0.1:${PORT}${path}`), numberOfRuns: 3, // Playwright's Chromium, which the e2e tests install anyway, rather than whichever Chrome the // machine has (or doesn't). diff --git a/apps/web/package.json b/apps/web/package.json index 2e9e4ae48..a7b0306db 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -32,6 +32,7 @@ "test:build": "vitest run --project build", "test:e2e": "playwright test", "test:lighthouse": "pnpm build && lhci autorun", + "test:lighthouse:full": "pnpm build && tsx scripts/lighthouse.ts && node scripts/lighthouse-history.ts --record", "lexicons:publish": "tsx scripts/publish-lexicons.ts", "fixtures:mix": "tsx scripts/mix-fixture.ts", "db:backup": "node scripts/db-backup.ts backup", diff --git a/apps/web/scripts/dev-network.ts b/apps/web/scripts/dev-network.ts index 485a8e5f2..2b75bb92d 100644 --- a/apps/web/scripts/dev-network.ts +++ b/apps/web/scripts/dev-network.ts @@ -381,7 +381,15 @@ async function main() { ); // The production build only reaches this network's loopback PDS and PLC with the SSRF guard off. - const production = { NODE_ENV: 'production', HOST: '127.0.0.1', ALLOW_PRIVATE_NETWORK: 'true' }; + // adapter-node takes a request to be https unless PROTOCOL_HEADER's header says otherwise, so a + // form posted from http://127.0.0.1 (signing in) fails SvelteKit's CSRF check without + // `X-Forwarded-Proto: http`, which scripts/lighthouse.ts sends while it signs in. + const production = { + NODE_ENV: 'production', + HOST: '127.0.0.1', + PROTOCOL_HEADER: 'x-forwarded-proto', + ALLOW_PRIVATE_NETWORK: 'true' + }; const app = START_APP ? spawn(PRODUCTION ? process.execPath : 'pnpm', PRODUCTION ? ['build'] : ['exec', 'vite', 'dev'], { diff --git a/apps/web/scripts/lib/lighthouse.test.ts b/apps/web/scripts/lib/lighthouse.test.ts new file mode 100644 index 000000000..300803770 --- /dev/null +++ b/apps/web/scripts/lib/lighthouse.test.ts @@ -0,0 +1,255 @@ +import { describe, expect, it } from 'vitest'; +import { + compare, + formatValue, + type HistoryEntry, + isFailed, + type Lhr, + METRICS, + median, + OLDER_RUNS, + RECENT_RUNS, + renderMarkdown, + sparkline, + summarizePage, + type Summary +} from './lighthouse.ts'; + +const metric = (id: string) => METRICS.find((candidate) => candidate.id === id)!; + +/** A run of `path` that scored `performance` with an LCP of `lcp` ms. */ +function lhr(path: string, { performance = 0.98, lcp = 1800, benchmarkIndex = 2000 } = {}): Lhr { + return { + lighthouseVersion: '12.6.1', + finalDisplayedUrl: `http://127.0.0.1:5198${path}`, + environment: { benchmarkIndex }, + categories: { performance: { score: performance }, accessibility: { score: 1 } }, + audits: { + 'largest-contentful-paint': { score: 0.9, numericValue: lcp }, + 'http-status-code': { score: 1 } + } + }; +} + +const summary = (pages: Summary['pages'], formFactor: Summary['formFactor'] = 'mobile'): Summary => ({ + formFactor, + lighthouseVersion: '12.6.1', + benchmarkIndex: 2000, + runs: 3, + pages +}); + +const page = (path: string, metrics: Record, problems: string[] = []) => ({ + path, + runs: 3, + problems, + metrics: Object.fromEntries( + Object.entries(metrics).map(([id, value]) => [id, { median: value, min: value, max: value }]) + ) +}); + +const entry = (results: HistoryEntry['results'], extra: Partial = {}): HistoryEntry => ({ + date: '2026-10-01T00:00:00.000Z', + commit: 'abc', + lighthouseVersion: '12.6.1', + benchmarkIndex: 2000, + results, + ...extra +}); + +const RUN = { date: '2026-10-31T00:00:00.000Z', commit: '0123456789abcdef' }; + +describe('median', () => { + it('takes the middle value, or the mean of the two middle ones', () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + }); +}); + +describe('summarizePage', () => { + it('keeps the median, lowest and highest of each metric over the runs', () => { + const result = summarizePage({ path: '/' }, [ + lhr('/', { lcp: 1700 }), + lhr('/', { lcp: 2100 }), + lhr('/', { lcp: 1800 }) + ]); + + expect(result.runs).toBe(3); + expect(result.problems).toEqual([]); + expect(result.metrics.lcp).toEqual({ median: 1800, min: 1700, max: 2100 }); + expect(result.metrics.accessibility).toEqual({ median: 1, min: 1, max: 1 }); + expect(result.metrics).not.toHaveProperty('tbt'); + }); + + it('reports failed runs and leaves them out of the numbers', () => { + const failed = { ...lhr('/', { lcp: 9000 }), runtimeError: { code: 'NO_FCP', message: 'The page did not paint' } }; + const result = summarizePage({ path: '/' }, [failed, lhr('/', { lcp: 1800 })]); + + expect(result.runs).toBe(1); + expect(result.problems).toEqual(['a run failed: NO_FCP (The page did not paint)']); + expect(result.metrics.lcp.median).toBe(1800); + }); + + it('reports a page that redirected, answered an error, or never ran', () => { + const redirected = { ...lhr('/login?next=%2Fdashboard'), audits: { 'http-status-code': { score: 0 } } }; + + expect(summarizePage({ path: '/dashboard', account: 'dj' }, [redirected]).problems).toEqual([ + 'ended up on /login?next=%2Fdashboard', + 'answered with an error status' + ]); + expect(summarizePage({ path: '/' }, []).problems).toEqual(['no run produced a result']); + }); + + it('adds lhci assertion failures to the problems', () => { + expect(summarizePage({ path: '/' }, [lhr('/')], ['failed largest-contentful-paint']).problems).toEqual([ + 'failed largest-contentful-paint' + ]); + }); +}); + +describe('compare', () => { + const history = Array.from({ length: RECENT_RUNS }, () => entry({ 'mobile /': { performance: 0.98, lcp: 1800 } })); + + it('flags a metric worse than the recent median by more than its tolerance', () => { + const result = compare([summary([page('/', { performance: 0.9, lcp: 2400 })])], history, RUN); + + expect(result.regressions.map(({ metric: { id }, baseline, current }) => [id, baseline, current])).toEqual([ + ['performance', 0.98, 0.9], + ['lcp', 1800, 2400] + ]); + expect(isFailed(result)).toBe(true); + }); + + it('leaves changes within tolerance alone, in both directions', () => { + // LCP's tolerance is the larger of 150 ms and 10%: 180 ms on 1.8 s. + const result = compare([summary([page('/', { performance: 0.96, lcp: 1970 })])], history, RUN); + + expect(result.regressions).toEqual([]); + expect(result.improvements).toEqual([]); + expect(isFailed(result)).toBe(false); + }); + + it('lists improvements past the tolerance', () => { + const result = compare([summary([page('/', { performance: 1, lcp: 1200 })])], history, RUN); + + expect(result.improvements.map(({ metric: { id } }) => id)).toEqual(['lcp']); + }); + + it('takes the median of the recent runs, so one outlier in them moves nothing', () => { + const withOutlier = [...history.slice(1), entry({ 'mobile /': { lcp: 6000 } })]; + + expect(compare([summary([page('/', { lcp: 1850 })])], withOutlier, RUN).regressions).toEqual([]); + }); + + it('flags a slow creep against the runs before the recent ones', () => { + const older = Array.from({ length: OLDER_RUNS }, () => entry({ 'mobile /': { lcp: 1500 } })); + const recent = [1600, 1700, 1750, 1800, 1850].map((lcp) => entry({ 'mobile /': { lcp } })); + const result = compare([summary([page('/', { lcp: 1800 })])], [...older, ...recent], RUN); + + expect(result.regressions).toEqual([]); + expect(result.drifts.map(({ metric: { id }, baseline }) => [id, baseline])).toEqual([['lcp', 1500]]); + expect(isFailed(result)).toBe(false); + }); + + it('keeps form factors apart', () => { + const result = compare([summary([page('/', { lcp: 2400 })], 'desktop')], history, RUN); + + expect(result.regressions).toEqual([]); + expect(result.added).toEqual(['desktop /']); + }); + + it('fails on a bad result even with nothing to compare against', () => { + const result = compare([summary([page('/dashboard', {}, ['ended up on /login'])])], [], RUN); + + expect(result.problems).toEqual([{ key: 'mobile /dashboard', problem: 'ended up on /login' }]); + expect(result.baselineRuns).toBe(0); + expect(isFailed(result)).toBe(true); + }); + + it('flags runs that spread wider than twice the tolerance', () => { + const spread = { ...page('/', {}), metrics: { lcp: { median: 1800, min: 1500, max: 2400 } } }; + + expect(compare([summary([spread])], history, RUN).unstable.map(({ key, metric: { id } }) => [key, id])).toEqual([ + ['mobile /', 'lcp'] + ]); + }); + + it('notes a new Lighthouse version and a machine of a different speed', () => { + const old = history.map((past) => ({ ...past, lighthouseVersion: '12.5.0', benchmarkIndex: 1000 })); + + expect(compare([summary([page('/', { lcp: 1800 })])], old, RUN).notes).toEqual([ + 'Lighthouse changed from 12.5.0 to 12.6.1 since the last run.', + "This machine benchmarked at 2000, against 1000 for the baseline's, so timings may have moved with it." + ]); + }); + + it('records each page’s medians under its form factor and path', () => { + const result = compare( + [summary([page('/', { lcp: 1800 })]), summary([page('/', { lcp: 900 })], 'desktop')], + [], + RUN + ); + + expect(result.entry).toEqual({ + ...RUN, + lighthouseVersion: '12.6.1', + benchmarkIndex: 2000, + results: { 'mobile /': { lcp: 1800 }, 'desktop /': { lcp: 900 } } + }); + }); +}); + +describe('formatValue', () => { + it('shows each unit the way Lighthouse does', () => { + expect(formatValue(metric('performance'), 0.973)).toBe('97'); + expect(formatValue(metric('lcp'), 1834)).toBe('1.83 s'); + expect(formatValue(metric('tbt'), 42.4)).toBe('42 ms'); + expect(formatValue(metric('bytes'), 204_800)).toBe('200 KiB'); + expect(formatValue(metric('cls'), 0.0123)).toBe('0.012'); + expect(formatValue(metric('dom'), 812)).toBe('812'); + }); +}); + +describe('sparkline', () => { + it('draws the lowest value lowest and the highest highest', () => { + expect(sparkline([1, 8, 4.5])).toBe('▁█▅'); + expect(sparkline([3, 3])).toBe('▁▁'); + }); +}); + +describe('renderMarkdown', () => { + it('puts regressions and bad results first, and every page in a table', () => { + const history = [entry({ 'mobile /': { performance: 0.98, lcp: 1800 } })]; + + const result = compare( + [summary([page('/', { performance: 0.98, lcp: 2400 }), page('/admin', {}, ['ended up on /login'])])], + history, + RUN + ); + + const markdown = renderMarkdown(result, history); + + expect(markdown).toContain('0123456789ab, Lighthouse 12.6.1, against the median of the last 1 recorded run.'); + expect(markdown).toContain( + '### Regressions (1)\n\n| Page | Metric | Baseline | Now |\n| --- | --- | ---: | ---: |\n| `mobile /` | LCP | 1.80 s | 2.40 s |' + ); + expect(markdown).toContain('### Bad results (1)\n\n- `mobile /admin`: ended up on /login'); + expect(markdown).toContain('New pages, with no history yet: `mobile /admin`'); + expect(markdown).toContain('| `mobile /` | 98 | | | | 2.40 s | | | | | ▁▁ | ▁█ |'); + }); + + it('shortens the commit but keeps a dirty marker', () => { + const dirty = { ...RUN, commit: `${RUN.commit}0123456789abcdef01234567-dirty` }; + + expect(renderMarkdown(compare([summary([page('/', { lcp: 1800 })])], [], dirty), [])).toContain( + '0123456789ab-dirty, Lighthouse' + ); + }); + + it('says when there is no history yet, without listing every page as new', () => { + const markdown = renderMarkdown(compare([summary([page('/', { lcp: 1800 })])], [], RUN), []); + + expect(markdown).toContain('No history yet: this run is the first baseline.'); + expect(markdown).not.toContain('New pages'); + }); +}); diff --git a/apps/web/scripts/lib/lighthouse.ts b/apps/web/scripts/lib/lighthouse.ts new file mode 100644 index 000000000..c2172b8a4 --- /dev/null +++ b/apps/web/scripts/lib/lighthouse.ts @@ -0,0 +1,444 @@ +// What scripts/lighthouse.ts keeps of each page's Lighthouse runs, and how scripts/lighthouse-history.ts +// holds that against the pages' history. No dependencies, so CI's history job runs it on plain Node. + +export type FormFactor = 'mobile' | 'desktop'; + +export type Account = 'dj' | 'admin'; + +/** A number tracked over time: a category's score, or an audit's numericValue. */ +export type Metric = { + id: string; + label: string; + category?: string; + audit?: string; + unit: 'score' | 'ms' | 'bytes' | 'count'; + higherIsBetter: boolean; + /** A change only counts past both: this much, and (when set) this share of the baseline. */ + tolerance: { absolute: number; relative?: number }; +}; + +// Tolerances sit above the run-to-run noise of Lighthouse's simulated throttling on a median of three: +// the category scores other than performance don't move between runs, so any drop counts. +export const METRICS: Metric[] = [ + { + id: 'performance', + label: 'Performance', + category: 'performance', + unit: 'score', + higherIsBetter: true, + tolerance: { absolute: 0.05 } + }, + { + id: 'accessibility', + label: 'Accessibility', + category: 'accessibility', + unit: 'score', + higherIsBetter: true, + tolerance: { absolute: 0.01 } + }, + { + id: 'best-practices', + label: 'Best practices', + category: 'best-practices', + unit: 'score', + higherIsBetter: true, + tolerance: { absolute: 0.01 } + }, + { id: 'seo', label: 'SEO', category: 'seo', unit: 'score', higherIsBetter: true, tolerance: { absolute: 0.01 } }, + { + id: 'fcp', + label: 'FCP', + audit: 'first-contentful-paint', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.1 } + }, + { + id: 'lcp', + label: 'LCP', + audit: 'largest-contentful-paint', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 150, relative: 0.1 } + }, + { + id: 'tbt', + label: 'TBT', + audit: 'total-blocking-time', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 50, relative: 0.25 } + }, + { + id: 'cls', + label: 'CLS', + audit: 'cumulative-layout-shift', + unit: 'count', + higherIsBetter: false, + tolerance: { absolute: 0.02 } + }, + { + id: 'si', + label: 'Speed Index', + audit: 'speed-index', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 200, relative: 0.15 } + }, + { + id: 'ttfb', + label: 'Server response', + audit: 'server-response-time', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.5 } + }, + { + id: 'js', + label: 'JS execution', + audit: 'bootup-time', + unit: 'ms', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.25 } + }, + { + id: 'bytes', + label: 'Transfer size', + audit: 'total-byte-weight', + unit: 'bytes', + higherIsBetter: false, + tolerance: { absolute: 10_240, relative: 0.05 } + }, + { + id: 'dom', + label: 'DOM elements', + audit: 'dom-size', + unit: 'count', + higherIsBetter: false, + tolerance: { absolute: 100, relative: 0.1 } + } +]; + +/** The parts of a Lighthouse result (LHR) read here. */ +export type Lhr = { + lighthouseVersion: string; + finalDisplayedUrl: string; + runtimeError?: { code: string; message: string }; + environment: { benchmarkIndex: number }; + categories: Record; + audits: Record; +}; + +export type Stat = { median: number; min: number; max: number }; + +export type PageSummary = { + path: string; + account?: Account; + /** Runs that produced a result. */ + runs: number; + /** Bad results: failed runs, a page that didn't load as itself, and lhci's failed assertions. */ + problems: string[]; + metrics: Record; +}; + +/** One form factor's pages from a run of scripts/lighthouse.ts (lighthouse-full/summary-.json). */ +export type Summary = { + formFactor: FormFactor; + lighthouseVersion: string; + benchmarkIndex: number; + runs: number; + pages: PageSummary[]; +}; + +/** A line of the history file: each page's medians, keyed ` `. */ +export type HistoryEntry = { + date: string; + commit: string; + lighthouseVersion: string; + benchmarkIndex: number; + results: Record>; +}; + +export const median = (values: number[]) => { + const sorted = values.toSorted((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + + return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +const metricValue = (lhr: Lhr, metric: Metric) => + metric.category ? lhr.categories[metric.category]?.score : lhr.audits[metric.audit ?? '']?.numericValue; + +/** One page's runs, with what went wrong in them. */ +export function summarizePage( + page: { path: string; account?: Account }, + lhrs: Lhr[], + assertionFailures: string[] = [] +): PageSummary { + const problems: string[] = []; + const ok = lhrs.filter((lhr) => !lhr.runtimeError); + + for (const lhr of lhrs.filter((lhr) => lhr.runtimeError)) { + problems.push(`a run failed: ${lhr.runtimeError?.code} (${lhr.runtimeError?.message})`); + } + + if (!lhrs.length) problems.push('no run produced a result'); + + // A signed-in page measured signed out lands on /login, and a missing one on the error page. + const landed = new Set( + ok.map(({ finalDisplayedUrl }) => new URL(finalDisplayedUrl)).map((url) => url.pathname + url.search) + ); + + for (const path of landed) if (path !== page.path) problems.push(`ended up on ${path}`); + + if (ok.some((lhr) => lhr.audits['http-status-code']?.score === 0)) problems.push('answered with an error status'); + + problems.push(...assertionFailures); + + const metrics: Record = {}; + + for (const metric of METRICS) { + const values = ok.flatMap((lhr) => metricValue(lhr, metric) ?? []); + + if (values.length) + metrics[metric.id] = { median: median(values), min: Math.min(...values), max: Math.max(...values) }; + } + + return { ...page, runs: ok.length, problems, metrics }; +} + +export type Change = { key: string; metric: Metric; baseline: number; current: number }; + +export type Comparison = { + entry: HistoryEntry; + /** Worse than the median of the last few runs by more than the metric's tolerance. */ + regressions: Change[]; + /** Not a regression against the last few runs, but worse than the runs before them: a slow creep. */ + drifts: Change[]; + improvements: Change[]; + problems: { key: string; problem: string }[]; + /** Runs of one page that spread wider than twice the metric's tolerance, so its median is shaky. */ + unstable: { key: string; metric: Metric; stat: Stat }[]; + /** Pages with no history yet. */ + added: string[]; + /** Why this run's numbers may not line up with its baseline's. */ + notes: string[]; + baselineRuns: number; +}; + +export const RECENT_RUNS = 5; + +export const OLDER_RUNS = 20; + +const limit = (metric: Metric, baseline: number) => + Math.max(metric.tolerance.absolute, (metric.tolerance.relative ?? 0) * Math.abs(baseline)); + +/** How much worse `current` is than `baseline`, positive when worse. */ +const worsening = (metric: Metric, baseline: number, current: number) => + metric.higherIsBetter ? baseline - current : current - baseline; + +const baselineOf = (entries: HistoryEntry[], key: string, metric: Metric) => { + const values = entries.flatMap((entry) => entry.results[key]?.[metric.id] ?? []); + + return values.length ? median(values) : undefined; +}; + +/** This run's summaries against the history file's entries (oldest first), and the entry to record. */ +export function compare( + summaries: Summary[], + history: HistoryEntry[], + run: { date: string; commit: string } +): Comparison { + const keyed = summaries.flatMap((summary) => + summary.pages.map((page) => ({ key: `${summary.formFactor} ${page.path}`, page })) + ); + + const entry: HistoryEntry = { + ...run, + lighthouseVersion: summaries[0]?.lighthouseVersion ?? '', + benchmarkIndex: median(summaries.map((summary) => summary.benchmarkIndex)), + results: Object.fromEntries( + keyed.map(({ key, page }) => [ + key, + Object.fromEntries(Object.entries(page.metrics).map(([id, stat]) => [id, stat.median])) + ]) + ) + }; + + const recent = history.slice(-RECENT_RUNS); + const older = history.slice(-RECENT_RUNS - OLDER_RUNS, -RECENT_RUNS); + + const comparison: Comparison = { + entry, + regressions: [], + drifts: [], + improvements: [], + problems: keyed.flatMap(({ key, page }) => page.problems.map((problem) => ({ key, problem }))), + unstable: [], + added: [], + notes: [], + baselineRuns: recent.length + }; + + for (const { key, page } of keyed) { + if (!recent.some((past) => past.results[key])) comparison.added.push(key); + + for (const metric of METRICS) { + const stat = page.metrics[metric.id]; + + if (!stat) continue; + + if (stat.max - stat.min > 2 * limit(metric, stat.median)) comparison.unstable.push({ key, metric, stat }); + + const baseline = baselineOf(recent, key, metric); + + if (baseline === undefined) continue; + + const change = { key, metric, baseline, current: stat.median }; + const worse = worsening(metric, baseline, stat.median); + + if (worse > limit(metric, baseline)) { + comparison.regressions.push(change); + continue; + } + + if (-worse > limit(metric, baseline)) comparison.improvements.push(change); + + const longBaseline = baselineOf(older, key, metric); + + if (longBaseline !== undefined && worsening(metric, longBaseline, stat.median) > limit(metric, longBaseline)) { + comparison.drifts.push({ ...change, baseline: longBaseline }); + } + } + } + + const last = recent.at(-1); + + if (last && last.lighthouseVersion !== entry.lighthouseVersion) { + comparison.notes.push( + `Lighthouse changed from ${last.lighthouseVersion} to ${entry.lighthouseVersion} since the last run.` + ); + } + + // Lighthouse's estimate of the machine's CPU speed: a slower runner moves every timing. + const benchmark = recent.length ? median(recent.map((past) => past.benchmarkIndex)) : undefined; + + if (benchmark && Math.abs(entry.benchmarkIndex - benchmark) > 0.25 * benchmark) { + comparison.notes.push( + `This machine benchmarked at ${Math.round(entry.benchmarkIndex)}, against ${Math.round(benchmark)} for the baseline's, so timings may have moved with it.` + ); + } + + return comparison; +} + +export const isFailed = (comparison: Comparison) => comparison.regressions.length > 0 || comparison.problems.length > 0; + +export function formatValue(metric: Metric, value: number) { + if (metric.unit === 'score') return String(Math.round(value * 100)); + + if (metric.unit === 'bytes') return `${(value / 1024).toFixed(0)} KiB`; + + if (metric.unit === 'ms') return value >= 1000 ? `${(value / 1000).toFixed(2)} s` : `${Math.round(value)} ms`; + + return Number.isInteger(value) ? String(value) : value.toFixed(3); +} + +const SPARKS = '▁▂▃▄▅▆▇█'; + +/** A row of block characters for `values`, from the lowest (▁) to the highest (█). */ +export function sparkline(values: number[]) { + const low = Math.min(...values); + const range = Math.max(...values) - low; + + return values.map((value) => SPARKS[range ? Math.round(((value - low) / range) * (SPARKS.length - 1)) : 0]).join(''); +} + +const changeRows = (changes: Change[]) => [ + '| Page | Metric | Baseline | Now |', + '| --- | --- | ---: | ---: |', + ...changes.map( + ({ key, metric, baseline, current }) => + `| \`${key}\` | ${metric.label} | ${formatValue(metric, baseline)} | ${formatValue(metric, current)} |` + ) +]; + +const TABLE_METRICS = ['performance', 'accessibility', 'best-practices', 'seo', 'lcp', 'tbt', 'cls', 'ttfb', 'bytes']; + +/** The comparison as Markdown, for the run's summary or the terminal. `trend` is how many past runs the sparklines show. */ +export function renderMarkdown(comparison: Comparison, history: HistoryEntry[], trend = 12) { + const { entry, regressions, drifts, improvements, problems, unstable, added, notes, baselineRuns } = comparison; + + const lines = [ + '## Lighthouse history', + '', + `${entry.commit.replace(/^[0-9a-f]{12}[0-9a-f]*/, (sha) => sha.slice(0, 12))}, Lighthouse ${entry.lighthouseVersion}, against the median of the last ${baselineRuns} recorded run${baselineRuns === 1 ? '' : 's'}.`, + ...notes.map((note) => `\n> ${note}`), + '' + ]; + + if (!baselineRuns) lines.push('No history yet: this run is the first baseline.', ''); + + if (regressions.length) lines.push(`### Regressions (${regressions.length})`, '', ...changeRows(regressions), ''); + + if (problems.length) + lines.push( + `### Bad results (${problems.length})`, + '', + ...problems.map(({ key, problem }) => `- \`${key}\`: ${problem}`), + '' + ); + + if (drifts.length) { + lines.push( + `### Slow creep (${drifts.length})`, + '', + `Worse than the ${OLDER_RUNS} runs before the baseline's.`, + '', + ...changeRows(drifts), + '' + ); + } + + if (unstable.length) { + lines.push( + `### Unstable (${unstable.length})`, + '', + ...unstable.map( + ({ key, metric, stat }) => + `- \`${key}\`: ${metric.label} ranged ${formatValue(metric, stat.min)} to ${formatValue(metric, stat.max)}` + ), + '' + ); + } + + if (improvements.length) lines.push(`### Improvements (${improvements.length})`, '', ...changeRows(improvements), ''); + + if (baselineRuns && added.length) + lines.push(`New pages, with no history yet: ${added.map((key) => `\`${key}\``).join(', ')}`, ''); + + const metrics = METRICS.filter((metric) => TABLE_METRICS.includes(metric.id)); + + const past = history.slice(-trend); + + const trendOf = (key: string, id: string) => + sparkline([...past.flatMap((old) => old.results[key]?.[id] ?? []), entry.results[key][id]]); + + lines.push( + '
Every page', + '', + `| Page | ${metrics.map((metric) => metric.label).join(' | ')} | Performance trend | LCP trend |`, + `| --- | ${metrics.map(() => '---:').join(' | ')} | --- | --- |`, + ...Object.entries(entry.results).map( + ([key, values]) => + `| \`${key}\` | ${metrics.map((metric) => (metric.id in values ? formatValue(metric, values[metric.id]) : '')).join(' | ')} | ${ + 'performance' in values ? trendOf(key, 'performance') : '' + } | ${'lcp' in values ? trendOf(key, 'lcp') : ''} |` + ), + '', + '
', + '' + ); + + return lines.join('\n'); +} diff --git a/apps/web/scripts/lighthouse-history.ts b/apps/web/scripts/lighthouse-history.ts new file mode 100644 index 000000000..4f965e66e --- /dev/null +++ b/apps/web/scripts/lighthouse-history.ts @@ -0,0 +1,62 @@ +/** + * Holds a run of scripts/lighthouse.ts (lighthouse-full/summary-*.json) against the pages' history: a + * JSON Lines file, one line per recorded run. Prints what regressed, crept, broke or improved, and + * exits 1 on a regression or a bad result. Plain Node (no dependencies), so CI runs it uninstalled. + * + * node scripts/lighthouse-history.ts # compare only, say while bisecting + * node scripts/lighthouse-history.ts --record # compare, then add this run to the history + * + * --results the summaries' directory (lighthouse-full) + * --history the history (lighthouse-history.jsonl); CI keeps it as a workflow artifact + * --markdown append the report to a file (CI's $GITHUB_STEP_SUMMARY) instead of printing it + * --regressions-only exit 1 on a regression alone, so bad results every commit has don't fail a bisect + */ +import { execFileSync } from 'node:child_process'; +import { appendFileSync, existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { compare, type HistoryEntry, isFailed, renderMarkdown, type Summary } from './lib/lighthouse.ts'; + +const { values: args } = parseArgs({ + options: { + results: { type: 'string', default: 'lighthouse-full' }, + history: { type: 'string', default: 'lighthouse-history.jsonl' }, + markdown: { type: 'string' }, + record: { type: 'boolean', default: false }, + 'regressions-only': { type: 'boolean', default: false } + } +}); + +const summaries: Summary[] = readdirSync(args.results) + .filter((name) => /^summary-.+\.json$/.test(name)) + .map((name) => JSON.parse(readFileSync(join(args.results, name), 'utf8'))); + +if (!summaries.length) { + console.error(`No summaries in ${args.results}: run scripts/lighthouse.ts first.`); + process.exit(1); +} + +const history: HistoryEntry[] = existsSync(args.history) + ? readFileSync(args.history, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) + : []; + +// A run of uncommitted changes says so, so it isn't mistaken for the commit's own numbers. +const git = (...gitArgs: string[]) => execFileSync('git', gitArgs, { encoding: 'utf8' }).trim(); + +const commit = process.env.GITHUB_SHA ?? `${git('rev-parse', 'HEAD')}${git('status', '--porcelain') ? '-dirty' : ''}`; + +const comparison = compare(summaries, history, { date: new Date().toISOString(), commit }); + +const report = renderMarkdown(comparison, history); + +if (args.markdown) appendFileSync(args.markdown, `${report}\n`); +else console.log(report); + +if (args.record) appendFileSync(args.history, `${JSON.stringify(comparison.entry)}\n`); + +const isFailing = args['regressions-only'] ? comparison.regressions.length > 0 : isFailed(comparison); + +if (isFailing) process.exit(1); diff --git a/apps/web/scripts/lighthouse.ts b/apps/web/scripts/lighthouse.ts new file mode 100644 index 000000000..ac2d01abd --- /dev/null +++ b/apps/web/scripts/lighthouse.ts @@ -0,0 +1,256 @@ +/** + * Lighthouse CI on every page in lighthouse/pages.json, on a phone and a desktop, signed in where a page + * needs it, then each page's runs summed up for scripts/lighthouse-history.ts (`pnpm test:lighthouse:full` + * builds first, then records the run in the history). Uses lighthouserc.cjs's network, Chromium and + * budgets, and starts the network once for every page. + * + * tsx scripts/lighthouse.ts # every page, both form factors + * tsx scripts/lighthouse.ts --form-factor desktop # one form factor (CI runs one per job) + * tsx scripts/lighthouse.ts --page /charts --runs 5 # pages whose path contains /charts, five runs each + * + * Writes lighthouse-full/summary-.json, and each run's report under lighthouse-full//. + */ +import { spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { chromium } from 'playwright'; +import { signIn } from '../tests/sign-in.ts'; +import { type Account, type FormFactor, type Lhr, median, summarizePage, type Summary } from './lib/lighthouse.ts'; + +type Page = { path: string; account?: Account; pr?: boolean }; + +const require = createRequire(import.meta.url); + +const { ci } = require('../lighthouserc.cjs'); + +const { pages }: { pages: Page[] } = require('../lighthouse/pages.json'); + +// The seeded accounts of scripts/dev-network.ts. +const ACCOUNTS: Record = { + dj: { handle: 'neon.test', password: 'neon-pass' }, + admin: { handle: 'ada.test', password: 'ada-pass' } +}; + +const OUT = 'lighthouse-full'; + +const LHCI_DIR = '.lighthouseci'; + +const { values: args } = parseArgs({ + options: { + 'form-factor': { type: 'string', multiple: true, default: ['mobile', 'desktop'] }, + page: { type: 'string', multiple: true }, + runs: { type: 'string', default: String(ci.collect.numberOfRuns) } + } +}); + +const formFactors = args['form-factor'].filter( + (value): value is FormFactor => value === 'mobile' || value === 'desktop' +); + +const selected = pages.filter(({ path }) => !args.page || args.page.some((filter) => path.includes(filter))); + +const origin = new URL(ci.collect.url[0]).origin; + +if (!selected.length || formFactors.length !== args['form-factor'].length) { + console.error( + 'Nothing to measure: check --page (a part of a path in lighthouse/pages.json) and --form-factor (mobile or desktop).' + ); + process.exit(1); +} + +const log = (message: string) => console.log(`\x1b[35m▸\x1b[0m ${message}`); + +/** The network and the production build, as lhci's startServerCommand would start them; resolves once it serves. */ +async function startServer() { + const server = spawn(ci.collect.startServerCommand, { + shell: true, + detached: true, + stdio: ['ignore', 'pipe', 'inherit'] + }); + + const exited = once(server, 'exit').then(([code]) => { + throw new Error(`The network exited (${code}) before the app was serving.`); + }); + + const ready = new Promise((resolve) => { + let output = ''; + server.stdout.on('data', (chunk: Buffer) => { + process.stdout.write(chunk); + output += chunk.toString(); + + if (output.includes(ci.collect.startServerReadyPattern)) resolve(); + }); + }); + + const timeout = new Promise((_, reject) => + setTimeout( + () => reject(new Error('The network took too long to start.')), + ci.collect.startServerReadyTimeout + ).unref() + ); + + await Promise.race([ready, exited, timeout]); + + return async () => { + // The whole group, so the shell, tsx and the network all get it; the network then stops the + // app and removes HappyView's container (scripts/dev-network.ts). + if (server.exitCode !== null || !server.pid) return; + const exit = once(server, 'exit'); + process.kill(-server.pid, 'SIGTERM'); + await exit; + }; +} + +/** A Cookie header for each account the selected pages are measured as, from a real sign-in. */ +async function sessionCookies() { + const accounts = [...new Set(selected.map((page) => page.account).filter((account) => account !== undefined))]; + const browser = await chromium.launch(); + const cookies: Partial> = {}; + + try { + for (const account of accounts) { + // The production build's origin is https without this header (scripts/dev-network.ts). + const context = await browser.newContext({ baseURL: origin, extraHTTPHeaders: { 'X-Forwarded-Proto': 'http' } }); + await signIn(await context.newPage(), ACCOUNTS[account].handle, ACCOUNTS[account].password); + cookies[account] = (await context.cookies(origin)).map(({ name, value }) => `${name}=${value}`).join('; '); + await context.close(); + } + } finally { + await browser.close(); + } + + return cookies; +} + +const lhci = (command: string, config: string, ...flags: string[]) => + spawnSync('lhci', [command, `--config=${config}`, ...flags], { stdio: 'inherit' }); + +/** Each failed lhci assertion of the last `lhci assert`, by page path. */ +function assertionFailures() { + const failures = new Map(); + + let results: { + url: string; + level: string; + auditId: string; + auditProperty?: string; + actual: number; + operator: string; + expected: number; + }[]; + + try { + results = JSON.parse(readFileSync(join(LHCI_DIR, 'assertion-results.json'), 'utf8')); + } catch { + return failures; + } + + for (const result of results.filter(({ level }) => level === 'error')) { + const url = new URL(result.url); + const audit = result.auditProperty ? `${result.auditId}.${result.auditProperty}` : result.auditId; + const failure = `failed ${audit}: ${Math.round(result.actual * 100) / 100}, expected ${result.operator} ${result.expected}`; + failures.set(url.pathname + url.search, [...(failures.get(url.pathname + url.search) ?? []), failure]); + } + + return failures; +} + +/** The results of the last `lhci collect`, by the page path each run asked for. */ +function collectedLhrs() { + const lhrs = new Map(); + + for (const name of readdirSync(LHCI_DIR).filter((file) => file.startsWith('lhr-') && file.endsWith('.json'))) { + const lhr: Lhr & { requestedUrl: string } = JSON.parse(readFileSync(join(LHCI_DIR, name), 'utf8')); + const url = new URL(lhr.requestedUrl); + lhrs.set(url.pathname + url.search, [...(lhrs.get(url.pathname + url.search) ?? []), lhr]); + } + + return lhrs; +} + +// Whole, so a summary from an earlier run of another form factor isn't compared or recorded again. +rmSync(OUT, { recursive: true, force: true }); + +mkdirSync(join(OUT, 'config'), { recursive: true }); + +const stopServer = await startServer(); + +try { + const cookies = await sessionCookies(); + + for (const formFactor of formFactors) { + const summary: Summary = { + formFactor, + lighthouseVersion: '', + benchmarkIndex: 0, + runs: Number(args.runs), + pages: [] + }; + + const benchmarks: number[] = []; + + for (const account of [undefined, 'dj', 'admin'] satisfies (Account | undefined)[]) { + const group = selected.filter((page) => page.account === account); + + if (!group.length) continue; + + const name = `${formFactor}-${account ?? 'signed-out'}`; + const config = join(OUT, 'config', `${name}.json`); + const cookie = account && cookies[account]; + log( + `${group.length} page${group.length === 1 ? '' : 's'}, ${formFactor}, ${account ? `signed in as ${ACCOUNTS[account].handle}` : 'signed out'}` + ); + + writeFileSync( + config, + JSON.stringify({ + ci: { + collect: { + numberOfRuns: Number(args.runs), + chromePath: ci.collect.chromePath, + settings: { + ...(formFactor === 'desktop' && { preset: 'desktop' }), + ...(cookie && { extraHeaders: { Cookie: cookie } }) + } + }, + // Search engines never see a signed-in page, so its SEO score is tracked but not held to a budget. + assert: { + assertions: Object.fromEntries( + Object.entries(ci.assert.assertions).filter(([id]) => !account || id !== 'categories:seo') + ) + }, + upload: { target: 'filesystem', outputDir: join(OUT, formFactor, account ?? 'signed-out') } + } + }) + ); + + // One page per collect, adding to the last: lhci gives up on the pages after one that fails to + // load, and here that page just has no runs in its summary. + for (const [index, page] of group.entries()) { + lhci('collect', config, `--url=${origin}${page.path}`, ...(index ? ['--additive'] : [])); + } + + lhci('assert', config); + lhci('upload', config); + + const lhrs = collectedLhrs(); + const failures = assertionFailures(); + + for (const page of group) { + const runs = lhrs.get(page.path) ?? []; + summary.lighthouseVersion ||= runs[0]?.lighthouseVersion ?? ''; + benchmarks.push(...runs.map((lhr) => lhr.environment.benchmarkIndex)); + summary.pages.push(summarizePage({ path: page.path, account: page.account }, runs, failures.get(page.path))); + } + } + + summary.benchmarkIndex = benchmarks.length ? median(benchmarks) : 0; + writeFileSync(join(OUT, `summary-${formFactor}.json`), `${JSON.stringify(summary, null, '\t')}\n`); + log(`${OUT}/summary-${formFactor}.json`); + } +} finally { + await stopServer(); +} diff --git a/apps/web/src/lib/server/atproto/oauth.test.ts b/apps/web/src/lib/server/atproto/oauth.test.ts index 1d48d8141..d435f2412 100644 --- a/apps/web/src/lib/server/atproto/oauth.test.ts +++ b/apps/web/src/lib/server/atproto/oauth.test.ts @@ -239,6 +239,36 @@ describe('OAuth scopes', () => { }); }); +describe('OAuthService', () => { + /** Signing in at an http PDS from a production build with ALLOW_PRIVATE_NETWORK as given. */ + const signInAtHttpPds = async (allowPrivateNetwork: string) => { + const oauth = new OAuthService({ + config: loadConfig({ + NODE_ENV: 'production', + ALLOW_PRIVATE_NETWORK: allowPrivateNetwork, + ROOT_IDENTIFIER: 'did:plc:root', + ROOT_PASSWORD: 'x', + PUBLIC_URL: 'https://localnetwork.test', + HAPPYVIEW_URL: 'http://127.0.0.1:8000' + }), + db: await openDatabase(':memory:'), + logger: silentLogger + }); + + // Port 9 is one fetch refuses to connect to, so nothing leaves the machine either way. + return oauth.client.authorize('http://127.0.0.1:9/'); + }; + + it('signs in at an http PDS in a production build only on a private network it may reach', async () => { + const refusedHttp = expect.stringMatching(/Unsecure resource metadata URL \(http:\)/); + + await expect(signInAtHttpPds('false')).rejects.toMatchObject({ cause: { message: refusedHttp } }); + await expect(signInAtHttpPds('true')).rejects.toMatchObject({ + cause: { message: expect.not.stringMatching(/Unsecure/) } + }); + }); +}); + describe('OAuthService.authorize', () => { const user = 'did:plc:user'; const page = new URL('https://pds.test/oauth/authorize'); diff --git a/apps/web/src/lib/server/atproto/oauth.ts b/apps/web/src/lib/server/atproto/oauth.ts index 7a499f365..487ee4edd 100644 --- a/apps/web/src/lib/server/atproto/oauth.ts +++ b/apps/web/src/lib/server/atproto/oauth.ts @@ -229,7 +229,9 @@ export class OAuthService { constructor(private readonly deps: { config: Config; db: Db; logger: Logger }) { this.client = new NodeOAuthClient({ clientMetadata: clientMetadata(deps.config, DECLARED_SCOPES), - allowHttp: deps.config.dev, + // An http PDS only lives on a private network, which ALLOW_PRIVATE_NETWORK already lets the + // production build reach (scripts/dev-network.ts --production, for Lighthouse's signed-in pages). + allowHttp: deps.config.dev || deps.config.allowPrivateNetwork, plcDirectoryUrl: deps.config.plcUrl, handleResolver: deps.config.handleResolverUrl, requestLock: requestLocalLock, diff --git a/apps/web/tests/e2e/app.e2e.ts b/apps/web/tests/e2e/app.e2e.ts index 798a5aaac..5d00ac534 100644 --- a/apps/web/tests/e2e/app.e2e.ts +++ b/apps/web/tests/e2e/app.e2e.ts @@ -1,6 +1,7 @@ import { expect, type Page, type Route, test } from '@playwright/test'; import headline from '../../src/lib/headline.json' with { type: 'json' }; import { z } from 'zod'; +import { signIn } from '../sign-in.ts'; import { expectAccessible } from './axe.ts'; /** @@ -29,20 +30,6 @@ function keepPdsCookies(page: Page) { test.beforeEach(({ page }) => keepPdsCookies(page)); -/** Signs in through the local PDS's OAuth screens with a seeded test account. */ -async function signIn(page: Page, handle: string, password: string, next = '/dashboard') { - await page.goto(`/login?next=${encodeURIComponent(next)}`); - await page.waitForLoadState('networkidle'); - await page.getByPlaceholder('alice.bsky.social').fill(handle); - await page.getByRole('button', { name: 'Continue' }).click(); - await page.locator('input[type="password"]').fill(password); - await page.getByRole('button', { name: 'Sign in' }).click(); - await page.getByRole('button', { name: 'Authorize' }).click(); - const target = new URL(next, 'http://127.0.0.1:5199'); - await page.waitForURL((url) => url.port === '5199' && url.pathname === target.pathname); - await page.waitForLoadState('networkidle'); -} - /** * Signs out from the dashboard and waits for the redirect home: /logout records the audit entry and * revokes the OAuth session first, and a sign-in started before it finishes would find the old session. diff --git a/apps/web/tests/sign-in.ts b/apps/web/tests/sign-in.ts new file mode 100644 index 000000000..622ad1b2c --- /dev/null +++ b/apps/web/tests/sign-in.ts @@ -0,0 +1,19 @@ +import type { Page } from 'playwright'; + +/** + * Signs in through the local PDS's OAuth screens with a seeded test account (scripts/dev-network.ts), + * for the end-to-end tests and Lighthouse's signed-in pages. The page's context needs a baseURL. + */ +export async function signIn(page: Page, handle: string, password: string, next = '/dashboard') { + await page.goto(`/login?next=${encodeURIComponent(next)}`); + const app = new URL(page.url()); + await page.waitForLoadState('networkidle'); + await page.getByPlaceholder('alice.bsky.social').fill(handle); + await page.getByRole('button', { name: 'Continue' }).click(); + await page.locator('input[type="password"]').fill(password); + await page.getByRole('button', { name: 'Sign in' }).click(); + await page.getByRole('button', { name: 'Authorize' }).click(); + const target = new URL(next, app); + await page.waitForURL((url) => url.origin === app.origin && url.pathname === target.pathname); + await page.waitForLoadState('networkidle'); +}