From 12bcb936f383047eb424178fc6e707cb2cbf084b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:35:29 +0000 Subject: [PATCH 1/2] Add WCAG 2.1 AA accessibility gate for test-app routes (GSA/sam-ui-elements#708) Mirror the playwright.a11y.config pattern from GSA/sam-styles and GSA/ngx-uswds: an axe-core-backed Playwright spec scans every route declared in test-app/src/app/app.module.ts with the wcag2a/wcag2aa/ wcag21a/wcag21aa tags and diffs violation fingerprints against a committed baseline (currently empty). New violations fail CI; resolved violations must be removed from the baseline. Runs as npm run test:a11y and as a step in e2e.yml. Co-Authored-By: Erik Rozi --- .github/workflows/e2e.yml | 3 + AGENTS.md | 3 +- CONTRIBUTING.md | 4 + package.json | 1 + test-app/README.md | 4 + .../e2e/accessibility/test-app-a11y.spec.ts | 146 ++++++++++++++++++ .../accessibility/wcag-2.1-aa-baseline.json | 1 + test-app/package-lock.json | 24 +++ test-app/package.json | 2 + test-app/playwright.a11y.config.ts | 15 ++ 10 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 test-app/e2e/accessibility/test-app-a11y.spec.ts create mode 100644 test-app/e2e/accessibility/wcag-2.1-aa-baseline.json create mode 100644 test-app/playwright.a11y.config.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 82720c529..085ae46d5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -35,6 +35,9 @@ jobs: - name: Run Playwright smoke test run: npm run test:e2e working-directory: test-app + - name: Enforce WCAG 2.1 AA on test-app routes + run: npm run test:a11y + working-directory: test-app - name: Upload Playwright test artifacts if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/AGENTS.md b/AGENTS.md index 39fc97ec7..7825e5b06 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ This is a **raw-source, dual-workspace** Angular library, not a normal Angular C - **`coverage-floor.json` is a ratchet — it only ever moves up, and only via a dedicated bump.** Feature and spec PRs must **not** edit `coverage-floor.json` directly; raising it is a separate, deliberate commit made with `npm run coverage:bump` (rewrites the floor file to the currently measured coverage — preserving istanbul's fractional `pct`, not floored to a whole percent — and never lowering an existing entry). `.github/workflows/ci.yml` also runs `scripts/check-coverage-floor-not-decreased.mjs` on PRs, comparing `coverage-floor.json` against the base branch and failing if any metric's floor decreased, so a floor can't be hand-lowered in the same PR that regresses coverage. This mirrors the ESLint baseline guard below. - The gate scripts are covered by `node --test scripts/check-coverage.test.mjs` and `node --test scripts/check-coverage-floor-not-decreased.test.mjs` (Node's built-in test runner, no extra deps). These are **not currently wired into any CI workflow** — run them manually after touching either script. - Playwright E2E smoke test (`test-app/e2e/`) runs via `npm --prefix test-app run test:e2e`, gated separately in `.github/workflows/e2e.yml`. +- WCAG 2.1 AA accessibility gate (`test-app/playwright.a11y.config.ts`, spec in `test-app/e2e/accessibility/`) runs via `npm run test:a11y` (root, delegates to `test-app`) and is enforced in the same `e2e.yml` job. It mirrors the `playwright.a11y.config.{mjs,ts}` pattern in `sam-styles` and `ngx-uswds` (GSA/sam-ui-elements#708): axe-core scans every route declared in `test-app/src/app/app.module.ts` with the `wcag2a`/`wcag2aa`/`wcag21a`/`wcag21aa` tags and diffs the violation fingerprints (` | | `) against the committed `test-app/e2e/accessibility/wcag-2.1-aa-baseline.json`. **The baseline is a ratchet like `coverage-floor.json` and `eslint-baseline.json`**: new violations fail CI and must be fixed, not baselined; resolved violations fail CI until they're removed from the baseline. `UPDATE_A11Y_BASELINE=1 npm run test:a11y` rewrites it. Because the spec reads the route table from `app.module.ts`, every new `-gallery` route is scanned automatically — adding a gallery route is how you put a component under the a11y gate. ### Choosing between Vitest and Playwright @@ -83,7 +84,7 @@ Two related traps worth knowing: - `.github/workflows/ci.yml` — installs both workspaces, runs `npm --prefix test-app test`, then `npm run coverage:check`. - `.github/workflows/lint.yml` — format check, baseline-not-increased guard, `ng lint` + baseline gate for both workspaces. -- `.github/workflows/e2e.yml` — Playwright smoke test. +- `.github/workflows/e2e.yml` — Playwright smoke test, then the WCAG 2.1 AA gate (`npm run test:a11y`). - `.github/workflows/security.yml` — DAST (OWASP ZAP) against `test-app` built in production mode; SAST is GitHub CodeQL default setup (no committed workflow). See `docs/security-scanning.md`. - `.github/workflows/publish.yml` — npm Trusted Publisher (OIDC) flow, gated on `validate:publish`; only trigger is a GitHub Release (or a dry-run `workflow_dispatch`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3461bcc8..7a8833a2b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,10 @@ If you've found a bug or have a feature request, please [open an issue](../../is - Does PR include a component from another library? - [ ] The developer added test cases to cover integrating with library - [ ] If extending a third-party component, you have validated that is the best design solution +- Accessibility + - [ ] `npm run test:a11y` (the WCAG 2.1 AA gate in `test-app/playwright.a11y.config.ts`) passes against the committed `test-app/e2e/accessibility/wcag-2.1-aa-baseline.json` + - [ ] No new violation fingerprints were added to the baseline; fix the markup instead. Entries are only removed (after a fix) via `UPDATE_A11Y_BASELINE=1 npm run test:a11y` + - [ ] A component rendered in a new `test-app` gallery route is scanned automatically — confirm the route is declared in `test-app/src/app/app.module.ts` - HTML/CSS/UI Controls - [ ] HTML markup and CSS classes are semantic and meaningful - [ ] Styles are applied by classes, not through style tags on elements diff --git a/package.json b/package.json index 6ca3c5132..2a7470c9f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "lint:baseline:bump": "npm run lint:report && node scripts/check-lint-baseline.mjs --bump root eslint-report.json", "coverage:check": "node scripts/check-coverage.mjs test-app/coverage/coverage-summary.json", "coverage:bump": "node scripts/check-coverage.mjs --bump test-app/coverage/coverage-summary.json", + "test:a11y": "npm --prefix test-app run test:a11y", "validate:publish": "node scripts/validate-publish-package.mjs", "validate:security-workflow": "node scripts/validate-security-workflow.mjs" }, diff --git a/test-app/README.md b/test-app/README.md index 21663a2fd..f314d173f 100644 --- a/test-app/README.md +++ b/test-app/README.md @@ -22,6 +22,10 @@ Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github. Run `npm run test:e2e` to execute the [Playwright](https://playwright.dev/) smoke test. It boots the dev server and asserts the app renders. In CI it runs headless; locally it will reuse a running dev server if one is already up. +## Running the accessibility gate + +Run `npm run test:a11y` to execute the WCAG 2.1 AA gate (`playwright.a11y.config.ts`, spec in `e2e/accessibility/`). It boots the dev server, visits every route declared in `src/app/app.module.ts`, scans each with [axe-core](https://github.com/dequelabs/axe-core) using the `wcag2a`, `wcag2aa`, `wcag21a`, and `wcag21aa` rule tags, and compares the violations against the committed baseline `e2e/accessibility/wcag-2.1-aa-baseline.json`. New violations fail the run; violations that no longer reproduce must be removed from the baseline. After an intentional change, regenerate the baseline with `UPDATE_A11Y_BASELINE=1 npm run test:a11y` and commit the result. + ## Further help To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI README](https://github.com/angular/angular-cli/blob/master/README.md). diff --git a/test-app/e2e/accessibility/test-app-a11y.spec.ts b/test-app/e2e/accessibility/test-app-a11y.spec.ts new file mode 100644 index 000000000..e035bead2 --- /dev/null +++ b/test-app/e2e/accessibility/test-app-a11y.spec.ts @@ -0,0 +1,146 @@ +import AxeBuilder from "@axe-core/playwright"; +import { expect, test } from "@playwright/test"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +// Runtime WCAG 2.1 AA gate for sam-ui-elements (GSA/sam-ui-elements#708). +// Walks every routed test-app gallery page with axe-core and compares the +// rendered violations against a committed baseline. New violations fail CI; +// resolved violations must be removed from the baseline. The baseline is the +// "triage, don't red-wall" mechanism borrowed from sam-styles and ngx-uswds: +// the pre-existing backlog is recorded, not blocking, and is burned down over +// time by deleting fingerprints as components are fixed. +// +// Regenerate the baseline after an intentional change with: +// UPDATE_A11Y_BASELINE=1 npm run test:a11y + +const WCAG_21_AA_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"]; +const BASELINE_PATH = join(__dirname, "wcag-2.1-aa-baseline.json"); +const APP_MODULE_PATH = join( + __dirname, + "..", + "..", + "src", + "app", + "app.module.ts" +); + +interface AxeNode { + target: string[]; +} + +interface AxeViolation { + id: string; + nodes: AxeNode[]; +} + +// test-app has no Storybook index; its pages are the gallery routes wired in +// `app.module.ts` (one route per component under test, per AGENTS.md). Read +// the route table from source so a new gallery route is scanned automatically +// and can never silently bypass the gate. +function testAppRoutes(): string[] { + const source = readFileSync(APP_MODULE_PATH, "utf8"); + const routes = [...source.matchAll(/\{\s*path:\s*"([^"]*)"/g)].map( + ([, path]) => `/${path}` + ); + return [...new Set(routes)].sort(); +} + +// Angular stamps view-encapsulation classes (`ng-tns-c-`) and +// content/host attributes (`_ngcontent-*`, `_nghost-*`) into rendered markup. +// Their hashes and indices shift with unrelated compilation or instance-order +// changes, so leaving them in a committed fingerprint makes the same violation +// churn as simultaneously "new" and "resolved". Strip them so a fingerprint is +// stable across builds. +function normalizeTarget(selector: string): string { + return selector + .replace(/\.ng-tns-c\d+-\d+/g, "") + .replace(/\[_ngcontent-[^\]]+\]/g, "") + .replace(/\[_nghost-[^\]]+\]/g, "") + .replace(/\s{2,}/g, " ") + .trim(); +} + +function violationFingerprints( + route: string, + violations: AxeViolation[] +): string[] { + return violations.flatMap((violation) => + violation.nodes.map((node) => + [route, violation.id, ...node.target.map(normalizeTarget)].join(" | ") + ) + ); +} + +test("axe detects a rendered WCAG 2.1 AA violation", async ({ page }) => { + // Sanity check: prove the harness actually reports contrast failures, so a + // green route run means "no violations" and never "axe silently did nothing". + await page.setContent( + '

Text

' + ); + + const results = await new AxeBuilder({ page }) + .withTags(WCAG_21_AA_TAGS) + .analyze(); + + expect(results.violations.map(({ id }) => id)).toContain("color-contrast"); +}); + +test("test-app routes have no unresolved WCAG 2.1 AA violations", async ({ + context, +}) => { + const routes = testAppRoutes(); + expect( + routes, + "app.module.ts should declare at least the home route" + ).toContain("/"); + + const fingerprints: string[] = []; + for (const route of routes) { + // Fresh page per route: a clean frame with no carried-over render/axe state + // from the previous iteration. + const routePage = await context.newPage(); + try { + await routePage.goto(route); + await expect( + routePage.locator("app-root h1").first(), + `${route} should finish rendering` + ).toBeVisible(); + + const results = await new AxeBuilder({ page: routePage }) + .withTags(WCAG_21_AA_TAGS) + .analyze(); + + fingerprints.push( + ...violationFingerprints(route, results.violations as AxeViolation[]) + ); + } finally { + await routePage.close(); + } + } + + const current = [...new Set(fingerprints)].sort(); + + if (process.env.UPDATE_A11Y_BASELINE === "1") { + writeFileSync(BASELINE_PATH, `${JSON.stringify(current, null, 2)}\n`); + } + + const baseline = JSON.parse(readFileSync(BASELINE_PATH, "utf8")) as string[]; + const baselineSet = new Set(baseline); + const currentSet = new Set(current); + const newViolations = current.filter( + (fingerprint) => !baselineSet.has(fingerprint) + ); + const resolvedViolations = baseline.filter( + (fingerprint) => !currentSet.has(fingerprint) + ); + + expect( + newViolations, + "New WCAG 2.1 A/AA violations (fix them; do not update the baseline)" + ).toEqual([]); + expect( + resolvedViolations, + "Resolved WCAG 2.1 A/AA violations (remove them from the baseline with UPDATE_A11Y_BASELINE=1)" + ).toEqual([]); +}); diff --git a/test-app/e2e/accessibility/wcag-2.1-aa-baseline.json b/test-app/e2e/accessibility/wcag-2.1-aa-baseline.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/test-app/e2e/accessibility/wcag-2.1-aa-baseline.json @@ -0,0 +1 @@ +[] diff --git a/test-app/package-lock.json b/test-app/package-lock.json index 1d3e3ca12..0c91f30f4 100644 --- a/test-app/package-lock.json +++ b/test-app/package-lock.json @@ -41,6 +41,7 @@ "@angular/cli": "21.2.26", "@angular/compiler-cli": "21.2.25", "@angular/language-service": "21.2.25", + "@axe-core/playwright": "4.13.0", "@playwright/test": "1.63.0", "@types/node": "26.6.5", "@vitest/coverage-istanbul": "4.1.11", @@ -1328,6 +1329,19 @@ "node": "20 || >=22" } }, + "node_modules/@axe-core/playwright": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.13.0.tgz", + "integrity": "sha512-6YLx+kxXu5GJceG4ozFg+33a2EMTdjYwWGloJ3sb9Kta5pp+ZNS53uxGVog5JetIY8s++P5UrtX+cri+u0VAVg==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.13.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -9667,6 +9681,16 @@ "postcss": "^8.1.0" } }, + "node_modules/axe-core": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", + "integrity": "sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", diff --git a/test-app/package.json b/test-app/package.json index d96282baf..fe9900f7b 100644 --- a/test-app/package.json +++ b/test-app/package.json @@ -9,6 +9,7 @@ "test": "vitest run --coverage --config vitest.config.mts", "test:watch": "vitest --config vitest.config.mts", "test:e2e": "playwright test", + "test:a11y": "playwright test --config=playwright.a11y.config.ts", "lint": "ng lint", "lint:report": "ng lint --format json --output-file eslint-report.json", "lint:baseline": "npm run lint:report && node ../scripts/check-lint-baseline.mjs test-app eslint-report.json", @@ -48,6 +49,7 @@ "@angular/cli": "21.2.26", "@angular/compiler-cli": "21.2.25", "@angular/language-service": "21.2.25", + "@axe-core/playwright": "4.13.0", "@playwright/test": "1.63.0", "@types/node": "26.6.5", "@vitest/coverage-istanbul": "4.1.11", diff --git a/test-app/playwright.a11y.config.ts b/test-app/playwright.a11y.config.ts new file mode 100644 index 000000000..eb6da0098 --- /dev/null +++ b/test-app/playwright.a11y.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "@playwright/test"; +import baseConfig from "./playwright.config"; + +// Dedicated config for the WCAG 2.1 AA gate (GSA/sam-ui-elements#708), +// mirroring `playwright.a11y.config.{mjs,ts}` in sam-styles and ngx-uswds. +// Reuses the base Playwright setup (same `ng serve` webServer) but points at +// the accessibility spec dir and runs serially with a generous timeout because +// axe walks every test-app route in a single worker. +export default defineConfig({ + ...baseConfig, + testDir: "./e2e/accessibility", + timeout: 600_000, + fullyParallel: false, + workers: 1, +}); From d49ce3a86672ec91bf7a6c20593d283fc27b46dc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 02:40:27 +0000 Subject: [PATCH 2/2] a11y gate: fail on new violations in update mode; exclude a11y spec from smoke config - UPDATE_A11Y_BASELINE=1 now only drops resolved fingerprints; new violations still fail, so the committed baseline can only shrink. - playwright.config.ts ignores e2e/accessibility/** so test:e2e no longer runs the route scan under its 30 s timeout; playwright.a11y.config.ts clears the ignore for the dedicated run. Co-Authored-By: Erik Rozi --- AGENTS.md | 2 +- test-app/README.md | 2 +- test-app/e2e/accessibility/test-app-a11y.spec.ts | 15 +++++++++------ test-app/playwright.a11y.config.ts | 1 + test-app/playwright.config.ts | 2 ++ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7825e5b06..23e52453f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ This is a **raw-source, dual-workspace** Angular library, not a normal Angular C - **`coverage-floor.json` is a ratchet — it only ever moves up, and only via a dedicated bump.** Feature and spec PRs must **not** edit `coverage-floor.json` directly; raising it is a separate, deliberate commit made with `npm run coverage:bump` (rewrites the floor file to the currently measured coverage — preserving istanbul's fractional `pct`, not floored to a whole percent — and never lowering an existing entry). `.github/workflows/ci.yml` also runs `scripts/check-coverage-floor-not-decreased.mjs` on PRs, comparing `coverage-floor.json` against the base branch and failing if any metric's floor decreased, so a floor can't be hand-lowered in the same PR that regresses coverage. This mirrors the ESLint baseline guard below. - The gate scripts are covered by `node --test scripts/check-coverage.test.mjs` and `node --test scripts/check-coverage-floor-not-decreased.test.mjs` (Node's built-in test runner, no extra deps). These are **not currently wired into any CI workflow** — run them manually after touching either script. - Playwright E2E smoke test (`test-app/e2e/`) runs via `npm --prefix test-app run test:e2e`, gated separately in `.github/workflows/e2e.yml`. -- WCAG 2.1 AA accessibility gate (`test-app/playwright.a11y.config.ts`, spec in `test-app/e2e/accessibility/`) runs via `npm run test:a11y` (root, delegates to `test-app`) and is enforced in the same `e2e.yml` job. It mirrors the `playwright.a11y.config.{mjs,ts}` pattern in `sam-styles` and `ngx-uswds` (GSA/sam-ui-elements#708): axe-core scans every route declared in `test-app/src/app/app.module.ts` with the `wcag2a`/`wcag2aa`/`wcag21a`/`wcag21aa` tags and diffs the violation fingerprints (` | | `) against the committed `test-app/e2e/accessibility/wcag-2.1-aa-baseline.json`. **The baseline is a ratchet like `coverage-floor.json` and `eslint-baseline.json`**: new violations fail CI and must be fixed, not baselined; resolved violations fail CI until they're removed from the baseline. `UPDATE_A11Y_BASELINE=1 npm run test:a11y` rewrites it. Because the spec reads the route table from `app.module.ts`, every new `-gallery` route is scanned automatically — adding a gallery route is how you put a component under the a11y gate. +- WCAG 2.1 AA accessibility gate (`test-app/playwright.a11y.config.ts`, spec in `test-app/e2e/accessibility/`) runs via `npm run test:a11y` (root, delegates to `test-app`) and is enforced in the same `e2e.yml` job. It mirrors the `playwright.a11y.config.{mjs,ts}` pattern in `sam-styles` and `ngx-uswds` (GSA/sam-ui-elements#708): axe-core scans every route declared in `test-app/src/app/app.module.ts` with the `wcag2a`/`wcag2aa`/`wcag21a`/`wcag21aa` tags and diffs the violation fingerprints (` | | `) against the committed `test-app/e2e/accessibility/wcag-2.1-aa-baseline.json`. **The baseline is a ratchet like `coverage-floor.json` and `eslint-baseline.json`**: new violations fail CI and must be fixed, not baselined; resolved violations fail CI until they're removed from the baseline. `UPDATE_A11Y_BASELINE=1 npm run test:a11y` drops resolved entries but still fails on new violations, so the file can only shrink. The smoke config (`test-app/playwright.config.ts`) sets `testIgnore: "**/accessibility/**"` so `test:e2e` doesn't re-run the scan under its 30 s budget. Because the spec reads the route table from `app.module.ts`, every new `-gallery` route is scanned automatically — adding a gallery route is how you put a component under the a11y gate. ### Choosing between Vitest and Playwright diff --git a/test-app/README.md b/test-app/README.md index f314d173f..73e2503a1 100644 --- a/test-app/README.md +++ b/test-app/README.md @@ -24,7 +24,7 @@ Run `npm run test:e2e` to execute the [Playwright](https://playwright.dev/) smok ## Running the accessibility gate -Run `npm run test:a11y` to execute the WCAG 2.1 AA gate (`playwright.a11y.config.ts`, spec in `e2e/accessibility/`). It boots the dev server, visits every route declared in `src/app/app.module.ts`, scans each with [axe-core](https://github.com/dequelabs/axe-core) using the `wcag2a`, `wcag2aa`, `wcag21a`, and `wcag21aa` rule tags, and compares the violations against the committed baseline `e2e/accessibility/wcag-2.1-aa-baseline.json`. New violations fail the run; violations that no longer reproduce must be removed from the baseline. After an intentional change, regenerate the baseline with `UPDATE_A11Y_BASELINE=1 npm run test:a11y` and commit the result. +Run `npm run test:a11y` to execute the WCAG 2.1 AA gate (`playwright.a11y.config.ts`, spec in `e2e/accessibility/`). It boots the dev server, visits every route declared in `src/app/app.module.ts`, scans each with [axe-core](https://github.com/dequelabs/axe-core) using the `wcag2a`, `wcag2aa`, `wcag21a`, and `wcag21aa` rule tags, and compares the violations against the committed baseline `e2e/accessibility/wcag-2.1-aa-baseline.json`. New violations fail the run; violations that no longer reproduce must be removed from the baseline. After fixing a violation, shrink the baseline with `UPDATE_A11Y_BASELINE=1 npm run test:a11y` and commit the result. Update mode still fails on new violations, so the baseline can only ever shrink. The smoke config (`playwright.config.ts`) ignores `e2e/accessibility/**`, so `npm run test:e2e` does not re-run the scan. ## Further help diff --git a/test-app/e2e/accessibility/test-app-a11y.spec.ts b/test-app/e2e/accessibility/test-app-a11y.spec.ts index e035bead2..58814fcdb 100644 --- a/test-app/e2e/accessibility/test-app-a11y.spec.ts +++ b/test-app/e2e/accessibility/test-app-a11y.spec.ts @@ -120,11 +120,6 @@ test("test-app routes have no unresolved WCAG 2.1 AA violations", async ({ } const current = [...new Set(fingerprints)].sort(); - - if (process.env.UPDATE_A11Y_BASELINE === "1") { - writeFileSync(BASELINE_PATH, `${JSON.stringify(current, null, 2)}\n`); - } - const baseline = JSON.parse(readFileSync(BASELINE_PATH, "utf8")) as string[]; const baselineSet = new Set(baseline); const currentSet = new Set(current); @@ -135,10 +130,18 @@ test("test-app routes have no unresolved WCAG 2.1 AA violations", async ({ (fingerprint) => !currentSet.has(fingerprint) ); + // New violations always fail, even in update mode: the baseline is a ratchet + // that only ever shrinks. UPDATE_A11Y_BASELINE=1 may only drop resolved entries. expect( newViolations, - "New WCAG 2.1 A/AA violations (fix them; do not update the baseline)" + "New WCAG 2.1 A/AA violations (fix them; the baseline only ever shrinks)" ).toEqual([]); + + if (process.env.UPDATE_A11Y_BASELINE === "1") { + writeFileSync(BASELINE_PATH, `${JSON.stringify(current, null, 2)}\n`); + return; + } + expect( resolvedViolations, "Resolved WCAG 2.1 A/AA violations (remove them from the baseline with UPDATE_A11Y_BASELINE=1)" diff --git a/test-app/playwright.a11y.config.ts b/test-app/playwright.a11y.config.ts index eb6da0098..5ccbeacef 100644 --- a/test-app/playwright.a11y.config.ts +++ b/test-app/playwright.a11y.config.ts @@ -9,6 +9,7 @@ import baseConfig from "./playwright.config"; export default defineConfig({ ...baseConfig, testDir: "./e2e/accessibility", + testIgnore: [], timeout: 600_000, fullyParallel: false, workers: 1, diff --git a/test-app/playwright.config.ts b/test-app/playwright.config.ts index 1dd0381a6..8089a7c14 100644 --- a/test-app/playwright.config.ts +++ b/test-app/playwright.config.ts @@ -2,6 +2,8 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./e2e", + // The WCAG gate has its own config (playwright.a11y.config.ts) and budget. + testIgnore: "**/accessibility/**", timeout: 30_000, expect: { timeout: 5_000,