diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 82720c529..085ae46d5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -35,6 +35,9 @@ jobs: - name: Run Playwright smoke test run: npm run test:e2e working-directory: test-app + - name: Enforce WCAG 2.1 AA on test-app routes + run: npm run test:a11y + working-directory: test-app - name: Upload Playwright test artifacts if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/AGENTS.md b/AGENTS.md index 39fc97ec7..23e52453f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ This is a **raw-source, dual-workspace** Angular library, not a normal Angular C - **`coverage-floor.json` is a ratchet — it only ever moves up, and only via a dedicated bump.** Feature and spec PRs must **not** edit `coverage-floor.json` directly; raising it is a separate, deliberate commit made with `npm run coverage:bump` (rewrites the floor file to the currently measured coverage — preserving istanbul's fractional `pct`, not floored to a whole percent — and never lowering an existing entry). `.github/workflows/ci.yml` also runs `scripts/check-coverage-floor-not-decreased.mjs` on PRs, comparing `coverage-floor.json` against the base branch and failing if any metric's floor decreased, so a floor can't be hand-lowered in the same PR that regresses coverage. This mirrors the ESLint baseline guard below. - The gate scripts are covered by `node --test scripts/check-coverage.test.mjs` and `node --test scripts/check-coverage-floor-not-decreased.test.mjs` (Node's built-in test runner, no extra deps). These are **not currently wired into any CI workflow** — run them manually after touching either script. - Playwright E2E smoke test (`test-app/e2e/`) runs via `npm --prefix test-app run test:e2e`, gated separately in `.github/workflows/e2e.yml`. +- WCAG 2.1 AA accessibility gate (`test-app/playwright.a11y.config.ts`, spec in `test-app/e2e/accessibility/`) runs via `npm run test:a11y` (root, delegates to `test-app`) and is enforced in the same `e2e.yml` job. It mirrors the `playwright.a11y.config.{mjs,ts}` pattern in `sam-styles` and `ngx-uswds` (GSA/sam-ui-elements#708): axe-core scans every route declared in `test-app/src/app/app.module.ts` with the `wcag2a`/`wcag2aa`/`wcag21a`/`wcag21aa` tags and diffs the violation fingerprints (` | | `) against the committed `test-app/e2e/accessibility/wcag-2.1-aa-baseline.json`. **The baseline is a ratchet like `coverage-floor.json` and `eslint-baseline.json`**: new violations fail CI and must be fixed, not baselined; resolved violations fail CI until they're removed from the baseline. `UPDATE_A11Y_BASELINE=1 npm run test:a11y` drops resolved entries but still fails on new violations, so the file can only shrink. The smoke config (`test-app/playwright.config.ts`) sets `testIgnore: "**/accessibility/**"` so `test:e2e` doesn't re-run the scan under its 30 s budget. Because the spec reads the route table from `app.module.ts`, every new `-gallery` route is scanned automatically — adding a gallery route is how you put a component under the a11y gate. ### Choosing between Vitest and Playwright @@ -83,7 +84,7 @@ Two related traps worth knowing: - `.github/workflows/ci.yml` — installs both workspaces, runs `npm --prefix test-app test`, then `npm run coverage:check`. - `.github/workflows/lint.yml` — format check, baseline-not-increased guard, `ng lint` + baseline gate for both workspaces. -- `.github/workflows/e2e.yml` — Playwright smoke test. +- `.github/workflows/e2e.yml` — Playwright smoke test, then the WCAG 2.1 AA gate (`npm run test:a11y`). - `.github/workflows/security.yml` — DAST (OWASP ZAP) against `test-app` built in production mode; SAST is GitHub CodeQL default setup (no committed workflow). See `docs/security-scanning.md`. - `.github/workflows/publish.yml` — npm Trusted Publisher (OIDC) flow, gated on `validate:publish`; only trigger is a GitHub Release (or a dry-run `workflow_dispatch`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b3461bcc8..7a8833a2b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,10 @@ If you've found a bug or have a feature request, please [open an issue](../../is - Does PR include a component from another library? - [ ] The developer added test cases to cover integrating with library - [ ] If extending a third-party component, you have validated that is the best design solution +- Accessibility + - [ ] `npm run test:a11y` (the WCAG 2.1 AA gate in `test-app/playwright.a11y.config.ts`) passes against the committed `test-app/e2e/accessibility/wcag-2.1-aa-baseline.json` + - [ ] No new violation fingerprints were added to the baseline; fix the markup instead. Entries are only removed (after a fix) via `UPDATE_A11Y_BASELINE=1 npm run test:a11y` + - [ ] A component rendered in a new `test-app` gallery route is scanned automatically — confirm the route is declared in `test-app/src/app/app.module.ts` - HTML/CSS/UI Controls - [ ] HTML markup and CSS classes are semantic and meaningful - [ ] Styles are applied by classes, not through style tags on elements diff --git a/package.json b/package.json index 6ca3c5132..2a7470c9f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "lint:baseline:bump": "npm run lint:report && node scripts/check-lint-baseline.mjs --bump root eslint-report.json", "coverage:check": "node scripts/check-coverage.mjs test-app/coverage/coverage-summary.json", "coverage:bump": "node scripts/check-coverage.mjs --bump test-app/coverage/coverage-summary.json", + "test:a11y": "npm --prefix test-app run test:a11y", "validate:publish": "node scripts/validate-publish-package.mjs", "validate:security-workflow": "node scripts/validate-security-workflow.mjs" }, diff --git a/test-app/README.md b/test-app/README.md index 21663a2fd..73e2503a1 100644 --- a/test-app/README.md +++ b/test-app/README.md @@ -22,6 +22,10 @@ Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github. Run `npm run test:e2e` to execute the [Playwright](https://playwright.dev/) smoke test. It boots the dev server and asserts the app renders. In CI it runs headless; locally it will reuse a running dev server if one is already up. +## Running the accessibility gate + +Run `npm run test:a11y` to execute the WCAG 2.1 AA gate (`playwright.a11y.config.ts`, spec in `e2e/accessibility/`). It boots the dev server, visits every route declared in `src/app/app.module.ts`, scans each with [axe-core](https://github.com/dequelabs/axe-core) using the `wcag2a`, `wcag2aa`, `wcag21a`, and `wcag21aa` rule tags, and compares the violations against the committed baseline `e2e/accessibility/wcag-2.1-aa-baseline.json`. New violations fail the run; violations that no longer reproduce must be removed from the baseline. After fixing a violation, shrink the baseline with `UPDATE_A11Y_BASELINE=1 npm run test:a11y` and commit the result. Update mode still fails on new violations, so the baseline can only ever shrink. The smoke config (`playwright.config.ts`) ignores `e2e/accessibility/**`, so `npm run test:e2e` does not re-run the scan. + ## Further help To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI README](https://github.com/angular/angular-cli/blob/master/README.md). diff --git a/test-app/e2e/accessibility/test-app-a11y.spec.ts b/test-app/e2e/accessibility/test-app-a11y.spec.ts new file mode 100644 index 000000000..58814fcdb --- /dev/null +++ b/test-app/e2e/accessibility/test-app-a11y.spec.ts @@ -0,0 +1,149 @@ +import AxeBuilder from "@axe-core/playwright"; +import { expect, test } from "@playwright/test"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +// Runtime WCAG 2.1 AA gate for sam-ui-elements (GSA/sam-ui-elements#708). +// Walks every routed test-app gallery page with axe-core and compares the +// rendered violations against a committed baseline. New violations fail CI; +// resolved violations must be removed from the baseline. The baseline is the +// "triage, don't red-wall" mechanism borrowed from sam-styles and ngx-uswds: +// the pre-existing backlog is recorded, not blocking, and is burned down over +// time by deleting fingerprints as components are fixed. +// +// Regenerate the baseline after an intentional change with: +// UPDATE_A11Y_BASELINE=1 npm run test:a11y + +const WCAG_21_AA_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"]; +const BASELINE_PATH = join(__dirname, "wcag-2.1-aa-baseline.json"); +const APP_MODULE_PATH = join( + __dirname, + "..", + "..", + "src", + "app", + "app.module.ts" +); + +interface AxeNode { + target: string[]; +} + +interface AxeViolation { + id: string; + nodes: AxeNode[]; +} + +// test-app has no Storybook index; its pages are the gallery routes wired in +// `app.module.ts` (one route per component under test, per AGENTS.md). Read +// the route table from source so a new gallery route is scanned automatically +// and can never silently bypass the gate. +function testAppRoutes(): string[] { + const source = readFileSync(APP_MODULE_PATH, "utf8"); + const routes = [...source.matchAll(/\{\s*path:\s*"([^"]*)"/g)].map( + ([, path]) => `/${path}` + ); + return [...new Set(routes)].sort(); +} + +// Angular stamps view-encapsulation classes (`ng-tns-c-`) and +// content/host attributes (`_ngcontent-*`, `_nghost-*`) into rendered markup. +// Their hashes and indices shift with unrelated compilation or instance-order +// changes, so leaving them in a committed fingerprint makes the same violation +// churn as simultaneously "new" and "resolved". Strip them so a fingerprint is +// stable across builds. +function normalizeTarget(selector: string): string { + return selector + .replace(/\.ng-tns-c\d+-\d+/g, "") + .replace(/\[_ngcontent-[^\]]+\]/g, "") + .replace(/\[_nghost-[^\]]+\]/g, "") + .replace(/\s{2,}/g, " ") + .trim(); +} + +function violationFingerprints( + route: string, + violations: AxeViolation[] +): string[] { + return violations.flatMap((violation) => + violation.nodes.map((node) => + [route, violation.id, ...node.target.map(normalizeTarget)].join(" | ") + ) + ); +} + +test("axe detects a rendered WCAG 2.1 AA violation", async ({ page }) => { + // Sanity check: prove the harness actually reports contrast failures, so a + // green route run means "no violations" and never "axe silently did nothing". + await page.setContent( + '

Text

' + ); + + const results = await new AxeBuilder({ page }) + .withTags(WCAG_21_AA_TAGS) + .analyze(); + + expect(results.violations.map(({ id }) => id)).toContain("color-contrast"); +}); + +test("test-app routes have no unresolved WCAG 2.1 AA violations", async ({ + context, +}) => { + const routes = testAppRoutes(); + expect( + routes, + "app.module.ts should declare at least the home route" + ).toContain("/"); + + const fingerprints: string[] = []; + for (const route of routes) { + // Fresh page per route: a clean frame with no carried-over render/axe state + // from the previous iteration. + const routePage = await context.newPage(); + try { + await routePage.goto(route); + await expect( + routePage.locator("app-root h1").first(), + `${route} should finish rendering` + ).toBeVisible(); + + const results = await new AxeBuilder({ page: routePage }) + .withTags(WCAG_21_AA_TAGS) + .analyze(); + + fingerprints.push( + ...violationFingerprints(route, results.violations as AxeViolation[]) + ); + } finally { + await routePage.close(); + } + } + + const current = [...new Set(fingerprints)].sort(); + const baseline = JSON.parse(readFileSync(BASELINE_PATH, "utf8")) as string[]; + const baselineSet = new Set(baseline); + const currentSet = new Set(current); + const newViolations = current.filter( + (fingerprint) => !baselineSet.has(fingerprint) + ); + const resolvedViolations = baseline.filter( + (fingerprint) => !currentSet.has(fingerprint) + ); + + // New violations always fail, even in update mode: the baseline is a ratchet + // that only ever shrinks. UPDATE_A11Y_BASELINE=1 may only drop resolved entries. + expect( + newViolations, + "New WCAG 2.1 A/AA violations (fix them; the baseline only ever shrinks)" + ).toEqual([]); + + if (process.env.UPDATE_A11Y_BASELINE === "1") { + writeFileSync(BASELINE_PATH, `${JSON.stringify(current, null, 2)}\n`); + return; + } + + expect( + resolvedViolations, + "Resolved WCAG 2.1 A/AA violations (remove them from the baseline with UPDATE_A11Y_BASELINE=1)" + ).toEqual([]); +}); diff --git a/test-app/e2e/accessibility/wcag-2.1-aa-baseline.json b/test-app/e2e/accessibility/wcag-2.1-aa-baseline.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/test-app/e2e/accessibility/wcag-2.1-aa-baseline.json @@ -0,0 +1 @@ +[] diff --git a/test-app/package-lock.json b/test-app/package-lock.json index 1d3e3ca12..0c91f30f4 100644 --- a/test-app/package-lock.json +++ b/test-app/package-lock.json @@ -41,6 +41,7 @@ "@angular/cli": "21.2.26", "@angular/compiler-cli": "21.2.25", "@angular/language-service": "21.2.25", + "@axe-core/playwright": "4.13.0", "@playwright/test": "1.63.0", "@types/node": "26.6.5", "@vitest/coverage-istanbul": "4.1.11", @@ -1328,6 +1329,19 @@ "node": "20 || >=22" } }, + "node_modules/@axe-core/playwright": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.13.0.tgz", + "integrity": "sha512-6YLx+kxXu5GJceG4ozFg+33a2EMTdjYwWGloJ3sb9Kta5pp+ZNS53uxGVog5JetIY8s++P5UrtX+cri+u0VAVg==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.13.0" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -9667,6 +9681,16 @@ "postcss": "^8.1.0" } }, + "node_modules/axe-core": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.13.0.tgz", + "integrity": "sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/axobject-query": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", diff --git a/test-app/package.json b/test-app/package.json index d96282baf..fe9900f7b 100644 --- a/test-app/package.json +++ b/test-app/package.json @@ -9,6 +9,7 @@ "test": "vitest run --coverage --config vitest.config.mts", "test:watch": "vitest --config vitest.config.mts", "test:e2e": "playwright test", + "test:a11y": "playwright test --config=playwright.a11y.config.ts", "lint": "ng lint", "lint:report": "ng lint --format json --output-file eslint-report.json", "lint:baseline": "npm run lint:report && node ../scripts/check-lint-baseline.mjs test-app eslint-report.json", @@ -48,6 +49,7 @@ "@angular/cli": "21.2.26", "@angular/compiler-cli": "21.2.25", "@angular/language-service": "21.2.25", + "@axe-core/playwright": "4.13.0", "@playwright/test": "1.63.0", "@types/node": "26.6.5", "@vitest/coverage-istanbul": "4.1.11", diff --git a/test-app/playwright.a11y.config.ts b/test-app/playwright.a11y.config.ts new file mode 100644 index 000000000..5ccbeacef --- /dev/null +++ b/test-app/playwright.a11y.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "@playwright/test"; +import baseConfig from "./playwright.config"; + +// Dedicated config for the WCAG 2.1 AA gate (GSA/sam-ui-elements#708), +// mirroring `playwright.a11y.config.{mjs,ts}` in sam-styles and ngx-uswds. +// Reuses the base Playwright setup (same `ng serve` webServer) but points at +// the accessibility spec dir and runs serially with a generous timeout because +// axe walks every test-app route in a single worker. +export default defineConfig({ + ...baseConfig, + testDir: "./e2e/accessibility", + testIgnore: [], + timeout: 600_000, + fullyParallel: false, + workers: 1, +}); diff --git a/test-app/playwright.config.ts b/test-app/playwright.config.ts index 1dd0381a6..8089a7c14 100644 --- a/test-app/playwright.config.ts +++ b/test-app/playwright.config.ts @@ -2,6 +2,8 @@ import { defineConfig, devices } from "@playwright/test"; export default defineConfig({ testDir: "./e2e", + // The WCAG gate has its own config (playwright.a11y.config.ts) and budget. + testIgnore: "**/accessibility/**", timeout: 30_000, expect: { timeout: 5_000,