Repository navigation
Add WCAG 2.1 AA accessibility gate for test-app routes (GSA/sam-ui-elements#708) #2
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| import AxeBuilder from "@axe-core/playwright"; | ||
| import { expect, test } from "@playwright/test"; | ||
| import { readFileSync, writeFileSync } from "node:fs"; | ||
| import { join } from "node:path"; | ||
|
|
||
| // Runtime WCAG 2.1 AA gate for sam-ui-elements (GSA/sam-ui-elements#708). | ||
| // Walks every routed test-app gallery page with axe-core and compares the | ||
| // rendered violations against a committed baseline. New violations fail CI; | ||
| // resolved violations must be removed from the baseline. The baseline is the | ||
| // "triage, don't red-wall" mechanism borrowed from sam-styles and ngx-uswds: | ||
| // the pre-existing backlog is recorded, not blocking, and is burned down over | ||
| // time by deleting fingerprints as components are fixed. | ||
| // | ||
| // Regenerate the baseline after an intentional change with: | ||
| // UPDATE_A11Y_BASELINE=1 npm run test:a11y | ||
|
|
||
| const WCAG_21_AA_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"]; | ||
| const BASELINE_PATH = join(__dirname, "wcag-2.1-aa-baseline.json"); | ||
| const APP_MODULE_PATH = join( | ||
| __dirname, | ||
| "..", | ||
| "..", | ||
| "src", | ||
| "app", | ||
| "app.module.ts" | ||
| ); | ||
|
|
||
| interface AxeNode { | ||
| target: string[]; | ||
| } | ||
|
|
||
| interface AxeViolation { | ||
| id: string; | ||
| nodes: AxeNode[]; | ||
| } | ||
|
|
||
| // test-app has no Storybook index; its pages are the gallery routes wired in | ||
| // `app.module.ts` (one route per component under test, per AGENTS.md). Read | ||
| // the route table from source so a new gallery route is scanned automatically | ||
| // and can never silently bypass the gate. | ||
| function testAppRoutes(): string[] { | ||
| const source = readFileSync(APP_MODULE_PATH, "utf8"); | ||
| const routes = [...source.matchAll(/\{\s*path:\s*"([^"]*)"/g)].map( | ||
| ([, path]) => `/${path}` | ||
| ); | ||
| return [...new Set(routes)].sort(); | ||
| } | ||
|
|
||
| // Angular stamps view-encapsulation classes (`ng-tns-c<hash>-<index>`) and | ||
| // content/host attributes (`_ngcontent-*`, `_nghost-*`) into rendered markup. | ||
| // Their hashes and indices shift with unrelated compilation or instance-order | ||
| // changes, so leaving them in a committed fingerprint makes the same violation | ||
| // churn as simultaneously "new" and "resolved". Strip them so a fingerprint is | ||
| // stable across builds. | ||
| function normalizeTarget(selector: string): string { | ||
| return selector | ||
| .replace(/\.ng-tns-c\d+-\d+/g, "") | ||
| .replace(/\[_ngcontent-[^\]]+\]/g, "") | ||
| .replace(/\[_nghost-[^\]]+\]/g, "") | ||
| .replace(/\s{2,}/g, " ") | ||
| .trim(); | ||
| } | ||
|
|
||
| function violationFingerprints( | ||
| route: string, | ||
| violations: AxeViolation[] | ||
| ): string[] { | ||
| return violations.flatMap((violation) => | ||
| violation.nodes.map((node) => | ||
| [route, violation.id, ...node.target.map(normalizeTarget)].join(" | ") | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| test("axe detects a rendered WCAG 2.1 AA violation", async ({ page }) => { | ||
| // Sanity check: prove the harness actually reports contrast failures, so a | ||
| // green route run means "no violations" and never "axe silently did nothing". | ||
| await page.setContent( | ||
| '<main><p style="color:#aaa;background:#fff">Text</p></main>' | ||
| ); | ||
|
|
||
| const results = await new AxeBuilder({ page }) | ||
| .withTags(WCAG_21_AA_TAGS) | ||
| .analyze(); | ||
|
|
||
| expect(results.violations.map(({ id }) => id)).toContain("color-contrast"); | ||
| }); | ||
|
|
||
| test("test-app routes have no unresolved WCAG 2.1 AA violations", async ({ | ||
| context, | ||
| }) => { | ||
| const routes = testAppRoutes(); | ||
| expect( | ||
| routes, | ||
| "app.module.ts should declare at least the home route" | ||
| ).toContain("/"); | ||
|
|
||
| const fingerprints: string[] = []; | ||
| for (const route of routes) { | ||
| // Fresh page per route: a clean frame with no carried-over render/axe state | ||
| // from the previous iteration. | ||
| const routePage = await context.newPage(); | ||
| try { | ||
| await routePage.goto(route); | ||
| await expect( | ||
| routePage.locator("app-root h1").first(), | ||
| `${route} should finish rendering` | ||
| ).toBeVisible(); | ||
|
|
||
| const results = await new AxeBuilder({ page: routePage }) | ||
| .withTags(WCAG_21_AA_TAGS) | ||
| .analyze(); | ||
|
|
||
| fingerprints.push( | ||
| ...violationFingerprints(route, results.violations as AxeViolation[]) | ||
| ); | ||
| } finally { | ||
| await routePage.close(); | ||
| } | ||
| } | ||
|
|
||
| const current = [...new Set(fingerprints)].sort(); | ||
| const baseline = JSON.parse(readFileSync(BASELINE_PATH, "utf8")) as string[]; | ||
| const baselineSet = new Set(baseline); | ||
| const currentSet = new Set(current); | ||
| const newViolations = current.filter( | ||
| (fingerprint) => !baselineSet.has(fingerprint) | ||
| ); | ||
| const resolvedViolations = baseline.filter( | ||
| (fingerprint) => !currentSet.has(fingerprint) | ||
| ); | ||
|
|
||
| // New violations always fail, even in update mode: the baseline is a ratchet | ||
| // that only ever shrinks. UPDATE_A11Y_BASELINE=1 may only drop resolved entries. | ||
| expect( | ||
| newViolations, | ||
| "New WCAG 2.1 A/AA violations (fix them; the baseline only ever shrinks)" | ||
| ).toEqual([]); | ||
|
|
||
| if (process.env.UPDATE_A11Y_BASELINE === "1") { | ||
| writeFileSync(BASELINE_PATH, `${JSON.stringify(current, null, 2)}\n`); | ||
| return; | ||
| } | ||
|
|
||
| expect( | ||
| resolvedViolations, | ||
| "Resolved WCAG 2.1 A/AA violations (remove them from the baseline with UPDATE_A11Y_BASELINE=1)" | ||
| ).toEqual([]); | ||
| }); | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| [] |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| import { defineConfig } from "@playwright/test"; | ||
| import baseConfig from "./playwright.config"; | ||
|
|
||
| // Dedicated config for the WCAG 2.1 AA gate (GSA/sam-ui-elements#708), | ||
| // mirroring `playwright.a11y.config.{mjs,ts}` in sam-styles and ngx-uswds. | ||
| // Reuses the base Playwright setup (same `ng serve` webServer) but points at | ||
| // the accessibility spec dir and runs serially with a generous timeout because | ||
| // axe walks every test-app route in a single worker. | ||
| export default defineConfig({ | ||
| ...baseConfig, | ||
| testDir: "./e2e/accessibility", | ||
| testIgnore: [], | ||
| timeout: 600_000, | ||
| fullyParallel: false, | ||
| workers: 1, | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.