diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 8c727c4a..16b8b8b9 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -329,7 +329,7 @@ searchCapability → describeCapability → invokeCapability | Tool | What it does | |---|---| -| `setupBrowserStackAutomateTests` | Integrate BrowserStack SDK and run web tests on BrowserStack. For visual testing/Percy, use the dedicated Percy tools. | +| `setupBrowserStackAutomateTests` | Integrate BrowserStack SDK and run web tests on BrowserStack. With `accessibility: true`, the run also performs Accessibility (WCAG) checks, turned on in `browserstack.yml` (Cypress: `browserstack.json`; WebdriverIO: the BrowserStack service's options), and the output says how to read the issues. For visual testing/Percy, use the dedicated Percy tools. | | `fetchAutomationScreenshots` | Fetch screenshots captured during a given Automate/App Automate session. | | `listSessions` | List the sessions in an Automate/App Automate build. Each record carries `sessionId`, `name`, `status`, `os`, `osVersion`, `browser`, `device`, `browserUrl` (dashboard link), and `videoUrl`, with optional `limit` / `offset` paging and a client-side `status` filter. Takes either the **hashed** build ID from the dashboard URL or the observability build id returned by `getBuildId` / `listBuildId` — an observability id is resolved to the hashed id automatically via the build's sessions. Returned `sessionId` values work with `getFailureLogs`, `fetchAutomationScreenshots`, and `fetchSelfHealedSelectors`. | @@ -340,7 +340,7 @@ searchCapability → describeCapability → invokeCapability | Tool | What it does | |---|---| -| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). | +| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Optional settings: `wcagVersion` (WCAG 2.0–2.2 at A/AA/AAA, or RGAA 4.1), `bestPractices`, `needsReview`, `advancedRules` and `experimentalRules`; the answer states the settings the scan ran with. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). | | `accessibilityExpert` | Ask the A11y Expert (WCAG 2.0/2.1/2.2, mobile/web usability, best practices). | | `createAccessibilityAuthConfig` | Create an authentication configuration (form-based or basic) for accessibility scans behind a login. | | `getAccessibilityAuthConfig` | Retrieve an existing accessibility authentication configuration by ID. | diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 1b5fad0c..2b147437 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -1011,6 +1011,59 @@ function pageNote(page: { ]; } +/** + * The pages of the newer run that did not load (an error page was scanned in their place), + * or whose earlier issues all gave way to new ones. Neither the scan nor the dashboard says + * so, and their real issues would otherwise read as fixed. + */ +function pagesNotLoadedNote( + diff: { + not_checked?: unknown[]; + failed_pages?: string[]; + not_checked_earlier?: unknown[]; + failed_pages_earlier?: string[]; + changed_pages?: string[]; + }, + runId: string, + baseRunId: string, +): string[] { + const pages = (urls: string[]) => + wrapUntrusted("report page addresses", JSON.stringify(urls)); + // Worded for one page or several, one issue or several. + const one = (count: number, singular: string, plural: string) => + count === 1 ? singular : plural; + const they = (n: number) => one(n, "it", "they"); + const didNotLoad = (urls: string[], run: string) => { + const n = urls.length; + return `⚠️ ${one(n, "1 page", `${n} pages`)} did not load in run ${run}: ${one(n, "an error page was", "error pages were")} scanned in ${one(n, "its", "their")} place. The ${one(n, "error page's", "error pages'")} issues are not counted as`; + }; + // A page that failed in both runs has no issues of its own in either. + const foundThere = (count: number, run: string, notCountedAs: string) => + count > 0 + ? `, and the ${one(count, "1 issue", `${count} issues`)} found there in run ${run} ${one(count, "is", "are")} not counted as ${notCountedAs}` + : ""; + const failed = diff.failed_pages ?? []; + const failedEarlier = diff.failed_pages_earlier ?? []; + const changed = diff.changed_pages ?? []; + return [ + ...(failed.length > 0 + ? [ + `${didNotLoad(failed, runId)} new${foundThere(diff.not_checked?.length ?? 0, baseRunId, "resolved (listed as not checked)")}. Scan again once ${they(failed.length)} ${one(failed.length, "loads", "load")}. ${pages(failed)}`, + ] + : []), + ...(failedEarlier.length > 0 + ? [ + `${didNotLoad(failedEarlier, baseRunId)} resolved${foundThere(diff.not_checked_earlier?.length ?? 0, runId, "new (listed as not checked against the earlier run)")}. Compare with a run in which ${they(failedEarlier.length)} loaded. ${pages(failedEarlier)}`, + ] + : []), + ...(changed.length > 0 + ? [ + `⚠️ None of the earlier issues on ${one(changed.length, "1 page remain, and it has", `${changed.length} pages remain, and they have`)} new ones: ${they(changed.length)} may not have loaded, or loaded something else, in run ${runId}. Check ${one(changed.length, "it", "them")} before calling those issues fixed. ${pages(changed)}`, + ] + : []), + ]; +} + /** At most this many new and resolved issues are listed in a comparison. */ const COMPARE_LIST_LIMIT = 50; @@ -1090,8 +1143,17 @@ async function compareScanRuns( changes ? `The dashboard's comparison: ${changes.newIssues} new, ${changes.resolvedIssues} resolved, ${changes.retainedIssues} still open. Matching the issues by rule, element and page gives ${matched}.` : `Matching the issues by rule, element and page: ${matched}.`, + ...pagesNotLoadedNote(diff, source.scanRunId, baseRunId), `New issues: ${listed(diff.added)}`, `Resolved issues: ${listed(diff.resolved)}`, + ...(diff.not_checked?.length + ? [`Not checked (page did not load): ${listed(diff.not_checked)}`] + : []), + ...(diff.not_checked_earlier?.length + ? [ + `Not checked against the earlier run (page did not load in run ${baseRunId}): ${listed(diff.not_checked_earlier)}`, + ] + : []), `CSV of each run, valid for about 12 hours: newer ${links[0]} , older ${links[1]}`, ]); } @@ -1134,6 +1196,9 @@ function scanSettingsNote( settings.advancedRules !== undefined ? `advanced rules ${onOff(settings.advancedRules)}` : "", + settings.experimentalRules !== undefined + ? `experimental rules ${onOff(settings.experimentalRules)}` + : "", ] .filter(Boolean) .join(", "); @@ -1173,6 +1238,7 @@ async function executeAccessibilityScan( bestPractices: args.bestPractices, needsReview: args.needsReview, advancedRules: args.advancedRules, + experimentalRules: args.experimentalRules, }, ); } catch (error) { @@ -1476,6 +1542,10 @@ export default function addAccessibilityTools( .describe( 'Also report "needs review" issues to check by hand (default true).', ), + experimentalRules: z + .boolean() + .optional() + .describe("Run the experimental accessibility rules (default true)."), }, { title: "Start Accessibility Scan", diff --git a/src/tools/accessiblity-utils/report-parser.ts b/src/tools/accessiblity-utils/report-parser.ts index 41598bdd..d151ae1c 100644 --- a/src/tools/accessiblity-utils/report-parser.ts +++ b/src/tools/accessiblity-utils/report-parser.ts @@ -221,8 +221,22 @@ export type RunIssue = { severity: string; component: string; affected_page: string; + /** Found on an error page scanned in the page's place: the page did not load. */ + on_error_page?: true; }; +/** + * Snippets only an error page shows: BrowserStack's Local connection page (its links carry + * ref=dashboard-privoxy-page), and Chrome's network error page. A page that fails to load + * is still "scanned", as one of these. A link to the Local docs, or a class merely starting + * with neterror, is not enough: a real page can have either. + */ +const ERROR_PAGE_SNIPPET = + /dashboard-privoxy-page|id="main-frame-error"|chrome-error:\/\/|class="neterror[" ]/i; + +/** A page whose earlier issues are all gone and new ones appeared: worth a second look. */ +const CHANGED_PAGE_MIN_ISSUES = 2; + /** A report's main issues that the dashboard counts (not hidden), keyed for comparing runs. */ export async function loadMainIssues(reportLink: string): Promise { const { parsed, columns } = await loadReport(reportLink); @@ -241,21 +255,107 @@ export async function loadMainIssues(reportLink: string): Promise { severity: (row["Severity"] || "unknown").trim(), component: row["Component"], affected_page: row["Affected page"], + ...(ERROR_PAGE_SNIPPET.test(row["HTML snippet"] ?? "") + ? { on_error_page: true as const } + : {}), }; }); } -/** Which issues of `current` are new since `base`, which of `base` are gone, and how many remain. */ +/** A run's issues by page, with whether an error page was scanned in the page's place. */ +function issuesByPage( + issues: RunIssue[], +): Map { + const pages = new Map< + string, + { url: string; issues: RunIssue[]; failed: boolean } + >(); + for (const issue of issues) { + const key = samePage(issue.affected_page ?? ""); + let page = pages.get(key); + if (!page) { + page = { url: issue.affected_page, issues: [], failed: false }; + pages.set(key, page); + } + page.issues.push(issue); + if (issue.on_error_page) page.failed = true; + } + return pages; +} + +/** + * Which issues of `current` are new since `base`, which of `base` are gone, and how many + * remain, decided page by page. A page that did not load in one of the runs (an error page + * was scanned under its address) is set apart. Not loaded in `current`: its earlier issues + * were not checked again, so they are `not_checked` rather than resolved, and the error + * page's issues are not new. Not loaded in `base`: the error page's issues are not resolved, + * and the page's issues now are `not_checked_earlier` rather than new. A page whose earlier + * issues are all gone while new ones appeared is named in `changed_pages`, as it may not have + * loaded either. + */ export function compareIssues( base: RunIssue[], current: RunIssue[], -): { added: RunIssue[]; resolved: RunIssue[]; retained: number } { - const before = new Set(base.map((issue) => issue.key)); - const after = new Set(current.map((issue) => issue.key)); +): { + added: RunIssue[]; + resolved: RunIssue[]; + retained: number; + not_checked: RunIssue[]; + failed_pages: string[]; + not_checked_earlier: RunIssue[]; + failed_pages_earlier: string[]; + changed_pages: string[]; +} { + const before = issuesByPage(base); + const after = issuesByPage(current); + const added: RunIssue[] = []; + const resolved: RunIssue[] = []; + const notChecked: RunIssue[] = []; + const notCheckedEarlier: RunIssue[] = []; + const failedPages: string[] = []; + const failedPagesEarlier: string[] = []; + const changedPages: string[] = []; + let retained = 0; + for (const key of new Set([...before.keys(), ...after.keys()])) { + const earlier = before.get(key); + const later = after.get(key); + if (later?.failed) failedPages.push(later.url); + if (earlier?.failed) failedPagesEarlier.push(earlier.url); + if (later?.failed || earlier?.failed) { + // Only the run in which the page loaded has its issues, and they cannot be matched. + if (!earlier?.failed) notChecked.push(...(earlier?.issues ?? [])); + if (!later?.failed) notCheckedEarlier.push(...(later?.issues ?? [])); + continue; + } + const earlierKeys = new Set(earlier?.issues.map((issue) => issue.key)); + const laterKeys = new Set(later?.issues.map((issue) => issue.key)); + const pageAdded = (later?.issues ?? []).filter( + (issue) => !earlierKeys.has(issue.key), + ); + const pageResolved = (earlier?.issues ?? []).filter( + (issue) => !laterKeys.has(issue.key), + ); + added.push(...pageAdded); + resolved.push(...pageResolved); + retained += (later?.issues.length ?? 0) - pageAdded.length; + if ( + earlier && + earlier.issues.length >= CHANGED_PAGE_MIN_ISSUES && + pageResolved.length === earlier.issues.length && + pageAdded.length > 0 + ) { + changedPages.push(earlier.url); + } + } return { - added: sortBySeverity(current.filter((issue) => !before.has(issue.key))), - resolved: sortBySeverity(base.filter((issue) => !after.has(issue.key))), - retained: current.filter((issue) => before.has(issue.key)).length, + added: sortBySeverity(added), + resolved: sortBySeverity(resolved), + retained, + not_checked: sortBySeverity(notChecked), + failed_pages: failedPages, + not_checked_earlier: sortBySeverity(notCheckedEarlier), + failed_pages_earlier: failedPagesEarlier, + changed_pages: changedPages, }; } diff --git a/src/tools/accessiblity-utils/scanner.ts b/src/tools/accessiblity-utils/scanner.ts index 67457d9e..029beb14 100644 --- a/src/tools/accessiblity-utils/scanner.ts +++ b/src/tools/accessiblity-utils/scanner.ts @@ -33,6 +33,8 @@ export interface ScanSettings { needsReview?: boolean; /** Run the advanced rules (API default true). */ advancedRules?: boolean; + /** Run the experimental rules (API default true). */ + experimentalRules?: boolean; } export class AccessibilityScanner { @@ -158,27 +160,37 @@ export class AccessibilityScanner { /** * The settings a run actually ran with, from its summary, or undefined when it cannot be * read. A setting left out takes the API's default, and the plan can turn advanced rules - * off, so what was asked for is not always what ran. + * off, so what was asked for is not always what ran. The summary has no experimental + * rules: they come from the scan's own settings, which a scan started here has for this + * run alone. */ async runSettings( scanId: string, scanRunId: string, ): Promise { - const resp = await apiClient - .get({ - url: `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${scanId}/scan_runs/${scanRunId}`, - headers: { - Authorization: - "Basic " + - Buffer.from( - `${this.auth?.username}:${this.auth?.password}`, - ).toString("base64"), - }, - raise_error: false, - }) - .catch(() => undefined); + const read = (path: string) => + apiClient + .get({ + url: `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${path}`, + headers: { + Authorization: + "Basic " + + Buffer.from( + `${this.auth?.username}:${this.auth?.password}`, + ).toString("base64"), + }, + raise_error: false, + }) + .catch(() => undefined); + const [resp, overview] = await Promise.all([ + read(`${scanId}/scan_runs/${scanRunId}`), + read(`${scanId}/overview`), + ]); const run = resp?.ok ? resp.data?.data : undefined; if (!run) return undefined; + const experimental = overview?.ok + ? overview.data?.data?.scanConfigInfo?.experimental + : undefined; return { ...(run.wcagVersion ? { wcagVersion: run.wcagVersion } : {}), ...(typeof run.bestPractices === "boolean" @@ -190,6 +202,9 @@ export class AccessibilityScanner { ...(typeof run.advanced === "boolean" ? { advancedRules: run.advanced } : {}), + ...(typeof experimental === "boolean" + ? { experimentalRules: experimental } + : {}), }; } diff --git a/src/tools/sdk-utils/bstack/configUtils.ts b/src/tools/sdk-utils/bstack/configUtils.ts index e577ce5e..3fde2caf 100644 --- a/src/tools/sdk-utils/bstack/configUtils.ts +++ b/src/tools/sdk-utils/bstack/configUtils.ts @@ -5,6 +5,8 @@ export function generateBrowserStackYMLInstructions(config: { platforms?: string[]; enablePercy?: boolean; projectName: string; + /** Run Accessibility checks during the tests. */ + accessibility?: boolean; }): string { const enablePercy = config.enablePercy || false; const projectName = config.projectName || "BrowserStack Automate Build"; @@ -12,8 +14,9 @@ export function generateBrowserStackYMLInstructions(config: { // Generate platform configurations using the utility function const platformConfigs = generatePlatformConfigs(config); + // An existing file holds the user's own settings, so it is added to, not replaced. const stepTitle = - "Create a browserstack.yml file in the project root with your validated device configurations:If already exists, update it with the following content for devices and project details."; + "Create a browserstack.yml file in the project root with your validated device configurations. If the file already exists, add these keys to it and keep its other settings."; const buildName = `${projectName}-Build`; @@ -60,6 +63,16 @@ browserstackLocal: true debug: true # Visual logs, text logs, etc. testObservability: true # For Test Observability`; + if (config.accessibility) { + ymlContent += ` + +# ====================== +# Accessibility Testing +# ====================== +# Runs Accessibility (WCAG) checks on the pages the tests visit. +accessibility: true`; + } + if (enablePercy) { ymlContent += ` diff --git a/src/tools/sdk-utils/bstack/sdkHandler.ts b/src/tools/sdk-utils/bstack/sdkHandler.ts index b9b43435..9546f66f 100644 --- a/src/tools/sdk-utils/bstack/sdkHandler.ts +++ b/src/tools/sdk-utils/bstack/sdkHandler.ts @@ -76,6 +76,18 @@ export async function runBstackSDKOnly( }); } + // Neither framework reads browserstack.yml: each turns accessibility on in its own file. + if (input.accessibility) { + steps.push({ + type: "instruction", + title: "Turn on Accessibility Testing", + content: + input.detectedBrowserAutomationFramework === "cypress" + ? 'In browserstack.json, add "accessibility": true under run_settings, keeping the file\'s other settings. It runs Accessibility (WCAG) checks on the pages the tests visit.' + : "In your WebdriverIO config, add accessibility: true to the browserstack service's options (services: [['browserstack', { accessibility: true, ... }]]), keeping its other options. It runs Accessibility (WCAG) checks on the pages the tests visit.", + }); + } + if (frameworkInstructions.run && !isPercyAutomate) { steps.push({ type: "instruction", @@ -83,6 +95,15 @@ export async function runBstackSDKOnly( content: frameworkInstructions.run, }); } + + if (input.accessibility) { + steps.push({ + type: "instruction", + title: "Read the accessibility issues", + content: + "When the run finishes, read its accessibility issues with fetchAccessibilityIssues: pass the Automate build's dashboard link, printed during the run, as reportUrl.", + }); + } } return { @@ -126,6 +147,7 @@ export async function runBstackSDKOnly( validatedEnvironments, enablePercy: false, projectName: input.projectName, + accessibility: input.accessibility, }); if (ymlInstructions) { @@ -146,6 +168,15 @@ export async function runBstackSDKOnly( } } + if (input.accessibility) { + steps.push({ + type: "instruction", + title: "Read the accessibility issues", + content: + "When the run finishes, read its accessibility issues with fetchAccessibilityIssues: pass the Automate build's dashboard link as reportUrl, or the build id the BrowserStack SDK prints at the end of the run as buildId.", + }); + } + return { steps, requiresPercy: false, diff --git a/src/tools/sdk-utils/common/schema.ts b/src/tools/sdk-utils/common/schema.ts index 1cb6159b..8d7660d1 100644 --- a/src/tools/sdk-utils/common/schema.ts +++ b/src/tools/sdk-utils/common/schema.ts @@ -90,6 +90,13 @@ export const RunTestsOnBrowserStackParamsShape = { .describe( "Device objects array. Use the object format directly - no transformation needed. Add only when user explicitly requests devices. Examples: [{ platform: 'windows', osVersion: '11', browser: 'chrome', browserVersion: 'latest' }] or [{ platform: 'android', deviceName: 'Samsung Galaxy S24', osVersion: '14', browser: 'chrome' }].", ), + accessibility: z + .boolean() + .optional() + .default(false) + .describe( + "Run Accessibility (WCAG) checks during the tests: writes accessibility: true in browserstack.yml.", + ), }; export const SetUpPercySchema = z.object(SetUpPercyParamsShape); diff --git a/src/tools/sdk-utils/handler.ts b/src/tools/sdk-utils/handler.ts index a75cf608..8240d097 100644 --- a/src/tools/sdk-utils/handler.ts +++ b/src/tools/sdk-utils/handler.ts @@ -89,6 +89,7 @@ export async function setUpPercyHandler( const percyWithBrowserstackSDKResult = runPercyWithBrowserstackSDK({ ...percyInput, devices: [], + accessibility: false, }); const hasPercySDKError = percyWithBrowserstackSDKResult.steps && @@ -125,6 +126,7 @@ export async function setUpPercyHandler( input.detectedBrowserAutomationFramework, detectedTestingFramework: input.detectedTestingFramework, devices: [], + accessibility: false, }; const sdkResult = await runBstackSDKOnly(sdkInput, config, true); // Percy Automate instructions diff --git a/tests/tools/accessibilityFetchPagesAndCompare.test.ts b/tests/tools/accessibilityFetchPagesAndCompare.test.ts index 72764c50..4ebb84ee 100644 --- a/tests/tools/accessibilityFetchPagesAndCompare.test.ts +++ b/tests/tools/accessibilityFetchPagesAndCompare.test.ts @@ -196,6 +196,54 @@ describe("fetchAccessibilityIssues comparing scan runs", () => { expect(text(result)).toMatch(/first run/); }); + it("says when a page did not load in the later run, and does not count its issues as resolved", async () => { + m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); + m.changesSinceLastRun.mockResolvedValue({ newIssues: 2, retainedIssues: 2, resolvedIssues: 6 }); + m.compareIssues.mockReturnValue({ + added: [], + resolved: [], + retained: 2, + not_checked: [{ issue_type: "label", severity: "critical", component: "input", affected_page: "https://shop.example/b" }], + failed_pages: ["https://shop.example/b"], + changed_pages: ["https://shop.example/c"], + }); + + const out = text(await fetchIssues()({ ...RUN, compareWith: "previous" })); + + expect(out).toContain( + "⚠️ 1 page did not load in run 5520318: an error page was scanned in its place. The error page's issues are not counted as new, and the 1 issue found there in run 5520296 is not counted as resolved (listed as not checked). Scan again once it loads.", + ); + expect(out).toContain( + "⚠️ None of the earlier issues on 1 page remain, and it has new ones: it may not have loaded, or loaded something else, in run 5520318. Check it before calling those issues fixed.", + ); + // The addresses come from the report, so they are quoted as data. + expect(out.indexOf("https://shop.example/b")).toBeGreaterThan(out.indexOf("«UNTRUSTED")); + expect(out).toMatch(/Not checked \(page did not load\)/); + }); + + it("says when pages did not load in the earlier run, and does not count their issues now as new", async () => { + m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); + m.changesSinceLastRun.mockResolvedValue({ newIssues: 3, retainedIssues: 2, resolvedIssues: 2 }); + const issue = (page: string) => ({ issue_type: "label", severity: "critical", component: "input", affected_page: page }); + m.compareIssues.mockReturnValue({ + added: [], + resolved: [], + retained: 2, + not_checked: [], + failed_pages: [], + changed_pages: [], + not_checked_earlier: [issue("https://shop.example/b"), issue("https://shop.example/c"), issue("https://shop.example/c")], + failed_pages_earlier: ["https://shop.example/b", "https://shop.example/c"], + }); + + const out = text(await fetchIssues()({ ...RUN, compareWith: "previous" })); + + expect(out).toContain( + "⚠️ 2 pages did not load in run 5520296: error pages were scanned in their place. The error pages' issues are not counted as resolved, and the 3 issues found there in run 5520318 are not counted as new (listed as not checked against the earlier run). Compare with a run in which they loaded.", + ); + expect(out).toMatch(/Not checked against the earlier run \(page did not load in run 5520296\)/); + }); + it("compares Website Scanner runs only", async () => { const result = await fetchIssues()({ buildId: "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja", compareWith: "previous" }); diff --git a/tests/tools/accessibilityReportPages.test.ts b/tests/tools/accessibilityReportPages.test.ts index 1a5713bb..e3034cf4 100644 --- a/tests/tools/accessibilityReportPages.test.ts +++ b/tests/tools/accessibilityReportPages.test.ts @@ -180,3 +180,149 @@ describe("compareIssues", () => { expect(diff.retained).toBe(2); }); }); + +// A page that fails to load in a re-run is still "scanned": what gets scanned is an error page +// (BrowserStack's Local connection page, or Chrome's network error page) filed under the +// page's address. Its real issues then read as resolved and the error page's as new. +describe("compareIssues and a page that did not load in the later run", () => { + beforeEach(() => vi.clearAllMocks()); + + const PAGE_A = "http://bs-local.com:3100/page-a.html"; + const PAGE_B = "http://bs-local.com:3100/page-b.html"; + const issueOn = (page: string, type: string, selector: string, snippet = "") => + row({ "Issue type": type, Component: selector, "CSS selector": selector, "Affected page": page, "HTML snippet": snippet, "Issue link": link(`${type}-${page}`) }); + + async function runs(laterB: Record[]) { + serve([ + issueOn(PAGE_A, "css-orientation-lock", "html"), + issueOn(PAGE_B, "label", "input#no-label"), + issueOn(PAGE_B, "button-name", "button#icon-btn"), + issueOn(PAGE_B, "link-name", "a#icon-link"), + ]); + const before = await loadMainIssues("https://csv-old"); + serve([issueOn(PAGE_A, "css-orientation-lock", "html"), ...laterB]); + const after = await loadMainIssues("https://csv-new"); + return compareIssues(before, after); + } + + it("sets apart, rather than resolves, the issues of a page that showed BrowserStack's connection error page", async () => { + const diff = await runs([ + issueOn(PAGE_B, "keyboard-focus-visible", "a.docs", ''), + issueOn(PAGE_B, "decorative-image", "img.logo", 'BrowserStack Logo'), + ]); + + expect(diff.failed_pages).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + expect(diff.not_checked.map((r) => r.issue_type).sort()).toEqual(["button-name", "label", "link-name"]); + expect(diff.added).toEqual([]); + expect(diff.retained).toBe(1); + }); + + it("knows Chrome's network error page too, as a public page that fails to load shows it", async () => { + const diff = await runs([ + issueOn(PAGE_B, "landmark-one-main", "div#main-frame-error", '
'), + ]); + + expect(diff.failed_pages).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + }); + + it("warns, without reclassifying, when every earlier issue on a page is gone and new ones appeared", async () => { + const diff = await runs([issueOn(PAGE_B, "region", "div.promo")]); + + expect(diff.failed_pages).toEqual([]); + expect(diff.changed_pages).toEqual([PAGE_B]); + expect(diff.resolved).toHaveLength(3); + expect(diff.added.map((r) => r.issue_type)).toEqual(["region"]); + }); + + it("flags no page when its issues were simply fixed", async () => { + const diff = await runs([]); + + expect(diff.failed_pages).toEqual([]); + expect(diff.changed_pages).toEqual([]); + expect(diff.resolved).toHaveLength(3); + }); +}); + +// The mirror case: the page failed to load in the earlier run. Its "issues" there were the +// error page's, so they are not resolved, and the page's real issues now are not new. +describe("compareIssues and a page that did not load in the earlier run", () => { + beforeEach(() => vi.clearAllMocks()); + + const PAGE_A = "http://bs-local.com:3100/page-a.html"; + const PAGE_B = "http://bs-local.com:3100/page-b.html"; + const issueOn = (page: string, type: string, selector: string, snippet = "") => + row({ "Issue type": type, Component: selector, "CSS selector": selector, "Affected page": page, "HTML snippet": snippet, "Issue link": link(`${type}-${page}`) }); + const errorPage = [ + issueOn(PAGE_B, "keyboard-focus-visible", "a.docs", ''), + issueOn(PAGE_B, "decorative-image", "img.logo", 'BrowserStack Logo'), + ]; + + async function runs(earlierB: Record[], laterB: Record[]) { + serve([issueOn(PAGE_A, "css-orientation-lock", "html"), ...earlierB]); + const before = await loadMainIssues("https://csv-old"); + serve([issueOn(PAGE_A, "css-orientation-lock", "html"), ...laterB]); + const after = await loadMainIssues("https://csv-new"); + return compareIssues(before, after); + } + + it("does not resolve the error page's issues, nor count the page's real issues as new", async () => { + const diff = await runs(errorPage, [ + issueOn(PAGE_B, "label", "input#no-label"), + issueOn(PAGE_B, "button-name", "button#icon-btn"), + ]); + + expect(diff.failed_pages).toEqual([]); + expect(diff.failed_pages_earlier).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + expect(diff.added).toEqual([]); + expect(diff.not_checked_earlier.map((r) => r.issue_type).sort()).toEqual(["button-name", "label"]); + expect(diff.retained).toBe(1); + }); + + it("compares nothing on a page that failed in both runs", async () => { + const diff = await runs(errorPage, errorPage); + + expect(diff.failed_pages).toEqual([PAGE_B]); + expect(diff.failed_pages_earlier).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + expect(diff.added).toEqual([]); + expect(diff.not_checked).toEqual([]); + expect(diff.not_checked_earlier).toEqual([]); + expect(diff.retained).toBe(1); + }); + + it("does not take a real page that links to the Local docs, or names a class neterror-*, for an error page", async () => { + const diff = await runs( + [issueOn(PAGE_B, "label", "input#no-label")], + [ + issueOn(PAGE_B, "link-name", "a.help", ''), + issueOn(PAGE_B, "color-contrast", "div.neterror-banner", '
Offline?
'), + ], + ); + + expect(diff.failed_pages).toEqual([]); + expect(diff.failed_pages_earlier).toEqual([]); + expect(diff.added).toHaveLength(2); + expect(diff.resolved).toHaveLength(1); + }); +}); + +describe("compareIssues on large runs", () => { + it("compares runs of thousands of issues without stalling", () => { + const issues = (n: number, rule: string) => + Array.from({ length: n }, (_, i) => { + const page = `https://shop.example/p${i % 300}`; + return { key: `${rule}|#e${i}|${page}`, issue_type: rule, severity: "serious", component: "input", affected_page: page }; + }); + + const started = performance.now(); + const diff = compareIssues(issues(3000, "label"), issues(3000, "region")); + + expect(performance.now() - started).toBeLessThan(500); + expect(diff.resolved).toHaveLength(3000); + expect(diff.added).toHaveLength(3000); + expect(diff.changed_pages).toHaveLength(300); + }); +}); diff --git a/tests/tools/accessibilityScanSettings.test.ts b/tests/tools/accessibilityScanSettings.test.ts index 32a62885..b3364da4 100644 --- a/tests/tools/accessibilityScanSettings.test.ts +++ b/tests/tools/accessibilityScanSettings.test.ts @@ -1,11 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; // A Website Scanner scan can be asked for a WCAG version and level, best practices, "needs -// review" issues and advanced rules: the create-scan API takes them under scanSettings, -// and anything left out takes the API's default. +// review" issues, advanced rules and experimental rules: the create-scan API takes them +// under scanSettings, and anything left out takes the API's default. const m = vi.hoisted(() => ({ post: vi.fn(), + get: vi.fn(), startScan: vi.fn(), waitUntilComplete: vi.fn(), runSettings: vi.fn(), @@ -13,7 +14,7 @@ const m = vi.hoisted(() => ({ parse: vi.fn(), })); -vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post } })); +vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post, get: m.get } })); vi.mock("../../src/lib/local.js", () => ({ isLocalURL: () => false, ensureLocalBinarySetup: vi.fn(), @@ -86,6 +87,14 @@ describe("AccessibilityScanner.startScan", () => { ).rejects.toThrow(/A scan with this name already exists/); }); + it("sends experimental rules off when asked, as false is a setting rather than a missing one", async () => { + await (await scanner()).startScan("checkout", ["https://shop.example/"], undefined, { + experimentalRules: false, + }); + + expect(m.post.mock.calls[0][0].body.scanSettings).toEqual({ experimentalRules: false }); + }); + it("sends no scanSettings when none are given", async () => { await (await scanner()).startScan("checkout", ["https://shop.example/"]); @@ -93,6 +102,59 @@ describe("AccessibilityScanner.startScan", () => { }); }); +// The run summary carries a run's WCAG version, best practices, needs review and advanced +// rules, but not experimental rules: those are read from the scan's own settings (its +// overview), which a scan started here has for this run alone. +describe("AccessibilityScanner.runSettings", () => { + const SCANS = "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans"; + const run = { ok: true, status: 200, data: { data: { wcagVersion: "wcag22aa", bestPractices: true, needsReview: true, advanced: true } } }; + + beforeEach(() => { + vi.resetModules(); + m.get.mockReset(); + }); + + async function scanner() { + const { AccessibilityScanner } = await vi.importActual< + typeof import("../../src/tools/accessiblity-utils/scanner.js") + >("../../src/tools/accessiblity-utils/scanner.js"); + const s = new AccessibilityScanner(); + s.setAuth({ username: "u", password: "p" }); + return s; + } + + it("reads experimental rules from the scan's own settings, as the run summary has none", async () => { + m.get.mockImplementation(async ({ url }: { url: string }) => + url === `${SCANS}/7310042/overview` + ? { ok: true, status: 200, data: { data: { scanConfigInfo: { experimental: false } } } } + : run, + ); + + await expect((await scanner()).runSettings("7310042", "5520318")).resolves.toEqual({ + wcagVersion: "wcag22aa", + bestPractices: true, + needsReview: true, + advancedRules: true, + experimentalRules: false, + }); + expect(m.get.mock.calls.map(([o]) => o.url).sort()).toEqual([ + `${SCANS}/7310042/overview`, + `${SCANS}/7310042/scan_runs/5520318`, + ]); + }); + + it("still reads the run's settings when the scan's own cannot be read", async () => { + m.get.mockImplementation(async ({ url }: { url: string }) => + url.endsWith("/overview") ? { ok: false, status: 500, data: {} } : run, + ); + + const settings = await (await scanner()).runSettings("7310042", "5520318"); + + expect(settings).toMatchObject({ wcagVersion: "wcag22aa", advancedRules: true }); + expect(settings).not.toHaveProperty("experimentalRules"); + }); +}); + describe("startAccessibilityScan scan settings", () => { beforeEach(() => { vi.resetModules(); @@ -174,6 +236,30 @@ describe("startAccessibilityScan scan settings", () => { ); }); + it("passes experimental rules to the scan and reports them as the scan ran", async () => { + m.runSettings.mockResolvedValue({ + wcagVersion: "wcag21aa", + bestPractices: false, + needsReview: true, + advancedRules: true, + experimentalRules: false, + }); + const { run } = await tool(); + + const result = await run({ + name: "checkout", + pageURL: "https://shop.example/", + experimentalRules: false, + }); + + expect(m.startScan).toHaveBeenCalledWith("checkout", ["https://shop.example/"], undefined, { + experimentalRules: false, + }); + expect(text(result)).toContain( + "Scan settings, as the scan ran: WCAG 2.1 AA, best practices off, needs review on, advanced rules on, experimental rules off.", + ); + }); + it("says only what was asked for when the run's settings cannot be read", async () => { m.runSettings.mockResolvedValue(undefined); const { run } = await tool(); @@ -191,6 +277,14 @@ describe("startAccessibilityScan scan settings", () => { expect(text(result)).not.toContain("(default)"); }); + it("takes experimental rules only as on or off", async () => { + const { schema } = await tool(); + + expect(schema.experimentalRules.safeParse(false).success).toBe(true); + expect(schema.experimentalRules.safeParse(true).success).toBe(true); + expect(schema.experimentalRules.safeParse("off").success).toBe(false); + }); + it("takes only the WCAG versions the API knows, RGAA 4.1 among them", async () => { const { schema } = await tool(); diff --git a/tests/tools/automateAccessibilitySetup.test.ts b/tests/tools/automateAccessibilitySetup.test.ts new file mode 100644 index 00000000..4586884f --- /dev/null +++ b/tests/tools/automateAccessibilitySetup.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; + +import { generateBrowserStackYMLInstructions } from "../../src/tools/sdk-utils/bstack/configUtils.js"; +import { RunTestsOnBrowserStackSchema } from "../../src/tools/sdk-utils/common/schema.js"; +import { runBstackSDKOnly } from "../../src/tools/sdk-utils/bstack/sdkHandler.js"; +import { BrowserStackConfig } from "../../src/lib/types.js"; + +// Asked to run Automate tests with accessibility testing on, the setup tool must write +// accessibility: true into browserstack.yml, and must not tell the user to replace a file +// they already have, which would drop their own settings. + +const yml = (accessibility?: boolean) => + generateBrowserStackYMLInstructions({ + validatedEnvironments: [], + projectName: "Checkout", + ...(accessibility === undefined ? {} : { accessibility }), + }); + +describe("setupBrowserStackAutomateTests and accessibility", () => { + it("takes accessibility as an option, off by default", () => { + const base = { + projectName: "Checkout", + detectedLanguage: "nodejs", + detectedBrowserAutomationFramework: "playwright", + detectedTestingFramework: "playwright", + }; + expect(RunTestsOnBrowserStackSchema.parse(base).accessibility).toBe(false); + expect(RunTestsOnBrowserStackSchema.parse({ ...base, accessibility: true }).accessibility).toBe(true); + }); + + it("writes accessibility: true into browserstack.yml when asked", () => { + expect(yml(true)).toContain("\naccessibility: true\n"); + }); + + it("leaves the file as it was when accessibility is not asked for", () => { + expect(yml()).not.toMatch(/^accessibility:/m); + }); + + it("says to add to an existing browserstack.yml rather than replace it", () => { + expect(yml(true)).toContain("If the file already exists, add these keys to it and keep its other settings"); + }); +}); + +// Cypress and WebdriverIO are set up without browserstack.yml: Cypress through the +// browserstack-cypress CLI's browserstack.json, WebdriverIO through its BrowserStack +// service's options. Each turns accessibility on in its own file. +describe("setupBrowserStackAutomateTests, accessibility and the frameworks without browserstack.yml", () => { + const setup = async (framework: "cypress" | "webdriverio", accessibility: boolean) => { + const { steps } = await runBstackSDKOnly( + RunTestsOnBrowserStackSchema.parse({ + projectName: "Checkout", + detectedLanguage: "nodejs", + detectedBrowserAutomationFramework: framework, + detectedTestingFramework: framework === "cypress" ? "cypress" : "mocha", + accessibility, + }), + {} as BrowserStackConfig, + ); + return steps.map((step) => `${step.title}\n${step.content}`).join("\n\n"); + }; + + it("adds accessibility: true under run_settings in Cypress's browserstack.json", async () => { + const text = await setup("cypress", true); + expect(text).toContain('"accessibility": true'); + expect(text).toMatch(/under run_settings/); + expect(text).toMatch(/fetchAccessibilityIssues/); + }); + + it("adds accessibility: true to WebdriverIO's BrowserStack service options", async () => { + const text = await setup("webdriverio", true); + expect(text).toContain("accessibility: true"); + expect(text).toMatch(/browserstack service's options/); + expect(text).toMatch(/fetchAccessibilityIssues/); + }); + + it("says nothing of accessibility when it is not asked for", async () => { + for (const framework of ["cypress", "webdriverio"] as const) { + const text = await setup(framework, false); + expect(text).not.toMatch(/accessibility: true|"accessibility"|fetchAccessibilityIssues/); + } + }); +});