From 8d1afbebd24f02f0c4bf052f3c9383618b212fb7 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Mon, 26 Oct 2026 15:27:44 +0530 Subject: [PATCH 1/3] feat(accessibility): turn a scan's experimental rules on or off startAccessibilityScan takes experimentalRules, sent to the create-scan API as scanSettings.experimentalRules (browserstack/accessibility#5649, now deployed). Left out, the API's default (on) applies; false is sent as a setting, not dropped. The settings line reports experimental rules as the scan has them. The run summary does not carry them, so they are read from the scan's own settings (its overview), which a scan started here has for this run alone; when that cannot be read, the rest of the line still comes from the run summary. Co-Authored-By: Claude Opus 5.5 --- docs/TOOLS.md | 2 +- src/tools/accessibility.ts | 8 ++ src/tools/accessiblity-utils/scanner.ts | 43 +++++--- tests/tools/accessibilityScanSettings.test.ts | 100 +++++++++++++++++- 4 files changed, 135 insertions(+), 18 deletions(-) diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 8c727c4a..8e17f610 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -340,7 +340,7 @@ searchCapability → describeCapability → invokeCapability | Tool | What it does | |---|---| -| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). | +| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Optional settings: `wcagVersion` (WCAG 2.0–2.2 at A/AA/AAA, or RGAA 4.1), `bestPractices`, `needsReview`, `advancedRules` and `experimentalRules`; the answer states the settings the scan ran with. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). | | `accessibilityExpert` | Ask the A11y Expert (WCAG 2.0/2.1/2.2, mobile/web usability, best practices). | | `createAccessibilityAuthConfig` | Create an authentication configuration (form-based or basic) for accessibility scans behind a login. | | `getAccessibilityAuthConfig` | Retrieve an existing accessibility authentication configuration by ID. | diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 1b5fad0c..d9d4755f 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -1134,6 +1134,9 @@ function scanSettingsNote( settings.advancedRules !== undefined ? `advanced rules ${onOff(settings.advancedRules)}` : "", + settings.experimentalRules !== undefined + ? `experimental rules ${onOff(settings.experimentalRules)}` + : "", ] .filter(Boolean) .join(", "); @@ -1173,6 +1176,7 @@ async function executeAccessibilityScan( bestPractices: args.bestPractices, needsReview: args.needsReview, advancedRules: args.advancedRules, + experimentalRules: args.experimentalRules, }, ); } catch (error) { @@ -1476,6 +1480,10 @@ export default function addAccessibilityTools( .describe( 'Also report "needs review" issues to check by hand (default true).', ), + experimentalRules: z + .boolean() + .optional() + .describe("Run the experimental accessibility rules (default true)."), }, { title: "Start Accessibility Scan", diff --git a/src/tools/accessiblity-utils/scanner.ts b/src/tools/accessiblity-utils/scanner.ts index 67457d9e..029beb14 100644 --- a/src/tools/accessiblity-utils/scanner.ts +++ b/src/tools/accessiblity-utils/scanner.ts @@ -33,6 +33,8 @@ export interface ScanSettings { needsReview?: boolean; /** Run the advanced rules (API default true). */ advancedRules?: boolean; + /** Run the experimental rules (API default true). */ + experimentalRules?: boolean; } export class AccessibilityScanner { @@ -158,27 +160,37 @@ export class AccessibilityScanner { /** * The settings a run actually ran with, from its summary, or undefined when it cannot be * read. A setting left out takes the API's default, and the plan can turn advanced rules - * off, so what was asked for is not always what ran. + * off, so what was asked for is not always what ran. The summary has no experimental + * rules: they come from the scan's own settings, which a scan started here has for this + * run alone. */ async runSettings( scanId: string, scanRunId: string, ): Promise { - const resp = await apiClient - .get({ - url: `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${scanId}/scan_runs/${scanRunId}`, - headers: { - Authorization: - "Basic " + - Buffer.from( - `${this.auth?.username}:${this.auth?.password}`, - ).toString("base64"), - }, - raise_error: false, - }) - .catch(() => undefined); + const read = (path: string) => + apiClient + .get({ + url: `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${path}`, + headers: { + Authorization: + "Basic " + + Buffer.from( + `${this.auth?.username}:${this.auth?.password}`, + ).toString("base64"), + }, + raise_error: false, + }) + .catch(() => undefined); + const [resp, overview] = await Promise.all([ + read(`${scanId}/scan_runs/${scanRunId}`), + read(`${scanId}/overview`), + ]); const run = resp?.ok ? resp.data?.data : undefined; if (!run) return undefined; + const experimental = overview?.ok + ? overview.data?.data?.scanConfigInfo?.experimental + : undefined; return { ...(run.wcagVersion ? { wcagVersion: run.wcagVersion } : {}), ...(typeof run.bestPractices === "boolean" @@ -190,6 +202,9 @@ export class AccessibilityScanner { ...(typeof run.advanced === "boolean" ? { advancedRules: run.advanced } : {}), + ...(typeof experimental === "boolean" + ? { experimentalRules: experimental } + : {}), }; } diff --git a/tests/tools/accessibilityScanSettings.test.ts b/tests/tools/accessibilityScanSettings.test.ts index 32a62885..b3364da4 100644 --- a/tests/tools/accessibilityScanSettings.test.ts +++ b/tests/tools/accessibilityScanSettings.test.ts @@ -1,11 +1,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; // A Website Scanner scan can be asked for a WCAG version and level, best practices, "needs -// review" issues and advanced rules: the create-scan API takes them under scanSettings, -// and anything left out takes the API's default. +// review" issues, advanced rules and experimental rules: the create-scan API takes them +// under scanSettings, and anything left out takes the API's default. const m = vi.hoisted(() => ({ post: vi.fn(), + get: vi.fn(), startScan: vi.fn(), waitUntilComplete: vi.fn(), runSettings: vi.fn(), @@ -13,7 +14,7 @@ const m = vi.hoisted(() => ({ parse: vi.fn(), })); -vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post } })); +vi.mock("../../src/lib/apiClient.js", () => ({ apiClient: { post: m.post, get: m.get } })); vi.mock("../../src/lib/local.js", () => ({ isLocalURL: () => false, ensureLocalBinarySetup: vi.fn(), @@ -86,6 +87,14 @@ describe("AccessibilityScanner.startScan", () => { ).rejects.toThrow(/A scan with this name already exists/); }); + it("sends experimental rules off when asked, as false is a setting rather than a missing one", async () => { + await (await scanner()).startScan("checkout", ["https://shop.example/"], undefined, { + experimentalRules: false, + }); + + expect(m.post.mock.calls[0][0].body.scanSettings).toEqual({ experimentalRules: false }); + }); + it("sends no scanSettings when none are given", async () => { await (await scanner()).startScan("checkout", ["https://shop.example/"]); @@ -93,6 +102,59 @@ describe("AccessibilityScanner.startScan", () => { }); }); +// The run summary carries a run's WCAG version, best practices, needs review and advanced +// rules, but not experimental rules: those are read from the scan's own settings (its +// overview), which a scan started here has for this run alone. +describe("AccessibilityScanner.runSettings", () => { + const SCANS = "https://api-accessibility.browserstack.com/api/website-scanner/v1/scans"; + const run = { ok: true, status: 200, data: { data: { wcagVersion: "wcag22aa", bestPractices: true, needsReview: true, advanced: true } } }; + + beforeEach(() => { + vi.resetModules(); + m.get.mockReset(); + }); + + async function scanner() { + const { AccessibilityScanner } = await vi.importActual< + typeof import("../../src/tools/accessiblity-utils/scanner.js") + >("../../src/tools/accessiblity-utils/scanner.js"); + const s = new AccessibilityScanner(); + s.setAuth({ username: "u", password: "p" }); + return s; + } + + it("reads experimental rules from the scan's own settings, as the run summary has none", async () => { + m.get.mockImplementation(async ({ url }: { url: string }) => + url === `${SCANS}/7310042/overview` + ? { ok: true, status: 200, data: { data: { scanConfigInfo: { experimental: false } } } } + : run, + ); + + await expect((await scanner()).runSettings("7310042", "5520318")).resolves.toEqual({ + wcagVersion: "wcag22aa", + bestPractices: true, + needsReview: true, + advancedRules: true, + experimentalRules: false, + }); + expect(m.get.mock.calls.map(([o]) => o.url).sort()).toEqual([ + `${SCANS}/7310042/overview`, + `${SCANS}/7310042/scan_runs/5520318`, + ]); + }); + + it("still reads the run's settings when the scan's own cannot be read", async () => { + m.get.mockImplementation(async ({ url }: { url: string }) => + url.endsWith("/overview") ? { ok: false, status: 500, data: {} } : run, + ); + + const settings = await (await scanner()).runSettings("7310042", "5520318"); + + expect(settings).toMatchObject({ wcagVersion: "wcag22aa", advancedRules: true }); + expect(settings).not.toHaveProperty("experimentalRules"); + }); +}); + describe("startAccessibilityScan scan settings", () => { beforeEach(() => { vi.resetModules(); @@ -174,6 +236,30 @@ describe("startAccessibilityScan scan settings", () => { ); }); + it("passes experimental rules to the scan and reports them as the scan ran", async () => { + m.runSettings.mockResolvedValue({ + wcagVersion: "wcag21aa", + bestPractices: false, + needsReview: true, + advancedRules: true, + experimentalRules: false, + }); + const { run } = await tool(); + + const result = await run({ + name: "checkout", + pageURL: "https://shop.example/", + experimentalRules: false, + }); + + expect(m.startScan).toHaveBeenCalledWith("checkout", ["https://shop.example/"], undefined, { + experimentalRules: false, + }); + expect(text(result)).toContain( + "Scan settings, as the scan ran: WCAG 2.1 AA, best practices off, needs review on, advanced rules on, experimental rules off.", + ); + }); + it("says only what was asked for when the run's settings cannot be read", async () => { m.runSettings.mockResolvedValue(undefined); const { run } = await tool(); @@ -191,6 +277,14 @@ describe("startAccessibilityScan scan settings", () => { expect(text(result)).not.toContain("(default)"); }); + it("takes experimental rules only as on or off", async () => { + const { schema } = await tool(); + + expect(schema.experimentalRules.safeParse(false).success).toBe(true); + expect(schema.experimentalRules.safeParse(true).success).toBe(true); + expect(schema.experimentalRules.safeParse("off").success).toBe(false); + }); + it("takes only the WCAG versions the API knows, RGAA 4.1 among them", async () => { const { schema } = await tool(); From 2fcf8f866c869f6b5a88b3cec674140f6865b942 Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Mon, 26 Oct 2026 16:31:03 +0530 Subject: [PATCH 2/3] fix(accessibility): Automate setup writes accessibility: true; a page that did not load is not "fixed" From the QA evals (ATA-1461, ATA-1465): - setupBrowserStackAutomateTests takes accessibility. When it is on, the browserstack.yml step writes accessibility: true, and a last step says how to read the issues. The step now says to add to an existing file and keep its settings, rather than to update it with this content. - Comparing two scan runs: a page that did not load in the newer run is scanned anyway, as an error page under its address (BrowserStack's Local connection page, or Chrome's network error page), so its real issues read as resolved and the error page's as new. Issues on such a page are now set apart: the earlier ones as "not checked", the error page's left out, with a warning naming the page. A page whose earlier issues all gave way to new ones gets a softer warning. Neither the scan's logs nor the dashboard flag either case. Co-Authored-By: Claude Opus 5.5 --- src/tools/accessibility.ts | 35 ++++++++++ src/tools/accessiblity-utils/report-parser.ts | 67 +++++++++++++++++-- src/tools/sdk-utils/bstack/configUtils.ts | 15 ++++- src/tools/sdk-utils/bstack/sdkHandler.ts | 10 +++ src/tools/sdk-utils/common/schema.ts | 7 ++ src/tools/sdk-utils/handler.ts | 2 + .../accessibilityFetchPagesAndCompare.test.ts | 23 +++++++ tests/tools/accessibilityReportPages.test.ts | 64 ++++++++++++++++++ .../tools/automateAccessibilitySetup.test.ts | 40 +++++++++++ 9 files changed, 258 insertions(+), 5 deletions(-) create mode 100644 tests/tools/automateAccessibilitySetup.test.ts diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index d9d4755f..f1cd6666 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -1011,6 +1011,37 @@ function pageNote(page: { ]; } +/** + * The pages of the newer run that did not load (an error page was scanned in their place), + * or whose earlier issues all gave way to new ones. Neither the scan nor the dashboard says + * so, and their real issues would otherwise read as fixed. + */ +function pagesNotLoadedNote( + diff: { + not_checked?: unknown[]; + failed_pages?: string[]; + changed_pages?: string[]; + }, + runId: string, +): string[] { + const pages = (urls: string[]) => + wrapUntrusted("report page addresses", JSON.stringify(urls)); + const failed = diff.failed_pages ?? []; + const changed = diff.changed_pages ?? []; + return [ + ...(failed.length > 0 + ? [ + `⚠️ ${failed.length === 1 ? "1 page" : `${failed.length} pages`} did not load in run ${runId}: an error page was scanned in its place. Their ${diff.not_checked?.length ?? 0} issues from the earlier run are not counted as resolved (listed as not checked), and the error page's issues are not counted as new. Scan again once they load. ${pages(failed)}`, + ] + : []), + ...(changed.length > 0 + ? [ + `⚠️ None of the earlier issues on ${changed.length === 1 ? "1 page" : `${changed.length} pages`} remain, and they have new ones: they may not have loaded, or loaded something else, in run ${runId}. Check them before calling those issues fixed. ${pages(changed)}`, + ] + : []), + ]; +} + /** At most this many new and resolved issues are listed in a comparison. */ const COMPARE_LIST_LIMIT = 50; @@ -1090,8 +1121,12 @@ async function compareScanRuns( changes ? `The dashboard's comparison: ${changes.newIssues} new, ${changes.resolvedIssues} resolved, ${changes.retainedIssues} still open. Matching the issues by rule, element and page gives ${matched}.` : `Matching the issues by rule, element and page: ${matched}.`, + ...pagesNotLoadedNote(diff, source.scanRunId), `New issues: ${listed(diff.added)}`, `Resolved issues: ${listed(diff.resolved)}`, + ...(diff.not_checked?.length + ? [`Not checked (page did not load): ${listed(diff.not_checked)}`] + : []), `CSV of each run, valid for about 12 hours: newer ${links[0]} , older ${links[1]}`, ]); } diff --git a/src/tools/accessiblity-utils/report-parser.ts b/src/tools/accessiblity-utils/report-parser.ts index 41598bdd..6beed8b7 100644 --- a/src/tools/accessiblity-utils/report-parser.ts +++ b/src/tools/accessiblity-utils/report-parser.ts @@ -221,8 +221,20 @@ export type RunIssue = { severity: string; component: string; affected_page: string; + /** Found on an error page scanned in the page's place: the page did not load. */ + on_error_page?: true; }; +/** + * Snippets only an error page shows: BrowserStack's Local connection page, and Chrome's + * network error page. A page that fails to load is still "scanned", as one of these. + */ +const ERROR_PAGE_SNIPPET = + /dashboard-privoxy-page|browserstack\.com\/docs\/local-testing\/|main-frame-error|chrome-error:\/\/|\bneterror\b/i; + +/** A page whose earlier issues are all gone and new ones appeared: worth a second look. */ +const CHANGED_PAGE_MIN_ISSUES = 2; + /** A report's main issues that the dashboard counts (not hidden), keyed for comparing runs. */ export async function loadMainIssues(reportLink: string): Promise { const { parsed, columns } = await loadReport(reportLink); @@ -241,21 +253,68 @@ export async function loadMainIssues(reportLink: string): Promise { severity: (row["Severity"] || "unknown").trim(), component: row["Component"], affected_page: row["Affected page"], + ...(ERROR_PAGE_SNIPPET.test(row["HTML snippet"] ?? "") + ? { on_error_page: true as const } + : {}), }; }); } -/** Which issues of `current` are new since `base`, which of `base` are gone, and how many remain. */ +/** + * Which issues of `current` are new since `base`, which of `base` are gone, and how many + * remain. A page that did not load in `current` (an error page was scanned under its + * address) is set apart: its earlier issues were not checked again, so they are + * `not_checked` rather than resolved, and the error page's issues are not new. A page whose + * earlier issues are all gone while new ones appeared is named in `changed_pages`, as it may + * not have loaded either. + */ export function compareIssues( base: RunIssue[], current: RunIssue[], -): { added: RunIssue[]; resolved: RunIssue[]; retained: number } { +): { + added: RunIssue[]; + resolved: RunIssue[]; + retained: number; + not_checked: RunIssue[]; + failed_pages: string[]; + changed_pages: string[]; +} { const before = new Set(base.map((issue) => issue.key)); const after = new Set(current.map((issue) => issue.key)); + const page = (issue: RunIssue) => samePage(issue.affected_page ?? ""); + const failed = new Map(); + for (const issue of current) { + if (issue.on_error_page) failed.set(page(issue), issue.affected_page); + } + const added = current.filter( + (issue) => !before.has(issue.key) && !failed.has(page(issue)), + ); + const gone = base.filter((issue) => !after.has(issue.key)); + const resolved = gone.filter((issue) => !failed.has(page(issue))); + const changed = [ + ...new Map( + resolved + .filter((issue) => { + const on = page(issue); + const earlier = base.filter((b) => page(b) === on); + return ( + earlier.length >= CHANGED_PAGE_MIN_ISSUES && + earlier.every((b) => !after.has(b.key)) && + added.some((a) => page(a) === on) + ); + }) + .map((issue) => [page(issue), issue.affected_page]), + ).values(), + ]; return { - added: sortBySeverity(current.filter((issue) => !before.has(issue.key))), - resolved: sortBySeverity(base.filter((issue) => !after.has(issue.key))), + added: sortBySeverity(added), + resolved: sortBySeverity(resolved), retained: current.filter((issue) => before.has(issue.key)).length, + not_checked: sortBySeverity( + gone.filter((issue) => failed.has(page(issue))), + ), + failed_pages: [...failed.values()], + changed_pages: changed, }; } diff --git a/src/tools/sdk-utils/bstack/configUtils.ts b/src/tools/sdk-utils/bstack/configUtils.ts index e577ce5e..3fde2caf 100644 --- a/src/tools/sdk-utils/bstack/configUtils.ts +++ b/src/tools/sdk-utils/bstack/configUtils.ts @@ -5,6 +5,8 @@ export function generateBrowserStackYMLInstructions(config: { platforms?: string[]; enablePercy?: boolean; projectName: string; + /** Run Accessibility checks during the tests. */ + accessibility?: boolean; }): string { const enablePercy = config.enablePercy || false; const projectName = config.projectName || "BrowserStack Automate Build"; @@ -12,8 +14,9 @@ export function generateBrowserStackYMLInstructions(config: { // Generate platform configurations using the utility function const platformConfigs = generatePlatformConfigs(config); + // An existing file holds the user's own settings, so it is added to, not replaced. const stepTitle = - "Create a browserstack.yml file in the project root with your validated device configurations:If already exists, update it with the following content for devices and project details."; + "Create a browserstack.yml file in the project root with your validated device configurations. If the file already exists, add these keys to it and keep its other settings."; const buildName = `${projectName}-Build`; @@ -60,6 +63,16 @@ browserstackLocal: true debug: true # Visual logs, text logs, etc. testObservability: true # For Test Observability`; + if (config.accessibility) { + ymlContent += ` + +# ====================== +# Accessibility Testing +# ====================== +# Runs Accessibility (WCAG) checks on the pages the tests visit. +accessibility: true`; + } + if (enablePercy) { ymlContent += ` diff --git a/src/tools/sdk-utils/bstack/sdkHandler.ts b/src/tools/sdk-utils/bstack/sdkHandler.ts index b9b43435..8eec1f0d 100644 --- a/src/tools/sdk-utils/bstack/sdkHandler.ts +++ b/src/tools/sdk-utils/bstack/sdkHandler.ts @@ -126,6 +126,7 @@ export async function runBstackSDKOnly( validatedEnvironments, enablePercy: false, projectName: input.projectName, + accessibility: input.accessibility, }); if (ymlInstructions) { @@ -146,6 +147,15 @@ export async function runBstackSDKOnly( } } + if (input.accessibility) { + steps.push({ + type: "instruction", + title: "Read the accessibility issues", + content: + "When the run finishes, read its accessibility issues with fetchAccessibilityIssues: pass the Automate build's dashboard link as reportUrl, or the build id the BrowserStack SDK prints at the end of the run as buildId.", + }); + } + return { steps, requiresPercy: false, diff --git a/src/tools/sdk-utils/common/schema.ts b/src/tools/sdk-utils/common/schema.ts index 1cb6159b..8d7660d1 100644 --- a/src/tools/sdk-utils/common/schema.ts +++ b/src/tools/sdk-utils/common/schema.ts @@ -90,6 +90,13 @@ export const RunTestsOnBrowserStackParamsShape = { .describe( "Device objects array. Use the object format directly - no transformation needed. Add only when user explicitly requests devices. Examples: [{ platform: 'windows', osVersion: '11', browser: 'chrome', browserVersion: 'latest' }] or [{ platform: 'android', deviceName: 'Samsung Galaxy S24', osVersion: '14', browser: 'chrome' }].", ), + accessibility: z + .boolean() + .optional() + .default(false) + .describe( + "Run Accessibility (WCAG) checks during the tests: writes accessibility: true in browserstack.yml.", + ), }; export const SetUpPercySchema = z.object(SetUpPercyParamsShape); diff --git a/src/tools/sdk-utils/handler.ts b/src/tools/sdk-utils/handler.ts index a75cf608..8240d097 100644 --- a/src/tools/sdk-utils/handler.ts +++ b/src/tools/sdk-utils/handler.ts @@ -89,6 +89,7 @@ export async function setUpPercyHandler( const percyWithBrowserstackSDKResult = runPercyWithBrowserstackSDK({ ...percyInput, devices: [], + accessibility: false, }); const hasPercySDKError = percyWithBrowserstackSDKResult.steps && @@ -125,6 +126,7 @@ export async function setUpPercyHandler( input.detectedBrowserAutomationFramework, detectedTestingFramework: input.detectedTestingFramework, devices: [], + accessibility: false, }; const sdkResult = await runBstackSDKOnly(sdkInput, config, true); // Percy Automate instructions diff --git a/tests/tools/accessibilityFetchPagesAndCompare.test.ts b/tests/tools/accessibilityFetchPagesAndCompare.test.ts index 72764c50..5e8098fc 100644 --- a/tests/tools/accessibilityFetchPagesAndCompare.test.ts +++ b/tests/tools/accessibilityFetchPagesAndCompare.test.ts @@ -196,6 +196,29 @@ describe("fetchAccessibilityIssues comparing scan runs", () => { expect(text(result)).toMatch(/first run/); }); + it("says when a page did not load in the later run, and does not count its issues as resolved", async () => { + m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); + m.changesSinceLastRun.mockResolvedValue({ newIssues: 2, retainedIssues: 2, resolvedIssues: 6 }); + m.compareIssues.mockReturnValue({ + added: [], + resolved: [], + retained: 2, + not_checked: [{ issue_type: "label", severity: "critical", component: "input", affected_page: "https://shop.example/b" }], + failed_pages: ["https://shop.example/b"], + changed_pages: ["https://shop.example/c"], + }); + + const out = text(await fetchIssues()({ ...RUN, compareWith: "previous" })); + + expect(out).toContain( + "⚠️ 1 page did not load in run 5520318: an error page was scanned in its place. Their 1 issues from the earlier run are not counted as resolved (listed as not checked), and the error page's issues are not counted as new. Scan again once they load.", + ); + expect(out).toContain("⚠️ None of the earlier issues on 1 page remain, and they have new ones"); + // The addresses come from the report, so they are quoted as data. + expect(out.indexOf("https://shop.example/b")).toBeGreaterThan(out.indexOf("«UNTRUSTED")); + expect(out).toMatch(/Not checked \(page did not load\)/); + }); + it("compares Website Scanner runs only", async () => { const result = await fetchIssues()({ buildId: "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja", compareWith: "previous" }); diff --git a/tests/tools/accessibilityReportPages.test.ts b/tests/tools/accessibilityReportPages.test.ts index 1a5713bb..91258d13 100644 --- a/tests/tools/accessibilityReportPages.test.ts +++ b/tests/tools/accessibilityReportPages.test.ts @@ -180,3 +180,67 @@ describe("compareIssues", () => { expect(diff.retained).toBe(2); }); }); + +// A page that fails to load in a re-run is still "scanned": what gets scanned is an error page +// (BrowserStack's Local connection page, or Chrome's network error page) filed under the +// page's address. Its real issues then read as resolved and the error page's as new. +describe("compareIssues and a page that did not load in the later run", () => { + beforeEach(() => vi.clearAllMocks()); + + const PAGE_A = "http://bs-local.com:3100/page-a.html"; + const PAGE_B = "http://bs-local.com:3100/page-b.html"; + const issueOn = (page: string, type: string, selector: string, snippet = "") => + row({ "Issue type": type, Component: selector, "CSS selector": selector, "Affected page": page, "HTML snippet": snippet, "Issue link": link(`${type}-${page}`) }); + + async function runs(laterB: Record[]) { + serve([ + issueOn(PAGE_A, "css-orientation-lock", "html"), + issueOn(PAGE_B, "label", "input#no-label"), + issueOn(PAGE_B, "button-name", "button#icon-btn"), + issueOn(PAGE_B, "link-name", "a#icon-link"), + ]); + const before = await loadMainIssues("https://csv-old"); + serve([issueOn(PAGE_A, "css-orientation-lock", "html"), ...laterB]); + const after = await loadMainIssues("https://csv-new"); + return compareIssues(before, after); + } + + it("sets apart, rather than resolves, the issues of a page that showed BrowserStack's connection error page", async () => { + const diff = await runs([ + issueOn(PAGE_B, "keyboard-focus-visible", "a.docs", ''), + issueOn(PAGE_B, "decorative-image", "img.logo", 'BrowserStack Logo'), + ]); + + expect(diff.failed_pages).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + expect(diff.not_checked.map((r) => r.issue_type).sort()).toEqual(["button-name", "label", "link-name"]); + expect(diff.added).toEqual([]); + expect(diff.retained).toBe(1); + }); + + it("knows Chrome's network error page too, as a public page that fails to load shows it", async () => { + const diff = await runs([ + issueOn(PAGE_B, "landmark-one-main", "div#main-frame-error", '
'), + ]); + + expect(diff.failed_pages).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + }); + + it("warns, without reclassifying, when every earlier issue on a page is gone and new ones appeared", async () => { + const diff = await runs([issueOn(PAGE_B, "region", "div.promo")]); + + expect(diff.failed_pages).toEqual([]); + expect(diff.changed_pages).toEqual([PAGE_B]); + expect(diff.resolved).toHaveLength(3); + expect(diff.added.map((r) => r.issue_type)).toEqual(["region"]); + }); + + it("flags no page when its issues were simply fixed", async () => { + const diff = await runs([]); + + expect(diff.failed_pages).toEqual([]); + expect(diff.changed_pages).toEqual([]); + expect(diff.resolved).toHaveLength(3); + }); +}); diff --git a/tests/tools/automateAccessibilitySetup.test.ts b/tests/tools/automateAccessibilitySetup.test.ts new file mode 100644 index 00000000..1144912c --- /dev/null +++ b/tests/tools/automateAccessibilitySetup.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; + +import { generateBrowserStackYMLInstructions } from "../../src/tools/sdk-utils/bstack/configUtils.js"; +import { RunTestsOnBrowserStackSchema } from "../../src/tools/sdk-utils/common/schema.js"; + +// Asked to run Automate tests with accessibility testing on, the setup tool must write +// accessibility: true into browserstack.yml, and must not tell the user to replace a file +// they already have, which would drop their own settings. + +const yml = (accessibility?: boolean) => + generateBrowserStackYMLInstructions({ + validatedEnvironments: [], + projectName: "Checkout", + ...(accessibility === undefined ? {} : { accessibility }), + }); + +describe("setupBrowserStackAutomateTests and accessibility", () => { + it("takes accessibility as an option, off by default", () => { + const base = { + projectName: "Checkout", + detectedLanguage: "nodejs", + detectedBrowserAutomationFramework: "playwright", + detectedTestingFramework: "playwright", + }; + expect(RunTestsOnBrowserStackSchema.parse(base).accessibility).toBe(false); + expect(RunTestsOnBrowserStackSchema.parse({ ...base, accessibility: true }).accessibility).toBe(true); + }); + + it("writes accessibility: true into browserstack.yml when asked", () => { + expect(yml(true)).toContain("\naccessibility: true\n"); + }); + + it("leaves the file as it was when accessibility is not asked for", () => { + expect(yml()).not.toMatch(/^accessibility:/m); + }); + + it("says to add to an existing browserstack.yml rather than replace it", () => { + expect(yml(true)).toContain("If the file already exists, add these keys to it and keep its other settings"); + }); +}); From cc1e8aca96175094976803ed02e07cadda38e07e Mon Sep 17 00:00:00 2001 From: Aaryan Choudhary Date: Mon, 26 Oct 2026 17:46:28 +0530 Subject: [PATCH 3/3] fix(accessibility): review of #519: fast run compare, pages that failed in the earlier run, Cypress/WebdriverIO accessibility - compareIssues groups both runs' issues by page once and decides page by page. The "changed page" check re-filtered the whole earlier run for each resolved issue, parsing a URL each time: 3,000 issues per run took 3.3 s and blocked the process; it now takes about 10 ms. - A page that did not load in the earlier run is set apart too: the error page's issues are not resolved, and the page's issues now are listed as not checked against the earlier run rather than new. A page that failed in both runs is not compared. Issues on a page that failed in either run are no longer counted as still open. - The error-page pattern no longer matches a link to the Local docs or a class merely starting with neterror: a real page can have either. The Local connection page is still caught by its ref=dashboard-privoxy-page links, and Chrome's by id="main-frame-error", class="neterror" or chrome-error://. - The warnings agree in number ("1 page ... its place ... the 1 issue ... is"), and the earlier-run case has its own. - setupBrowserStackAutomateTests with accessibility: true now covers Cypress ("accessibility": true under run_settings in browserstack.json) and WebdriverIO (accessibility: true in the browserstack service's options), which return before the browserstack.yml step, and says how to read the issues. Both flags checked in browserstack-cypress-cli 1.37.1 and @wdio/browserstack-service 9.41.0. Co-Authored-By: Claude Opus 5.5 --- docs/TOOLS.md | 2 +- src/tools/accessibility.ts | 33 ++++- src/tools/accessiblity-utils/report-parser.ts | 121 ++++++++++++------ src/tools/sdk-utils/bstack/sdkHandler.ts | 21 +++ .../accessibilityFetchPagesAndCompare.test.ts | 29 ++++- tests/tools/accessibilityReportPages.test.ts | 82 ++++++++++++ .../tools/automateAccessibilitySetup.test.ts | 42 ++++++ 7 files changed, 284 insertions(+), 46 deletions(-) diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 8e17f610..16b8b8b9 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -329,7 +329,7 @@ searchCapability → describeCapability → invokeCapability | Tool | What it does | |---|---| -| `setupBrowserStackAutomateTests` | Integrate BrowserStack SDK and run web tests on BrowserStack. For visual testing/Percy, use the dedicated Percy tools. | +| `setupBrowserStackAutomateTests` | Integrate BrowserStack SDK and run web tests on BrowserStack. With `accessibility: true`, the run also performs Accessibility (WCAG) checks, turned on in `browserstack.yml` (Cypress: `browserstack.json`; WebdriverIO: the BrowserStack service's options), and the output says how to read the issues. For visual testing/Percy, use the dedicated Percy tools. | | `fetchAutomationScreenshots` | Fetch screenshots captured during a given Automate/App Automate session. | | `listSessions` | List the sessions in an Automate/App Automate build. Each record carries `sessionId`, `name`, `status`, `os`, `osVersion`, `browser`, `device`, `browserUrl` (dashboard link), and `videoUrl`, with optional `limit` / `offset` paging and a client-side `status` filter. Takes either the **hashed** build ID from the dashboard URL or the observability build id returned by `getBuildId` / `listBuildId` — an observability id is resolved to the hashed id automatically via the build's sessions. Returned `sessionId` values work with `getFailureLogs`, `fetchAutomationScreenshots`, and `fetchSelfHealedSelectors`. | diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index f1cd6666..2b147437 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -1020,23 +1020,45 @@ function pagesNotLoadedNote( diff: { not_checked?: unknown[]; failed_pages?: string[]; + not_checked_earlier?: unknown[]; + failed_pages_earlier?: string[]; changed_pages?: string[]; }, runId: string, + baseRunId: string, ): string[] { const pages = (urls: string[]) => wrapUntrusted("report page addresses", JSON.stringify(urls)); + // Worded for one page or several, one issue or several. + const one = (count: number, singular: string, plural: string) => + count === 1 ? singular : plural; + const they = (n: number) => one(n, "it", "they"); + const didNotLoad = (urls: string[], run: string) => { + const n = urls.length; + return `⚠️ ${one(n, "1 page", `${n} pages`)} did not load in run ${run}: ${one(n, "an error page was", "error pages were")} scanned in ${one(n, "its", "their")} place. The ${one(n, "error page's", "error pages'")} issues are not counted as`; + }; + // A page that failed in both runs has no issues of its own in either. + const foundThere = (count: number, run: string, notCountedAs: string) => + count > 0 + ? `, and the ${one(count, "1 issue", `${count} issues`)} found there in run ${run} ${one(count, "is", "are")} not counted as ${notCountedAs}` + : ""; const failed = diff.failed_pages ?? []; + const failedEarlier = diff.failed_pages_earlier ?? []; const changed = diff.changed_pages ?? []; return [ ...(failed.length > 0 ? [ - `⚠️ ${failed.length === 1 ? "1 page" : `${failed.length} pages`} did not load in run ${runId}: an error page was scanned in its place. Their ${diff.not_checked?.length ?? 0} issues from the earlier run are not counted as resolved (listed as not checked), and the error page's issues are not counted as new. Scan again once they load. ${pages(failed)}`, + `${didNotLoad(failed, runId)} new${foundThere(diff.not_checked?.length ?? 0, baseRunId, "resolved (listed as not checked)")}. Scan again once ${they(failed.length)} ${one(failed.length, "loads", "load")}. ${pages(failed)}`, + ] + : []), + ...(failedEarlier.length > 0 + ? [ + `${didNotLoad(failedEarlier, baseRunId)} resolved${foundThere(diff.not_checked_earlier?.length ?? 0, runId, "new (listed as not checked against the earlier run)")}. Compare with a run in which ${they(failedEarlier.length)} loaded. ${pages(failedEarlier)}`, ] : []), ...(changed.length > 0 ? [ - `⚠️ None of the earlier issues on ${changed.length === 1 ? "1 page" : `${changed.length} pages`} remain, and they have new ones: they may not have loaded, or loaded something else, in run ${runId}. Check them before calling those issues fixed. ${pages(changed)}`, + `⚠️ None of the earlier issues on ${one(changed.length, "1 page remain, and it has", `${changed.length} pages remain, and they have`)} new ones: ${they(changed.length)} may not have loaded, or loaded something else, in run ${runId}. Check ${one(changed.length, "it", "them")} before calling those issues fixed. ${pages(changed)}`, ] : []), ]; @@ -1121,12 +1143,17 @@ async function compareScanRuns( changes ? `The dashboard's comparison: ${changes.newIssues} new, ${changes.resolvedIssues} resolved, ${changes.retainedIssues} still open. Matching the issues by rule, element and page gives ${matched}.` : `Matching the issues by rule, element and page: ${matched}.`, - ...pagesNotLoadedNote(diff, source.scanRunId), + ...pagesNotLoadedNote(diff, source.scanRunId, baseRunId), `New issues: ${listed(diff.added)}`, `Resolved issues: ${listed(diff.resolved)}`, ...(diff.not_checked?.length ? [`Not checked (page did not load): ${listed(diff.not_checked)}`] : []), + ...(diff.not_checked_earlier?.length + ? [ + `Not checked against the earlier run (page did not load in run ${baseRunId}): ${listed(diff.not_checked_earlier)}`, + ] + : []), `CSV of each run, valid for about 12 hours: newer ${links[0]} , older ${links[1]}`, ]); } diff --git a/src/tools/accessiblity-utils/report-parser.ts b/src/tools/accessiblity-utils/report-parser.ts index 6beed8b7..d151ae1c 100644 --- a/src/tools/accessiblity-utils/report-parser.ts +++ b/src/tools/accessiblity-utils/report-parser.ts @@ -226,11 +226,13 @@ export type RunIssue = { }; /** - * Snippets only an error page shows: BrowserStack's Local connection page, and Chrome's - * network error page. A page that fails to load is still "scanned", as one of these. + * Snippets only an error page shows: BrowserStack's Local connection page (its links carry + * ref=dashboard-privoxy-page), and Chrome's network error page. A page that fails to load + * is still "scanned", as one of these. A link to the Local docs, or a class merely starting + * with neterror, is not enough: a real page can have either. */ const ERROR_PAGE_SNIPPET = - /dashboard-privoxy-page|browserstack\.com\/docs\/local-testing\/|main-frame-error|chrome-error:\/\/|\bneterror\b/i; + /dashboard-privoxy-page|id="main-frame-error"|chrome-error:\/\/|class="neterror[" ]/i; /** A page whose earlier issues are all gone and new ones appeared: worth a second look. */ const CHANGED_PAGE_MIN_ISSUES = 2; @@ -260,13 +262,36 @@ export async function loadMainIssues(reportLink: string): Promise { }); } +/** A run's issues by page, with whether an error page was scanned in the page's place. */ +function issuesByPage( + issues: RunIssue[], +): Map { + const pages = new Map< + string, + { url: string; issues: RunIssue[]; failed: boolean } + >(); + for (const issue of issues) { + const key = samePage(issue.affected_page ?? ""); + let page = pages.get(key); + if (!page) { + page = { url: issue.affected_page, issues: [], failed: false }; + pages.set(key, page); + } + page.issues.push(issue); + if (issue.on_error_page) page.failed = true; + } + return pages; +} + /** * Which issues of `current` are new since `base`, which of `base` are gone, and how many - * remain. A page that did not load in `current` (an error page was scanned under its - * address) is set apart: its earlier issues were not checked again, so they are - * `not_checked` rather than resolved, and the error page's issues are not new. A page whose - * earlier issues are all gone while new ones appeared is named in `changed_pages`, as it may - * not have loaded either. + * remain, decided page by page. A page that did not load in one of the runs (an error page + * was scanned under its address) is set apart. Not loaded in `current`: its earlier issues + * were not checked again, so they are `not_checked` rather than resolved, and the error + * page's issues are not new. Not loaded in `base`: the error page's issues are not resolved, + * and the page's issues now are `not_checked_earlier` rather than new. A page whose earlier + * issues are all gone while new ones appeared is named in `changed_pages`, as it may not have + * loaded either. */ export function compareIssues( base: RunIssue[], @@ -277,44 +302,60 @@ export function compareIssues( retained: number; not_checked: RunIssue[]; failed_pages: string[]; + not_checked_earlier: RunIssue[]; + failed_pages_earlier: string[]; changed_pages: string[]; } { - const before = new Set(base.map((issue) => issue.key)); - const after = new Set(current.map((issue) => issue.key)); - const page = (issue: RunIssue) => samePage(issue.affected_page ?? ""); - const failed = new Map(); - for (const issue of current) { - if (issue.on_error_page) failed.set(page(issue), issue.affected_page); + const before = issuesByPage(base); + const after = issuesByPage(current); + const added: RunIssue[] = []; + const resolved: RunIssue[] = []; + const notChecked: RunIssue[] = []; + const notCheckedEarlier: RunIssue[] = []; + const failedPages: string[] = []; + const failedPagesEarlier: string[] = []; + const changedPages: string[] = []; + let retained = 0; + for (const key of new Set([...before.keys(), ...after.keys()])) { + const earlier = before.get(key); + const later = after.get(key); + if (later?.failed) failedPages.push(later.url); + if (earlier?.failed) failedPagesEarlier.push(earlier.url); + if (later?.failed || earlier?.failed) { + // Only the run in which the page loaded has its issues, and they cannot be matched. + if (!earlier?.failed) notChecked.push(...(earlier?.issues ?? [])); + if (!later?.failed) notCheckedEarlier.push(...(later?.issues ?? [])); + continue; + } + const earlierKeys = new Set(earlier?.issues.map((issue) => issue.key)); + const laterKeys = new Set(later?.issues.map((issue) => issue.key)); + const pageAdded = (later?.issues ?? []).filter( + (issue) => !earlierKeys.has(issue.key), + ); + const pageResolved = (earlier?.issues ?? []).filter( + (issue) => !laterKeys.has(issue.key), + ); + added.push(...pageAdded); + resolved.push(...pageResolved); + retained += (later?.issues.length ?? 0) - pageAdded.length; + if ( + earlier && + earlier.issues.length >= CHANGED_PAGE_MIN_ISSUES && + pageResolved.length === earlier.issues.length && + pageAdded.length > 0 + ) { + changedPages.push(earlier.url); + } } - const added = current.filter( - (issue) => !before.has(issue.key) && !failed.has(page(issue)), - ); - const gone = base.filter((issue) => !after.has(issue.key)); - const resolved = gone.filter((issue) => !failed.has(page(issue))); - const changed = [ - ...new Map( - resolved - .filter((issue) => { - const on = page(issue); - const earlier = base.filter((b) => page(b) === on); - return ( - earlier.length >= CHANGED_PAGE_MIN_ISSUES && - earlier.every((b) => !after.has(b.key)) && - added.some((a) => page(a) === on) - ); - }) - .map((issue) => [page(issue), issue.affected_page]), - ).values(), - ]; return { added: sortBySeverity(added), resolved: sortBySeverity(resolved), - retained: current.filter((issue) => before.has(issue.key)).length, - not_checked: sortBySeverity( - gone.filter((issue) => failed.has(page(issue))), - ), - failed_pages: [...failed.values()], - changed_pages: changed, + retained, + not_checked: sortBySeverity(notChecked), + failed_pages: failedPages, + not_checked_earlier: sortBySeverity(notCheckedEarlier), + failed_pages_earlier: failedPagesEarlier, + changed_pages: changedPages, }; } diff --git a/src/tools/sdk-utils/bstack/sdkHandler.ts b/src/tools/sdk-utils/bstack/sdkHandler.ts index 8eec1f0d..9546f66f 100644 --- a/src/tools/sdk-utils/bstack/sdkHandler.ts +++ b/src/tools/sdk-utils/bstack/sdkHandler.ts @@ -76,6 +76,18 @@ export async function runBstackSDKOnly( }); } + // Neither framework reads browserstack.yml: each turns accessibility on in its own file. + if (input.accessibility) { + steps.push({ + type: "instruction", + title: "Turn on Accessibility Testing", + content: + input.detectedBrowserAutomationFramework === "cypress" + ? 'In browserstack.json, add "accessibility": true under run_settings, keeping the file\'s other settings. It runs Accessibility (WCAG) checks on the pages the tests visit.' + : "In your WebdriverIO config, add accessibility: true to the browserstack service's options (services: [['browserstack', { accessibility: true, ... }]]), keeping its other options. It runs Accessibility (WCAG) checks on the pages the tests visit.", + }); + } + if (frameworkInstructions.run && !isPercyAutomate) { steps.push({ type: "instruction", @@ -83,6 +95,15 @@ export async function runBstackSDKOnly( content: frameworkInstructions.run, }); } + + if (input.accessibility) { + steps.push({ + type: "instruction", + title: "Read the accessibility issues", + content: + "When the run finishes, read its accessibility issues with fetchAccessibilityIssues: pass the Automate build's dashboard link, printed during the run, as reportUrl.", + }); + } } return { diff --git a/tests/tools/accessibilityFetchPagesAndCompare.test.ts b/tests/tools/accessibilityFetchPagesAndCompare.test.ts index 5e8098fc..4ebb84ee 100644 --- a/tests/tools/accessibilityFetchPagesAndCompare.test.ts +++ b/tests/tools/accessibilityFetchPagesAndCompare.test.ts @@ -211,14 +211,39 @@ describe("fetchAccessibilityIssues comparing scan runs", () => { const out = text(await fetchIssues()({ ...RUN, compareWith: "previous" })); expect(out).toContain( - "⚠️ 1 page did not load in run 5520318: an error page was scanned in its place. Their 1 issues from the earlier run are not counted as resolved (listed as not checked), and the error page's issues are not counted as new. Scan again once they load.", + "⚠️ 1 page did not load in run 5520318: an error page was scanned in its place. The error page's issues are not counted as new, and the 1 issue found there in run 5520296 is not counted as resolved (listed as not checked). Scan again once it loads.", + ); + expect(out).toContain( + "⚠️ None of the earlier issues on 1 page remain, and it has new ones: it may not have loaded, or loaded something else, in run 5520318. Check it before calling those issues fixed.", ); - expect(out).toContain("⚠️ None of the earlier issues on 1 page remain, and they have new ones"); // The addresses come from the report, so they are quoted as data. expect(out.indexOf("https://shop.example/b")).toBeGreaterThan(out.indexOf("«UNTRUSTED")); expect(out).toMatch(/Not checked \(page did not load\)/); }); + it("says when pages did not load in the earlier run, and does not count their issues now as new", async () => { + m.previousScanRun.mockResolvedValue({ id: 5520296, scanDate: "2026-10-21T11:31:00.584Z" }); + m.changesSinceLastRun.mockResolvedValue({ newIssues: 3, retainedIssues: 2, resolvedIssues: 2 }); + const issue = (page: string) => ({ issue_type: "label", severity: "critical", component: "input", affected_page: page }); + m.compareIssues.mockReturnValue({ + added: [], + resolved: [], + retained: 2, + not_checked: [], + failed_pages: [], + changed_pages: [], + not_checked_earlier: [issue("https://shop.example/b"), issue("https://shop.example/c"), issue("https://shop.example/c")], + failed_pages_earlier: ["https://shop.example/b", "https://shop.example/c"], + }); + + const out = text(await fetchIssues()({ ...RUN, compareWith: "previous" })); + + expect(out).toContain( + "⚠️ 2 pages did not load in run 5520296: error pages were scanned in their place. The error pages' issues are not counted as resolved, and the 3 issues found there in run 5520318 are not counted as new (listed as not checked against the earlier run). Compare with a run in which they loaded.", + ); + expect(out).toMatch(/Not checked against the earlier run \(page did not load in run 5520296\)/); + }); + it("compares Website Scanner runs only", async () => { const result = await fetchIssues()({ buildId: "q7m2k9x4c1v8b3n6z0w5r2t9y4p1s8d3f6g0h2ja", compareWith: "previous" }); diff --git a/tests/tools/accessibilityReportPages.test.ts b/tests/tools/accessibilityReportPages.test.ts index 91258d13..e3034cf4 100644 --- a/tests/tools/accessibilityReportPages.test.ts +++ b/tests/tools/accessibilityReportPages.test.ts @@ -244,3 +244,85 @@ describe("compareIssues and a page that did not load in the later run", () => { expect(diff.resolved).toHaveLength(3); }); }); + +// The mirror case: the page failed to load in the earlier run. Its "issues" there were the +// error page's, so they are not resolved, and the page's real issues now are not new. +describe("compareIssues and a page that did not load in the earlier run", () => { + beforeEach(() => vi.clearAllMocks()); + + const PAGE_A = "http://bs-local.com:3100/page-a.html"; + const PAGE_B = "http://bs-local.com:3100/page-b.html"; + const issueOn = (page: string, type: string, selector: string, snippet = "") => + row({ "Issue type": type, Component: selector, "CSS selector": selector, "Affected page": page, "HTML snippet": snippet, "Issue link": link(`${type}-${page}`) }); + const errorPage = [ + issueOn(PAGE_B, "keyboard-focus-visible", "a.docs", ''), + issueOn(PAGE_B, "decorative-image", "img.logo", 'BrowserStack Logo'), + ]; + + async function runs(earlierB: Record[], laterB: Record[]) { + serve([issueOn(PAGE_A, "css-orientation-lock", "html"), ...earlierB]); + const before = await loadMainIssues("https://csv-old"); + serve([issueOn(PAGE_A, "css-orientation-lock", "html"), ...laterB]); + const after = await loadMainIssues("https://csv-new"); + return compareIssues(before, after); + } + + it("does not resolve the error page's issues, nor count the page's real issues as new", async () => { + const diff = await runs(errorPage, [ + issueOn(PAGE_B, "label", "input#no-label"), + issueOn(PAGE_B, "button-name", "button#icon-btn"), + ]); + + expect(diff.failed_pages).toEqual([]); + expect(diff.failed_pages_earlier).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + expect(diff.added).toEqual([]); + expect(diff.not_checked_earlier.map((r) => r.issue_type).sort()).toEqual(["button-name", "label"]); + expect(diff.retained).toBe(1); + }); + + it("compares nothing on a page that failed in both runs", async () => { + const diff = await runs(errorPage, errorPage); + + expect(diff.failed_pages).toEqual([PAGE_B]); + expect(diff.failed_pages_earlier).toEqual([PAGE_B]); + expect(diff.resolved).toEqual([]); + expect(diff.added).toEqual([]); + expect(diff.not_checked).toEqual([]); + expect(diff.not_checked_earlier).toEqual([]); + expect(diff.retained).toBe(1); + }); + + it("does not take a real page that links to the Local docs, or names a class neterror-*, for an error page", async () => { + const diff = await runs( + [issueOn(PAGE_B, "label", "input#no-label")], + [ + issueOn(PAGE_B, "link-name", "a.help", ''), + issueOn(PAGE_B, "color-contrast", "div.neterror-banner", '
Offline?
'), + ], + ); + + expect(diff.failed_pages).toEqual([]); + expect(diff.failed_pages_earlier).toEqual([]); + expect(diff.added).toHaveLength(2); + expect(diff.resolved).toHaveLength(1); + }); +}); + +describe("compareIssues on large runs", () => { + it("compares runs of thousands of issues without stalling", () => { + const issues = (n: number, rule: string) => + Array.from({ length: n }, (_, i) => { + const page = `https://shop.example/p${i % 300}`; + return { key: `${rule}|#e${i}|${page}`, issue_type: rule, severity: "serious", component: "input", affected_page: page }; + }); + + const started = performance.now(); + const diff = compareIssues(issues(3000, "label"), issues(3000, "region")); + + expect(performance.now() - started).toBeLessThan(500); + expect(diff.resolved).toHaveLength(3000); + expect(diff.added).toHaveLength(3000); + expect(diff.changed_pages).toHaveLength(300); + }); +}); diff --git a/tests/tools/automateAccessibilitySetup.test.ts b/tests/tools/automateAccessibilitySetup.test.ts index 1144912c..4586884f 100644 --- a/tests/tools/automateAccessibilitySetup.test.ts +++ b/tests/tools/automateAccessibilitySetup.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { generateBrowserStackYMLInstructions } from "../../src/tools/sdk-utils/bstack/configUtils.js"; import { RunTestsOnBrowserStackSchema } from "../../src/tools/sdk-utils/common/schema.js"; +import { runBstackSDKOnly } from "../../src/tools/sdk-utils/bstack/sdkHandler.js"; +import { BrowserStackConfig } from "../../src/lib/types.js"; // Asked to run Automate tests with accessibility testing on, the setup tool must write // accessibility: true into browserstack.yml, and must not tell the user to replace a file @@ -38,3 +40,43 @@ describe("setupBrowserStackAutomateTests and accessibility", () => { expect(yml(true)).toContain("If the file already exists, add these keys to it and keep its other settings"); }); }); + +// Cypress and WebdriverIO are set up without browserstack.yml: Cypress through the +// browserstack-cypress CLI's browserstack.json, WebdriverIO through its BrowserStack +// service's options. Each turns accessibility on in its own file. +describe("setupBrowserStackAutomateTests, accessibility and the frameworks without browserstack.yml", () => { + const setup = async (framework: "cypress" | "webdriverio", accessibility: boolean) => { + const { steps } = await runBstackSDKOnly( + RunTestsOnBrowserStackSchema.parse({ + projectName: "Checkout", + detectedLanguage: "nodejs", + detectedBrowserAutomationFramework: framework, + detectedTestingFramework: framework === "cypress" ? "cypress" : "mocha", + accessibility, + }), + {} as BrowserStackConfig, + ); + return steps.map((step) => `${step.title}\n${step.content}`).join("\n\n"); + }; + + it("adds accessibility: true under run_settings in Cypress's browserstack.json", async () => { + const text = await setup("cypress", true); + expect(text).toContain('"accessibility": true'); + expect(text).toMatch(/under run_settings/); + expect(text).toMatch(/fetchAccessibilityIssues/); + }); + + it("adds accessibility: true to WebdriverIO's BrowserStack service options", async () => { + const text = await setup("webdriverio", true); + expect(text).toContain("accessibility: true"); + expect(text).toMatch(/browserstack service's options/); + expect(text).toMatch(/fetchAccessibilityIssues/); + }); + + it("says nothing of accessibility when it is not asked for", async () => { + for (const framework of ["cypress", "webdriverio"] as const) { + const text = await setup(framework, false); + expect(text).not.toMatch(/accessibility: true|"accessibility"|fetchAccessibilityIssues/); + } + }); +});