feat(accessibility): experimental rules; Automate accessibility setup; failed pages in scan compares

docs/TOOLS.md

@@ -329,7 +329,7 @@ searchCapability  →  describeCapability  →  invokeCapability
 
 | Tool | What it does |
 |---|---|
-| `setupBrowserStackAutomateTests` | Integrate BrowserStack SDK and run web tests on BrowserStack. For visual testing/Percy, use the dedicated Percy tools. |
+| `setupBrowserStackAutomateTests` | Integrate BrowserStack SDK and run web tests on BrowserStack. With `accessibility: true`, the run also performs Accessibility (WCAG) checks, turned on in `browserstack.yml` (Cypress: `browserstack.json`; WebdriverIO: the BrowserStack service's options), and the output says how to read the issues. For visual testing/Percy, use the dedicated Percy tools. |
 | `fetchAutomationScreenshots` | Fetch screenshots captured during a given Automate/App Automate session. |
 | `listSessions` | List the sessions in an Automate/App Automate build. Each record carries `sessionId`, `name`, `status`, `os`, `osVersion`, `browser`, `device`, `browserUrl` (dashboard link), and `videoUrl`, with optional `limit` / `offset` paging and a client-side `status` filter. Takes either the **hashed** build ID from the dashboard URL or the observability build id returned by `getBuildId` / `listBuildId` — an observability id is resolved to the hashed id automatically via the build's sessions. Returned `sessionId` values work with `getFailureLogs`, `fetchAutomationScreenshots`, and `fetchSelfHealedSelectors`. |
 
@@ -340,7 +340,7 @@ searchCapability  →  describeCapability  →  invokeCapability
 
 | Tool | What it does |
 |---|---|
-| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). |
+| `startAccessibilityScan` | Start a web accessibility scan of a page URL and retrieve a local CSV report path. Optional settings: `wcagVersion` (WCAG 2.0–2.2 at A/AA/AAA, or RGAA 4.1), `bestPractices`, `needsReview`, `advancedRules` and `experimentalRules`; the answer states the settings the scan ran with. Mobile apps are scanned through an App Automate run instead (`setupBrowserStackAppAutomateTests` with `accessibility: true`). |
 | `accessibilityExpert` | Ask the A11y Expert (WCAG 2.0/2.1/2.2, mobile/web usability, best practices). |
 | `createAccessibilityAuthConfig` | Create an authentication configuration (form-based or basic) for accessibility scans behind a login. |
 | `getAccessibilityAuthConfig` | Retrieve an existing accessibility authentication configuration by ID. |

src/tools/accessibility.ts

@@ -1011,6 +1011,59 @@ function pageNote(page: {
   ];
 }
 
+/**
+ * The pages of the newer run that did not load (an error page was scanned in their place),
+ * or whose earlier issues all gave way to new ones. Neither the scan nor the dashboard says
+ * so, and their real issues would otherwise read as fixed.
+ */
+function pagesNotLoadedNote(
+  diff: {
+    not_checked?: unknown[];
+    failed_pages?: string[];
+    not_checked_earlier?: unknown[];
+    failed_pages_earlier?: string[];
+    changed_pages?: string[];
+  },
+  runId: string,
+  baseRunId: string,
+): string[] {
+  const pages = (urls: string[]) =>
+    wrapUntrusted("report page addresses", JSON.stringify(urls));
+  // Worded for one page or several, one issue or several.
+  const one = (count: number, singular: string, plural: string) =>
+    count === 1 ? singular : plural;
+  const they = (n: number) => one(n, "it", "they");
+  const didNotLoad = (urls: string[], run: string) => {
+    const n = urls.length;
+    return `⚠️ ${one(n, "1 page", `${n} pages`)} did not load in run ${run}: ${one(n, "an error page was", "error pages were")} scanned in ${one(n, "its", "their")} place. The ${one(n, "error page's", "error pages'")} issues are not counted as`;
+  };
+  // A page that failed in both runs has no issues of its own in either.
+  const foundThere = (count: number, run: string, notCountedAs: string) =>
+    count > 0
+      ? `, and the ${one(count, "1 issue", `${count} issues`)} found there in run ${run} ${one(count, "is", "are")} not counted as ${notCountedAs}`
+      : "";
+  const failed = diff.failed_pages ?? [];
+  const failedEarlier = diff.failed_pages_earlier ?? [];
+  const changed = diff.changed_pages ?? [];
+  return [
+    ...(failed.length > 0
+      ? [
+          `${didNotLoad(failed, runId)} new${foundThere(diff.not_checked?.length ?? 0, baseRunId, "resolved (listed as not checked)")}. Scan again once ${they(failed.length)} ${one(failed.length, "loads", "load")}. ${pages(failed)}`,
+        ]
+      : []),
+    ...(failedEarlier.length > 0
+      ? [
+          `${didNotLoad(failedEarlier, baseRunId)} resolved${foundThere(diff.not_checked_earlier?.length ?? 0, runId, "new (listed as not checked against the earlier run)")}. Compare with a run in which ${they(failedEarlier.length)} loaded. ${pages(failedEarlier)}`,
+        ]
+      : []),
+    ...(changed.length > 0
+      ? [
+          `⚠️ None of the earlier issues on ${one(changed.length, "1 page remain, and it has", `${changed.length} pages remain, and they have`)} new ones: ${they(changed.length)} may not have loaded, or loaded something else, in run ${runId}. Check ${one(changed.length, "it", "them")} before calling those issues fixed. ${pages(changed)}`,
+        ]
+      : []),
+  ];
+}
+
 /** At most this many new and resolved issues are listed in a comparison. */
 const COMPARE_LIST_LIMIT = 50;
 
@@ -1090,8 +1143,17 @@ async function compareScanRuns(
     changes
       ? `The dashboard's comparison: ${changes.newIssues} new, ${changes.resolvedIssues} resolved, ${changes.retainedIssues} still open. Matching the issues by rule, element and page gives ${matched}.`
       : `Matching the issues by rule, element and page: ${matched}.`,
+    ...pagesNotLoadedNote(diff, source.scanRunId, baseRunId),
     `New issues: ${listed(diff.added)}`,
     `Resolved issues: ${listed(diff.resolved)}`,
+    ...(diff.not_checked?.length
+      ? [`Not checked (page did not load): ${listed(diff.not_checked)}`]
+      : []),
+    ...(diff.not_checked_earlier?.length
+      ? [
+          `Not checked against the earlier run (page did not load in run ${baseRunId}): ${listed(diff.not_checked_earlier)}`,
+        ]
+      : []),
     `CSV of each run, valid for about 12 hours: newer ${links[0]} , older ${links[1]}`,
   ]);
 }
@@ -1134,6 +1196,9 @@ function scanSettingsNote(
       settings.advancedRules !== undefined
         ? `advanced rules ${onOff(settings.advancedRules)}`
         : "",
+      settings.experimentalRules !== undefined
+        ? `experimental rules ${onOff(settings.experimentalRules)}`
+        : "",
     ]
       .filter(Boolean)
       .join(", ");
@@ -1173,6 +1238,7 @@ async function executeAccessibilityScan(
         bestPractices: args.bestPractices,
         needsReview: args.needsReview,
         advancedRules: args.advancedRules,
+        experimentalRules: args.experimentalRules,
       },
     );
   } catch (error) {
@@ -1476,6 +1542,10 @@ export default function addAccessibilityTools(
         .describe(
           'Also report "needs review" issues to check by hand (default true).',
         ),
+      experimentalRules: z
+        .boolean()
+        .optional()
+        .describe("Run the experimental accessibility rules (default true)."),
     },
     {
       title: "Start Accessibility Scan",

src/tools/accessiblity-utils/report-parser.ts

@@ -221,8 +221,22 @@ export type RunIssue = {
   severity: string;
   component: string;
   affected_page: string;
+  /** Found on an error page scanned in the page's place: the page did not load. */
+  on_error_page?: true;
 };
 
+/**
+ * Snippets only an error page shows: BrowserStack's Local connection page (its links carry
+ * ref=dashboard-privoxy-page), and Chrome's network error page. A page that fails to load
+ * is still "scanned", as one of these. A link to the Local docs, or a class merely starting
+ * with neterror, is not enough: a real page can have either.
+ */
+const ERROR_PAGE_SNIPPET =
+  /dashboard-privoxy-page|id="main-frame-error"|chrome-error:\/\/|class="neterror[" ]/i;
+
+/** A page whose earlier issues are all gone and new ones appeared: worth a second look. */
+const CHANGED_PAGE_MIN_ISSUES = 2;
+
 /** A report's main issues that the dashboard counts (not hidden), keyed for comparing runs. */
 export async function loadMainIssues(reportLink: string): Promise<RunIssue[]> {
   const { parsed, columns } = await loadReport(reportLink);
@@ -241,21 +255,107 @@ export async function loadMainIssues(reportLink: string): Promise<RunIssue[]> {
         severity: (row["Severity"] || "unknown").trim(),
         component: row["Component"],
         affected_page: row["Affected page"],
+        ...(ERROR_PAGE_SNIPPET.test(row["HTML snippet"] ?? "")
+          ? { on_error_page: true as const }
+          : {}),
       };
     });
 }
 
-/** Which issues of `current` are new since `base`, which of `base` are gone, and how many remain. */
+/** A run's issues by page, with whether an error page was scanned in the page's place. */
+function issuesByPage(
+  issues: RunIssue[],
+): Map<string, { url: string; issues: RunIssue[]; failed: boolean }> {
+  const pages = new Map<
+    string,
+    { url: string; issues: RunIssue[]; failed: boolean }
+  >();
+  for (const issue of issues) {
+    const key = samePage(issue.affected_page ?? "");
+    let page = pages.get(key);
+    if (!page) {
+      page = { url: issue.affected_page, issues: [], failed: false };
+      pages.set(key, page);
+    }
+    page.issues.push(issue);
+    if (issue.on_error_page) page.failed = true;
+  }
+  return pages;
+}
+
+/**
+ * Which issues of `current` are new since `base`, which of `base` are gone, and how many
+ * remain, decided page by page. A page that did not load in one of the runs (an error page
+ * was scanned under its address) is set apart. Not loaded in `current`: its earlier issues
+ * were not checked again, so they are `not_checked` rather than resolved, and the error
+ * page's issues are not new. Not loaded in `base`: the error page's issues are not resolved,
+ * and the page's issues now are `not_checked_earlier` rather than new. A page whose earlier
+ * issues are all gone while new ones appeared is named in `changed_pages`, as it may not have
+ * loaded either.
+ */
 export function compareIssues(
   base: RunIssue[],
   current: RunIssue[],
-): { added: RunIssue[]; resolved: RunIssue[]; retained: number } {
-  const before = new Set(base.map((issue) => issue.key));
-  const after = new Set(current.map((issue) => issue.key));
+): {
+  added: RunIssue[];
+  resolved: RunIssue[];
+  retained: number;
+  not_checked: RunIssue[];
+  failed_pages: string[];
+  not_checked_earlier: RunIssue[];
+  failed_pages_earlier: string[];
+  changed_pages: string[];
+} {
+  const before = issuesByPage(base);
+  const after = issuesByPage(current);
+  const added: RunIssue[] = [];
+  const resolved: RunIssue[] = [];
+  const notChecked: RunIssue[] = [];
+  const notCheckedEarlier: RunIssue[] = [];
+  const failedPages: string[] = [];
+  const failedPagesEarlier: string[] = [];
+  const changedPages: string[] = [];
+  let retained = 0;
+  for (const key of new Set([...before.keys(), ...after.keys()])) {
+    const earlier = before.get(key);
+    const later = after.get(key);
+    if (later?.failed) failedPages.push(later.url);
+    if (earlier?.failed) failedPagesEarlier.push(earlier.url);
+    if (later?.failed || earlier?.failed) {
+      // Only the run in which the page loaded has its issues, and they cannot be matched.
+      if (!earlier?.failed) notChecked.push(...(earlier?.issues ?? []));
+      if (!later?.failed) notCheckedEarlier.push(...(later?.issues ?? []));
+      continue;
+    }
+    const earlierKeys = new Set(earlier?.issues.map((issue) => issue.key));
+    const laterKeys = new Set(later?.issues.map((issue) => issue.key));
+    const pageAdded = (later?.issues ?? []).filter(
+      (issue) => !earlierKeys.has(issue.key),
+    );
+    const pageResolved = (earlier?.issues ?? []).filter(
+      (issue) => !laterKeys.has(issue.key),
+    );
+    added.push(...pageAdded);
+    resolved.push(...pageResolved);
+    retained += (later?.issues.length ?? 0) - pageAdded.length;
+    if (
+      earlier &&
+      earlier.issues.length >= CHANGED_PAGE_MIN_ISSUES &&
+      pageResolved.length === earlier.issues.length &&
+      pageAdded.length > 0
+    ) {
+      changedPages.push(earlier.url);
+    }
+  }
   return {
-    added: sortBySeverity(current.filter((issue) => !before.has(issue.key))),
-    resolved: sortBySeverity(base.filter((issue) => !after.has(issue.key))),
-    retained: current.filter((issue) => before.has(issue.key)).length,
+    added: sortBySeverity(added),
+    resolved: sortBySeverity(resolved),
+    retained,
+    not_checked: sortBySeverity(notChecked),
+    failed_pages: failedPages,
+    not_checked_earlier: sortBySeverity(notCheckedEarlier),
+    failed_pages_earlier: failedPagesEarlier,
+    changed_pages: changedPages,
   };
 }
 

src/tools/accessiblity-utils/scanner.ts

@@ -33,6 +33,8 @@ export interface ScanSettings {
   needsReview?: boolean;
   /** Run the advanced rules (API default true). */
   advancedRules?: boolean;
+  /** Run the experimental rules (API default true). */
+  experimentalRules?: boolean;
 }
 
 export class AccessibilityScanner {
@@ -158,27 +160,37 @@ export class AccessibilityScanner {
   /**
    * The settings a run actually ran with, from its summary, or undefined when it cannot be
    * read. A setting left out takes the API's default, and the plan can turn advanced rules
-   * off, so what was asked for is not always what ran.
+   * off, so what was asked for is not always what ran. The summary has no experimental
+   * rules: they come from the scan's own settings, which a scan started here has for this
+   * run alone.
    */
   async runSettings(
     scanId: string,
     scanRunId: string,
   ): Promise<ScanSettings | undefined> {
-    const resp = await apiClient
-      .get({
-        url: `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${scanId}/scan_runs/${scanRunId}`,
-        headers: {
-          Authorization:
-            "Basic " +
-            Buffer.from(
-              `${this.auth?.username}:${this.auth?.password}`,
-            ).toString("base64"),
-        },
-        raise_error: false,
-      })
-      .catch(() => undefined);
+    const read = (path: string) =>
+      apiClient
+        .get({
+          url: `https://api-accessibility.browserstack.com/api/website-scanner/v1/scans/${path}`,
+          headers: {
+            Authorization:
+              "Basic " +
+              Buffer.from(
+                `${this.auth?.username}:${this.auth?.password}`,
+              ).toString("base64"),
+          },
+          raise_error: false,
+        })
+        .catch(() => undefined);
+    const [resp, overview] = await Promise.all([
+      read(`${scanId}/scan_runs/${scanRunId}`),
+      read(`${scanId}/overview`),
+    ]);
     const run = resp?.ok ? resp.data?.data : undefined;
     if (!run) return undefined;
+    const experimental = overview?.ok
+      ? overview.data?.data?.scanConfigInfo?.experimental
+      : undefined;
     return {
       ...(run.wcagVersion ? { wcagVersion: run.wcagVersion } : {}),
       ...(typeof run.bestPractices === "boolean"
@@ -190,6 +202,9 @@ export class AccessibilityScanner {
       ...(typeof run.advanced === "boolean"
         ? { advancedRules: run.advanced }
         : {}),
+      ...(typeof experimental === "boolean"
+        ? { experimentalRules: experimental }
+        : {}),
     };
   }
 

src/tools/sdk-utils/bstack/configUtils.ts

@@ -5,15 +5,18 @@ export function generateBrowserStackYMLInstructions(config: {
   platforms?: string[];
   enablePercy?: boolean;
   projectName: string;
+  /** Run Accessibility checks during the tests. */
+  accessibility?: boolean;
 }): string {
   const enablePercy = config.enablePercy || false;
   const projectName = config.projectName || "BrowserStack Automate Build";
 
   // Generate platform configurations using the utility function
   const platformConfigs = generatePlatformConfigs(config);
 
+  // An existing file holds the user's own settings, so it is added to, not replaced.
   const stepTitle =
-    "Create a browserstack.yml file in the project root with your validated device configurations:If already exists, update it with the following content for devices and project details.";
+    "Create a browserstack.yml file in the project root with your validated device configurations. If the file already exists, add these keys to it and keep its other settings.";
 
   const buildName = `${projectName}-Build`;
 
@@ -60,6 +63,16 @@ browserstackLocal: true
 debug: true # Visual logs, text logs, etc.
 testObservability: true # For Test Observability`;
 
+  if (config.accessibility) {
+    ymlContent += `
+
+# ======================
+# Accessibility Testing
+# ======================
+# Runs Accessibility (WCAG) checks on the pages the tests visit.
+accessibility: true`;
+  }
+
   if (enablePercy) {
     ymlContent += `