test(regression): 100% E2E coverage for CLI configs (PER-8250)

.github/workflows/test.yml

@@ -175,7 +175,14 @@ jobs:
           yarn global:link
           yarn link `echo $PERCY_PACKAGES`
           npx percy --version
-      - name: Run regression tests
+      - name: Run CLI config validation (token-free)
+        run: yarn test:regression:config
+      - name: Run functional discovery tests (token-free)
+        run: yarn test:regression:functional
+      # Visual track runs last and is the ONLY step that creates a Percy build,
+      # so the PR's single build carries all visual snapshots (no stray build
+      # superseding it on the same commit).
+      - name: Run visual regression tests
         run: yarn test:regression
         env:
           PERCY_TOKEN: ${{ secrets.PERCY_REGRESSION_TOKEN }}

.gitignore

@@ -15,6 +15,9 @@ docs/
 # spec-level retry bookkeeping (CI only, see PER-9011)
 .cli-test-failures.json
 
+# regression config-validation archive artifact (PER-8250)
+test/regression/.percy-archive
+
 # bstack-ai-harness:begin (managed — do not edit between markers)
 bstack-ai-harness.yml
 .harness-docs.json

package.json

@@ -24,7 +24,9 @@
     "test:types": "lerna run --parallel test:types",
     "global:link": "lerna exec -- yarn link",
     "global:unlink": "lerna exec -- yarn unlink",
-    "test:regression": "node test/regression/regression.test.js"
+    "test:regression": "node test/regression/regression.test.js",
+    "test:regression:config": "node test/regression/config-validation.test.js",
+    "test:regression:functional": "node test/regression/functional.test.js"
   },
   "devDependencies": {
     "@babel/cli": "^7.11.6",

test/regression/COVERAGE.md

@@ -0,0 +1,121 @@
+# Percy CLI Config — E2E Coverage Matrix (PER-8250)
+
+Tracks which `percy snapshot` config options the E2E regression suite exercises,
+and how. The option source of truth is `packages/core/src/config.js`
+(`configSchema` + `snapshotSchema`) plus `packages/cli-snapshot/src/config.js`
+(`static` / `sitemap`).
+
+## Tracks
+
+| Track | File(s) | Token? | What it proves |
+|-------|---------|--------|----------------|
+| **C — Config validation** | `config-validation.test.js`, `configs/`, `per-snapshot-options.yml` | no | The CLI parses, validates & loads **every** option with no `Invalid config:` output. Runs token-free + discovery-free (`--dry-run`), so it gates every PR including forks. The literal-100% option backbone. |
+| **V — Visual** | `regression.test.js`, `snapshots.yml`, `pages/config-*.html` | yes | Render-affecting options produce the correct snapshot (reviewed via Percy's dashboard). The **only** track that creates a Percy build. |
+| **F — Functional** | `functional.test.js`, `configs/functional-config.yml`, `server.js` gated routes | no | Discovery options behave correctly — asserted on what the test servers observed, not on log text. Runs `percy snapshot --debug` (skipUploads): discovery runs but **no build is created**, so it stays token-free and never adds a stray build to the visual project. |
+
+Run:
+
+```bash
+yarn test:regression:config        # Track C — no token needed
+yarn test:regression:functional    # Track F — no token needed (--debug, no build)
+PERCY_TOKEN=… yarn test:regression  # Track V — token-gated, creates the build
+```
+
+## Scope boundaries (explicit non-goals)
+
+- **`onlyAutomate` snapshot options are excluded** — unreachable via the
+  `percy snapshot` web flow (they belong to the SDK / Percy-on-Automate path,
+  covered separately under PER-8249): `fullPage`, `freezeAnimation`,
+  `freezeAnimatedImage`, `freezeAnimatedImageOptions`, `ignoreRegions`,
+  `considerRegions`.
+- **`comparisonSchema`** (the upload/comparison wire format) is not user-facing
+  CLI config — out of scope.
+- **`/snapshot/server` `port`** is not reachable via `percy snapshot`: there is
+  no `--port` flag on the command and it isn't in the `static` config namespace
+  (the command only sets `serve`/`cleanUrls`/`baseUrl` for a directory — see
+  `cli-snapshot/src/snapshot.js`). It's a programmatic/SDK option only. `serve`
+  itself IS covered (server mode via `percy snapshot static-site/`).
+- All additions are test-only; no production code changes.
+
+## Coverage by namespace
+
+Track key: **C** validated · **V** visual · **F** functional · **X** excluded.
+Every listed option is at minimum **C** (in a config the CLI loads & validates).
+
+### `percy`
+| Option | Tracks | Where |
+|--------|--------|-------|
+| deferUploads, archiveDir, useSystemProxy, labels, skipBaseBuild, platforms[] | C | `configs/all-config.yml` |
+| token | C | supplied via `PERCY_TOKEN` env (string field validated implicitly) |
+
+### `snapshot`
+| Option | Tracks | Where |
+|--------|--------|-------|
+| widths, minHeight | C, V | `all-config.yml`; `snapshots.yml` (responsive + defaults) |
+| percyCSS | C, V | global `.percy.yml`; per-snapshot override "Config - percyCSS Override" |
+| enableJavaScript | C, V | `snapshots.yml` "JavaScript Enabled" |
+| cliEnableJavaScript | C | `all-config.yml` |
+| disableShadowDOM | C, V | "Config - Disable Shadow DOM" |
+| forceShadowAsLightDOM | C, V | "Config - Force Shadow As Light DOM" |
+| enableLayout | C | `all-config.yml` |
+| domTransformation | C, V | "Config - DOM Transformation" |
+| reshuffleInvalidTags | C | `all-config.yml` |
+| scope, scopeOptions.scroll | C, V | "Config - Scope" |
+| sync | C | `all-config.yml` |
+| readiness.* (preset, *WindowMs, timeoutMs, domStability, imageReady, fontReady, jsIdle, readySelectors, notPresentSelectors, maxTimeoutMs) | C | `all-config.yml` |
+| responsiveSnapshotCapture | C, V | "Config - Responsive Snapshot Capture" |
+| testCase, labels, thTestCaseExecutionId, browsers | C | `all-config.yml` |
+| regions[] (elementSelector: boundingBox / elementCSS / elementXpath; padding, algorithm, configuration, assertion) | C | `all-config.yml` (all 3 selector forms) + `per-snapshot-options.yml` |
+| algorithm (standard / layout / intelliignore / ignore), algorithmConfiguration.* | C | `all-config.yml` (incl. `layout`); `ignore` via region in `per-snapshot-options.yml` |
+| ignoreCanvasSerializationErrors, ignoreStyleSheetSerializationErrors | C | `all-config.yml` |
+| ignoreIframeSelectors | C, V | `snapshots.yml` "DOM Structures Coverage" |
+| pseudoClassEnabledElements (id, className, xpath, selectors) | C, V | `all-config.yml` (all forms); "Interactive States" (selectors) |
+| fullPage, freezeAnimation, freezeAnimatedImage, freezeAnimatedImageOptions, ignoreRegions, considerRegions | **X** | onlyAutomate — excluded |
+
+### `discovery`
+| Option | Tracks | Where |
+|--------|--------|-------|
+| allowedHostnames | C, V | global `.percy.yml` (CORS resources) |
+| disallowedHostnames | C, F | `functional-config.yml` (9101 probe aborted) |
+| networkIdleTimeout | C | global `.percy.yml` |
+| waitForSelector, waitForTimeout | C | `all-config.yml` |
+| scrollToBottom | C, V | "Config - Scroll To Bottom" |
+| disableCache, maxCacheRam (int + null) | C | `all-config.yml` + `alt-forms.yml` |
+| captureMockedServiceWorker | C | `all-config.yml` |
+| captureSrcset | C, F | `functional-config.yml` (both srcset candidates fetched) |
+| requestHeaders | C, F | `functional-config.yml` (header observed) |
+| authorization | C, F | `functional-config.yml` (Basic auth observed) |
+| cookies (object + array) | C, F | `all-config.yml` (object) + `alt-forms.yml` (array); F observes Cookie header |
+| userAgent | C, F | `functional-config.yml` (UA observed) |
+| devicePixelRatio | C | `all-config.yml` |
+| concurrency | C | global `.percy.yml` |
+| snapshotConcurrency, retry, autoConfigureAllowedHostnames | C | `all-config.yml` |
+| launchOptions (executable, timeout, args, headless, closeBrowser) | C | `all-config.yml` (timeout also in global `.percy.yml`) |
+| fontDomains | C | `all-config.yml` |
+
+### `project`
+| Option | Tracks | Where |
+|--------|--------|-------|
+| id, name | C | `all-config.yml` |
+
+### Per-snapshot / list / static / sitemap (cli-snapshot)
+| Option | Tracks | Where |
+|--------|--------|-------|
+| name | C, V | every snapshot |
+| execute (string / lifecycle-object / array) | C, V | `per-snapshot-options.yml`; "Config - Execute" |
+| additionalSnapshots (name / prefix / suffix / execute) | C, V | `per-snapshot-options.yml`; "Config - Execute" |
+| precapture waitForSelector / waitForTimeout | C | `per-snapshot-options.yml` |
+| include / exclude (filter; also `--include`/`--exclude`) | C | `per-snapshot-options.yml` |
+| list baseUrl, options | C | `per-snapshot-options.yml` |
+| static cleanUrls, rewrites, baseUrl, options | C | `all-config.yml` (`static:` section); `serve` + `cleanUrls` also via server mode `percy snapshot static-site/` |
+| server mode: serve | C | `static-site/` via `percy snapshot <dir> --dry-run` (`port` is not CLI-reachable — see Scope) |
+| sitemap options | C | `all-config.yml` (`sitemap:` section) |
+
+## Adding coverage for a new option
+
+1. Add it to `configs/all-config.yml` (or a focused fixture) and bump the
+   expected count in `config-validation.test.js` if it adds snapshots — that
+   alone gives Track C coverage.
+2. If it changes the render, add a page + `snapshots.yml` entry (Track V).
+3. If its effect is behavioral (a request/header/resource decision), add a
+   gated route to `server.js` and an assertion to `functional.test.js` (Track F).

test/regression/README.md

@@ -1,6 +1,24 @@
 # Percy CLI E2E Regression Tests
 
-Visual regression tests that upload real snapshots to Percy, covering all asset discovery features.
+E2E regression coverage for the `percy snapshot` CLI and its full config
+surface. Three complementary tracks (see [`COVERAGE.md`](./COVERAGE.md) for the
+per-option matrix):
+
+- **Config validation** (`config-validation.test.js`) — token-free `--dry-run`
+  that loads fixtures setting **every** CLI config option and asserts the CLI
+  accepts them all (no `Invalid config:`). Runs on every PR, no token needed.
+- **Visual** (`regression.test.js`) — uploads real snapshots; render-affecting
+  options are reviewed as visual diffs in the Percy dashboard.
+- **Functional** (`functional.test.js`) — runs `percy snapshot --debug`
+  (discovery runs but no build is uploaded) and asserts discovery options
+  against what the test servers observed (headers, auth, cookies, user-agent,
+  blocked hosts). Token-free, and creates no build — so it never adds a stray
+  build to the visual project.
+
+> **Scope:** `onlyAutomate` snapshot options (`fullPage`, `freezeAnimation`,
+> `freezeAnimatedImage`, `freezeAnimatedImageOptions`, `ignoreRegions`,
+> `considerRegions`) are out of scope here — they are unreachable via the
+> `percy snapshot` web flow and are covered on the SDK/Automate path.
 
 ## Setup
 
@@ -12,34 +30,50 @@ Visual regression tests that upload real snapshots to Percy, covering all asset
 ## Running Locally
 
 ```bash
-# Requires PERCY_TOKEN — skips gracefully without it
+# Config validation + functional — no token required, run anywhere
+yarn test:regression:config
+yarn test:regression:functional
+
+# Visual — requires PERCY_TOKEN (creates the build); skips gracefully without it
 PERCY_TOKEN=your_token_here yarn test:regression
 ```
 
 ## How It Works
 
 1. Starts two local servers: main (port 9100) and CORS (port 9101)
-2. Runs `percy snapshot` against all pages defined in `snapshots.yml`
+2. Runs `percy snapshot` against the pages defined in `snapshots.yml`
 3. Percy uploads snapshots and creates a build
 4. Visual diffs are reviewed in the Percy dashboard
 5. Percy's GitHub VCS integration gates PRs via checks
 
+The config-validation track instead runs `percy snapshot --dry-run` (no
+discovery, no upload) and asserts the CLI loads every config fixture cleanly.
+
 ## Adding a New Test
 
 1. Create a new HTML page in `pages/`
 2. Add an entry to `snapshots.yml`
 3. (Optional) Add assets to `assets/`
 4. (Optional) Add special server routes to `server.js`
 
-No changes to `regression.test.js` needed.
+No changes to `regression.test.js` needed. To cover a new **config option**, see
+the "Adding coverage" section of [`COVERAGE.md`](./COVERAGE.md).
 
 ## Configuration
 
-- `snapshots.yml` — Snapshot definitions (URLs, names, per-snapshot options)
+- `snapshots.yml` — Visual snapshot definitions (URLs, names, per-snapshot options)
 - `.percy.yml` — Percy project config (discovery settings, anti-flakiness CSS)
+- `configs/` — Config-validation fixtures (`all-config.yml`, `alt-forms.yml`,
+  `functional-config.yml`, `invalid-example.yml`)
+- `per-snapshot-options.yml` — List-mode fixture covering per-snapshot options
+- `functional-snapshots.yml` — Functional discovery snapshot
+- [`COVERAGE.md`](./COVERAGE.md) — Per-option coverage matrix
 
 Each snapshot entry supports all Percy options: `widths`, `enableJavaScript`, `discovery.allowedHostnames`, `waitForSelector`, `execute`, `percyCSS`, etc.
 
 ## CI
 
-Runs automatically on PRs and pushes to master via `.github/workflows/regression.yml` (Linux only).
+Runs automatically on PRs and pushes to master via the `regression` job in
+`.github/workflows/test.yml` (Linux only): config validation and functional
+run token-free; the visual track runs with `PERCY_REGRESSION_TOKEN` and is the
+only step that creates a Percy build.

test/regression/config-validation.test.js

@@ -0,0 +1,115 @@
+// Track C — CLI config validation (token-free, --dry-run).
+//
+// `percy snapshot --dry-run` skips discovery + upload and only enumerates
+// snapshots, so it runs without a PERCY_TOKEN and without the test servers.
+// Percy validates the FULL config (every registered namespace) at load time
+// regardless of command mode, logging "Invalid config:" for any unknown or
+// out-of-range option. This harness loads fixtures that set every non-excluded
+// CLI config option and asserts the CLI accepts them all — the literal-100%
+// option-coverage backbone for PER-8250.
+//
+// Run: node test/regression/config-validation.test.js  (or yarn test:regression:config)
+
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { rmSync } from 'fs';
+import { runPercy, snapshotCount, hasInvalidConfig } from './lib/percy-cli.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// all-config.yml sets `percy.archiveDir`, which makes the CLI write a snapshot
+// archive to disk. Keep the working tree clean by removing it around the run.
+const archiveDir = join(__dirname, '.percy-archive');
+const cleanArchive = () => rmSync(archiveDir, { recursive: true, force: true });
+
+// Each case asserts: exits 0, the expected "Invalid config:" presence, and the
+// expected snapshot count (catches snapshots silently dropping).
+const CASES = [
+  {
+    name: 'all-config.yml — every global / static / sitemap option',
+    args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100',
+      '--config', 'configs/all-config.yml', '--dry-run'],
+    expectCount: 25,
+    expectValid: true
+  },
+  {
+    name: 'alt-forms.yml — cookies array form + maxCacheRam null',
+    args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100',
+      '--config', 'configs/alt-forms.yml', '--dry-run'],
+    expectCount: 25,
+    expectValid: true
+  },
+  {
+    name: 'per-snapshot-options.yml — capture-level options (execute, additionalSnapshots, discovery subset)',
+    args: ['snapshot', 'per-snapshot-options.yml', '--dry-run'],
+    expectCount: 8,
+    expectValid: true
+  },
+  {
+    // Server mode: `percy snapshot <dir>` exercises the /snapshot/server schema
+    // (serve) plus the static cleanUrls flag, which list/base-url mode doesn't.
+    name: 'static-site/ — server mode (serve) + cleanUrls via percy snapshot <dir>',
+    args: ['snapshot', 'static-site', '--dry-run', '--clean-urls'],
+    expectCount: 2,
+    expectValid: true
+  },
+  {
+    // Negative guard: proves the "Invalid config:" detector actually fires, so
+    // the assertion protecting the valid fixtures above is not a no-op.
+    name: 'invalid-example.yml — detector self-test (Invalid config EXPECTED)',
+    args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100',
+      '--config', 'configs/invalid-example.yml', '--dry-run'],
+    expectCount: 25,
+    expectValid: false
+  }
+];
+
+async function run() {
+  // Track C is token-free by design — ensure no token leaks build creation in.
+  delete process.env.PERCY_TOKEN;
+  const env = { PERCY_CLIENT_ERROR_LOGS: 'false' };
+
+  let failures = 0;
+  const check = (cond, msg) => {
+    if (cond) {
+      console.log(`  ✓ ${msg}`);
+    } else {
+      failures++;
+      console.error(`  ✗ ${msg}`);
+    }
+  };
+
+  console.log('Track C — CLI config validation (token-free, --dry-run)\n');
+
+  for (const c of CASES) {
+    console.log(`• ${c.name}`);
+    const { code, output } = await runPercy(c.args, { env, cwd: __dirname });
+    check(code === 0, `exits 0 (got ${code})`);
+
+    const invalid = hasInvalidConfig(output);
+    if (c.expectValid) {
+      check(!invalid, 'no "Invalid config:" output — every option accepted');
+    } else {
+      check(invalid, '"Invalid config:" detected on intentionally-invalid fixture');
+    }
+
+    const count = snapshotCount(output);
+    check(count === c.expectCount, `found ${c.expectCount} snapshots (got ${count})`);
+    console.log('');
+  }
+
+  cleanArchive();
+
+  if (failures) {
+    console.error(`TRACK C FAILED: ${failures} assertion(s) failed`);
+    process.exit(1);
+  }
+  console.log('TRACK C PASSED');
+  process.exit(0);
+}
+
+run().catch(err => {
+  cleanArchive();
+  console.error('Config-validation runner error:', err);
+  process.exit(1);
+});