From 395e9e53b3563aff3f82fb1cad242d9b370eb227 Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Tue, 23 Jun 2026 14:18:39 +0530 Subject: [PATCH 1/8] test(regression): add token-free CLI config validation harness (Track C) Covers every non-excluded CLI config option (percy/snapshot/discovery/ project + cli-snapshot static/sitemap + per-snapshot capture options) via `percy snapshot --dry-run`, asserting the CLI loads & validates each with no "Invalid config:" output. Runs token-free and discovery-free, so it can gate every PR in CI. Includes a negative self-test fixture proving the validator detector fires. PER-8250 --- .gitignore | 3 + package.json | 3 +- test/regression/config-validation.test.js | 107 ++++++++++++ test/regression/configs/all-config.yml | 177 ++++++++++++++++++++ test/regression/configs/alt-forms.yml | 11 ++ test/regression/configs/invalid-example.yml | 11 ++ test/regression/lib/percy-cli.js | 39 +++++ test/regression/per-snapshot-options.yml | 104 ++++++++++++ 8 files changed, 454 insertions(+), 1 deletion(-) create mode 100644 test/regression/config-validation.test.js create mode 100644 test/regression/configs/all-config.yml create mode 100644 test/regression/configs/alt-forms.yml create mode 100644 test/regression/configs/invalid-example.yml create mode 100644 test/regression/lib/percy-cli.js create mode 100644 test/regression/per-snapshot-options.yml diff --git a/.gitignore b/.gitignore index 2db4ac1d6..4b8ef9148 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ docs/ # spec-level retry bookkeeping (CI only, see PER-9011) .cli-test-failures.json +# regression config-validation archive artifact (PER-8250) +test/regression/.percy-archive + # bstack-ai-harness:begin (managed — do not edit between markers) bstack-ai-harness.yml .harness-docs.json diff --git a/package.json b/package.json index a9a14942f..f0230277c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,8 @@ "test:types": "lerna run --parallel test:types", "global:link": "lerna exec -- yarn link", "global:unlink": "lerna exec -- yarn unlink", - "test:regression": "node test/regression/regression.test.js" + "test:regression": "node test/regression/regression.test.js", + "test:regression:config": "node test/regression/config-validation.test.js" }, "devDependencies": { "@babel/cli": "^7.11.6", diff --git a/test/regression/config-validation.test.js b/test/regression/config-validation.test.js new file mode 100644 index 000000000..1347d500b --- /dev/null +++ b/test/regression/config-validation.test.js @@ -0,0 +1,107 @@ +// Track C — CLI config validation (token-free, --dry-run). +// +// `percy snapshot --dry-run` skips discovery + upload and only enumerates +// snapshots, so it runs without a PERCY_TOKEN and without the test servers. +// Percy validates the FULL config (every registered namespace) at load time +// regardless of command mode, logging "Invalid config:" for any unknown or +// out-of-range option. This harness loads fixtures that set every non-excluded +// CLI config option and asserts the CLI accepts them all — the literal-100% +// option-coverage backbone for PER-8250. +// +// Run: node test/regression/config-validation.test.js (or yarn test:regression:config) + +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { rmSync } from 'fs'; +import { runPercy, snapshotCount, hasInvalidConfig } from './lib/percy-cli.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// all-config.yml sets `percy.archiveDir`, which makes the CLI write a snapshot +// archive to disk. Keep the working tree clean by removing it around the run. +const archiveDir = join(__dirname, '.percy-archive'); +const cleanArchive = () => rmSync(archiveDir, { recursive: true, force: true }); + +// Each case asserts: exits 0, the expected "Invalid config:" presence, and the +// expected snapshot count (catches snapshots silently dropping). +const CASES = [ + { + name: 'all-config.yml — every global / static / sitemap option', + args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100', + '--config', 'configs/all-config.yml', '--dry-run'], + expectCount: 16, + expectValid: true + }, + { + name: 'alt-forms.yml — cookies array form + maxCacheRam null', + args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100', + '--config', 'configs/alt-forms.yml', '--dry-run'], + expectCount: 16, + expectValid: true + }, + { + name: 'per-snapshot-options.yml — capture-level options (execute, additionalSnapshots, discovery subset)', + args: ['snapshot', 'per-snapshot-options.yml', '--dry-run'], + expectCount: 8, + expectValid: true + }, + { + // Negative guard: proves the "Invalid config:" detector actually fires, so + // the assertion protecting the valid fixtures above is not a no-op. + name: 'invalid-example.yml — detector self-test (Invalid config EXPECTED)', + args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100', + '--config', 'configs/invalid-example.yml', '--dry-run'], + expectCount: 16, + expectValid: false + } +]; + +async function run() { + // Track C is token-free by design — ensure no token leaks build creation in. + delete process.env.PERCY_TOKEN; + const env = { PERCY_CLIENT_ERROR_LOGS: 'false' }; + + let failures = 0; + const check = (cond, msg) => { + if (cond) { + console.log(` ✓ ${msg}`); + } else { + failures++; + console.error(` ✗ ${msg}`); + } + }; + + console.log('Track C — CLI config validation (token-free, --dry-run)\n'); + + for (const c of CASES) { + console.log(`• ${c.name}`); + const { code, output } = await runPercy(c.args, { env, cwd: __dirname }); + check(code === 0, `exits 0 (got ${code})`); + + const invalid = hasInvalidConfig(output); + if (c.expectValid) { + check(!invalid, 'no "Invalid config:" output — every option accepted'); + } else { + check(invalid, '"Invalid config:" detected on intentionally-invalid fixture'); + } + + const count = snapshotCount(output); + check(count === c.expectCount, `found ${c.expectCount} snapshots (got ${count})`); + console.log(''); + } + + cleanArchive(); + + if (failures) { + console.error(`TRACK C FAILED: ${failures} assertion(s) failed`); + process.exit(1); + } + console.log('TRACK C PASSED'); + process.exit(0); +} + +run().catch(err => { + cleanArchive(); + console.error('Config-validation runner error:', err); + process.exit(1); +}); diff --git a/test/regression/configs/all-config.yml b/test/regression/configs/all-config.yml new file mode 100644 index 000000000..3a7b2c51b --- /dev/null +++ b/test/regression/configs/all-config.yml @@ -0,0 +1,177 @@ +# Track C — Config validation fixture (token-free, --dry-run) +# +# Exercises EVERY non-excluded CLI config option across the percy / snapshot / +# discovery / project namespaces plus the cli-snapshot static & sitemap config +# namespaces. The config-validation harness runs this with `percy snapshot +# ... --dry-run` and asserts the CLI loads & validates it with NO "Invalid +# config:" output — proving every option below is schema-recognized and valid. +# +# onlyAutomate options (fullPage, freezeAnimation, freezeAnimatedImage, +# freezeAnimatedImageOptions, ignoreRegions, considerRegions) are intentionally +# absent — they are unreachable via the `percy snapshot` web flow. See README. + +version: 2 + +# ── percy namespace ────────────────────────────────────────────────────────── +percy: + deferUploads: false + archiveDir: .percy-archive + useSystemProxy: false + labels: regression,cli-config + skipBaseBuild: false + platforms: + - browserName: chrome + browserVersion: latest + osVersion: '14' + deviceName: Desktop + os: macos + percyBrowserCustomName: chrome-mac + # token is supplied via the PERCY_TOKEN env var at runtime, not the config file + +# ── snapshot namespace ─────────────────────────────────────────────────────── +snapshot: + widths: [375, 768, 1280] + minHeight: 1024 + percyCSS: | + * { caret-color: transparent !important; } + enableJavaScript: false + cliEnableJavaScript: true + disableShadowDOM: false + forceShadowAsLightDOM: false + enableLayout: false + domTransformation: | + (documentElement) => { + const body = documentElement.querySelector('body'); + if (body) body.setAttribute('data-percy-transformed', 'true'); + } + reshuffleInvalidTags: false + scope: '#root' + scopeOptions: + scroll: true + sync: false + responsiveSnapshotCapture: false + testCase: regression-cli-config + labels: snapshot-label + thTestCaseExecutionId: th-exec-123 + browsers: + - chrome + - firefox + readiness: + preset: balanced + stabilityWindowMs: 200 + jsIdleWindowMs: 200 + networkIdleWindowMs: 200 + timeoutMs: 5000 + domStability: true + imageReady: true + fontReady: true + jsIdle: true + readySelectors: + - '#ready' + - css: '.ready' + xpath: '//div[@id="ready"]' + notPresentSelectors: + - '.spinner' + - css: '.loading' + maxTimeoutMs: 30000 + regions: + - algorithm: standard + elementSelector: + elementCSS: '#region' + padding: + top: 5 + bottom: 5 + left: 5 + right: 5 + assertion: + diffIgnoreThreshold: 0.1 + - algorithm: intelliignore + elementSelector: + boundingBox: + x: 0 + y: 0 + width: 100 + height: 100 + configuration: + diffSensitivity: 2 + imageIgnoreThreshold: 0.2 + carouselsEnabled: true + bannersEnabled: true + adsEnabled: true + algorithm: standard + algorithmConfiguration: + diffSensitivity: 3 + imageIgnoreThreshold: 0.5 + carouselsEnabled: false + bannersEnabled: false + adsEnabled: false + ignoreCanvasSerializationErrors: false + ignoreStyleSheetSerializationErrors: false + ignoreIframeSelectors: + - '.ad-frame' + pseudoClassEnabledElements: + id: + - hover-by-id + className: + - hover-by-class + xpath: + - '//button[@data-hover]' + selectors: + - '.hover-test' + - '.active-test' + +# ── discovery namespace ────────────────────────────────────────────────────── +discovery: + allowedHostnames: + - localhost:9101 + disallowedHostnames: + - blocked.example.com + networkIdleTimeout: 150 + waitForSelector: '#ready' + waitForTimeout: 1000 + scrollToBottom: false + disableCache: false + maxCacheRam: 524288000 + captureMockedServiceWorker: false + captureSrcset: false + requestHeaders: + X-Percy-Regression: 'true' + authorization: + username: percy + password: secret + cookies: + session: abc123 + userAgent: PercyRegression/1.0 + devicePixelRatio: 2 + concurrency: 5 + snapshotConcurrency: 2 + retry: true + launchOptions: + executable: /usr/bin/google-chrome + timeout: 120000 + headless: true + closeBrowser: true + args: + - --no-sandbox + fontDomains: + - localhost:9101 + autoConfigureAllowedHostnames: true + +# ── project namespace ──────────────────────────────────────────────────────── +project: + id: 12345 + name: percy-cli-regression + +# ── static mode config (cli-snapshot) ──────────────────────────────────────── +static: + cleanUrls: true + rewrites: + /old-path: /new-path + baseUrl: / + include: '*' + exclude: 'ignore-*' + +# ── sitemap mode config (cli-snapshot) ─────────────────────────────────────── +sitemap: + include: '*' + exclude: 'ignore-*' diff --git a/test/regression/configs/alt-forms.yml b/test/regression/configs/alt-forms.yml new file mode 100644 index 000000000..a4c9aa4bb --- /dev/null +++ b/test/regression/configs/alt-forms.yml @@ -0,0 +1,11 @@ +# Track C — alternate schema shapes that cannot coexist with all-config.yml. +# Covers the discovery.cookies ARRAY form (all-config.yml uses the object form) +# and the discovery.maxCacheRam null form (all-config.yml uses the integer form). +version: 2 +discovery: + cookies: + - name: session + value: abc123 + - name: theme + value: dark + maxCacheRam: null diff --git a/test/regression/configs/invalid-example.yml b/test/regression/configs/invalid-example.yml new file mode 100644 index 000000000..081f550ce --- /dev/null +++ b/test/regression/configs/invalid-example.yml @@ -0,0 +1,11 @@ +# Negative-guard fixture for the config-validation harness self-test. +# Intentionally invalid: an unknown property and out-of-range values. The +# harness asserts the CLI emits "Invalid config:" for this file, proving the +# detector that protects the valid fixtures actually fires (Track C is not a +# no-op). NOT representative of a real config. +version: 2 +snapshot: + widths: [99999] + bogusUnknownOption: true +discovery: + networkIdleTimeout: 99999 diff --git a/test/regression/lib/percy-cli.js b/test/regression/lib/percy-cli.js new file mode 100644 index 000000000..18aa63a60 --- /dev/null +++ b/test/regression/lib/percy-cli.js @@ -0,0 +1,39 @@ +import { spawn } from 'child_process'; + +// Runs the local `percy` CLI with the given args, capturing combined +// stdout+stderr. Resolves with { code, output }; never rejects on non-zero +// exit (callers assert on code). Shared by the config-validation and +// functional regression harnesses. +export function runPercy(args, { env = {}, cwd } = {}) { + return new Promise((resolve, reject) => { + let output = ''; + const child = spawn('npx', ['percy', ...args], { + env: { ...process.env, ...env }, + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + shell: true + }); + + const onData = data => { + const text = data.toString(); + output += text; + process.stdout.write(text); + }; + + child.stdout.on('data', onData); + child.stderr.on('data', onData); + child.on('error', reject); + child.on('close', code => resolve({ code, output })); + }); +} + +// Extracts the "Found N snapshots" count from CLI output, or null if absent. +export function snapshotCount(output) { + const match = output.match(/Found (\d+) snapshots/); + return match ? Number(match[1]) : null; +} + +// True when the CLI reported a config validation failure. +export function hasInvalidConfig(output) { + return /Invalid config:/.test(output); +} diff --git a/test/regression/per-snapshot-options.yml b/test/regression/per-snapshot-options.yml new file mode 100644 index 000000000..8a26a6cc9 --- /dev/null +++ b/test/regression/per-snapshot-options.yml @@ -0,0 +1,104 @@ +# Track C — per-snapshot capture options (list-mode snapshots file). +# +# Validated token-free via `percy snapshot per-snapshot-options.yml --dry-run`. +# Exercises the cli-snapshot LIST schema (baseUrl, include/exclude filter, +# shared options) plus every per-snapshot capture option shape: execute as a +# string / lifecycle-object / array, additionalSnapshots (name / prefix / +# suffix / execute), precapture waitForSelector & waitForTimeout, and the +# per-snapshot discovery subset. + +baseUrl: http://localhost:9100 + +# filter — also available as the --include / --exclude CLI flags +include: '*' +exclude: 'skip-*' + +# shared options merged into every snapshot below +options: + widths: [1280] + enableJavaScript: false + +snapshots: + # execute: string function body + precapture waits + - url: /comprehensive.html + name: exec-string + execute: "() => { document.body.classList.add('ready'); }" + waitForSelector: '#root' + waitForTimeout: 500 + + # execute: lifecycle-object form (all four hooks) + - url: /comprehensive.html + name: exec-object + execute: + afterNavigation: "() => window.scrollTo(0, 0)" + beforeResize: "() => {}" + afterResize: "() => {}" + beforeSnapshot: "() => document.body.setAttribute('data-ready', '1')" + + # execute: array of function bodies + - url: /comprehensive.html + name: exec-array + execute: + - "() => {}" + - "() => {}" + + # additionalSnapshots: name / prefix / suffix / execute + - url: /comprehensive.html + name: additional-base + additionalSnapshots: + - name: additional-by-name + execute: "() => {}" + - prefix: 'prefixed-' + - suffix: '-suffixed' + + # per-snapshot common overrides + per-snapshot discovery subset + - url: /comprehensive.html + name: per-snapshot-overrides + minHeight: 800 + scope: '#root' + scopeOptions: + scroll: true + percyCSS: '* { color: black !important; }' + disableShadowDOM: false + forceShadowAsLightDOM: false + domTransformation: "(d) => d" + enableLayout: false + sync: false + responsiveSnapshotCapture: false + testCase: per-snap + labels: per-snap-label + thTestCaseExecutionId: th-1 + browsers: [chrome] + reshuffleInvalidTags: false + algorithm: standard + algorithmConfiguration: + diffSensitivity: 1 + ignoreCanvasSerializationErrors: false + ignoreStyleSheetSerializationErrors: false + ignoreIframeSelectors: ['.ad-frame'] + pseudoClassEnabledElements: + selectors: ['.x'] + readiness: + preset: fast + regions: + - algorithm: ignore + elementSelector: + elementCSS: '#r' + discovery: + allowedHostnames: [localhost:9101] + disallowedHostnames: [blocked.example.com] + requestHeaders: + X-Test: '1' + waitForSelector: '#root' + waitForTimeout: 100 + authorization: + username: u + password: p + disableCache: false + captureMockedServiceWorker: false + captureSrcset: false + userAgent: PerSnap/1.0 + devicePixelRatio: 2 + retry: true + scrollToBottom: false + fontDomains: [localhost:9101] From 624bb6f705cca33f42a954de5799cf8fd33f1e1b Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Tue, 23 Jun 2026 14:22:49 +0530 Subject: [PATCH 2/8] test(regression): add visual coverage for render-affecting CLI configs (Track V) New pages + snapshots.yml entries exercise scope/scopeOptions, per-snapshot percyCSS override, execute + additionalSnapshots, domTransformation, discovery.scrollToBottom, disableShadowDOM, forceShadowAsLightDOM, and responsiveSnapshotCapture. Visual diffs are reviewed via Percy's dashboard on token-gated runs; the suite skips cleanly without PERCY_TOKEN. Bumped Track C expected snapshot count 16 -> 25 to match. PER-8250 --- test/regression/config-validation.test.js | 6 +- .../pages/config-dom-transformation.html | 15 ++++ test/regression/pages/config-execute.html | 27 +++++++ test/regression/pages/config-scope.html | 37 ++++++++++ .../pages/config-scroll-to-bottom.html | 34 +++++++++ test/regression/snapshots.yml | 73 +++++++++++++++++++ 6 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 test/regression/pages/config-dom-transformation.html create mode 100644 test/regression/pages/config-execute.html create mode 100644 test/regression/pages/config-scope.html create mode 100644 test/regression/pages/config-scroll-to-bottom.html diff --git a/test/regression/config-validation.test.js b/test/regression/config-validation.test.js index 1347d500b..712758bca 100644 --- a/test/regression/config-validation.test.js +++ b/test/regression/config-validation.test.js @@ -29,14 +29,14 @@ const CASES = [ name: 'all-config.yml — every global / static / sitemap option', args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100', '--config', 'configs/all-config.yml', '--dry-run'], - expectCount: 16, + expectCount: 25, expectValid: true }, { name: 'alt-forms.yml — cookies array form + maxCacheRam null', args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100', '--config', 'configs/alt-forms.yml', '--dry-run'], - expectCount: 16, + expectCount: 25, expectValid: true }, { @@ -51,7 +51,7 @@ const CASES = [ name: 'invalid-example.yml — detector self-test (Invalid config EXPECTED)', args: ['snapshot', 'snapshots.yml', '--base-url', 'http://localhost:9100', '--config', 'configs/invalid-example.yml', '--dry-run'], - expectCount: 16, + expectCount: 25, expectValid: false } ]; diff --git a/test/regression/pages/config-dom-transformation.html b/test/regression/pages/config-dom-transformation.html new file mode 100644 index 000000000..19db124ed --- /dev/null +++ b/test/regression/pages/config-dom-transformation.html @@ -0,0 +1,15 @@ + + + + + + DOM Transformation Regression Test + + + +

DOM Transformation

+

A banner element is injected at the top of the body by the + domTransformation hook before the DOM is serialized, so it + appears in the snapshot but not in the source page.

+ + diff --git a/test/regression/pages/config-execute.html b/test/regression/pages/config-execute.html new file mode 100644 index 000000000..dc9cf5381 --- /dev/null +++ b/test/regression/pages/config-execute.html @@ -0,0 +1,27 @@ + + + + + + Execute Regression Test + + + + +

Execute

+

The panel below is hidden by default. The execute capture hook + reveals it (and mutates the counter) before the snapshot is serialized.

+ + + +
+

Revealed Panel

+

Shown by the execute hook prior to capture.

+
+ +
state: initial
+ + diff --git a/test/regression/pages/config-scope.html b/test/regression/pages/config-scope.html new file mode 100644 index 000000000..3aeec2a34 --- /dev/null +++ b/test/regression/pages/config-scope.html @@ -0,0 +1,37 @@ + + + + + + Scope Regression Test + + + + +

Scope

+ + +
Surrounding content OUTSIDE the scoped element — excluded when scope is applied.
+ +
+

Scoped Region

+

Only this element is captured when scope: '#scope-target' is set.

+
+

Scroll line 1 — scopeOptions.scroll scrolls within this region during capture.

+

Scroll line 2

+

Scroll line 3

+

Scroll line 4

+

Scroll line 5

+

Scroll line 6

+

Scroll line 7 (only visible after scrolling)

+
+
+ +
More surrounding content OUTSIDE the scoped element — also excluded.
+ + diff --git a/test/regression/pages/config-scroll-to-bottom.html b/test/regression/pages/config-scroll-to-bottom.html new file mode 100644 index 000000000..ec45df844 --- /dev/null +++ b/test/regression/pages/config-scroll-to-bottom.html @@ -0,0 +1,34 @@ + + + + + + Scroll To Bottom Regression Test + + + + +

Scroll To Bottom

+

Content at the bottom of this page is injected only after the page is + scrolled. With discovery.scrollToBottom: true the CLI scrolls + the page during capture so the lazy content is serialized.

+ +
Tall spacer — forces the lazy region below the fold.
+ +
lazy content not yet loaded
+ + + + diff --git a/test/regression/snapshots.yml b/test/regression/snapshots.yml index cb1c86b78..b09ff0e24 100644 --- a/test/regression/snapshots.yml +++ b/test/regression/snapshots.yml @@ -83,3 +83,76 @@ - name: Sandbox & Nested Iframes url: /sandbox-iframes.html widths: [1280] + +# ───────────────────────────────────────────────────────────────────────────── +# CLI config visual coverage (PER-8250, Track V) — each entry exercises a +# render-affecting config option. See COVERAGE.md for the full option matrix. +# ───────────────────────────────────────────────────────────────────────────── + +# scope + scopeOptions.scroll — capture only the scoped element (surrounding +# .noise blocks must be excluded; the inner region is scrolled during capture). +- name: Config - Scope + url: /config-scope.html + widths: [1280] + scope: '#scope-target' + scopeOptions: + scroll: true + +# percyCSS per-snapshot override — recolors the page via snapshot-level CSS, +# proving per-snapshot percyCSS overrides the global anti-flake CSS. +- name: Config - percyCSS Override + url: /config-scope.html + widths: [1280] + percyCSS: | + body { background: #102a43 !important; } + h1 { color: #7ee787 !important; } + +# execute — pre-capture JS reveals a hidden panel; additionalSnapshots takes a +# second snapshot from the same URL in a different state (suffix-named). +- name: Config - Execute + url: /config-execute.html + widths: [1280] + execute: "() => { document.getElementById('panel').style.display = 'block'; document.getElementById('counter').textContent = 'state: executed'; }" + additionalSnapshots: + - suffix: ' (alt state)' + execute: "() => { var p = document.getElementById('panel'); p.style.display = 'block'; p.style.background = '#fde8e8'; document.getElementById('counter').textContent = 'state: alt'; }" + +# domTransformation — inject a banner into the serialized DOM before capture. +- name: Config - DOM Transformation + url: /config-dom-transformation.html + widths: [1280] + domTransformation: | + (documentElement) => { + const doc = documentElement.ownerDocument; + const banner = doc.createElement('div'); + banner.textContent = 'Injected by domTransformation'; + banner.setAttribute('style', 'background:#b91c1c;color:#fff;padding:12px;font-weight:bold;'); + documentElement.querySelector('body').prepend(banner); + } + +# discovery.scrollToBottom — scroll the page during capture so lazy content is +# serialized into the snapshot. +- name: Config - Scroll To Bottom + url: /config-scroll-to-bottom.html + widths: [1280] + discovery: + scrollToBottom: true + +# disableShadowDOM — shadow roots are not serialized as declarative shadow DOM. +- name: Config - Disable Shadow DOM + url: /shadow-dom.html + widths: [1280] + disableShadowDOM: true + +# forceShadowAsLightDOM — shadow content is flattened into light DOM. +- name: Config - Force Shadow As Light DOM + url: /shadow-dom.html + widths: [1280] + forceShadowAsLightDOM: true + +# responsiveSnapshotCapture — capture each width via CDP responsive emulation +# instead of reloading per width. +- name: Config - Responsive Snapshot Capture + url: /responsive.html + widths: [375, 768, 1280] + responsiveSnapshotCapture: true From d4c41cf634aa870e147f3487f1d6ea5922418c55 Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Tue, 23 Jun 2026 14:30:10 +0530 Subject: [PATCH 3/8] test(regression): add functional discovery coverage harness (Track F) Adds gated server routes that record the headers/auth/cookies/user-agent Percy sends during discovery plus a cross-origin disallowed-hostname probe. functional.test.js runs one real snapshot and asserts on those server observations (robust to log-format changes) for discovery.requestHeaders, authorization, cookies, userAgent, captureSrcset and disallowedHostnames. Token-gated; skips cleanly without PERCY_TOKEN. Server-route behavior is verified token-free; the end-to-end Percy assertions run when a token is set. PER-8250 --- package.json | 3 +- test/regression/configs/functional-config.yml | 17 +++ test/regression/functional-snapshots.yml | 7 ++ test/regression/functional.test.js | 103 ++++++++++++++++++ .../pages/functional-discovery.html | 33 ++++++ test/regression/server.js | 77 +++++++++++++ 6 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 test/regression/configs/functional-config.yml create mode 100644 test/regression/functional-snapshots.yml create mode 100644 test/regression/functional.test.js create mode 100644 test/regression/pages/functional-discovery.html diff --git a/package.json b/package.json index f0230277c..6626c0f8b 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "global:link": "lerna exec -- yarn link", "global:unlink": "lerna exec -- yarn unlink", "test:regression": "node test/regression/regression.test.js", - "test:regression:config": "node test/regression/config-validation.test.js" + "test:regression:config": "node test/regression/config-validation.test.js", + "test:regression:functional": "node test/regression/functional.test.js" }, "devDependencies": { "@babel/cli": "^7.11.6", diff --git a/test/regression/configs/functional-config.yml b/test/regression/configs/functional-config.yml new file mode 100644 index 000000000..d19f10620 --- /dev/null +++ b/test/regression/configs/functional-config.yml @@ -0,0 +1,17 @@ +# Track F — global discovery config for the functional harness. +# These options are applied during a real discovery run; functional.test.js +# asserts on what the gated server routes observed. cookies is a global-only +# discovery option (not valid per-snapshot), so the whole set lives here. +version: 2 +discovery: + requestHeaders: + X-Percy-Regression: present + authorization: + username: percy + password: secret + cookies: + session: regression-cookie + userAgent: PercyRegressionUA/1.0 + captureSrcset: true + disallowedHostnames: + - localhost:9101 diff --git a/test/regression/functional-snapshots.yml b/test/regression/functional-snapshots.yml new file mode 100644 index 000000000..fc545b45f --- /dev/null +++ b/test/regression/functional-snapshots.yml @@ -0,0 +1,7 @@ +# Track F — functional discovery coverage (token-gated). +# One real snapshot of a page that references gated resources. The discovery.* +# behavior under test is configured globally in configs/functional-config.yml; +# functional.test.js asserts on what the test servers observed during discovery. +- name: Functional - Discovery Config + url: /functional-discovery.html + widths: [1280] diff --git a/test/regression/functional.test.js b/test/regression/functional.test.js new file mode 100644 index 000000000..ac1bf6b9b --- /dev/null +++ b/test/regression/functional.test.js @@ -0,0 +1,103 @@ +// Track F — functional discovery coverage (token-gated). +// +// Some discovery options have no reviewable visual effect — their correctness +// is in behavior (which headers/auth/cookies/user-agent reach the server, which +// resources are fetched or blocked). This harness runs ONE real `percy snapshot` +// against gated server routes and asserts on what those routes observed, so the +// assertions verify Percy's actual behavior rather than fragile debug-log text. +// +// Requires PERCY_TOKEN (real capture + upload); skips cleanly without it, like +// the visual regression harness. +// +// Run: node test/regression/functional.test.js (or yarn test:regression:functional) + +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import { startServers, stopServers, getObservations, resetObservations } from './server.js'; +import { runPercy } from './lib/percy-cli.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +if (!process.env.PERCY_TOKEN) { + console.log('Skipping functional regression tests (PERCY_TOKEN not set)'); + process.exit(0); +} + +async function run() { + console.log('Track F — functional discovery coverage (token-gated)\n'); + await startServers(); + console.log('Servers listening on 127.0.0.1:9100 and :9101'); + resetObservations(); + + let result; + try { + result = await runPercy([ + 'snapshot', join(__dirname, 'functional-snapshots.yml'), + '--base-url', 'http://localhost:9100', + '--config', join(__dirname, 'configs/functional-config.yml'), + '--verbose' + ], { cwd: __dirname }); + } finally { + console.log('\nStopping test servers...'); + await stopServers(); + } + + const { code, output } = result; + const obs = getObservations(); + + let failures = 0; + const check = (cond, msg) => { + if (cond) { + console.log(` ✓ ${msg}`); + } else { + failures++; + console.error(` ✗ ${msg}`); + } + }; + + console.log(''); + check(code === 0, `percy snapshot exits 0 (got ${code})`); + check(/Finalized build/.test(output), 'build finalized'); + + // discovery.requestHeaders — custom header injected on discovery requests + check(obs.requestHeader === 'present', + `discovery.requestHeaders sent (X-Percy-Regression=${obs.requestHeader})`); + + // discovery.authorization — Basic auth injected (route 401s without it) + const decodedAuth = obs.authorization?.startsWith('Basic ') + ? Buffer.from(obs.authorization.slice(6), 'base64').toString() + : null; + check(decodedAuth === 'percy:secret', + `discovery.authorization sent (decoded=${decodedAuth})`); + + // discovery.cookies — Cookie header injected + check(!!obs.cookie && obs.cookie.includes('session=regression-cookie'), + `discovery.cookies sent (cookie=${obs.cookie})`); + + // discovery.userAgent — custom UA injected + check(!!obs.userAgent && obs.userAgent.includes('PercyRegressionUA/1.0'), + `discovery.userAgent sent (ua=${obs.userAgent})`); + + // discovery.captureSrcset — every srcset candidate fetched + check(obs.srcset.includes('1x') && obs.srcset.includes('2x'), + `discovery.captureSrcset fetched all candidates (got [${obs.srcset.join(', ')}])`); + + // discovery.disallowedHostnames — request to the disallowed 9101 host aborted + // before it left the browser, so the server never saw it. + check(obs.disallowedProbeRequested === false, + 'discovery.disallowedHostnames blocked the 9101 probe (server never hit)'); + check(/Skipping disallowed hostname/.test(output), + 'log confirms the disallowed-hostname skip'); + + if (failures) { + console.error(`\nTRACK F FAILED: ${failures} assertion(s) failed`); + process.exit(1); + } + console.log('\nTRACK F PASSED'); + process.exit(0); +} + +run().catch(err => { + console.error('Functional regression runner error:', err); + stopServers().finally(() => process.exit(1)); +}); diff --git a/test/regression/pages/functional-discovery.html b/test/regression/pages/functional-discovery.html new file mode 100644 index 000000000..fd4dcdd91 --- /dev/null +++ b/test/regression/pages/functional-discovery.html @@ -0,0 +1,33 @@ + + + + + + Functional Discovery Regression Test + + + + + + + + + + + + +

Functional Discovery Config

+

Exercises discovery.requestHeaders, authorization, cookies, userAgent, + captureSrcset and disallowedHostnames. The functional harness asserts on + what the test servers observed during discovery, not on log text.

+ + + srcset probe + + diff --git a/test/regression/server.js b/test/regression/server.js index b5a01f859..a0dd45ab1 100644 --- a/test/regression/server.js +++ b/test/regression/server.js @@ -28,6 +28,34 @@ const MIME_TYPES = { const wrongMimeFont = existsSync(join(assetsDir, 'fonts/test-font.woff2')) ? readFileSync(join(assetsDir, 'fonts/test-font.woff2')) : Buffer.alloc(0); +const logoPng = existsSync(join(assetsDir, 'images/logo.png')) + ? readFileSync(join(assetsDir, 'images/logo.png')) + : Buffer.alloc(0); + +// ── Track F (functional) observation state ─────────────────────────────────── +// The functional regression harness asserts on what the servers actually +// observed during a real `percy snapshot` discovery run, rather than on Percy's +// internal debug-log text. resetObservations() is called before each run; the +// /gated/* routes below populate this as discovery fetches their resources. +let observations = {}; +function freshObservations() { + return { + requestHeader: null, // value of the configured discovery.requestHeaders entry + authorization: null, // Authorization header sent for discovery.authorization + cookie: null, // Cookie header sent for discovery.cookies + userAgent: null, // User-Agent sent for discovery.userAgent + srcset: [], // srcset candidate paths fetched (discovery.captureSrcset) + disallowedProbeRequested: false // true if the disallowed 9101 host was hit + }; +} +observations = freshObservations(); +export function getObservations() { return observations; } +export function resetObservations() { observations = freshObservations(); } + +const css = (res, body = '/* gated regression resource */') => { + res.writeHead(200, { 'content-type': 'text/css' }); + res.end(body); +}; // Special routes for the main server (add new entries to extend) const mainRoutes = { @@ -44,6 +72,45 @@ const mainRoutes = { // Tests Percy's font MIME detection (magic byte sniffing) res.writeHead(200, { 'content-type': 'text/html' }); res.end(wrongMimeFont); + }, + + // ── Track F gated resources — record the request, then serve ─────────────── + // discovery.requestHeaders — records the custom header Percy injected. + '/gated/header.css': (req, res) => { + observations.requestHeader = req.headers['x-percy-regression'] ?? null; + css(res); + }, + // discovery.authorization — 401 without Basic auth so a missing header is a + // hard failure; records the Authorization header Percy sent. + '/gated/auth.css': (req, res) => { + observations.authorization = req.headers.authorization ?? null; + if (!req.headers.authorization) { + res.writeHead(401, { 'WWW-Authenticate': 'Basic realm="regression"' }); + res.end('Unauthorized'); + return; + } + css(res); + }, + // discovery.cookies — records the Cookie header Percy sent. + '/gated/cookie.css': (req, res) => { + observations.cookie = req.headers.cookie ?? null; + css(res); + }, + // discovery.userAgent — records the User-Agent Percy sent. + '/gated/ua.css': (req, res) => { + observations.userAgent = req.headers['user-agent'] ?? null; + css(res); + }, + // discovery.captureSrcset — records which srcset candidates were fetched. + '/gated/srcset-1x.png': (req, res) => { + observations.srcset.push('1x'); + res.writeHead(200, { 'content-type': 'image/png' }); + res.end(logoPng); + }, + '/gated/srcset-2x.png': (req, res) => { + observations.srcset.push('2x'); + res.writeHead(200, { 'content-type': 'image/png' }); + res.end(logoPng); } }; @@ -119,6 +186,16 @@ function createCorsServer() { return; } + // Track F: discovery.disallowedHostnames probe. If this 9101 resource is + // requested at all, the disallowed-hostname block did NOT take effect. The + // functional harness asserts this stays false when the host is disallowed. + if (pathname === '/disallowed-probe.css') { + observations.disallowedProbeRequested = true; + res.writeHead(200, { 'content-type': 'text/css', ...corsHeaders }); + res.end('/* disallowed probe */'); + return; + } + // Serve assets with CORS headers if (pathname.startsWith('/css/') || pathname.startsWith('/images/') || pathname.startsWith('/fonts/') || pathname.startsWith('/js/')) { From faf18c5bcdc68039df118883992dec92a6ad4e7d Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Tue, 23 Jun 2026 14:33:23 +0530 Subject: [PATCH 4/8] docs(regression): add COVERAGE.md matrix + wire config/functional tracks in CI Adds COVERAGE.md (per-option E2E coverage matrix with explicit onlyAutomate exclusions), rewrites the regression README around the three tracks, and adds config-validation (token-free) + functional (token-gated) steps to the regression job in test.yml. PER-8250 --- .github/workflows/test.yml | 6 ++ test/regression/COVERAGE.md | 115 ++++++++++++++++++++++++++++++++++++ test/regression/README.md | 43 ++++++++++++-- 3 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 test/regression/COVERAGE.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3045d61cc..134bf2e04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -175,7 +175,13 @@ jobs: yarn global:link yarn link `echo $PERCY_PACKAGES` npx percy --version + - name: Run CLI config validation (token-free) + run: yarn test:regression:config - name: Run regression tests run: yarn test:regression env: PERCY_TOKEN: ${{ secrets.PERCY_REGRESSION_TOKEN }} + - name: Run functional discovery tests + run: yarn test:regression:functional + env: + PERCY_TOKEN: ${{ secrets.PERCY_REGRESSION_TOKEN }} diff --git a/test/regression/COVERAGE.md b/test/regression/COVERAGE.md new file mode 100644 index 000000000..afd6c0376 --- /dev/null +++ b/test/regression/COVERAGE.md @@ -0,0 +1,115 @@ +# Percy CLI Config — E2E Coverage Matrix (PER-8250) + +Tracks which `percy snapshot` config options the E2E regression suite exercises, +and how. The option source of truth is `packages/core/src/config.js` +(`configSchema` + `snapshotSchema`) plus `packages/cli-snapshot/src/config.js` +(`static` / `sitemap`). + +## Tracks + +| Track | File(s) | Token? | What it proves | +|-------|---------|--------|----------------| +| **C — Config validation** | `config-validation.test.js`, `configs/`, `per-snapshot-options.yml` | no | The CLI parses, validates & loads **every** option with no `Invalid config:` output. Runs token-free + discovery-free (`--dry-run`), so it gates every PR including forks. The literal-100% option backbone. | +| **V — Visual** | `regression.test.js`, `snapshots.yml`, `pages/config-*.html` | yes | Render-affecting options produce the correct snapshot (reviewed via Percy's dashboard). | +| **F — Functional** | `functional.test.js`, `configs/functional-config.yml`, `server.js` gated routes | yes | Discovery options behave correctly — asserted on what the test servers observed, not on log text. | + +Run: + +```bash +yarn test:regression:config # Track C — no token needed +PERCY_TOKEN=… yarn test:regression # Track V +PERCY_TOKEN=… yarn test:regression:functional # Track F +``` + +## Scope boundaries (explicit non-goals) + +- **`onlyAutomate` snapshot options are excluded** — unreachable via the + `percy snapshot` web flow (they belong to the SDK / Percy-on-Automate path, + covered separately under PER-8249): `fullPage`, `freezeAnimation`, + `freezeAnimatedImage`, `freezeAnimatedImageOptions`, `ignoreRegions`, + `considerRegions`. +- **`comparisonSchema`** (the upload/comparison wire format) is not user-facing + CLI config — out of scope. +- All additions are test-only; no production code changes. + +## Coverage by namespace + +Track key: **C** validated · **V** visual · **F** functional · **X** excluded. +Every listed option is at minimum **C** (in a config the CLI loads & validates). + +### `percy` +| Option | Tracks | Where | +|--------|--------|-------| +| deferUploads, archiveDir, useSystemProxy, labels, skipBaseBuild, platforms[] | C | `configs/all-config.yml` | +| token | C | supplied via `PERCY_TOKEN` env (string field validated implicitly) | + +### `snapshot` +| Option | Tracks | Where | +|--------|--------|-------| +| widths, minHeight | C, V | `all-config.yml`; `snapshots.yml` (responsive + defaults) | +| percyCSS | C, V | global `.percy.yml`; per-snapshot override "Config - percyCSS Override" | +| enableJavaScript | C, V | `snapshots.yml` "JavaScript Enabled" | +| cliEnableJavaScript | C | `all-config.yml` | +| disableShadowDOM | C, V | "Config - Disable Shadow DOM" | +| forceShadowAsLightDOM | C, V | "Config - Force Shadow As Light DOM" | +| enableLayout | C | `all-config.yml` | +| domTransformation | C, V | "Config - DOM Transformation" | +| reshuffleInvalidTags | C | `all-config.yml` | +| scope, scopeOptions.scroll | C, V | "Config - Scope" | +| sync | C | `all-config.yml` | +| readiness.* (preset, *WindowMs, timeoutMs, domStability, imageReady, fontReady, jsIdle, readySelectors, notPresentSelectors, maxTimeoutMs) | C | `all-config.yml` | +| responsiveSnapshotCapture | C, V | "Config - Responsive Snapshot Capture" | +| testCase, labels, thTestCaseExecutionId, browsers | C | `all-config.yml` | +| regions[] (elementSelector, padding, algorithm, configuration, assertion) | C | `all-config.yml` + `per-snapshot-options.yml` | +| algorithm, algorithmConfiguration.* | C | `all-config.yml` | +| ignoreCanvasSerializationErrors, ignoreStyleSheetSerializationErrors | C | `all-config.yml` | +| ignoreIframeSelectors | C, V | `snapshots.yml` "DOM Structures Coverage" | +| pseudoClassEnabledElements (id, className, xpath, selectors) | C, V | `all-config.yml` (all forms); "Interactive States" (selectors) | +| fullPage, freezeAnimation, freezeAnimatedImage, freezeAnimatedImageOptions, ignoreRegions, considerRegions | **X** | onlyAutomate — excluded | + +### `discovery` +| Option | Tracks | Where | +|--------|--------|-------| +| allowedHostnames | C, V | global `.percy.yml` (CORS resources) | +| disallowedHostnames | C, F | `functional-config.yml` (9101 probe aborted) | +| networkIdleTimeout | C | global `.percy.yml` | +| waitForSelector, waitForTimeout | C | `all-config.yml` | +| scrollToBottom | C, V | "Config - Scroll To Bottom" | +| disableCache, maxCacheRam (int + null) | C | `all-config.yml` + `alt-forms.yml` | +| captureMockedServiceWorker | C | `all-config.yml` | +| captureSrcset | C, F | `functional-config.yml` (both srcset candidates fetched) | +| requestHeaders | C, F | `functional-config.yml` (header observed) | +| authorization | C, F | `functional-config.yml` (Basic auth observed) | +| cookies (object + array) | C, F | `all-config.yml` (object) + `alt-forms.yml` (array); F observes Cookie header | +| userAgent | C, F | `functional-config.yml` (UA observed) | +| devicePixelRatio | C | `all-config.yml` | +| concurrency | C | global `.percy.yml` | +| snapshotConcurrency, retry, autoConfigureAllowedHostnames | C | `all-config.yml` | +| launchOptions (executable, timeout, args, headless, closeBrowser) | C | `all-config.yml` (timeout also in global `.percy.yml`) | +| fontDomains | C | `all-config.yml` | + +### `project` +| Option | Tracks | Where | +|--------|--------|-------| +| id, name | C | `all-config.yml` | + +### Per-snapshot / list / static / sitemap (cli-snapshot) +| Option | Tracks | Where | +|--------|--------|-------| +| name | C, V | every snapshot | +| execute (string / lifecycle-object / array) | C, V | `per-snapshot-options.yml`; "Config - Execute" | +| additionalSnapshots (name / prefix / suffix / execute) | C, V | `per-snapshot-options.yml`; "Config - Execute" | +| precapture waitForSelector / waitForTimeout | C | `per-snapshot-options.yml` | +| include / exclude (filter; also `--include`/`--exclude`) | C | `per-snapshot-options.yml` | +| list baseUrl, options | C | `per-snapshot-options.yml` | +| static cleanUrls, rewrites, baseUrl, options | C | `all-config.yml` (`static:` section) | +| sitemap options | C | `all-config.yml` (`sitemap:` section) | + +## Adding coverage for a new option + +1. Add it to `configs/all-config.yml` (or a focused fixture) and bump the + expected count in `config-validation.test.js` if it adds snapshots — that + alone gives Track C coverage. +2. If it changes the render, add a page + `snapshots.yml` entry (Track V). +3. If its effect is behavioral (a request/header/resource decision), add a + gated route to `server.js` and an assertion to `functional.test.js` (Track F). diff --git a/test/regression/README.md b/test/regression/README.md index d7d2cfd69..d22e2e0d2 100644 --- a/test/regression/README.md +++ b/test/regression/README.md @@ -1,6 +1,22 @@ # Percy CLI E2E Regression Tests -Visual regression tests that upload real snapshots to Percy, covering all asset discovery features. +E2E regression coverage for the `percy snapshot` CLI and its full config +surface. Three complementary tracks (see [`COVERAGE.md`](./COVERAGE.md) for the +per-option matrix): + +- **Config validation** (`config-validation.test.js`) — token-free `--dry-run` + that loads fixtures setting **every** CLI config option and asserts the CLI + accepts them all (no `Invalid config:`). Runs on every PR, no token needed. +- **Visual** (`regression.test.js`) — uploads real snapshots; render-affecting + options are reviewed as visual diffs in the Percy dashboard. +- **Functional** (`functional.test.js`) — one real run whose discovery options + are asserted against what the test servers observed (headers, auth, cookies, + user-agent, blocked hosts). + +> **Scope:** `onlyAutomate` snapshot options (`fullPage`, `freezeAnimation`, +> `freezeAnimatedImage`, `freezeAnimatedImageOptions`, `ignoreRegions`, +> `considerRegions`) are out of scope here — they are unreachable via the +> `percy snapshot` web flow and are covered on the SDK/Automate path. ## Setup @@ -12,18 +28,25 @@ Visual regression tests that upload real snapshots to Percy, covering all asset ## Running Locally ```bash -# Requires PERCY_TOKEN — skips gracefully without it +# Config validation — no token required, runs anywhere +yarn test:regression:config + +# Visual + functional — require PERCY_TOKEN, skip gracefully without it PERCY_TOKEN=your_token_here yarn test:regression +PERCY_TOKEN=your_token_here yarn test:regression:functional ``` ## How It Works 1. Starts two local servers: main (port 9100) and CORS (port 9101) -2. Runs `percy snapshot` against all pages defined in `snapshots.yml` +2. Runs `percy snapshot` against the pages defined in `snapshots.yml` 3. Percy uploads snapshots and creates a build 4. Visual diffs are reviewed in the Percy dashboard 5. Percy's GitHub VCS integration gates PRs via checks +The config-validation track instead runs `percy snapshot --dry-run` (no +discovery, no upload) and asserts the CLI loads every config fixture cleanly. + ## Adding a New Test 1. Create a new HTML page in `pages/` @@ -31,15 +54,23 @@ PERCY_TOKEN=your_token_here yarn test:regression 3. (Optional) Add assets to `assets/` 4. (Optional) Add special server routes to `server.js` -No changes to `regression.test.js` needed. +No changes to `regression.test.js` needed. To cover a new **config option**, see +the "Adding coverage" section of [`COVERAGE.md`](./COVERAGE.md). ## Configuration -- `snapshots.yml` — Snapshot definitions (URLs, names, per-snapshot options) +- `snapshots.yml` — Visual snapshot definitions (URLs, names, per-snapshot options) - `.percy.yml` — Percy project config (discovery settings, anti-flakiness CSS) +- `configs/` — Config-validation fixtures (`all-config.yml`, `alt-forms.yml`, + `functional-config.yml`, `invalid-example.yml`) +- `per-snapshot-options.yml` — List-mode fixture covering per-snapshot options +- `functional-snapshots.yml` — Functional discovery snapshot +- [`COVERAGE.md`](./COVERAGE.md) — Per-option coverage matrix Each snapshot entry supports all Percy options: `widths`, `enableJavaScript`, `discovery.allowedHostnames`, `waitForSelector`, `execute`, `percyCSS`, etc. ## CI -Runs automatically on PRs and pushes to master via `.github/workflows/regression.yml` (Linux only). +Runs automatically on PRs and pushes to master via the `regression` job in +`.github/workflows/test.yml` (Linux only): config validation runs token-free, +visual + functional run with `PERCY_REGRESSION_TOKEN`. From f054912e2e7178fdc2aa098beb238f22c2459786 Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Tue, 23 Jun 2026 19:12:27 +0530 Subject: [PATCH 5/8] test(regression): resolve semgrep findings in new regression files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - lib/percy-cli.js: justify spawn shell:true with a nosemgrep comment (static, test-controlled args — no injection surface; mirrors regression.test.js). - functional-discovery.html: load the cross-origin disallowed-host probe via @import instead of a so it needs no Subresource Integrity hash (meaningless for a dynamic localhost test resource); discovery still attempts the request. PER-8250 --- test/regression/lib/percy-cli.js | 4 ++++ test/regression/pages/functional-discovery.html | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/regression/lib/percy-cli.js b/test/regression/lib/percy-cli.js index 18aa63a60..325b9446d 100644 --- a/test/regression/lib/percy-cli.js +++ b/test/regression/lib/percy-cli.js @@ -7,6 +7,10 @@ import { spawn } from 'child_process'; export function runPercy(args, { env = {}, cwd } = {}) { return new Promise((resolve, reject) => { let output = ''; + // shell:true lets `npx` resolve cross-platform (matches regression.test.js). + // Args are static and test-controlled — no untrusted input — so there is no + // command-injection surface here. + // nosemgrep: javascript.lang.security.audit.spawn-shell-true.spawn-shell-true const child = spawn('npx', ['percy', ...args], { env: { ...process.env, ...env }, cwd, diff --git a/test/regression/pages/functional-discovery.html b/test/regression/pages/functional-discovery.html index fd4dcdd91..6a3a7b91a 100644 --- a/test/regression/pages/functional-discovery.html +++ b/test/regression/pages/functional-discovery.html @@ -14,10 +14,12 @@ - - + +

Functional Discovery Config

From fb9ffceccfc0052ed0009c80fccd37ee6cda854d Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Wed, 24 Jun 2026 15:47:15 +0530 Subject: [PATCH 6/8] test(regression): drop shell:true in percy-cli helper to clear Semgrep OSS alert The nosemgrep comment suppressed spawn-shell-true for the workflow exit code, but GitHub code-scanning (Semgrep OSS) still surfaced the suppressed finding as an alert and failed the check. Eliminate it instead: npx resolves via PATH without a shell (this suite is Linux-only) and args are passed as an array, so shell:true was unnecessary and there is no command-injection surface. PER-8250 --- test/regression/lib/percy-cli.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/regression/lib/percy-cli.js b/test/regression/lib/percy-cli.js index 325b9446d..0da5a7e9c 100644 --- a/test/regression/lib/percy-cli.js +++ b/test/regression/lib/percy-cli.js @@ -7,15 +7,13 @@ import { spawn } from 'child_process'; export function runPercy(args, { env = {}, cwd } = {}) { return new Promise((resolve, reject) => { let output = ''; - // shell:true lets `npx` resolve cross-platform (matches regression.test.js). - // Args are static and test-controlled — no untrusted input — so there is no - // command-injection surface here. - // nosemgrep: javascript.lang.security.audit.spawn-shell-true.spawn-shell-true + // No shell:true — `npx` resolves via PATH directly (this regression suite is + // Linux-only), and args are passed as an array, so there is no shell parsing + // or command-injection surface. const child = spawn('npx', ['percy', ...args], { env: { ...process.env, ...env }, cwd, - stdio: ['ignore', 'pipe', 'pipe'], - shell: true + stdio: ['ignore', 'pipe', 'pipe'] }); const onData = data => { From 529b1bf5e78da50de3bacbe033e05449018182cf Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Thu, 25 Jun 2026 17:39:20 +0530 Subject: [PATCH 7/8] test(regression): run functional track with --debug (no build, token-free) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer feedback: the regression project's builds showed only 1 snapshot and read as baseline-only. Root cause: the functional track was a second `percy snapshot` invocation that uploaded its own 1-snapshot build on the same commit, superseding the 25-snapshot visual build. Fix: run the functional track with `--debug` (skipUploads). Discovery still runs — the browser fetches every gated resource so the servers verify the headers/auth/cookies/user-agent/srcset/disallowed-host behavior — but no Percy build is created. This makes the track token-free (now runs on every PR like the config track) and leaves the visual track as the sole build creator, so the PR's single build carries all visual snapshots and compares against the master baseline as a proper head build. Also tightened the captureSrcset assertion to the discriminating 2x candidate. PER-8250 --- .github/workflows/test.yml | 11 +++++----- test/regression/COVERAGE.md | 8 +++---- test/regression/README.md | 19 ++++++++++------- test/regression/functional.test.js | 34 ++++++++++++++++++------------ 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 134bf2e04..86c2af143 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -177,11 +177,12 @@ jobs: npx percy --version - name: Run CLI config validation (token-free) run: yarn test:regression:config - - name: Run regression tests - run: yarn test:regression - env: - PERCY_TOKEN: ${{ secrets.PERCY_REGRESSION_TOKEN }} - - name: Run functional discovery tests + - name: Run functional discovery tests (token-free) run: yarn test:regression:functional + # Visual track runs last and is the ONLY step that creates a Percy build, + # so the PR's single build carries all visual snapshots (no stray build + # superseding it on the same commit). + - name: Run visual regression tests + run: yarn test:regression env: PERCY_TOKEN: ${{ secrets.PERCY_REGRESSION_TOKEN }} diff --git a/test/regression/COVERAGE.md b/test/regression/COVERAGE.md index afd6c0376..4c7302499 100644 --- a/test/regression/COVERAGE.md +++ b/test/regression/COVERAGE.md @@ -10,15 +10,15 @@ and how. The option source of truth is `packages/core/src/config.js` | Track | File(s) | Token? | What it proves | |-------|---------|--------|----------------| | **C — Config validation** | `config-validation.test.js`, `configs/`, `per-snapshot-options.yml` | no | The CLI parses, validates & loads **every** option with no `Invalid config:` output. Runs token-free + discovery-free (`--dry-run`), so it gates every PR including forks. The literal-100% option backbone. | -| **V — Visual** | `regression.test.js`, `snapshots.yml`, `pages/config-*.html` | yes | Render-affecting options produce the correct snapshot (reviewed via Percy's dashboard). | -| **F — Functional** | `functional.test.js`, `configs/functional-config.yml`, `server.js` gated routes | yes | Discovery options behave correctly — asserted on what the test servers observed, not on log text. | +| **V — Visual** | `regression.test.js`, `snapshots.yml`, `pages/config-*.html` | yes | Render-affecting options produce the correct snapshot (reviewed via Percy's dashboard). The **only** track that creates a Percy build. | +| **F — Functional** | `functional.test.js`, `configs/functional-config.yml`, `server.js` gated routes | no | Discovery options behave correctly — asserted on what the test servers observed, not on log text. Runs `percy snapshot --debug` (skipUploads): discovery runs but **no build is created**, so it stays token-free and never adds a stray build to the visual project. | Run: ```bash yarn test:regression:config # Track C — no token needed -PERCY_TOKEN=… yarn test:regression # Track V -PERCY_TOKEN=… yarn test:regression:functional # Track F +yarn test:regression:functional # Track F — no token needed (--debug, no build) +PERCY_TOKEN=… yarn test:regression # Track V — token-gated, creates the build ``` ## Scope boundaries (explicit non-goals) diff --git a/test/regression/README.md b/test/regression/README.md index d22e2e0d2..357b68f52 100644 --- a/test/regression/README.md +++ b/test/regression/README.md @@ -9,9 +9,11 @@ per-option matrix): accepts them all (no `Invalid config:`). Runs on every PR, no token needed. - **Visual** (`regression.test.js`) — uploads real snapshots; render-affecting options are reviewed as visual diffs in the Percy dashboard. -- **Functional** (`functional.test.js`) — one real run whose discovery options - are asserted against what the test servers observed (headers, auth, cookies, - user-agent, blocked hosts). +- **Functional** (`functional.test.js`) — runs `percy snapshot --debug` + (discovery runs but no build is uploaded) and asserts discovery options + against what the test servers observed (headers, auth, cookies, user-agent, + blocked hosts). Token-free, and creates no build — so it never adds a stray + build to the visual project. > **Scope:** `onlyAutomate` snapshot options (`fullPage`, `freezeAnimation`, > `freezeAnimatedImage`, `freezeAnimatedImageOptions`, `ignoreRegions`, @@ -28,12 +30,12 @@ per-option matrix): ## Running Locally ```bash -# Config validation — no token required, runs anywhere +# Config validation + functional — no token required, run anywhere yarn test:regression:config +yarn test:regression:functional -# Visual + functional — require PERCY_TOKEN, skip gracefully without it +# Visual — requires PERCY_TOKEN (creates the build); skips gracefully without it PERCY_TOKEN=your_token_here yarn test:regression -PERCY_TOKEN=your_token_here yarn test:regression:functional ``` ## How It Works @@ -72,5 +74,6 @@ Each snapshot entry supports all Percy options: `widths`, `enableJavaScript`, `d ## CI Runs automatically on PRs and pushes to master via the `regression` job in -`.github/workflows/test.yml` (Linux only): config validation runs token-free, -visual + functional run with `PERCY_REGRESSION_TOKEN`. +`.github/workflows/test.yml` (Linux only): config validation and functional +run token-free; the visual track runs with `PERCY_REGRESSION_TOKEN` and is the +only step that creates a Percy build. diff --git a/test/regression/functional.test.js b/test/regression/functional.test.js index ac1bf6b9b..219e2356d 100644 --- a/test/regression/functional.test.js +++ b/test/regression/functional.test.js @@ -1,13 +1,16 @@ -// Track F — functional discovery coverage (token-gated). +// Track F — functional discovery coverage (token-free). // // Some discovery options have no reviewable visual effect — their correctness // is in behavior (which headers/auth/cookies/user-agent reach the server, which -// resources are fetched or blocked). This harness runs ONE real `percy snapshot` +// resources are fetched or blocked). This harness runs `percy snapshot --debug` // against gated server routes and asserts on what those routes observed, so the // assertions verify Percy's actual behavior rather than fragile debug-log text. // -// Requires PERCY_TOKEN (real capture + upload); skips cleanly without it, like -// the visual regression harness. +// `--debug` sets skipUploads: discovery still runs (the browser fetches every +// resource, so the servers observe the requests) but NO Percy build is created +// or uploaded. That keeps this track token-free AND stops it from creating a +// stray 1-snapshot build that would otherwise supersede the visual build on the +// same commit. It needs no PERCY_TOKEN and runs on every PR. // // Run: node test/regression/functional.test.js (or yarn test:regression:functional) @@ -18,13 +21,8 @@ import { runPercy } from './lib/percy-cli.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); -if (!process.env.PERCY_TOKEN) { - console.log('Skipping functional regression tests (PERCY_TOKEN not set)'); - process.exit(0); -} - async function run() { - console.log('Track F — functional discovery coverage (token-gated)\n'); + console.log('Track F — functional discovery coverage (token-free, --debug)\n'); await startServers(); console.log('Servers listening on 127.0.0.1:9100 and :9101'); resetObservations(); @@ -35,6 +33,9 @@ async function run() { 'snapshot', join(__dirname, 'functional-snapshots.yml'), '--base-url', 'http://localhost:9100', '--config', join(__dirname, 'configs/functional-config.yml'), + // --debug => skipUploads: discovery runs (servers observe the requests) + // but no build is created, so this stays token-free and build-free. + '--debug', '--verbose' ], { cwd: __dirname }); } finally { @@ -57,7 +58,10 @@ async function run() { console.log(''); check(code === 0, `percy snapshot exits 0 (got ${code})`); - check(/Finalized build/.test(output), 'build finalized'); + // --debug => skipUploads: confirm we ran discovery WITHOUT creating a build + // (so this track never pollutes the visual project with a stray build). + check(!/\/builds\/[0-9]/.test(output) && !/Finalized build/.test(output), + 'no Percy build created (--debug / skipUploads)'); // discovery.requestHeaders — custom header injected on discovery requests check(obs.requestHeader === 'present', @@ -78,9 +82,11 @@ async function run() { check(!!obs.userAgent && obs.userAgent.includes('PercyRegressionUA/1.0'), `discovery.userAgent sent (ua=${obs.userAgent})`); - // discovery.captureSrcset — every srcset candidate fetched - check(obs.srcset.includes('1x') && obs.srcset.includes('2x'), - `discovery.captureSrcset fetched all candidates (got [${obs.srcset.join(', ')}])`); + // discovery.captureSrcset — the 2x candidate is the discriminating signal: + // 1x is the and would load regardless, but 2x is only fetched when + // srcset candidates are captured. + check(obs.srcset.includes('2x'), + `discovery.captureSrcset fetched the srcset-only 2x candidate (got [${obs.srcset.join(', ')}])`); // discovery.disallowedHostnames — request to the disallowed 9101 host aborted // before it left the browser, so the server never saw it. From b04f42f670e5330f10c76b6e9b477127f9b6de57 Mon Sep 17 00:00:00 2001 From: Akash Sinha Date: Thu, 25 Jun 2026 17:47:16 +0530 Subject: [PATCH 8/8] test(regression): close coverage gaps from PR review (elementXpath, layout, serve) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses reviewer's coverage audit: 1. regions[].elementSelector.elementXpath — add a region using the xpath selector form (all-config.yml); previously only elementCSS + boundingBox. 2. algorithm: layout — the same new region uses the layout enum value, which no fixture/region exercised before. 3. server mode (serve) — add a static-site/ fixture covered by a new Track C case (percy snapshot --dry-run --clean-urls). The /snapshot/server 'port' option is not reachable via the CLI (no --port flag; not set for directories), so it's documented as a non-goal in COVERAGE.md. Track C now 5 cases; all-config.yml stays at 25 snapshots. PR-2312 review follow-up. PER-8250 --- test/regression/COVERAGE.md | 12 +++++++++--- test/regression/config-validation.test.js | 8 ++++++++ test/regression/configs/all-config.yml | 5 +++++ test/regression/static-site/about.html | 12 ++++++++++++ test/regression/static-site/index.html | 13 +++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 test/regression/static-site/about.html create mode 100644 test/regression/static-site/index.html diff --git a/test/regression/COVERAGE.md b/test/regression/COVERAGE.md index 4c7302499..d840ef631 100644 --- a/test/regression/COVERAGE.md +++ b/test/regression/COVERAGE.md @@ -30,6 +30,11 @@ PERCY_TOKEN=… yarn test:regression # Track V — token-gated, creates the bui `considerRegions`. - **`comparisonSchema`** (the upload/comparison wire format) is not user-facing CLI config — out of scope. +- **`/snapshot/server` `port`** is not reachable via `percy snapshot`: there is + no `--port` flag on the command and it isn't in the `static` config namespace + (the command only sets `serve`/`cleanUrls`/`baseUrl` for a directory — see + `cli-snapshot/src/snapshot.js`). It's a programmatic/SDK option only. `serve` + itself IS covered (server mode via `percy snapshot static-site/`). - All additions are test-only; no production code changes. ## Coverage by namespace @@ -60,8 +65,8 @@ Every listed option is at minimum **C** (in a config the CLI loads & validates). | readiness.* (preset, *WindowMs, timeoutMs, domStability, imageReady, fontReady, jsIdle, readySelectors, notPresentSelectors, maxTimeoutMs) | C | `all-config.yml` | | responsiveSnapshotCapture | C, V | "Config - Responsive Snapshot Capture" | | testCase, labels, thTestCaseExecutionId, browsers | C | `all-config.yml` | -| regions[] (elementSelector, padding, algorithm, configuration, assertion) | C | `all-config.yml` + `per-snapshot-options.yml` | -| algorithm, algorithmConfiguration.* | C | `all-config.yml` | +| regions[] (elementSelector: boundingBox / elementCSS / elementXpath; padding, algorithm, configuration, assertion) | C | `all-config.yml` (all 3 selector forms) + `per-snapshot-options.yml` | +| algorithm (standard / layout / intelliignore / ignore), algorithmConfiguration.* | C | `all-config.yml` (incl. `layout`); `ignore` via region in `per-snapshot-options.yml` | | ignoreCanvasSerializationErrors, ignoreStyleSheetSerializationErrors | C | `all-config.yml` | | ignoreIframeSelectors | C, V | `snapshots.yml` "DOM Structures Coverage" | | pseudoClassEnabledElements (id, className, xpath, selectors) | C, V | `all-config.yml` (all forms); "Interactive States" (selectors) | @@ -102,7 +107,8 @@ Every listed option is at minimum **C** (in a config the CLI loads & validates). | precapture waitForSelector / waitForTimeout | C | `per-snapshot-options.yml` | | include / exclude (filter; also `--include`/`--exclude`) | C | `per-snapshot-options.yml` | | list baseUrl, options | C | `per-snapshot-options.yml` | -| static cleanUrls, rewrites, baseUrl, options | C | `all-config.yml` (`static:` section) | +| static cleanUrls, rewrites, baseUrl, options | C | `all-config.yml` (`static:` section); `serve` + `cleanUrls` also via server mode `percy snapshot static-site/` | +| server mode: serve | C | `static-site/` via `percy snapshot --dry-run` (`port` is not CLI-reachable — see Scope) | | sitemap options | C | `all-config.yml` (`sitemap:` section) | ## Adding coverage for a new option diff --git a/test/regression/config-validation.test.js b/test/regression/config-validation.test.js index 712758bca..6333ad736 100644 --- a/test/regression/config-validation.test.js +++ b/test/regression/config-validation.test.js @@ -45,6 +45,14 @@ const CASES = [ expectCount: 8, expectValid: true }, + { + // Server mode: `percy snapshot ` exercises the /snapshot/server schema + // (serve) plus the static cleanUrls flag, which list/base-url mode doesn't. + name: 'static-site/ — server mode (serve) + cleanUrls via percy snapshot ', + args: ['snapshot', 'static-site', '--dry-run', '--clean-urls'], + expectCount: 2, + expectValid: true + }, { // Negative guard: proves the "Invalid config:" detector actually fires, so // the assertion protecting the valid fixtures above is not a no-op. diff --git a/test/regression/configs/all-config.yml b/test/regression/configs/all-config.yml index 3a7b2c51b..24c48d27f 100644 --- a/test/regression/configs/all-config.yml +++ b/test/regression/configs/all-config.yml @@ -98,6 +98,11 @@ snapshot: carouselsEnabled: true bannersEnabled: true adsEnabled: true + # Third selector form (elementXpath) + the `layout` algorithm enum value, + # which the other regions / top-level algorithm don't exercise. + - algorithm: layout + elementSelector: + elementXpath: '//div[@id="region"]' algorithm: standard algorithmConfiguration: diffSensitivity: 3 diff --git a/test/regression/static-site/about.html b/test/regression/static-site/about.html new file mode 100644 index 000000000..0d4bcc880 --- /dev/null +++ b/test/regression/static-site/about.html @@ -0,0 +1,12 @@ + + + + + + Static Server Mode — About + + +

About

+

Second page so the static directory enumerates more than one snapshot.

+ + diff --git a/test/regression/static-site/index.html b/test/regression/static-site/index.html new file mode 100644 index 000000000..8b0dfb1d3 --- /dev/null +++ b/test/regression/static-site/index.html @@ -0,0 +1,13 @@ + + + + + + Static Server Mode — Home + + +

Static server mode

+

Fixture for exercising the `percy snapshot <dir>` server/serve mode + (the /snapshot/server schema) in the config-validation track.

+ +