diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c5f0247..983cc8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,8 @@ jobs: timeout-minutes: 10 strategy: matrix: - # 20 is the supported floor; 22 exercises live streaming and CDP - # attach mode (global WebSocket client). - node-version: [20, 22] + # 20 is the supported floor; 22/24 support the global WebSocket client. + node-version: [20, 22, 24] steps: - name: Check out repository # v4.2.2 @@ -30,11 +29,64 @@ jobs: uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b with: node-version: ${{ matrix.node-version }} - - name: Run tests - run: npm test - - name: Validate shell syntax - run: bash -n scripts/*.sh + - name: Validate source, manifest, shell scripts, and tests + run: npm run validate - name: Run ShellCheck run: shellcheck scripts/*.sh - - name: Validate plugin manifest - run: node scripts/check-manifest.mjs + + browser: + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + node-version: [22, 24] + steps: + - name: Check out repository + # v4.2.2 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Set up Node.js for agent-browser installation + # agent-browser 0.33.2 requires Node 24 to install its native CLI. + # v4.0.3 + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b + with: + node-version: 24 + - name: Install pinned agent-browser + run: | + npm install --global --prefix "$RUNNER_TEMP/agent-browser" agent-browser@0.33.2 + echo "$RUNNER_TEMP/agent-browser/bin" >> "$GITHUB_PATH" + - name: Install Chromium and platform prerequisites + shell: bash + run: | + if [[ "$RUNNER_OS" == "Linux" ]]; then + agent-browser install --with-deps + else + agent-browser install + fi + - name: Export installed Chromium executable + shell: bash + run: | + python3 - <<'PY' >> "$GITHUB_ENV" + import os + from pathlib import Path + + cache = Path.home() / ".agent-browser" / "browsers" + binaries = [p for p in cache.rglob("*") + if p.name in ("chrome", "Google Chrome for Testing") + and p.is_file() and os.access(p, os.X_OK)] + if len(binaries) != 1: + raise SystemExit(f"Expected one installed Chromium executable, found {binaries}") + print(f"HERDR_BROWSER_CHROMIUM={binaries[0]}") + PY + - name: Set up Node.js under test + # v4.0.3 + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b + with: + node-version: ${{ matrix.node-version }} + - name: Require real-browser integration coverage + run: npm run test:integration + - name: Require saved QA browser coverage + run: npm run test:qa diff --git a/CHANGELOG.md b/CHANGELOG.md index 19cfed6..da0fcbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Add an on-demand page text view: press `s` for the accessibility-tree + snapshot (interactive elements carry `@eN` refs), `n` for the full recent + request table, and `d` for a ref-normalized diff against the last snapshot. + Works on both backends (agent-browser `snapshot`/`network requests`, CDP + `Accessibility`/`Network`); any page-driving action dismisses it. - Select attached or locally launched browser tabs explicitly from a title/URL picker. Clear stale frames and pending input when the selected tab closes; require another selection before forwarding input. diff --git a/README.md b/README.md index 873489f..9d53922 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ guides to this plugin and its three siblings (Guard, Swarm, Conductor). guesses; keyboard input, history, reload, and wheel scrolling are supported. - **Adaptive rendering** — Kitty graphics when available, ANSI symbols through chafa otherwise, and a text-only last resort. +- **Read the page as text** — press `s` for the accessibility-tree snapshot + (interactive elements get `@eN` refs), `n` for the full recent request table, + and `d` to diff the page against your last snapshot; the screenshot returns + on any page-driving action. - **Built-in recording** — capture the workspace session as WebM. - **Localhost integration** — Cmd/Ctrl+click a local development URL in Herdr to open it in the workspace browser pane. @@ -160,6 +164,9 @@ Use these controls to drive the shared session directly: | `j` / `k` | Scroll down / up | | Space | Scroll down | | Mouse wheel | Scroll the page | +| `s` | Toggle the accessibility-tree snapshot (interactive elements carry `@eN` refs) | +| `n` | Toggle the recent request table | +| `d` | Diff the current page against the last snapshot (press `s` first) | | `Esc` | Cancel the active prompt | | `q` | Close the viewer pane | @@ -499,6 +506,7 @@ npm run doctor npm run build npm run validate npm run test:integration +npm run test:qa herdr plugin link . ``` @@ -514,7 +522,15 @@ browser coverage; optional browser tests skip when their prerequisites are unavailable. `npm run test:integration` requires both an installed Chrome or Chromium and agent-browser with its engine installed, on Node 22+. It fails instead of silently skipping either real-browser path. Tests use a local HTTP -fixture and isolated browser sessions. +fixture and isolated browser sessions. `npm run test:qa` requires agent-browser +with its engine installed and runs saved desktop/mobile scenarios, including +assertion failures and browser error evidence. CI installs pinned agent-browser +0.33.2 using Node 24, then tests the plugin on Node 22/24 on Linux and macOS. + +Before release, require local validation, both strict browser commands, and the +hosted CI matrix to pass for the reviewed commit. A passing suite with browser +coverage skipped is insufficient. Manually verify Kitty graphics and symbol +rendering in Herdr on macOS and Linux. For backend choices and the remaining readiness work, see the [readiness assessment](docs/readiness.md). diff --git a/bin/cdp.mjs b/bin/cdp.mjs index 4959ad1..3d41eed 100644 --- a/bin/cdp.mjs +++ b/bin/cdp.mjs @@ -4,6 +4,7 @@ // creates or closes targets and exposes no listening port of its own. import http from "node:http"; import net from "node:net"; +import { formatAxNodes, formatRequests } from "./page-text.mjs"; // Attach mode needs a WebSocket client; Node grew a global one in 22. // On Node 20 the pane keeps working in agent-browser mode — callers gate on @@ -262,6 +263,10 @@ export function makeCdpBrowser(endpointInput, opts = {}) { let attachTimeMs = 0; let connectedOnce = false; let selection = 0; + let accessibilitySessionId = null; + let networkSessionId = null; + const networkRows = []; + const networkById = new Map(); const targetError = () => Object.assign(new Error("select a tab with t before sending input"), { code: "TARGET_GONE" }); const requireTarget = (expected = pageSessionId) => { if (!expected || expected !== pageSessionId || !pinnedTargetId || session?.dead) throw targetError(); @@ -285,6 +290,10 @@ export function makeCdpBrowser(endpointInput, opts = {}) { pinnedTargetId = null; pageSessionId = null; lastMeta = null; + accessibilitySessionId = null; + networkSessionId = null; + networkRows.length = 0; + networkById.clear(); emit({ type }); }; @@ -352,7 +361,37 @@ export function makeCdpBrowser(endpointInput, opts = {}) { } }; + const recordNetworkEvent = (m) => { + if (!networkSessionId || m.sessionId !== pageSessionId || m.sessionId !== networkSessionId) return; + const p = m.params ?? {}; + if (m.method === "Network.requestWillBeSent") { + const row = { + method: p.request?.method ?? "", + status: null, + type: p.type ?? "", + url: p.request?.url ?? "", + }; + networkRows.push(row); + networkById.set(p.requestId, row); + if (networkRows.length > 200) { + const removed = networkRows.shift(); + for (const [requestId, current] of networkById) { + if (current === removed) networkById.delete(requestId); + } + } + return; + } + const row = networkById.get(p.requestId); + if (!row) return; + if (m.method === "Network.responseReceived") row.status = p.response?.status ?? null; + else if (m.method === "Network.loadingFailed") row.status = p.errorText || "failed"; + }; + const onCdpEvent = (m) => { + if (m.method?.startsWith("Network.")) { + recordNetworkEvent(m); + return; + } if (m.method === "Page.screencastFrame" && pageSessionId && m.sessionId === pageSessionId) { lastMeta = m.params.metadata ?? null; emit({ @@ -516,6 +555,33 @@ export function makeCdpBrowser(endpointInput, opts = {}) { try { await pinTarget(next); } catch (err) { clearTarget("target_gone"); throw err; } }, + // Accessibility and Network stay opt-in: enabling either domain is the + // one honest footprint made only after the user requests its view. + async snapshotText() { + const sid = requireTarget(); + try { + if (accessibilitySessionId !== sid) { + await session.send("Accessibility.enable", {}, sid); + requireTarget(sid); + accessibilitySessionId = sid; + } + const { nodes = [] } = await session.send("Accessibility.getFullAXTree", {}, requireTarget(sid)); + return formatAxNodes(nodes.map(({ nodeId, role, name, childIds }) => ({ nodeId, role, name, childIds }))); + } catch (err) { + if (/wasn't found|not found|not supported|unknown method/i.test(err?.message ?? "")) + throw new Error("browser has no Accessibility domain"); + throw err; + } + }, + async networkRequests() { + const sid = requireTarget(); + if (networkSessionId !== sid) { + await session.send("Network.enable", {}, sid); + requireTarget(sid); + networkSessionId = sid; + } + return formatRequests(networkRows); + }, async open(u) { await session.send("Page.navigate", { url: u }, requireTarget()); }, diff --git a/bin/page-text.mjs b/bin/page-text.mjs new file mode 100644 index 0000000..8576353 --- /dev/null +++ b/bin/page-text.mjs @@ -0,0 +1,121 @@ +// --- Page snapshots --------------------------------------------------------- + +const INTERACTIVE_ROLES = new Set([ + "button", "link", "textbox", "checkbox", "combobox", "radio", + "menuitem", "option", "tab", "switch", "slider", "spinbutton", "searchbox", +]); + +function axValue(value) { + if (value && typeof value === "object" && "value" in value) return String(value.value ?? ""); + return String(value ?? ""); +} + +/** Format a CDP accessibility tree as indented, referenceable text. */ +export function formatAxNodes(nodes) { + const list = Array.isArray(nodes) ? nodes : []; + const byId = new Map(list.map((node) => [node.nodeId, node])); + const children = new Set(list.flatMap((node) => Array.isArray(node.childIds) ? node.childIds : [])); + const roots = list.filter((node) => !children.has(node.nodeId)); + const lines = []; + const visited = new Set(); + let interactive = 0; + + const walk = (node, depth) => { + if (!node || visited.has(node.nodeId)) return; + visited.add(node.nodeId); + const role = axValue(node.role) || "unknown"; + const name = axValue(node.name); + let line = `${" ".repeat(depth)}${role}`; + if (name) line += ` ${JSON.stringify(name)}`; + if (INTERACTIVE_ROLES.has(role.toLowerCase())) line += ` @e${++interactive}`; + lines.push(line); + for (const childId of Array.isArray(node.childIds) ? node.childIds : []) { + walk(byId.get(childId), depth + 1); + } + }; + + for (const root of roots) walk(root, 0); + for (const node of list) walk(node, 0); + return { title: `page snapshot (${interactive} interactive)`, lines }; +} + +/** Parse an agent-browser interactive snapshot without changing its refs. */ +export function parseAgentSnapshot(text) { + const input = String(text ?? "").replace(/\r\n?/g, "\n").split("\n"); + const headers = []; + while (input.length && /^(?:Page|URL)\s*:/i.test(input[0])) { + headers.push(input.shift().replace(/^(?:Page|URL)\s*:\s*/i, "")); + } + while (input.length && input[0] === "") input.shift(); // blank separator line + while (input.length && input.at(-1) === "") input.pop(); + return { title: headers.join(" — ") || "page snapshot", lines: input }; +} + +// --- Network requests ------------------------------------------------------- + +function redactUrl(value) { + try { + const url = new URL(String(value)); + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + return url.href; + } catch { + return String(value ?? "").replace(/([a-z][a-z\d+.-]*:\/\/)[^/@\s]+@/i, "$1").split(/[?#]/, 1)[0]; + } +} + +/** Format network request rows as a compact, redacted table. */ +export function formatRequests(rows) { + const values = (Array.isArray(rows) ? rows : []).map((row) => [ + String(row.method ?? ""), + row.status == null ? "…" : String(row.status), + String(row.type ?? ""), + redactUrl(row.url), + ]); + const headings = ["METHOD", "STATUS", "TYPE"]; + const widths = headings.map((heading, index) => Math.max(heading.length, ...values.map((row) => row[index].length))); + const lines = [ + [headings[0].padEnd(widths[0]), headings[1].padEnd(widths[1]), headings[2].padEnd(widths[2]), "URL"].join(" "), + ...values.map((row) => [row[0].padEnd(widths[0]), row[1].padEnd(widths[1]), row[2].padEnd(widths[2]), row[3]].join(" ")), + ]; + return { title: `requests (${values.length})`, lines }; +} + +// --- Snapshot differences -------------------------------------------------- + +function normalizeRefs(line) { + return String(line) + .replace(/@e\d+\b/g, "") // CDP formatAxNodes refs + .replace(/,?\s*ref=e\d+\b/g, "") // agent-browser snapshot refs + .replace(/\[\s*\]/g, "") // empty attribute list left behind + .replace(/[ \t]+$/g, ""); +} + +/** Diff snapshots while ignoring volatile interactive-element references. */ +export function diffSnapshot(prevLines, nextLines) { + const previous = (Array.isArray(prevLines) ? prevLines : []).map(normalizeRefs); + const next = (Array.isArray(nextLines) ? nextLines : []).map(normalizeRefs); + const previousCounts = new Map(); + for (const line of previous) previousCounts.set(line, (previousCounts.get(line) || 0) + 1); + const added = []; + for (const line of next) { + const count = previousCounts.get(line) || 0; + if (count) previousCounts.set(line, count - 1); + else added.push(line); + } + const nextCounts = new Map(); + for (const line of next) nextCounts.set(line, (nextCounts.get(line) || 0) + 1); + const removed = []; + for (const line of previous) { + const count = nextCounts.get(line) || 0; + if (count) nextCounts.set(line, count - 1); + else removed.push(line); + } + if (!added.length && !removed.length) return { title: "snapshot diff (no change)", lines: [] }; + return { + title: `snapshot diff (+${added.length} -${removed.length})`, + lines: [...added.map((line) => `+ ${line}`), ...removed.map((line) => `- ${line}`)], + }; +} diff --git a/bin/renderer.mjs b/bin/renderer.mjs index 6783e22..d757ecc 100644 --- a/bin/renderer.mjs +++ b/bin/renderer.mjs @@ -9,6 +9,7 @@ import { pathToFileURL } from "node:url"; import fs from "node:fs"; import path from "node:path"; import { makeCdpBrowser, cdpSupported, redactWsUrl } from "./cdp.mjs"; +import { formatRequests, parseAgentSnapshot, diffSnapshot } from "./page-text.mjs"; const pExecFile = promisify(execFile); const ESC = "\x1b"; @@ -423,7 +424,7 @@ export function formatNetworkFailure({ method, url, status }) { // --- agent-browser access --- -export function makeBrowser(session, bin = "agent-browser") { +export function makeBrowser(session, bin = "agent-browser", env = process.env) { // Console rings on noisy pages reach several MB — Node's 1 MiB default // maxBuffer would throw on every tick and freeze the pane for good. const maxBuffer = 16 * 1024 * 1024; @@ -446,7 +447,7 @@ export function makeBrowser(session, bin = "agent-browser") { const { stdout } = await pExecFile( bin, ["--session", session, "batch", "--bail", "--json", ...cmds], - { timeout, maxBuffer }, + { timeout, maxBuffer, env }, ); const arr = parse(stdout, "batch output"); for (const r of arr) { @@ -464,6 +465,7 @@ export function makeBrowser(session, bin = "agent-browser") { { timeout: 10_000, maxBuffer, + env, }, ); const parsed = parse(stdout, "output"); @@ -536,11 +538,33 @@ export function makeBrowser(session, bin = "agent-browser") { if (Array.isArray(data?.requests)) return data.requests; return Array.isArray(data) ? data : []; }, + // Page text view: the accessibility tree with @eN refs (agent-browser + // `snapshot`, the same primitive Playwright CLI and Stagehand expose). + snapshotText: async () => { + const data = await run("snapshot", "-i", "-c"); + const text = typeof data === "string" ? data : data?.snapshot ?? ""; + return parseAgentSnapshot(text); + }, + // Full recent-request table (not just the xhr/fetch/document failures + // the poll feed watches), formatted for the pane's text view. + networkRequests: async () => { + const data = await run("network", "requests"); + const rows = Array.isArray(data?.requests) ? data.requests : []; + return formatRequests( + rows.map((r) => ({ + method: r.method ?? "", + status: r.status ?? null, + type: r.resourceType ?? "", + url: r.url ?? "", + })), + ); + }, sessionExists: async () => { try { const { stdout } = await pExecFile(bin, ["session", "list", "--json"], { timeout: 10_000, maxBuffer, + env, }); // Exact match only: a substring hit (herdr-ws-w2 vs herdr-ws-w22) // would make the pane attach and auto-create a session it must not. @@ -592,7 +616,7 @@ export class Renderer { this.browser = this.backend === "attach" ? makeCdpBrowser(this.cdpEndpoint) - : makeBrowser(this.session, this.bin); + : makeBrowser(this.session, this.bin, this.env); // Attach mode observes a browser someone else owns: ownership is never // claimed, so the quit path can never close a stranger's session. this.ownershipEnabled = this.backend !== "attach"; @@ -638,6 +662,10 @@ export class Renderer { this.launchAttempted = false; this.promptState = null; this.targetPicker = null; + // Page text view (snapshot/network/diff) replaces the screenshot region + // while set; lastSnapshot backs the diff key. + this.textView = null; + this.lastSnapshot = null; this.pasteState = null; this.inputRevision = 0; this.cdpTargetId = null; @@ -715,13 +743,15 @@ export class Renderer { let consoleRows = 0; if (this.targetPicker) { consoleRows = 0; - } else if (this.mode === "text") { + } else if (this.mode === "text" && !this.textView) { consoleRows = Math.max(0, rows - 3); } else if (this.consoleLines.length > 0) { consoleRows = Math.max(4, Math.floor(rows * 0.3)); } const imageRows = - this.mode === "text" && !this.targetPicker ? 0 : Math.max(0, rows - consoleRows - 4); + this.mode === "text" && !this.targetPicker && !this.textView + ? 0 + : Math.max(0, rows - consoleRows - 4); return { cols, rows, @@ -1310,7 +1340,7 @@ export class Renderer { return; } await this.redrawAll(); - if (this.lastImageDims) { + if (!this.textView && this.lastImageDims) { this.lastViewportRequest = ""; await this.fitViewport(this.lastImageDims.w, this.lastImageDims.h); } @@ -1381,10 +1411,10 @@ export class Renderer { // Every advertised key is handled in both backends (a and l work // everywhere); t only moves targets on an attach backend. const help = this.observeOnly - ? " observe-only t:tabs o:enable-input q:quit" + ? " observe-only t:tabs s:snap n:reqs d:diff o:enable-input q:quit" : this.backend === "attach" - ? " t:tabs i:text u:url a:attach l:launch o:observe q:quit b/f:hist r:reload j/k:scroll" - : " u:url a:attach l:launch i:type b/f:hist r:reload j/k:scroll o:observe q:quit"; + ? " t:tabs i:text u:url a:attach l:launch o:observe s:snap n:reqs d:diff b/f:hist r:reload j/k:scroll q:quit" + : " u:url a:attach l:launch i:type b/f:hist r:reload j/k:scroll s:snap n:reqs d:diff o:observe q:quit"; process.stdout.write( `${ESC}[${bottomRow};1H${ESC}[2m${truncate(help, cols)}${ESC}[K${ESC}[0m`, ); @@ -1393,6 +1423,7 @@ export class Renderer { async renderImage() { if (this.targetPicker) return this.renderTargetPicker(); + if (this.textView) return this.renderTextView(); const { cols, imageRows } = this.size(); const shotPath = this.shotFormat === "jpg" ? this.shotJpg : this.shot; if (this.mode === "text" || imageRows < 3 || !fs.existsSync(shotPath)) @@ -1496,6 +1527,28 @@ export class Renderer { } } + // Page text view: the screenshot region is replaced by a bounded text + // readout (snapshot tree, request table, or a snapshot diff) while + // this.textView is set. Mirrors renderTargetPicker's per-line erase/write. + renderTextView() { + const tv = this.textView; + if (!tv) return; + const { cols, imageRows, imageTopRow } = this.size(); + if (imageRows < 3) return; + const modeLabel = + tv.mode === "diff" ? "diff" : tv.mode === "network" ? "requests" : "snapshot"; + const head = [` ${modeLabel} — ${sanitizeText(tv.title)}`]; + const shown = tv.lines.slice(0, Math.max(0, imageRows - head.length)); + const overflow = tv.lines.length - shown.length; + const lines = head.concat(shown); + if (overflow > 0) lines.push(` … ${overflow} more`); + for (let i = 0; i < imageRows; i++) { + process.stdout.write( + `${ESC}[${imageTopRow + i};1H${truncate(lines[i] ?? "", cols)}${ESC}[K`, + ); + } + } + openTargetPicker() { if (this.backend !== "attach" || !this.attached) return; const picker = { targets: [], index: 0, loading: true }; @@ -1670,6 +1723,16 @@ export class Renderer { const snap = await this.browser.snapshot(tmp); this.lastUrl = sanitizeText(snap.url); this.lastTitle = sanitizeText(snap.title); + // A navigation invalidates a captured snapshot/diff (the network + // list is a session log and survives); drop it so a stale page is + // never shown as the current one. + if ( + this.textView && + this.textView.mode !== "network" && + this.textView.url !== this.lastUrl + ) { + this.textView = null; + } const { newEntries, marker } = reconcileConsole( this.consoleState, snap.entries, @@ -1698,14 +1761,14 @@ export class Renderer { this.shotFormat = "png"; // heal after a dropped stream's jpg frames this.lastHash = hash; await this.renderImage(); - await this.fitViewport(dims.w, dims.h); + if (!this.textView) await this.fitViewport(dims.w, dims.h); if (consoleLayoutChanged) this.renderConsole(); } else { fs.unlinkSync(tmp); if (consoleLayoutChanged) { await this.redrawAll(); this.lastViewportRequest = ""; - await this.fitViewport(dims.w, dims.h); + if (!this.textView) await this.fitViewport(dims.w, dims.h); } } this.banner = ""; @@ -1883,6 +1946,7 @@ export class Renderer { try { spawnSync(this.bin, ["--session", this.session, "close"], { timeout: 2_000, + env: this.env, }); } catch { /* already gone */ @@ -1907,6 +1971,78 @@ export class Renderer { this.header(); } + // s/n/d: on-demand page text views (snapshot tree, request table, snapshot + // diff). Read-only queries — they never drive the page, so they are allowed + // in observe-only mode and never claim page input. + textViewKey(ch) { + if (!this.attached) { + this.banner = "no browser attached — press l or open a localhost link"; + this.header(); + return; + } + if (ch === "d") this.showSnapshotDiff(); + else this.toggleTextView(ch === "s" ? "snapshot" : "network"); + } + + toggleTextView(mode) { + // Re-pressing the active key dismisses the view back to the screenshot. + if (this.textView?.mode === mode) { + this.textView = null; + this.enqueue(() => this.redrawAll()); + return; + } + this.userAction(async () => { + try { + const fn = + mode === "snapshot" + ? this.browser.snapshotText + : this.browser.networkRequests; + if (typeof fn !== "function") + throw new Error(`${mode} not supported in ${this.backendName}`); + const res = await fn.call(this.browser); + if (!res || !Array.isArray(res.lines)) throw new Error("no data"); + this.textView = { + mode, + title: res.title ?? "", + lines: res.lines, + url: this.lastUrl, + }; + if (mode === "snapshot") this.lastSnapshot = res.lines; + await this.redrawAll(); + } catch (err) { + this.banner = sanitizeText(err?.message ?? `${mode} unavailable`); + this.header(); + } + }, { pageInput: false }); + } + + showSnapshotDiff() { + if (!this.lastSnapshot) { + this.banner = "press s first to capture a snapshot"; + this.header(); + return; + } + this.userAction(async () => { + try { + if (typeof this.browser.snapshotText !== "function") + throw new Error(`snapshot not supported in ${this.backendName}`); + const res = await this.browser.snapshotText(); + if (!res || !Array.isArray(res.lines)) throw new Error("no data"); + const diff = diffSnapshot(this.lastSnapshot, res.lines); + this.textView = { + mode: "diff", + title: diff.title ?? "", + lines: diff.lines, + url: this.lastUrl, + }; + await this.redrawAll(); + } catch (err) { + this.banner = sanitizeText(err?.message ?? "diff unavailable"); + this.header(); + } + }, { pageInput: false }); + } + onKey(ch) { if (this.targetPicker) { this.targetPickerInput(ch); @@ -1923,6 +2059,12 @@ export class Renderer { this.toggleObserveOnly(); return; } + // Page text views are read-only queries — safe in observe-only mode, + // and they need no page target beyond what the backend already guards. + if (ch === "s" || ch === "n" || ch === "d") { + this.textViewKey(ch); + return; + } if ( this.observeOnly && ["u", "i", "b", "f", "r", "j", "k", " "].includes(ch) @@ -2157,7 +2299,7 @@ export class Renderer { this.networkIdleTicks = 0; // page activity: keep failure polls prompt this.enqueue(async () => { await this.renderImage(); - await this.fitViewport(dims.w, dims.h); + if (!this.textView) await this.fitViewport(dims.w, dims.h); }); break; } @@ -2225,6 +2367,9 @@ export class Renderer { } } try { + // A page-driving action invalidates any text view captured from + // the pre-action page: drop it before the page changes. + if (pageInput) this.textView = null; await fn(); } catch (err) { // Poll mode reports daemon failures via the tick failure counter; @@ -2429,7 +2574,7 @@ export class Renderer { () => this.enqueue(async () => { this.lastViewportRequest = ""; - if (this.lastImageDims) { + if (!this.textView && this.lastImageDims) { await this.fitViewport( this.lastImageDims.w, this.lastImageDims.h, diff --git a/docs/plans/inspect-v0.9.md b/docs/plans/inspect-v0.9.md new file mode 100644 index 0000000..c9c8e88 --- /dev/null +++ b/docs/plans/inspect-v0.9.md @@ -0,0 +1,71 @@ +# Browser 0.9 — "Inspect": read the page as text, not pixels + +## Problem + +The pane is pixel-only: screenshots, a console strip, and a failed-request feed. +A human watching an agent — or a text-only terminal — cannot *read* the page, +inspect an element, see the full request log, or tell what an agent's last +action changed. Competitors (Playwright CLI, Stagehand, Browser Use) all ship an +accessibility-tree snapshot; the engine already does too (`agent-browser +snapshot`, `find`, `network requests`, `diff snapshot`, `get box/styles/text`, +`highlight`). The pane uses none of them. + +## Decision + +Add an on-demand **page text view** that renders in the screenshot region, with +three sources, uniform across both backends (agent-browser + CDP attach/launch). + +1. **Snapshot (a11y tree)** — key `s`. Role/name tree with `@eN` refs on + interactive elements (Playwright CLI cherry-pick; engine `snapshot` / + CDP `Accessibility.getFullAXTree`). +2. **Network requests** — key `n`. Bounded `METHOD STATUS TYPE URL` rows, not + just failures (DevTools / `network requests` cherry-pick). +3. **Snapshot diff** — key `d`. Ref-normalized line diff vs the last snapshot: + "what did the agent just change?" (new pane differentiator; `diff snapshot`). + +Any page-driving action (navigate/reload/back/forward/scroll/click/type) or +re-pressing the active key returns to the screenshot. + +## Contract + +New pure module `bin/page-text.mjs` (unit-testable, no browser): + +```js +formatAxNodes(nodes) -> { title, lines } // CDP AX node list -> indented role/name + @eN refs +parseAgentSnapshot(text) -> { title, lines } // agent-browser `snapshot -i -c` text +formatRequests(rows) -> { title, lines } // [{method,status,type,url}] -> rows +parseAgentRequests(text) -> { title, lines } // agent-browser `network requests` text +diffSnapshot(prev, next) -> { title, lines } // @eN refs stripped, +/- line diff +``` + +Both backends expose identical adapter methods (renderer `typeof`-guards absence): + +```js +snapshotText() -> Promise<{ title, lines }> +networkRequests() -> Promise<{ title, lines }> +``` + +- `bin/cdp.mjs` `makeCdpBrowser`: `Accessibility.enable` + `Accessibility.getFullAXTree` + (tree via `childIds`) and `Network.enable` + `requestWillBeSent` / + `responseReceived` / `loadingFailed` (bounded ring by `requestId`). +- `bin/renderer.mjs` `makeBrowser`: `snapshot -i -c` and `network requests`. + +## Renderer integration + +- `this.textView = null | { mode: 'snapshot'|'network'|'diff', title, lines }`. +- `this.lastSnapshot = null | string[]`. +- `s`/`n` fetch + set `textView`; `d` re-snapshots and diffs via `diffSnapshot`. +- Text view paints in the image region (header/controls/console unchanged); + `renderBottom()` help gains `s/n/d`. + +## Checks + +- `npm run build`, `npm test` green; new `page-text.test.mjs` + `cdp.test.mjs` + + `renderer.test.mjs` cases using existing local fixtures/test doubles. +- Manual: `s`/`n`/`d` in Herdr against a local page, both backends. +- README viewer-controls + highlights + CHANGELOG entry. + +## Non-goals (this loop) + +LLM-powered act/extract (model dependency), full DevTools network panel, +second renderer engine, emulation on attached sessions, full-page capture. diff --git a/docs/readiness.md b/docs/readiness.md index 38afc7f..2dac49a 100644 --- a/docs/readiness.md +++ b/docs/readiness.md @@ -36,13 +36,35 @@ own Chrome engine; a second bundled distribution is unnecessary. - Limit test discovery to this checkout's `tests/` directory; Node 20 otherwise traverses hidden nested worktrees and runs unrelated, stale test copies. +## Automated readiness gates + +CI runs `npm run validate` on Linux with Node 20, 22, and 24. This checks source +and shell syntax, the plugin manifest, ShellCheck, and the complete test suite. +Separate browser jobs run on Linux and macOS with Node 22 and 24. They install +pinned agent-browser 0.33.2 under Node 24, install its Chromium engine (including +Linux system dependencies), and export the installed executable through +`HERDR_BROWSER_CHROMIUM` before switching to the Node version under test. + +Every browser job runs `npm run test:integration` and `npm run test:qa`. +`npm run doctor` is not run there: it also requires the Herdr CLI, which the +tests do not use and hosted runners do not have. The strict test commands +require real local Chromium, shared-session streaming, and desktop/mobile saved +QA coverage; missing +prerequisites and installation failures fail the job instead of skipping that +coverage. Existing tests use local fixtures and clean up their browser sessions. + +Readiness requires local validation and both strict browser commands to pass, +plus a passing hosted matrix for the exact commit being reviewed. Configuring +these gates does not establish that they have passed. Before release, manually +verify Kitty graphics and symbol rendering in Herdr on macOS and Linux; hosted +browser tests do not verify terminal display quality or pane interaction. + ## Next recommendations, in priority order -1. **Release compatibility matrix.** Exercise macOS and Linux, Node 22/24, - supported agent-browser versions, and Kitty/symbol rendering in Herdr. - Current CI has Node 20/22 but does not install agent-browser, so its passing - status alone cannot prove shared-session integration. Adopting the strict - integration command in hosted CI is a separate workflow change. +1. **Release compatibility checks.** Complete the manual Kitty/symbol checks + in Herdr on macOS and Linux. Hosted CI now covers both platforms on Node + 22/24 with agent-browser 0.33.2; expand agent-browser version coverage when + claiming support for additional versions. 2. **CDP transport correctness.** HTTP discovery currently uses `node:http` even for an HTTPS input and assumes the local debugging port. Implement actual HTTPS discovery with transport tests before advertising secured diff --git a/tests/cdp.test.mjs b/tests/cdp.test.mjs index 06c406f..d029f95 100644 --- a/tests/cdp.test.mjs +++ b/tests/cdp.test.mjs @@ -1,4 +1,4 @@ -import { test } from "node:test"; +import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; import { @@ -252,10 +252,100 @@ test("adapter surface: forbidden methods are absent, required ones present", asy const { b } = await attachBrowser(fake); for (const missing of ["setViewport", "network", "snapshot", "streamEnable", "streamStatus"]) assert.equal(b[missing], undefined, `${missing} must not exist — duck-type guards depend on it`); - for (const required of ["open", "back", "forward", "reload", "click", "scroll", "type", "sessionExists", "screenshot", "listTargets", "selectTarget"]) + for (const required of ["open", "back", "forward", "reload", "click", "scroll", "type", "sessionExists", "screenshot", "listTargets", "selectTarget", "snapshotText", "networkRequests"]) assert.equal(typeof b[required], "function", `${required} missing — a key handler calls it unguarded`); }); +test("adapter snapshotText enables Accessibility lazily and formats the full AX tree", async () => { + const fake = makeFakeCdp({ + results: { + "Accessibility.getFullAXTree": { + nodes: [ + { + nodeId: "root", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "Example" }, + childIds: ["button"], + backendDOMNodeId: 99, + }, + { + nodeId: "button", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Submit" }, + childIds: [], + }, + ], + }, + }, + }); + const { b } = await attachBrowser(fake); + const snapshot = await b.snapshotText(); + assert.equal(typeof snapshot.title, "string"); + assert.ok(snapshot.lines.some((line) => line.includes("RootWebArea") && line.includes("Example"))); + assert.ok(snapshot.lines.some((line) => line.includes("button") && line.includes("Submit") && /@e1\b/.test(line))); + assert.equal(fake.calls("Accessibility.enable").length, 1); + assert.equal(fake.calls("Accessibility.getFullAXTree")[0].sessionId, "sess-T1"); + await b.snapshotText(); + assert.equal(fake.calls("Accessibility.enable").length, 1, "domain enabled once for the pinned session"); +}); + +test("adapter networkRequests correlates pinned-page lifecycle events", async () => { + const fake = makeFakeCdp(); + const { b } = await attachBrowser(fake); + await b.networkRequests(); + fake.deliver({ + method: "Network.requestWillBeSent", + sessionId: "sess-T1", + params: { requestId: "ok", type: "Document", request: { method: "GET", url: "https://user:pass@example.test/ok?q=secret#part" } }, + }); + fake.deliver({ + method: "Network.responseReceived", + sessionId: "sess-T1", + params: { requestId: "ok", response: { status: 200 } }, + }); + fake.deliver({ + method: "Network.requestWillBeSent", + sessionId: "sess-T1", + params: { requestId: "pending", type: "Fetch", request: { method: "POST", url: "https://example.test/pending" } }, + }); + fake.deliver({ + method: "Network.requestWillBeSent", + sessionId: "sess-T1", + params: { requestId: "failed", type: "Script", request: { method: "GET", url: "https://example.test/app.js" } }, + }); + fake.deliver({ + method: "Network.loadingFailed", + sessionId: "sess-T1", + params: { requestId: "failed", errorText: "net::ERR_CONNECTION_RESET" }, + }); + fake.deliver({ + method: "Network.requestWillBeSent", + sessionId: "sess-OTHER", + params: { requestId: "noise", type: "Image", request: { method: "GET", url: "https://noise.test/private" } }, + }); + const requests = await b.networkRequests(); + const text = requests.lines.join("\n"); + assert.match(text, /GET\s+200\s+Document\s+https:\/\/example\.test\/ok/); + assert.match(text, /POST\s+(?:…|-|—|null)?\s*Fetch\s+https:\/\/example\.test\/pending/); + assert.match(text, /GET\s+net::ERR_CONNECTION_RESET\s+Script\s+https:\/\/example\.test\/app\.js/); + assert.doesNotMatch(text, /secret|user:pass|#part|noise\.test/); + assert.equal(fake.calls("Network.enable").length, 1, "domain enabled once for the pinned session"); +}); + +test("adapter text views reject before a target is pinned", async () => { + const fake = makeFakeCdp({ + pages: [ + { targetId: "T1", type: "page", url: "https://one/", title: "One" }, + { targetId: "T2", type: "page", url: "https://two/", title: "Two" }, + ], + }); + const { b } = await attachBrowser(fake); + await assert.rejects(b.snapshotText(), /select a tab/); + await assert.rejects(b.networkRequests(), /select a tab/); + assert.equal(fake.calls("Accessibility.enable").length, 0); + assert.equal(fake.calls("Network.enable").length, 0); +}); + test("adapter requires a choice among multiple pages, then pins the exact target", async () => { const fake = makeFakeCdp({ pages: [ diff --git a/tests/page-text.test.mjs b/tests/page-text.test.mjs new file mode 100644 index 0000000..a9f6d75 --- /dev/null +++ b/tests/page-text.test.mjs @@ -0,0 +1,124 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + diffSnapshot, + formatAxNodes, + formatRequests, + parseAgentSnapshot, +} from "../bin/page-text.mjs"; + +test("formatAxNodes flattens nesting and refs only interactive roles", () => { + const result = formatAxNodes([ + { nodeId: "root", role: { value: "document" }, name: { value: "Example" }, childIds: ["heading", "group"] }, + { nodeId: "heading", role: { value: "heading" }, name: { value: "Welcome" }, childIds: [] }, + { nodeId: "group", role: "group", name: "", childIds: ["button", "link"] }, + { nodeId: "button", role: "button", name: "Save", childIds: [] }, + { nodeId: "link", role: "link", name: "More information", childIds: [] }, + ]); + + assert.deepEqual(result, { + title: "page snapshot (2 interactive)", + lines: [ + 'document "Example"', + ' heading "Welcome"', + " group", + ' button "Save" @e1', + ' link "More information" @e2', + ], + }); +}); + +test("formatAxNodes handles empty input and an interactive node without a name", () => { + assert.deepEqual(formatAxNodes([]), { title: "page snapshot (0 interactive)", lines: [] }); + assert.deepEqual(formatAxNodes([{ nodeId: "1", role: "checkbox", name: "", childIds: [] }]), { + title: "page snapshot (1 interactive)", + lines: ["checkbox @e1"], + }); +}); + +test("parseAgentSnapshot moves leading page headers into the title", () => { + const text = "pAgE: Example site\r\nuRl: https://example.test/path\r\n- button \"Save\" @e17\r\n - text: body\r\n"; + assert.deepEqual(parseAgentSnapshot(text), { + title: "Example site — https://example.test/path", + lines: ['- button "Save" @e17', " - text: body"], + }); +}); + +test("parseAgentSnapshot preserves a headerless body and handles empty text", () => { + assert.deepEqual(parseAgentSnapshot("button @e9"), { title: "page snapshot", lines: ["button @e9"] }); + assert.deepEqual(parseAgentSnapshot(""), { title: "page snapshot", lines: [] }); +}); + +test("parseAgentSnapshot drops the blank separator between header and body", () => { + const text = "Page: Home\nURL: https://x/\n\n@e1 [button] \"Go\"\n"; + assert.deepEqual(parseAgentSnapshot(text), { + title: "Home — https://x/", + lines: ['@e1 [button] "Go"'], + }); +}); + +test("formatRequests pads columns and redacts sensitive URL components", () => { + const result = formatRequests([ + { method: "GET", status: null, type: "Document", url: "https://alice:secret@example.test/path?q=secret#token" }, + { method: "POST", status: "net::ERR_CONNECTION_REFUSED", type: "Fetch", url: "https://example.test/api?key=value" }, + { method: "GET", status: 204, type: "XHR", url: "https://example.test/done#result" }, + ]); + + assert.equal(result.title, "requests (3)"); + assert.deepEqual(result.lines, [ + "METHOD STATUS TYPE URL", + "GET … Document https://example.test/path", + "POST net::ERR_CONNECTION_REFUSED Fetch https://example.test/api", + "GET 204 XHR https://example.test/done", + ]); + assert.ok(result.lines.every((line) => !/secret|token|alice|key=value/.test(line))); +}); + +test("formatRequests handles no rows", () => { + assert.deepEqual(formatRequests([]), { + title: "requests (0)", + lines: ["METHOD STATUS TYPE URL"], + }); +}); + +test("diffSnapshot ignores refs and emits additions before removals", () => { + assert.deepEqual( + diffSnapshot(['button "Save" @e1', 'link "Old" @e2'], ['button "Save" @e99', 'textbox "Name" @e4']), + { + title: "snapshot diff (+1 -1)", + lines: ['+ textbox "Name"', '- link "Old"'], + }, + ); +}); + +test("diffSnapshot handles duplicate lines and empty or unchanged input", () => { + assert.deepEqual(diffSnapshot(["button @e1", "button @e2"], ["button @e8"]), { + title: "snapshot diff (+0 -1)", + lines: ["- button"], + }); + assert.deepEqual(diffSnapshot(["link @e1"], ["link @e42"]), { + title: "snapshot diff (no change)", + lines: [], + }); + assert.deepEqual(diffSnapshot([], []), { title: "snapshot diff (no change)", lines: [] }); +}); + +test("diffSnapshot normalizes agent-browser ref=eN refs as well", () => { + // agent-browser `snapshot -i -c` emits `ref=eN` inside an attribute list, + // and refs shift on every snapshot; role/name/level stay stable. + assert.deepEqual(diffSnapshot(['- link "Go" [ref=e2]'], ['- link "Go" [ref=e9]']), { + title: "snapshot diff (no change)", + lines: [], + }); + assert.deepEqual( + diffSnapshot( + ['- heading "Hello" [level=1, ref=e1]', '- button "Click" [ref=e3]'], + ['- heading "Hello" [level=1, ref=e7]', '- textbox "Name" [ref=e9]'], + ), + { + title: "snapshot diff (+1 -1)", + lines: ['+ - textbox "Name"', '- - button "Click"'], + }, + ); +}); diff --git a/tests/renderer.test.mjs b/tests/renderer.test.mjs index 1212e07..520ce93 100644 --- a/tests/renderer.test.mjs +++ b/tests/renderer.test.mjs @@ -752,7 +752,7 @@ test("cleanup closes only self-created sessions and removes shot files", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "hb-close-")); const stub = path.join(dir, "ab-stub"); const logf = path.join(dir, "log"); - fs.writeFileSync(stub, `#!/usr/bin/env bash\necho "$@" >> "${logf}"\n`); + fs.writeFileSync(stub, `#!/bin/sh\necho "$@" >> "${logf}"\n`); fs.chmodSync(stub, 0o755); const mk = () => quiet(mkRenderer()); const own = mk(); @@ -1397,13 +1397,20 @@ test("e2e: goLive receives pushed frames and console from a real session", { }, async () => { assert.ok(hasAgentBrowser && canCdp, "integration requires agent-browser and Node 22+"); const session = `hb-itest-${process.pid}`; + // Keep Unix socket paths below macOS's limit even with a long checkout HOME. + const socketDir = fs.mkdtempSync("/tmp/hb-s-"); + const env = { + ...process.env, + HERDR_BROWSER_SESSION: session, + AGENT_BROWSER_SOCKET_DIR: socketDir, + }; const { execFile: ef } = await import("node:child_process"); const ab = (args) => new Promise((res, rej) => ef( "agent-browser", ["--session", session, ...args], - { timeout: 15_000 }, + { timeout: 15_000, env }, (e, so) => (e ? rej(e) : res(so)), ), ); @@ -1412,7 +1419,7 @@ test("e2e: goLive receives pushed frames and console from a real session", { res.end("