From 8884e5d9c7639dd0aa50e5c0dabff95c86e55896 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Thu, 15 Oct 2026 01:53:05 +0700 Subject: [PATCH 1/7] feat: add an Advanced build that drives agent tabs through the debugger One codebase, two build profiles. Standard is unchanged and has no debugger permission or code. DEWEE_PROFILE=advanced adds the debugger permission and a CDP adapter behind the same page-driver interface: accessibility-tree snapshots with the shared redaction rule, trusted click/type/press input and viewport JPEG screenshots under 1.5 MiB. The person turns the debugger on in the side panel, per browser session. It attaches only to agent-owned tabs while a command runs, calls only a fixed CDP method list, re-checks the live policy before every command and detaches on Stop, Pause, revoke, expiry, disconnect and opt-in off. When another debugger holds the tab it answers debugger_unavailable, or reads the standard way when the person allowed that; screenshots never fall back. The side panel now shows agent tabs, site scope and TTL, and a timeline of recent outcomes. Agent tabs get a badge, targets are highlighted before a click, and the person's own input in an agent tab pauses control. Refs #18 Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + eslint.config.js | 2 +- package.json | 6 +- scripts/build-advanced.mjs | 6 + scripts/build-e2e.mjs | 11 +- src/browser/adapters/cdp/cdp-adapter.ts | 427 ++++++++++++++++++ src/browser/adapters/cdp/cdp-client.ts | 186 ++++++++ src/browser/adapters/cdp/cdp-outline.ts | 205 +++++++++ src/browser/adapters/cdp/index.ts | 29 ++ src/browser/adapters/cdp/screenshot.ts | 93 ++++ src/browser/adapters/page-driver.ts | 52 +++ src/browser/chrome-api.ts | 41 +- src/browser/command-runner.ts | 7 +- src/browser/engine-router.ts | 156 +++++++ src/browser/executor.ts | 59 ++- src/browser/panel-state.ts | 40 +- src/browser/policy.ts | 9 +- src/browser/sensitive.ts | 40 ++ src/browser/session.ts | 296 +++++++++++- src/browser/snapshot.ts | 30 +- src/browser/tab-marker.ts | 57 +++ src/env.d.ts | 3 + src/protocol/messages.ts | 2 + src/shared/build-profile.ts | 42 ++ src/shared/i18n.ts | 26 ++ src/sidepanel/browser/BrowserActivity.tsx | 36 +- .../browser/BrowserControlDrawer.tsx | 107 ++++- src/sidepanel/sidepanel.css | 43 ++ tests/unit/browser/cdp-adapter.test.ts | 346 ++++++++++++++ tests/unit/browser/cdp-client.test.ts | 140 ++++++ tests/unit/browser/engine-router.test.ts | 117 +++++ tests/unit/browser/executor.test.ts | 74 ++- tests/unit/browser/screenshot.test.ts | 59 +++ tests/unit/browser/sensitive.test.ts | 68 +++ tests/unit/browser/session-advanced.test.ts | 315 +++++++++++++ tests/unit/browser/session.test.ts | 1 + tests/unit/browser/tab-marker.test.ts | 44 ++ tests/unit/protocol/messages.test.ts | 2 + tests/unit/shared/build-profile.test.ts | 26 ++ wxt.config.ts | 20 +- 40 files changed, 3132 insertions(+), 93 deletions(-) create mode 100644 scripts/build-advanced.mjs create mode 100644 src/browser/adapters/cdp/cdp-adapter.ts create mode 100644 src/browser/adapters/cdp/cdp-client.ts create mode 100644 src/browser/adapters/cdp/cdp-outline.ts create mode 100644 src/browser/adapters/cdp/index.ts create mode 100644 src/browser/adapters/cdp/screenshot.ts create mode 100644 src/browser/adapters/page-driver.ts create mode 100644 src/browser/engine-router.ts create mode 100644 src/browser/sensitive.ts create mode 100644 src/browser/tab-marker.ts create mode 100644 src/shared/build-profile.ts create mode 100644 tests/unit/browser/cdp-adapter.test.ts create mode 100644 tests/unit/browser/cdp-client.test.ts create mode 100644 tests/unit/browser/engine-router.test.ts create mode 100644 tests/unit/browser/screenshot.test.ts create mode 100644 tests/unit/browser/sensitive.test.ts create mode 100644 tests/unit/browser/session-advanced.test.ts create mode 100644 tests/unit/browser/tab-marker.test.ts create mode 100644 tests/unit/shared/build-profile.test.ts diff --git a/.gitignore b/.gitignore index a9a9569..78f6d42 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ node_modules/ .output/ .output-e2e/ .output-store/ +.output-advanced/ +.output-e2e-advanced/ .wxt/ dist/ coverage/ diff --git a/eslint.config.js b/eslint.config.js index 0c6c686..736ba67 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -4,7 +4,7 @@ import reactHooks from 'eslint-plugin-react-hooks'; import globals from 'globals'; export default tseslint.config( - { ignores: ['.output/', '.output-e2e/', '.output-store/', '.wxt/', 'node_modules/', 'coverage/', 'test-results/', 'playwright-report/'] }, + { ignores: ['.output/', '.output-e2e/', '.output-store/', '.output-advanced/', '.output-e2e-advanced/', '.wxt/', 'node_modules/', 'coverage/', 'test-results/', 'playwright-report/'] }, js.configs.recommended, ...tseslint.configs.recommended, { diff --git a/package.json b/package.json index 6ce5b01..e34533d 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,9 @@ "scripts": { "dev": "wxt", "build": "wxt build", + "build:advanced": "node scripts/build-advanced.mjs", "zip": "wxt zip", + "zip:advanced": "node scripts/build-advanced.mjs zip", "zip:store": "node scripts/zip-store.mjs", "postinstall": "wxt prepare", "lint": "eslint .", @@ -20,7 +22,9 @@ "test": "vitest run", "test:watch": "vitest", "build:e2e": "node scripts/build-e2e.mjs", - "test:e2e": "node scripts/build-e2e.mjs && playwright test" + "build:e2e:advanced": "node scripts/build-e2e.mjs --profile advanced", + "test:e2e": "node scripts/build-e2e.mjs && playwright test", + "bench:browser": "node scripts/bench-browser-control.mjs" }, "dependencies": { "dompurify": "3.4.16", diff --git a/scripts/build-advanced.mjs b/scripts/build-advanced.mjs new file mode 100644 index 0000000..feedd3a --- /dev/null +++ b/scripts/build-advanced.mjs @@ -0,0 +1,6 @@ +// Builds (or zips, with `zip`) the Advanced profile into .output-advanced: same code, plus the +// `debugger` permission (see src/shared/build-profile.ts). Sideload only; it keeps the pinned key. +import { execSync } from 'node:child_process'; + +const command = process.argv[2] === 'zip' ? 'zip' : 'build'; +execSync(`pnpm exec wxt ${command}`, { stdio: 'inherit', env: { ...process.env, DEWEE_PROFILE: 'advanced' } }); diff --git a/scripts/build-e2e.mjs b/scripts/build-e2e.mjs index 73496ea..b907f00 100644 --- a/scripts/build-e2e.mjs +++ b/scripts/build-e2e.mjs @@ -1,4 +1,11 @@ -// Builds the extension for Playwright into .output-e2e/chrome-mv3 (see wxt.config.ts). +// Builds the extension for Playwright: Standard into .output-e2e/chrome-mv3 and Advanced into +// .output-e2e-advanced/chrome-mv3 (see wxt.config.ts). `--profile standard|advanced` builds one only. import { execSync } from 'node:child_process'; -execSync('pnpm exec wxt build', { stdio: 'inherit', env: { ...process.env, DEWEE_E2E: '1' } }); +const flag = process.argv.indexOf('--profile'); +const only = flag === -1 ? null : process.argv[flag + 1]; +if (only !== null && only !== 'standard' && only !== 'advanced') throw new Error(`Unknown profile "${only}"`); + +for (const profile of only ? [only] : ['standard', 'advanced']) { + execSync('pnpm exec wxt build', { stdio: 'inherit', env: { ...process.env, DEWEE_E2E: '1', DEWEE_PROFILE: profile } }); +} diff --git a/src/browser/adapters/cdp/cdp-adapter.ts b/src/browser/adapters/cdp/cdp-adapter.ts new file mode 100644 index 0000000..1ce578a --- /dev/null +++ b/src/browser/adapters/cdp/cdp-adapter.ts @@ -0,0 +1,427 @@ +/** + * Advanced page adapter: the semantic page actions over the Chrome DevTools Protocol, on an + * agent-owned tab, while one command runs there. The executor has already applied the grant, + * approvals and redaction rules; this adapter only carries the action out, through `CdpClient` + * (whitelisted methods, policy check before every command). + * + * - snapshot: Chrome's accessibility tree, written in the Standard outline format (cdp-outline.ts). + * - refs: `eN` handles bound to a DOM backend node id, the frame it lives in, and the tab's + * snapshot generation. Navigation moves the generation; a node that no longer exists is + * `element_not_found`. Refs are never shared with the Standard adapter. + * - click/type/press: trusted input (`Input.*`) after a hit test at the element's centre. + * - screenshot: the visible viewport only, sensitive fields and ungranted frames blacked out. + * - frames: in-process frames whose origin is in the grant are read and driven; out-of-process + * (cross-site) frames are left out and their elements cannot be targeted. + */ +import type { PressKey } from '../../protocol'; +import { urlOrigin } from '../../policy'; +import type { DriverErrorCode, DriverRequest, DriverResponse, PageDriver } from '../page-driver'; +import { CdpFailure, type CdpClient } from './cdp-client'; +import { formatOutline, indexDom, isEditableNode, isSensitiveNode, nameOf, property, roleOf, type AXNode, type DomInfo, type DomNode } from './cdp-outline'; +import { bytesToBase64, fitScreenshot, quadsToRects, type DecodedImage, type ImageCodec, type Rect } from './screenshot'; + +/** How long trusted input from the debugger is not taken for the person's own input. */ +export const DISPATCH_GRACE_MS = 600; +/** Frames read per snapshot besides the main one. */ +const MAX_FRAMES = 10; + +interface RefTarget { + backend: number; + frame: string; +} + +interface TabRefs { + generation: number; + next: number; + byRef: Map; + byBackend: Map; +} + +interface FrameTreeNode { + frame: { id: string; url: string; urlFragment?: string }; + childFrames?: FrameTreeNode[]; +} + +interface Frames { + main: string; + mainUrl: string; + origins: Map; +} + +const KEYS: Record = { + Enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' }, + Tab: { key: 'Tab', code: 'Tab', keyCode: 9 }, + Escape: { key: 'Escape', code: 'Escape', keyCode: 27 }, + ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', keyCode: 38 }, + ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', keyCode: 40 }, + ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', keyCode: 37 }, + ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', keyCode: 39 }, + Backspace: { key: 'Backspace', code: 'Backspace', keyCode: 8 }, + PageUp: { key: 'PageUp', code: 'PageUp', keyCode: 33 }, + PageDown: { key: 'PageDown', code: 'PageDown', keyCode: 34 }, + Home: { key: 'Home', code: 'Home', keyCode: 36 }, + End: { key: 'End', code: 'End', keyCode: 35 }, +}; + +const fail = (code: DriverErrorCode): DriverResponse => ({ ok: false, code }); + +/** Chrome's message for a node id that no longer resolves. */ +const GONE = /No node|Could not find node|does not exist|not found/i; + +export interface CdpAdapterDeps { + client: CdpClient; + codec: ImageCodec; + now?(): number; +} + +export class CdpAdapter implements PageDriver { + private readonly refs = new Map(); + private readonly dispatching = new Map(); + private readonly overlay = new Set(); + private readonly now: () => number; + + constructor(private readonly deps: CdpAdapterDeps) { + this.now = deps.now ?? Date.now; + } + + get attachedTabs(): number[] { + return this.deps.client.attachedTabs; + } + + /** Attaches for the command about to run. Null when ready; otherwise why not (`debugger_unavailable` allows Standard to take over). */ + async prepare(tabId: number): Promise { + try { + await this.deps.client.attach(tabId); + return null; + } catch (error) { + return error instanceof CdpFailure ? error.code : 'debugger_unavailable'; + } + } + + async release(tabId: number): Promise { + this.overlay.delete(tabId); + await this.deps.client.detach(tabId); + } + + /** The tab is gone (closed, or no longer owned): its refs go too. */ + forget(tabId: number): void { + this.refs.delete(tabId); + this.dispatching.delete(tabId); + this.overlay.delete(tabId); + } + + async detachAll(): Promise { + this.overlay.clear(); + await this.deps.client.detachAll(); + } + + /** Detaches a tab this worker may have attached before it restarted. */ + async forceDetach(tabId: number): Promise { + await this.deps.client.detach(tabId, true); + } + + /** True while (and shortly after) the adapter sends trusted input to the tab. */ + isDispatching(tabId: number): boolean { + return (this.dispatching.get(tabId) ?? 0) > this.now(); + } + + async run(tabId: number, request: DriverRequest): Promise { + try { + await this.deps.client.attach(tabId); + switch (request.op) { + case 'snapshot': + return await this.snapshot(tabId, request.origins, request.generation, request.maxChars); + case 'describe': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + const ax = await this.axNode(tabId, target); + const dom = await this.describeDom(tabId, target); + return { ok: true, op: 'describe', role: roleOf(ax), name: nameOf(ax), sensitive: isSensitiveNode(dom), editable: isEditableNode(dom) }; + } + case 'highlight': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + await this.deps.client.send(tabId, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backend }); + if (!this.overlay.has(tabId)) { + await this.deps.client.send(tabId, 'DOM.enable'); + await this.deps.client.send(tabId, 'Overlay.enable'); + this.overlay.add(tabId); + } + await this.deps.client.send(tabId, 'Overlay.highlightNode', { + backendNodeId: target.backend, + highlightConfig: { showInfo: false, contentColor: { r: 124, g: 58, b: 237, a: 0.15 }, borderColor: { r: 124, g: 58, b: 237, a: 0.9 } }, + }); + return { ok: true, op: 'highlight' }; + } + case 'click': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + await this.axNode(tabId, target); + const point = await this.reachable(tabId, target); + await this.hideHighlight(tabId); + await this.input(tabId, async () => { + const base = { x: point.x, y: point.y, button: 'left', clickCount: 1 }; + await this.deps.client.send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: point.x, y: point.y }); + await this.deps.client.send(tabId, 'Input.dispatchMouseEvent', { ...base, type: 'mousePressed', buttons: 1 }); + await this.deps.client.send(tabId, 'Input.dispatchMouseEvent', { ...base, type: 'mouseReleased', buttons: 0 }); + }); + return { ok: true, op: 'click' }; + } + case 'type': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + await this.axNode(tabId, target); + const dom = await this.describeDom(tabId, target); + if (isSensitiveNode(dom)) return fail('sensitive_field'); + if (!isEditableNode(dom)) return fail('element_not_interactable'); + await this.reachable(tabId, target); + await this.input(tabId, async () => { + await this.deps.client.send(tabId, 'DOM.focus', { backendNodeId: target.backend }); + // Replace what is there, like the Standard adapter: select all, then insert (or delete). + await this.key(tabId, { key: 'a', code: 'KeyA', keyCode: 65 }, ['selectAll']); + if (request.text) await this.deps.client.send(tabId, 'Input.insertText', { text: request.text }); + else await this.key(tabId, { key: 'Delete', code: 'Delete', keyCode: 46 }); + if (request.submit) await this.key(tabId, KEYS.Enter); + }); + return { ok: true, op: 'type' }; + } + case 'press': { + const frames = await this.frames(tabId); + if (!request.origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); + await this.input(tabId, () => this.key(tabId, KEYS[request.key])); + return { ok: true, op: 'press' }; + } + case 'screenshot': + return await this.screenshot(tabId, request.origins); + default: + // Scrolling, waiting for text, the focused element and the input watcher stay with the page script. + return fail('internal'); + } + } catch (error) { + if (error instanceof CdpFailure) return fail(error.code); + return fail('internal'); + } + } + + // ---------- Refs ---------- + + private refsFor(tabId: number, generation: number): TabRefs { + const existing = this.refs.get(tabId); + if (existing && existing.generation === generation) return existing; + const fresh: TabRefs = { generation, next: 1, byRef: new Map(), byBackend: new Map() }; + this.refs.set(tabId, fresh); + return fresh; + } + + /** Resolves a ref of this generation and checks the page and the element's frame are in the grant. */ + private async target(tabId: number, origins: string[], generation: number, ref: string): Promise { + const state = this.refs.get(tabId); + if (!state || state.generation !== generation) throw new CdpFailure('stale_ref', false); + const target = state.byRef.get(ref); + if (!target) throw new CdpFailure('element_not_found', false); + const frames = await this.frames(tabId); + if (!origins.includes(frames.origins.get(frames.main) ?? '')) throw new CdpFailure('origin_not_granted', false); + // The frame the element lives in must still exist and still show a granted origin. + if (!origins.includes(frames.origins.get(target.frame) ?? '')) throw new CdpFailure('origin_not_granted', false); + return target; + } + + private async frames(tabId: number): Promise { + const { frameTree } = await this.deps.client.send<{ frameTree: FrameTreeNode }>(tabId, 'Page.getFrameTree'); + const origins = new Map(); + const visit = (node: FrameTreeNode) => { + origins.set(node.frame.id, urlOrigin(node.frame.url)); + for (const child of node.childFrames ?? []) visit(child); + }; + visit(frameTree); + return { main: frameTree.frame.id, mainUrl: frameTree.frame.url + (frameTree.frame.urlFragment ?? ''), origins }; + } + + private async document(tabId: number, mainFrame: string): Promise> { + const { root } = await this.deps.client.send<{ root: DomNode }>(tabId, 'DOM.getDocument', { depth: -1, pierce: true }); + return indexDom(root, mainFrame); + } + + /** The element's accessibility node: it must still exist, be rendered and not disabled. */ + private async axNode(tabId: number, target: RefTarget): Promise { + let nodes: AXNode[]; + try { + ({ nodes } = await this.deps.client.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getPartialAXTree', { backendNodeId: target.backend, fetchRelatives: false })); + } catch (error) { + throw this.nodeError(error); + } + const node = nodes.find((n) => n.backendDOMNodeId === target.backend) ?? nodes[0]; + if (!node) throw new CdpFailure('element_not_found', false); + if (node.ignored || property(node, 'disabled') === true || property(node, 'hidden') === true) throw new CdpFailure('element_not_interactable', false); + return node; + } + + private async describeDom(tabId: number, target: RefTarget): Promise { + try { + const { node } = await this.deps.client.send<{ node: DomNode }>(tabId, 'DOM.describeNode', { backendNodeId: target.backend }); + return indexDom(node, target.frame).get(node.backendNodeId) ?? { tag: node.nodeName.toUpperCase(), attrs: new Map(), frame: target.frame }; + } catch (error) { + throw this.nodeError(error); + } + } + + private nodeError(error: unknown): CdpFailure { + if (error instanceof CdpFailure && error.code !== 'internal') return error; + const detail = error instanceof CdpFailure ? error.detail : ''; + return new CdpFailure(GONE.test(detail) ? 'element_not_found' : 'element_not_interactable', false); + } + + /** + * Scrolls the element into view and checks that its centre is inside the viewport and that a + * pointer there lands on the element itself (or a label for it), as the Standard adapter does. + */ + private async reachable(tabId: number, target: RefTarget): Promise<{ x: number; y: number }> { + let quads: number[][]; + try { + await this.deps.client.send(tabId, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backend }); + ({ quads } = await this.deps.client.send<{ quads: number[][] }>(tabId, 'DOM.getContentQuads', { backendNodeId: target.backend })); + } catch (error) { + throw this.nodeError(error); + } + const quad = quads.find((q) => q.length >= 8 && Math.abs((q[4] as number) - (q[0] as number)) * Math.abs((q[5] as number) - (q[1] as number)) >= 1); + if (!quad) throw new CdpFailure('element_not_interactable', false); + const x = ((quad[0] as number) + (quad[2] as number) + (quad[4] as number) + (quad[6] as number)) / 4; + const y = ((quad[1] as number) + (quad[3] as number) + (quad[5] as number) + (quad[7] as number)) / 4; + const { cssLayoutViewport } = await this.deps.client.send<{ cssLayoutViewport: { clientWidth: number; clientHeight: number } }>(tabId, 'Page.getLayoutMetrics'); + if (x < 0 || y < 0 || x > cssLayoutViewport.clientWidth || y > cssLayoutViewport.clientHeight) throw new CdpFailure('element_not_interactable', false); + let hit: number | undefined; + try { + ({ backendNodeId: hit } = await this.deps.client.send<{ backendNodeId: number }>(tabId, 'DOM.getNodeForLocation', { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: false, ignorePointerEventsNone: false })); + } catch { + hit = undefined; + } + if (hit === undefined) throw new CdpFailure('element_not_interactable', false); + if (hit !== target.backend) { + const frames = await this.frames(tabId); + const dom = await this.document(tabId, frames.main); + if (!this.covers(dom, hit, target.backend)) throw new CdpFailure('element_not_interactable', false); + } + return { x, y }; + } + + /** The node at the point is the element, inside it, or a label naming or wrapping it. */ + private covers(dom: Map, hit: number, target: number): boolean { + const ancestors = (id: number): number[] => { + const list: number[] = []; + for (let node: number | undefined = id, depth = 0; node !== undefined && depth < 512; node = dom.get(node)?.parent, depth += 1) list.push(node); + return list; + }; + const hitChain = ancestors(hit); + if (hitChain.includes(target)) return true; + const targetId = dom.get(target)?.attrs.get('id'); + const targetChain = ancestors(target); + return hitChain.some((node) => { + const info = dom.get(node); + if (info?.tag !== 'LABEL') return false; + return (targetId !== undefined && info.attrs.get('for') === targetId) || targetChain.includes(node); + }); + } + + // ---------- Input ---------- + + private async input(tabId: number, send: () => Promise): Promise { + this.dispatching.set(tabId, Number.POSITIVE_INFINITY); + try { + await send(); + } finally { + this.dispatching.set(tabId, this.now() + DISPATCH_GRACE_MS); + } + } + + private async key(tabId: number, key: { key: string; code: string; keyCode: number; text?: string }, commands?: string[]): Promise { + const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode }; + await this.deps.client.send(tabId, 'Input.dispatchKeyEvent', { ...base, type: key.text ? 'keyDown' : 'rawKeyDown', ...(key.text ? { text: key.text, unmodifiedText: key.text } : {}), ...(commands ? { commands } : {}) }); + await this.deps.client.send(tabId, 'Input.dispatchKeyEvent', { ...base, type: 'keyUp' }); + } + + private async hideHighlight(tabId: number): Promise { + if (this.overlay.has(tabId)) await this.deps.client.send(tabId, 'Overlay.hideHighlight').catch(() => undefined); + } + + // ---------- Snapshot ---------- + + private async snapshot(tabId: number, origins: string[], generation: number, maxChars: number): Promise { + const frames = await this.frames(tabId); + if (!origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); + const dom = await this.document(tabId, frames.main); + const trees = new Map(); + trees.set(frames.main, (await this.deps.client.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree')).nodes); + const granted = new Set([...frames.origins].filter(([, origin]) => origin !== null && origins.includes(origin)).map(([id]) => id)); + for (const frame of granted) { + if (frame === frames.main || trees.size > MAX_FRAMES) continue; + try { + trees.set(frame, (await this.deps.client.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree', { frameId: frame })).nodes); + } catch (error) { + // An out-of-process frame cannot be read through this tab's session: it is left out. + if (error instanceof CdpFailure && error.code !== 'internal') throw error; + } + } + const state = this.refsFor(tabId, generation); + const outline = formatOutline({ + trees, + mainFrame: frames.main, + dom, + grantedFrames: granted, + maxChars, + refFor: (backend, frame) => { + const known = state.byBackend.get(backend); + if (known) return known; + const ref = `e${state.next++}`; + state.byBackend.set(backend, ref); + state.byRef.set(ref, { backend, frame }); + return ref; + }, + }); + // The page must not have moved to another site while it was read. + const after = await this.frames(tabId); + if (!origins.includes(after.origins.get(after.main) ?? '')) return fail('origin_not_granted'); + return { ok: true, op: 'snapshot', text: outline.text, url: frames.mainUrl, title: outline.title }; + } + + // ---------- Screenshot ---------- + + private async screenshot(tabId: number, origins: string[]): Promise { + const frames = await this.frames(tabId); + if (!origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); + const dom = await this.document(tabId, frames.main); + const { cssVisualViewport } = await this.deps.client.send<{ cssVisualViewport: { clientWidth: number; clientHeight: number } }>(tabId, 'Page.getLayoutMetrics'); + const { data } = await this.deps.client.send<{ data: string }>(tabId, 'Page.captureScreenshot', { format: 'png', fromSurface: true, captureBeyondViewport: false }); + // A navigation while capturing could have put another site in the picture. + const after = await this.frames(tabId); + if (after.main !== frames.main || !origins.includes(after.origins.get(after.main) ?? '')) return fail('origin_not_granted'); + + const image = await this.deps.codec.decode(data); + try { + const ratio = cssVisualViewport.clientWidth > 0 ? image.width / cssVisualViewport.clientWidth : 1; + const redact = await this.redactions(tabId, dom, after, origins, ratio, image); + const bytes = await fitScreenshot(this.deps.codec, image, redact); + if (!bytes) return fail('internal'); + return { ok: true, op: 'screenshot', data: bytesToBase64(bytes) }; + } finally { + image.close?.(); + } + } + + /** Boxes to black out: every sensitive field, and every frame whose origin is not in the grant. */ + private async redactions(tabId: number, dom: Map, frames: Frames, origins: string[], ratio: number, size: { width: number; height: number }): Promise { + const nodes: number[] = []; + for (const [backend, info] of dom) { + const field = info.tag === 'INPUT' || info.tag === 'TEXTAREA' || info.tag === 'SELECT' || info.attrs.has('contenteditable'); + if (field && isSensitiveNode(info)) nodes.push(backend); + else if (info.childFrame !== undefined && !origins.includes(frames.origins.get(info.childFrame) ?? '')) nodes.push(backend); + else if ((info.tag === 'IFRAME' || info.tag === 'FRAME') && info.childFrame === undefined) nodes.push(backend); + } + const rects: Rect[] = []; + for (const backendNodeId of nodes) { + try { + const { quads } = await this.deps.client.send<{ quads: number[][] }>(tabId, 'DOM.getContentQuads', { backendNodeId }); + rects.push(...quadsToRects(quads, ratio, size)); + } catch (error) { + // Not rendered (no box): nothing to hide. Any refusal from the guard still stops the screenshot. + if (error instanceof CdpFailure && error.code !== 'internal') throw error; + } + } + return rects; + } +} diff --git a/src/browser/adapters/cdp/cdp-client.ts b/src/browser/adapters/cdp/cdp-client.ts new file mode 100644 index 0000000..6679037 --- /dev/null +++ b/src/browser/adapters/cdp/cdp-client.ts @@ -0,0 +1,186 @@ +/** + * The only path to `chrome.debugger` in the extension. It attaches to one tab target at a time + * (`{ tabId }`; never a browser-level target, never a remote debugging port) and sends only the + * Chrome DevTools Protocol methods listed in `CDP_METHODS`, with parameters built by the adapter. + * Nothing that arrives over the wire is ever a method name or a parameter object. + * + * Every command first passes the local policy check for the tab (grant, Pause/Stop, the page's + * origin, the person's debugger opt-in); a refusal detaches at once. + */ +import type { DriverErrorCode } from '../page-driver'; + +/** + * The whitelist. No `Runtime.*` (no script evaluation), no `Network.*`, `Storage.*`, `Fetch.*` or + * cookies, no `Target.*` (no other targets), no `Page.navigate` (navigation goes through + * `chrome.tabs` and the guard), no `Page.bringToFront` (never steal focus). + */ +export const CDP_METHODS = [ + 'Accessibility.getFullAXTree', + 'Accessibility.getPartialAXTree', + 'DOM.enable', + 'DOM.getDocument', + 'DOM.describeNode', + 'DOM.getContentQuads', + 'DOM.scrollIntoViewIfNeeded', + 'DOM.focus', + 'DOM.getNodeForLocation', + 'Page.getFrameTree', + 'Page.getLayoutMetrics', + 'Page.captureScreenshot', + 'Input.dispatchMouseEvent', + 'Input.dispatchKeyEvent', + 'Input.insertText', + 'Overlay.enable', + 'Overlay.highlightNode', + 'Overlay.hideHighlight', +] as const; +export type CdpMethod = (typeof CDP_METHODS)[number]; + +const ALLOWED: ReadonlySet = new Set(CDP_METHODS); + +export function isAllowedCdpMethod(method: string): method is CdpMethod { + return ALLOWED.has(method); +} + +/** CDP protocol version requested on attach. */ +export const CDP_VERSION = '1.3'; +/** Longest a single CDP command may take before it counts as failed. */ +export const CDP_COMMAND_TIMEOUT_MS = 10_000; + +/** A tab target. There is deliberately no way to name a browser, worker or extension target. */ +export interface TabTarget { + tabId: number; +} + +/** The parts of `chrome.debugger` the client uses, so tests can provide a fake. */ +export interface DebuggerApi { + attach(target: TabTarget, version: string): Promise; + detach(target: TabTarget): Promise; + sendCommand(target: TabTarget, method: string, params?: Record): Promise; + onDetach: { addListener(callback: (source: { tabId?: number }, reason: string) => void): void }; +} + +/** + * A debugger step that did not happen or failed. `beforeEffect` is true only when nothing reached + * the page (attach itself failed), which is the only case where Standard may take over. + */ +export class CdpFailure extends Error { + constructor( + readonly code: DriverErrorCode, + readonly beforeEffect: boolean, + /** Chrome's own error text, kept for mapping only; never sent to the Gateway. */ + readonly detail = '', + ) { + super(code); + this.name = 'CdpFailure'; + } +} + +export interface CdpClientDeps { + debugger: DebuggerApi; + /** Local policy for this tab right now: null when allowed, otherwise the refusal. */ + check(tabId: number): Promise; + /** Chrome ended a session we held (tab closed, the person pressed Cancel on Chrome's bar, ...). */ + onDetached?(tabId: number, reason: string): void; + /** The set of attached tabs changed (for the side panel). */ + onChange?(): void; + timeoutMs?: number; +} + +function withTimeout(promise: Promise, ms: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new CdpFailure('timeout', false, 'CDP command timed out')), ms); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + clearTimeout(timer); + reject(error instanceof Error ? error : new Error(String(error))); + }, + ); + }); +} + +export class CdpClient { + private readonly attached = new Set(); + private readonly pending = new Map>(); + private readonly timeoutMs: number; + + constructor(private readonly deps: CdpClientDeps) { + this.timeoutMs = deps.timeoutMs ?? CDP_COMMAND_TIMEOUT_MS; + deps.debugger.onDetach.addListener((source, reason) => { + const tabId = source.tabId; + if (tabId === undefined || !this.attached.delete(tabId)) return; + deps.onDetached?.(tabId, reason); + deps.onChange?.(); + }); + } + + get attachedTabs(): number[] { + return [...this.attached]; + } + + isAttached(tabId: number): boolean { + return this.attached.has(tabId); + } + + /** Attaches to the tab after the policy check. Throws `CdpFailure` (`beforeEffect: true`). */ + async attach(tabId: number): Promise { + if (this.attached.has(tabId)) return; + const running = this.pending.get(tabId); + if (running) return running; + const attaching = (async () => { + const refused = await this.deps.check(tabId); + if (refused) throw new CdpFailure(refused, true); + try { + await withTimeout(this.deps.debugger.attach({ tabId }, CDP_VERSION), this.timeoutMs); + } catch (error) { + // DevTools or another extension holds the tab, the page is one Chrome never lets extensions + // debug, or the tab went away. + throw new CdpFailure('debugger_unavailable', true, error instanceof Error ? error.message : ''); + } + this.attached.add(tabId); + this.deps.onChange?.(); + })(); + this.pending.set(tabId, attaching); + try { + await attaching; + } finally { + this.pending.delete(tabId); + } + } + + /** Sends one whitelisted command to an attached tab, after the policy check. */ + async send>(tabId: number, method: CdpMethod, params: Record = {}): Promise { + // The type already restricts `method`; this runtime check holds even against a cast. + if (!isAllowedCdpMethod(method)) throw new CdpFailure('internal', false, `method not allowed: ${String(method)}`); + if (!this.attached.has(tabId)) throw new CdpFailure('debugger_unavailable', false); + const refused = await this.deps.check(tabId); + if (refused) { + await this.detach(tabId); + throw new CdpFailure(refused, false); + } + try { + return (await withTimeout(this.deps.debugger.sendCommand({ tabId }, method, params), this.timeoutMs)) as T; + } catch (error) { + if (error instanceof CdpFailure) throw error; + // Detached meanwhile (Stop, revoke, the person cancelled Chrome's bar, the tab closed). + if (!this.attached.has(tabId)) throw new CdpFailure('debugger_unavailable', false); + throw new CdpFailure('internal', false, error instanceof Error ? error.message : ''); + } + } + + /** Lets go of the tab. `force` also asks Chrome when this worker does not remember attaching (after a restart). */ + async detach(tabId: number, force = false): Promise { + const held = this.attached.delete(tabId); + if (!held && !force) return; + await this.deps.debugger.detach({ tabId }).catch(() => undefined); + if (held) this.deps.onChange?.(); + } + + async detachAll(): Promise { + await Promise.all([...this.attached].map((tabId) => this.detach(tabId))); + } +} diff --git a/src/browser/adapters/cdp/cdp-outline.ts b/src/browser/adapters/cdp/cdp-outline.ts new file mode 100644 index 0000000..a0d4f9d --- /dev/null +++ b/src/browser/adapters/cdp/cdp-outline.ts @@ -0,0 +1,205 @@ +/** + * Turns Chrome's accessibility tree (`Accessibility.getFullAXTree`) plus the DOM attributes + * (`DOM.getDocument`) into the same compact outline the Standard adapter writes: + * `[e12] button "Sign in"`, `heading "Account" level=1`, `text "..."`. + * + * The rules match the page script: values of sensitive fields (password, PIN, payment card, + * one-time code; see sensitive.ts) are written as `value=[redacted]` whatever role the page gives + * them; ignored nodes (hidden, `aria-hidden`) are skipped; a frame is walked only when its origin + * is in the grant and Chrome could read its tree here (in-process frames). Out-of-process frames + * are left out, like cross-origin frames in Standard. + */ +import { isEditableField, isSensitiveField, type FieldAttributes } from '../../sensitive'; + +export interface AXValue { + type: string; + value?: unknown; +} + +export interface AXNode { + nodeId: string; + ignored: boolean; + role?: AXValue; + name?: AXValue; + value?: AXValue; + properties?: Array<{ name: string; value: AXValue }>; + childIds?: string[]; + parentId?: string; + backendDOMNodeId?: number; +} + +/** A `DOM.Node` from `DOM.getDocument({ depth: -1, pierce: true })`, reduced to what is read. */ +export interface DomNode { + nodeId: number; + backendNodeId: number; + nodeType: number; + nodeName: string; + attributes?: string[]; + children?: DomNode[]; + contentDocument?: DomNode; + shadowRoots?: DomNode[]; + /** Set on frame owner elements (` + +`; + const inner = ``; + const foreign = `

Foreign text e2e

`; + let port = 0; const server = createServer((req, res) => { - if (req.url === '/favicon.ico') { - res.writeHead(404).end(); - return; - } - res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }).end(page); + const html = (body: string) => res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }).end(body); + if (req.url === '/favicon.ico') return void res.writeHead(404).end(); + if (req.url === '/frames') return void html(frames(port)); + if (req.url === '/inner') return void html(inner); + if (req.url === '/foreign') return void html(foreign); + html(page); }); + // Loopback only; Chrome tries 127.0.0.1 for `localhost` when ::1 refuses, so the cross-site frame loads too. await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); - const port = (server.address() as AddressInfo).port; + port = (server.address() as AddressInfo).port; return { port, stop: () => diff --git a/tests/e2e/support/connect-browser.ts b/tests/e2e/support/connect-browser.ts new file mode 100644 index 0000000..20afadf --- /dev/null +++ b/tests/e2e/support/connect-browser.ts @@ -0,0 +1,69 @@ +/** Side panel steps shared by the connected-browser specs (Standard and Advanced builds). */ +import type { Page, Worker } from '@playwright/test'; +import { expect } from '../fixtures'; +import { AGENT, RELAY_API_KEY, type BrowserRelay, type CommandOutcome } from './browser-relay'; + +export function connectedBrowser(relay: BrowserRelay, siteOrigin: () => string) { + /** A self-hosted profile with its API key and "Connected browser" on, written where the extension keeps them. */ + async function seedProfile(worker: Worker): Promise { + await worker.evaluate( + async ({ origin, key }) => { + await chrome.storage.session.set({ 'credential:p_e2e': { kind: 'api_key', apiKey: key } }); + await chrome.storage.local.set({ + settings: { + version: 1, + profiles: [{ id: 'p_e2e', name: 'Local gateway', kind: 'gateway', origin, remember: false }], + activeProfileId: 'p_e2e', + selections: {}, + prefs: { theme: 'system', locale: 'en', showReasoning: false, showToolDetails: false, askOnSelection: false, connectedBrowser: true }, + }, + }); + }, + { origin: relay.origin, key: RELAY_API_KEY }, + ); + } + + /** Registers this browser and allows the test agent on the loopback site, through the side panel. */ + async function connectBrowser(worker: Worker, openPanel: () => Promise, capability: 'read' | 'control' = 'control'): Promise { + await seedProfile(worker); + const panel = await openPanel(); + await panel.getByTestId('browser-toggle').click(); + const drawer = panel.getByRole('dialog', { name: 'Browser control' }); + await expect(drawer.getByTestId('browser-status')).toHaveText('Connected'); + await expect(drawer.getByLabel('Device name')).not.toHaveValue(''); + await drawer.getByRole('button', { name: 'Register this browser' }).click(); + await expect(drawer.getByTestId('browser-device-name')).toBeVisible(); + await drawer.getByRole('checkbox', { name: AGENT.name }).check(); + await drawer.getByLabel('Sites, one per line').fill(siteOrigin()); + if (capability === 'control') await drawer.getByRole('radio', { name: /Read and act/ }).check(); + await drawer.getByRole('button', { name: 'Allow', exact: true }).click(); + await expect(drawer.getByTestId('browser-grants')).toContainText(siteOrigin()); + expect(relay.activeGrant()).toMatchObject({ origins: [siteOrigin()], agent_ids: [AGENT.id] }); + return panel; + } + + async function run(action: string, params: Record, options?: Parameters[2]): Promise { + const { commandId, outcome } = relay.command(action, params, options); + const result = await outcome; + // Acknowledged after validation, before the result. + expect(relay.acks).toContain(commandId); + return result; + } + + async function openSite(path = '/account', title = 'E2E Shop'): Promise<{ tab: string; text: string; generation: number }> { + const opened = await run('open', { url: `${siteOrigin()}${path}` }); + expect(opened).toMatchObject({ ok: true, result: { tab: 't1', url: `${siteOrigin()}${path}`, title } }); + const snap = await run('snapshot', { tab: 't1' }); + expect(snap.ok).toBe(true); + const result = snap.result as { text: string; generation: number }; + return { tab: 't1', text: result.text, generation: result.generation }; + } + + return { connectBrowser, run, openSite }; +} + +export function refOf(text: string, name: string): string { + const ref = text.split('\n').find((line) => /\[e\d+\]/.test(line) && line.includes(`"${name}"`))?.match(/\[(e\d+)\]/)?.[1]; + if (!ref) throw new Error(`no ref for ${name} in:\n${text}`); + return ref; +} From fbafe983dc35a956e587d33710cbafa156fc83d5 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Thu, 15 Oct 2026 01:53:05 +0700 Subject: [PATCH 3/7] docs: describe the Advanced build and its debugger safeguards Threat model section for the debugger, build profiles and scripts, Chrome Web Store review notes for the debugger permission, the live control UX, and an AGENTS.md security rule for the CDP method list. Refs #18 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 ++ README.md | 3 ++- docs/api-contract.md | 2 +- docs/architecture.md | 13 ++++++++---- docs/browser-control-threat-model.md | 25 ++++++++++++++++++++---- docs/connected-browser-relay-contract.md | 2 +- docs/development.md | 25 ++++++++++++++++++++++-- 7 files changed, 59 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1def44b..719cdb0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,7 @@ pnpm typecheck pnpm test # unit + integration (Vitest) pnpm test:e2e # builds, then runs Playwright against the unpacked build pnpm build # .output/chrome-mv3 +pnpm build:advanced # .output-advanced/chrome-mv3, adds the `debugger` permission (zip with `pnpm zip:advanced`) pnpm zip # sideload zip (keeps the pinned extension key) pnpm zip:store # Chrome Web Store zip (manifest without `key`) ``` @@ -54,6 +55,7 @@ Rules: - The side panel never reads the active page or the person's other tabs. There are two opt-in exceptions, both off by default: - The "Ask Dewee" page script sends the text the person selected (never form fields) when they click its button, and only as an editable draft. - "Connected browser" (self-hosted only) lets an agent read and act only in tabs it opened itself, only on exact origins the person allowed for that agent in the panel, only until the grant expires or is revoked, and only after the person granted Chrome host permission for those origins. Password, hidden, payment and one-time-code values are never read or typed, page content is never stored, and Pause/Stop/Revoke work locally. See [docs/browser-control-threat-model.md](docs/browser-control-threat-model.md). +- The `debugger` permission exists only in the Advanced build profile. Its adapter attaches only to agent-owned tabs, only while a command runs and the person has Advanced mode on, and calls only the fixed CDP method list in `src/browser/adapters/cdp/cdp-client.ts`. Never add `Runtime.evaluate`, `Network`, `Storage`, cookie or `Target` methods, a browser-level target, a remote debugging port, or a CDP method or params taken from a command. Screenshots are viewport-only, never stored, and never fall back to another path. ## Code layout diff --git a/README.md b/README.md index c252dd0..5d78306 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ Chat with your Dewee agents, with Super Agent pinned first, in Chrome's side pan - **Workspaces and agents:** pick a workspace, search agents, and keep separate sessions per agent. - **Reliable chat:** streaming, stop, retry, history, new chat. If the connection drops, the panel checks the server instead of sending your message twice. - **Credential cards:** secrets stay in the form and go only to Dewee's respond endpoint. Nothing secret is stored by the extension. -- **Connected browser (self-hosted, opt-in):** turn on *Connected browser* in Settings, register this browser, and allow chosen agents on chosen sites for a limited time. Agents can then open, read, and act in tabs they open themselves, never in your other tabs. Risky clicks wait for your approval, passwords and payment fields are never read or typed, and *Pause*, *Stop* and *Revoke* are always one click away. +- **Connected browser (self-hosted, opt-in):** turn on *Connected browser* in Settings, register this browser, and allow chosen agents on chosen sites for a limited time. Agents can then open, read, and act in tabs they open themselves, never in your other tabs. Risky clicks wait for your approval, passwords and payment fields are never read or typed, and *Pause*, *Stop* and *Revoke* are always one click away. Agent tabs carry an "AI" badge, and using one yourself pauses the agents. +- **Advanced build (sideload, opt-in):** `pnpm build:advanced` adds Chrome's `debugger` permission. After you turn on *Advanced mode* in the browser drawer, agents read pages through the accessibility tree, use real input and can take viewport screenshots, in their own tabs only. It shares the extension ID with the standard build, so install one or the other. See the [threat model](docs/browser-control-threat-model.md#advanced-mode-debugger). - **Pending badge**, English and Vietnamese, light/dark/system theme, keyboard shortcuts. ## Install (development build) diff --git a/docs/api-contract.md b/docs/api-contract.md index eae31f6..6e90d31 100644 --- a/docs/api-contract.md +++ b/docs/api-contract.md @@ -197,7 +197,7 @@ What the extension adds on its side of the contract: - Every command is checked again locally before it runs: grant present, not expired or revoked, agent in the grant, capability, exact origin of both the tab's current page and any requested URL, restricted pages (every `about:` page included, so `open about:blank` is refused), owned tab, Chrome host permission for that exact host, and local Pause/Stop. - `ack` is sent after validation and before any work; exactly one `browser.command.result` follows. A command id is never executed twice while the service worker lives; after a worker restart this relies on the Gateway not resending a dispatched command. Results are not queued across a reconnect; commands in flight when the socket drops are cancelled locally, and the Gateway's deadline rules apply. A command runs for at most 120 s locally, whatever its deadline. - `type` with `submit`, `press` with `Enter`, and clicks on high-risk elements wait for the person's "Allow once" in the side panel and fail with `approval_denied` otherwise; if the element changed while the person decided, they fail with `stale_ref`. -- `type` is refused (`sensitive_field`) for password, PIN, payment-card and one-time-code fields, detected by input type, `autocomplete` (`cc-*`, `one-time-code`, `current-password`, `new-password`) and name/id hints; the snapshot redacts their values. `screenshot` answers `unsupported_capability` (no advanced build yet). +- `type` is refused (`sensitive_field`) for password, PIN, payment-card and one-time-code fields, detected by input type, `autocomplete` (`cc-*`, `one-time-code`, `current-password`, `new-password`) and name/id hints; the snapshot redacts their values. `screenshot` answers `unsupported_capability` in the Standard build and while Advanced mode is off; in the Advanced build with Advanced mode on it returns a viewport JPEG of at most 1.5 MiB, or `debugger_unavailable` when another debugger holds the tab. - Grants and devices are read by `grant_id` / `device_id`, also accepting `id`, since v1 does not pin the object shape beyond those fields. - A command whose grant is revoked while it runs, or that arrives for a revoked grant, fails with `grant_revoked`. Page results (`open`, `navigate`, `snapshot`, `click`, `type`, `screenshot`) always carry the page `url`; a result without a valid one is reported as `internal`, or as `outcome_unknown` after an interaction (the action may have run). A refresh (`attach`, `browser.grant.list`) that no longer lists a grant cancels its in-flight commands, approvals included, with `grant_revoked`. - The person's last Pause/Stop/Resume is kept per profile and device in `storage.local` and re-sent after every `attach` (`browser.control.state`), so a Chrome restart, a profile switch or a Gateway restart cannot lift a Pause. `active` is re-sent only when the person's last choice was Resume; with no stored choice nothing is sent and the Gateway keeps its state. A stored choice that cannot be read leaves the panel paused locally and sends nothing (see [browser-control-threat-model.md](browser-control-threat-model.md)). diff --git a/docs/architecture.md b/docs/architecture.md index b0f71bc..ecb89ca 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,16 +57,21 @@ Gateway ──/ws──► service worker executor.ts policy re-check before every side effect, then the action policy.ts grant, agent, capability, exact origin, restricted pages, owned tab, host permission tabs.ts owned tabs t1, t2…, generation per document, single writer, user-input hold + engine-router.ts Standard or debugger per action; refs stay with the engine that issued them chrome-api.ts ──► chrome.tabs + chrome.scripting.executeScript({ func: pageRun, args }) snapshot.ts (bundled page runtime, isolated world) -side panel ◄─port "browser-control"─ state, approval cards; Pause/Resume/Stop, enroll, grants + adapters/cdp/ ──► chrome.debugger (Advanced build only, opt-in, owned tabs, method whitelist) + tab-marker.ts "AI" badge and title on agent tabs +side panel ◄─port "browser-control"─ state, approval cards, agent tabs, timeline; Pause/Resume/Stop, enroll, grants, Advanced mode ``` - **Lifecycle.** The socket exists only while the "Connected browser" preference is on, the active profile is a self-hosted Gateway and it holds an API key. A profile switch, sign-out or turning the preference off closes it, cancels running commands, denies open approvals and forgets grants and owned tabs. Dewee Cloud shows "not available yet" and opens nothing. A one-minute `chrome.alarms` alarm revives a suspended worker, which reconnects and re-attaches with the stored device id; the 20 s heartbeat keeps it alive while connected (Chrome 116+). -- **Stored.** `storage.local`: the server-generated device id and name, the ids of grants created here and the person's last Pause/Stop/Resume, per profile (not secrets; `storage.local` stays locked to trusted contexts), so Pause/Stop survives a Chrome restart and a profile switch. `storage.session`: owned-tab handles, so a worker restart keeps them. Never stored: the API key outside the credential store, page content, snapshots, typed text, screenshots. +- **Stored.** `storage.local`: the server-generated device id and name, the ids of grants created here and the person's last Pause/Stop/Resume, per profile (not secrets; `storage.local` stays locked to trusted contexts), so Pause/Stop survives a Chrome restart and a profile switch. `storage.session`: owned-tab handles, so a worker restart keeps them, and the Advanced mode opt-in. Never stored: the API key outside the credential store, page content, snapshots, typed text, screenshots. - **Commands.** Each `browser.command` is validated strictly (unknown keys, actions or keys for `press` are refused), acknowledged, executed and answered with exactly one result. A command id is executed at most once. Results are never queued across a reconnect: a dropped link cancels the commands it carried. An interrupted command that may already have changed the page reports `outcome_unknown`. -- **Pages.** Only `chrome.tabs` and `chrome.scripting.executeScript` with the bundled `pageRun` function (func + args, never a code string), top frame only. Same-origin iframes are read only when their origin is granted; cross-origin iframes are omitted. No `tabs` and no `debugger` permission; screenshots answer `unsupported_capability`. -- **The person stays in charge.** High-risk clicks and `type` with `submit` wait for an "Allow once" card in an open side panel; no panel or a denial gives `approval_denied`. Pause and Stop cancel what is running and refuse new commands (Stop also releases the agent's tabs). A click or key press by the person in an owned tab cancels the agent's command there and holds the tab for 5 s. +- **Pages.** Only `chrome.tabs` and `chrome.scripting.executeScript` with the bundled `pageRun` function (func + args, never a code string), top frame only. Same-origin iframes are read only when their origin is granted; cross-origin iframes are omitted. No `tabs` permission. The Standard build has no `debugger` permission; screenshots answer `unsupported_capability`. +- **Build profiles.** One codebase, two builds. Standard (`pnpm build`) is the default. Advanced (`DEWEE_PROFILE=advanced`, `pnpm build:advanced`) adds the `debugger` permission and the CDP adapter; the `__DEWEE_ADVANCED__` define drops that adapter from Standard bundles. A device enrolls with what the build can do (`["standard"]` or `["standard","advanced"]`) and each attach announces what is usable now: `advanced` only while the person has Advanced mode on. +- **Advanced mode.** Off by default, per browser session, turned on in the drawer. `EngineRouter` sends snapshot, describe, highlight, click, type and press to the CDP adapter (accessibility tree, `Input.dispatch*`) and screenshots only there; scroll, wait, find and tab actions stay on the page script. The adapter attaches to the owned tab when a debugger command starts and detaches when it ends, and on Stop, Pause, revoke, expiry, disconnect, opt-in off and tab close. If attaching fails (DevTools open), it answers `debugger_unavailable`, or uses the Standard path for snapshot and press when the person allowed that fallback. Same-process frames on granted origins are read and driven; cross-site frames are left out and blacked out in screenshots. Details: [browser-control-threat-model.md](browser-control-threat-model.md#advanced-mode-debugger). +- **The person stays in charge.** High-risk clicks and `type` with `submit` wait for an "Allow once" card in an open side panel; no panel or a denial gives `approval_denied`. Pause and Stop cancel what is running and refuse new commands (Stop also releases the agent's tabs). A click or key press by the person in an owned tab cancels the agent's command there, holds the tab for 5 s and pauses control until Resume. Agent tabs carry an "AI" badge; the drawer lists them with agent, site and expiry, shows the active command's agent, channel and site scope, and keeps a short in-memory timeline of outcomes. Before a click the target is highlighted briefly. ## Decisions diff --git a/docs/browser-control-threat-model.md b/docs/browser-control-threat-model.md index 409ac72..90a9b55 100644 --- a/docs/browser-control-threat-model.md +++ b/docs/browser-control-threat-model.md @@ -18,20 +18,37 @@ The person's signed-in web sessions. An agent that can read and click in a tab c | Threat | Control | |---|---| | A compromised or buggy Gateway sends commands or grants the person did not allow | Grants are not taken on the Gateway's word. The extension uses a grant only if this browser created it (its id is remembered in `storage.local` when the person creates it), it names this device, every origin in it is an exact origin, and, on creation, its origins match what the person asked for. Any other grant from attach, list or events is ignored. Every command is then re-checked locally against that grant: present, unexpired, unrevoked; agent listed; capability; exact origin of the tab's current page and of any requested URL; owned tab. Before every page action Chrome must hold host permission for that exact host (`permissions.getAll`); a broad pattern held for another feature, such as Ask Dewee's `https://*/*`, does not count. | -| Code injection from the wire | No remote code, `eval` or CDP. Actions are a fixed list validated with Zod; the page function (`pageRun`) is bundled and receives only data (op, ref, generation, text). Selectors and scripts are never accepted. The page function is injected only after `storage.local` is confirmed locked to trusted extension contexts. | +| Code injection from the wire | No remote code or `eval`; no CDP in the Standard build, and in the Advanced build only the fixed method list below. Actions are a fixed list validated with Zod; the page function (`pageRun`) is bundled and receives only data (op, ref, generation, text). Selectors and scripts are never accepted. The page function is injected only after `storage.local` is confirmed locked to trusted extension contexts. | | Agent reaches the person's other tabs | Agents drive only tabs they opened ("owned tabs"), addressed by opaque handles (`t1`, `t2`); raw tab ids never leave the extension. `tabs` lists only owned tabs. The extension has no `tabs` permission, so other tabs' URLs and titles are invisible to it. A tab Chrome creates after the command was cancelled (Stop, revoke, profile switch) is closed, never registered. | | Navigation or redirect to an ungranted site | The URL is checked before and after every navigation and click; a page that lands outside the grant fails the command, and every later command on that tab is refused while it stays there. `navigate` also needs the page currently in the tab to be in the grant. `chrome://`, extension pages, the Chrome Web Store, `file:`, `data:`, `javascript:` and every `about:` page (`about:blank` included, since a blank tab has no granted origin) are always refused. | | Secrets read from the page | The snapshot redacts (`value=[redacted]`) any sensitive field, whatever role the page gives it: `type=password`; `autocomplete` `cc-*`, `one-time-code`, `current-password` or `new-password` (so a password revealed by a "show password" button stays redacted); a name or id containing a password, PIN, OTP, MFA or verification-code word (`userPin`, `otp_code`); payment card, CVC, expiry and IBAN hints. Hidden and `aria-hidden` nodes, hidden inputs, and frames from an origin outside the grant are left out. | | Agent types into a secret field | `type` into any field the rule above calls sensitive is refused (`sensitive_field`), and `Backspace` never edits one. | | Destructive or irreversible actions | `type` with `submit`, `press Enter` (which submits forms and sends chat messages), and clicks on elements named like submit, publish, delete, pay or similar wait for a per-action "Allow once" in the side panel. The card names the agent, the site and the target element, and for `type` with `submit` shows the start of the text to be sent (clipped, held only while the card is open, never stored or logged). With no side panel open the action is denied, never auto-approved. After approval the element (or the focused element, for Enter) is described again; if its role, name or sensitivity changed, the command fails with `stale_ref`. | | Stale refs act on the wrong element | Refs belong to one snapshot generation; navigation or a large DOM change bumps it and old refs fail with `stale_ref`. Clicks check that the element is visible, enabled and actually under the click point. | -| Agent fights the person for the tab | Best effort: real clicks and key presses by the person in an owned tab cancel the agent's running command there and hold new ones for 5 s (`busy`). The watcher listens in the capture phase on the page's window; a page can still stop the event before it arrives, so this is a courtesy, not a security boundary. One command at a time per tab. | +| Agent fights the person for the tab | Best effort: real clicks and key presses by the person in an owned tab cancel the agent's running command there, hold new ones for 5 s (`busy`) and pause control (a persisted Pause; the panel says why and the person presses Resume). Input the debugger itself dispatches is not taken for the person's (it is ignored while a debugger input runs and for 600 ms after). The watcher listens in the capture phase on the page's window; a page can still stop the event before it arrives, so this is a courtesy, not a security boundary. One command at a time per tab. | | Person loses control | Pause, Stop and Revoke work locally even if the Gateway does not cooperate: Pause/Stop refuse commands locally (`device_paused`), cancel the running ones and deny pending approvals, including commands waiting for text; Stop also releases owned tabs. Revoking a grant or the device, or a refresh that no longer lists a grant, cancels in-flight commands under it, including ones waiting for approval. The person's last Pause/Stop/Resume is kept per profile and device in `storage.local`, so it survives a Chrome restart and a profile switch, and is re-sent to the Gateway after every attach. Fail-safe: `active` is re-sent only when the person's last choice was Resume; with no stored choice nothing is sent and the Gateway keeps whatever state it holds; if the stored choice cannot be read or is damaged, the panel shows Paused, commands are refused locally and nothing is sent, so a Pause is never lifted by accident and the person can press Resume. | | Replayed side effects | A command id runs at most once while the service worker lives (seen ids are kept in memory). After a worker restart, not running a command twice relies on the Gateway, which does not resend a dispatched command. Results are not replayed across reconnects; a lost result becomes `outcome_unknown` on the Gateway and the agent must re-snapshot. A command runs for at most 120 s locally, whatever deadline it carries. | -| Page data persisted | Snapshots, typed text and page content are never stored. `storage.local` keeps only the device id and name, the ids of grants created here and the Pause/Stop/Resume choice, per profile; `storage.session` keeps the tab handle map. Storage stays locked to trusted extension contexts. | +| Page data persisted | Snapshots, screenshots, typed text and page content are never stored. `storage.local` keeps only the device id and name, the ids of grants created here and the Pause/Stop/Resume choice, per profile; `storage.session` keeps the tab handle map and, in the Advanced build, the debugger opt-in. The drawer's action timeline lives in memory and holds only agent, action, domain and outcome code. Storage stays locked to trusted extension contexts. | | Master token or key leak | Only the personal API key is used, only in the `connect` frame to the profile's own Gateway origin. It is never logged or put in a URL. | | A web page talks to the worker | Content scripts in web pages may send only the Ask Dewee selection and the owned-tab input notice; `browser.*` requests from a web page (or any non-extension sender) are refused. | -| Over-broad Chrome permission | No new install-time permission. Host permission is requested per exact origin at grant time, never ``, and only an exact-host permission satisfies the per-action check. | +| Over-broad Chrome permission | The Standard build adds no install-time permission. Host permission is requested per exact origin at grant time, never ``, and only an exact-host permission satisfies the per-action check. The Advanced build adds only `debugger` (see below). | + +## Advanced mode (debugger) + +A separate build profile, Advanced (`pnpm build:advanced`, `pnpm zip:advanced`), adds the `debugger` permission so agents can read the accessibility tree, dispatch real input and take screenshots. The Standard build has no `debugger` permission and no debugger code (the adapter is dropped at build time; a unit test pins the manifest permissions and an e2e test checks the built manifest). Code: `src/browser/adapters/cdp/`, `src/browser/engine-router.ts`. + +| Threat | Control | +|---|---| +| The debugger used without the person knowing | Off by default. The person turns it on in the drawer ("Advanced mode") for this browser session only (`storage.session`; a Chrome restart turns it off). Chrome shows its own "is debugging this browser" bar while attached; cancelling that bar turns the opt-in off. The Gateway is told `advanced` only while the opt-in is on, so it refuses screenshot commands otherwise. | +| Attaching wider than needed | Only `chrome.debugger.attach({ tabId })` on a tab the agent owns, never a browser-level target, another tab, an extension page or a remote debugging port. It attaches when a debugger command starts and detaches when that command ends. It detaches at once on Stop, Pause, grant revoke or expiry (a timer, not the next command), disconnect, profile switch, opt-in off and tab close; on a worker restart any tab still attached from before is detached. | +| Arbitrary CDP from the wire | Commands never carry a CDP method or params. The adapter calls one `send` wrapper that refuses any method outside a fixed list (unit-tested): `Accessibility.getFullAXTree`/`getPartialAXTree`, `DOM.enable`/`getDocument`/`describeNode`/`getContentQuads`/`getNodeForLocation`/`focus`/`scrollIntoViewIfNeeded`, `Page.getFrameTree`/`getLayoutMetrics`/`captureScreenshot`, `Input.dispatchMouseEvent`/`dispatchKeyEvent`/`insertText`, `Overlay.enable`/`highlightNode`/`hideHighlight`. No `Runtime.evaluate`, no `Network`, `Storage`, cookies, `Fetch` or `Target`. | +| Policy bypass through the debugger | The same executor runs before the adapter: grant, agent, capability, exact origin, owned tab, host permission, approvals for risky actions. In addition the adapter asks the session before attaching and before every command (opt-in on, control active, owned tab, live grant, tab URL not restricted and in the grant), checks the main frame's origin again after reading and before every input, and checks each element's frame origin. A navigation during a command fails it (`origin_not_granted` or `stale_ref`). | +| Secrets read through the accessibility tree | The AX snapshot uses the same `[eN] role "name"` format, generations and redaction as the page script: the sensitive-field rule is shared code (`src/browser/sensitive.ts`) applied to each node's DOM attributes, so a sensitive field is `value=[redacted]` whatever role the page gives it. `type` into a sensitive field and `Backspace` on one are refused. Hidden nodes are left out. | +| Secrets in screenshots | Viewport only (`captureBeyondViewport: false`), JPEG, at most 1.5 MiB after a quality and size step-down; nothing larger is sent. Sensitive fields and every frame whose origin is not granted are painted black before encoding. The image goes only into the command result; it is never stored, logged or kept in the timeline. Screenshots never fall back to the Standard path. | +| Frames | Same-process frames whose origin is in the grant are read and driven. Cross-site (out-of-process) frames and frames from origins outside the grant are left out of the snapshot, refused for actions and blacked out in screenshots. | +| Debugger busy (DevTools or another extension) | Attaching fails with `debugger_unavailable`. If the person allowed it (the "read and act the standard way" box, on by default) snapshot and `press` go the Standard way; refs then belong to that engine, and a ref from the other engine is `stale_ref`. Screenshots never fall back. | + +Residual risks of Advanced mode: debugger input is trusted input, so sites that accept only trusted events will accept the agent's clicks; the accessibility tree can expose text a page script would have treated as hidden when the page labels it badly; Chrome's debugging bar is the only browser-level indication beyond the extension's own tab badge. ## Residual risks diff --git a/docs/connected-browser-relay-contract.md b/docs/connected-browser-relay-contract.md index 4dfc1c9..e524475 100644 --- a/docs/connected-browser-relay-contract.md +++ b/docs/connected-browser-relay-contract.md @@ -60,7 +60,7 @@ All params are strict objects; unknown fields are rejected. | Method | Params | Result | |---|---|---| | `browser.device.enroll` | `{name, platform, capabilities:["standard"] \| ["standard","advanced"]}` | `{device_id}` (server-generated, random) | -| `browser.device.attach` | `{device_id, capabilities}` | `{device_id, grants:[Grant]}`. Binds this socket as the device's live connection. Refused (`NOT_FOUND`) unless the device belongs to the authenticated principal and tenant and is not revoked. A newer attach replaces an older socket. | +| `browser.device.attach` | `{device_id, capabilities}` | `{device_id, grants:[Grant]}`. Binds this socket as the device's live connection. The extension enrolls with what its build can do and sends on every attach what is usable now (`advanced` only while the person has Advanced mode on); it re-attaches on the same socket when that changes. Refused (`NOT_FOUND`) unless the device belongs to the authenticated principal and tenant and is not revoked. A newer attach replaces an older socket. | | `browser.device.heartbeat` | `{device_id}` | `{}` | | `browser.device.list` | `{}` | `{devices:[Device]}` (the principal's devices in this tenant) | | `browser.device.rename` | `{device_id, name}` | `{device}` | diff --git a/docs/development.md b/docs/development.md index ecd7c68..1e33fbc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,13 +38,26 @@ The unpacked extension id is pinned by the public key in `extension-public-key.j ```bash pnpm lint && pnpm typecheck && pnpm test && pnpm build -pnpm test:e2e # builds, then Playwright loads .output/chrome-mv3 +pnpm build:advanced # Advanced profile into .output-advanced/chrome-mv3 +pnpm test:e2e # builds both profiles into .output-e2e*/, then Playwright loads them pnpm zip # .output/dewee-chrome-ext--chrome.zip (keeps the pinned key, for sideloading) -pnpm zip:store # .output-store/… without `key`, for the Chrome Web Store +pnpm zip:advanced # .output-advanced/… Advanced sideload zip (keeps the pinned key) +pnpm zip:store # .output-store/… without `key`, for the Chrome Web Store (Standard only) ``` The Chrome Web Store rejects a manifest that carries `key` and assigns its own extension ID. Register that store ID's `chromiumapp.org` redirect URI and `chrome-extension://` origin next to the unpacked ones. +### Build profiles + +`DEWEE_PROFILE` picks the profile at build time (`src/shared/build-profile.ts`): `standard` (default) or `advanced`. Advanced adds the `debugger` permission, names the extension "Dewee (Advanced)" and compiles in the CDP adapter (`src/browser/adapters/cdp/`); Standard bundles leave it out (`__DEWEE_ADVANCED__` is `false`, so the bundler drops it). Unit tests run as Standard. Both profiles share the pinned sideload key, so they have the same extension ID and only one of them can be installed in a Chrome profile at a time; remove one before loading the other. Advanced has no Chrome Web Store listing yet: `DEWEE_STORE=1` with the Advanced profile fails the build on purpose. + +Chrome Web Store review notes for a future Advanced listing (`debugger` is a sensitive permission and needs a justification and in-depth review): + +- Single purpose: the connected browser drives only tabs the agent opened, on sites the person allowed, after the person turned on Advanced mode in the side panel; Chrome's debugging bar shows while attached. +- Justification: the accessibility tree gives agents a more faithful page outline than DOM heuristics, `Input.dispatch*` produces the input sites expect, and `Page.captureScreenshot` gives the agent a viewport image; none of this is possible with `scripting` alone. +- Limits to state: fixed CDP method whitelist (no `Runtime.evaluate`, network, storage or cookies), tab targets only, attached only during a command, nothing stored. +- Policies: [Use of permissions](https://developer.chrome.com/docs/webstore/program-policies/permissions), [Privacy fields and permission justification](https://developer.chrome.com/docs/webstore/cws-dashboard-privacy), [Limited use of user data](https://developer.chrome.com/docs/webstore/program-policies/limited-use), [`chrome.debugger`](https://developer.chrome.com/docs/extensions/reference/api/debugger). + ### Connected browser end-to-end `tests/e2e/browser-control.spec.ts` needs no Dewee Gateway. Each test starts two loopback servers on random ports: `tests/e2e/support/browser-relay.ts` plays the Gateway (the `/ws` relay with `connect`, `browser.device.*`, `browser.grant.*` and the `browser.command` events, plus the small `/v1/extension` HTTP API the panel needs to sign in), and a page server serves a test shop page with a password field, a hidden input and a "Delete account" button. The person's side (enroll, allow, approve, Stop, revoke) runs through the real side panel; the relay sends commands and records acks, results and control states. @@ -55,6 +68,14 @@ pnpm test:e2e -- tests/e2e/browser-control.spec.ts Both servers close in `afterEach`. The relay's API key is a fixed test value, not a secret. +`tests/e2e/browser-control-advanced.spec.ts` runs the same setup on the Advanced build (`test.use({ buildProfile: 'advanced' })` in `tests/e2e/fixtures.ts`): the opt-in, AX snapshot redaction, click and type, the JPEG screenshot, same-site and cross-site frames, attach and detach as Chrome reports them, revoke and opt-in off mid-action, and a debugger conflict. The conflict is simulated by attaching `chrome.debugger` to the agent tab from the service worker first; Chrome allows one client per extension per tab, so the adapter's attach fails as it does when DevTools holds the tab (real DevTools cannot be opened from Playwright). + +```bash +pnpm build:e2e:advanced +pnpm exec playwright test tests/e2e/browser-control-advanced.spec.ts +pnpm bench:browser # Standard vs Advanced snapshot/click latency (DEWEE_BENCH_ROUNDS, default 20) +``` + ### Live end-to-end The `tests/e2e/live-*.spec.ts` specs drive the built extension against a real local Dewee Gateway: connect, agents, chat, stop, key revocation, reopening the panel, reconciling a lost response (`live-selfhosted`), stream resilience (`live-resilience`), credential cards and admin policy (`live-credentials`), the connected browser driven by the agent's `browser` tool with the workspace feature flag, grants, approval, revoke and Stop (`live-connected-browser`), and Cloud sign-in through a local Dewee app (`live-cloud`). Each is skipped unless its environment is set (shell or `.env`). The model is the deterministic stub `tests/e2e/support/stub-llm.mjs`; `tests/e2e/support/setup-live-gateway.mjs` creates the tenant, agents and personal key and writes them to `.env`. Steps: [tests/e2e/support/live-gateway.md](../tests/e2e/support/live-gateway.md). From e0a43fe76e10721a6559c784850e051bf98b9d29 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Thu, 15 Oct 2026 02:22:00 +0700 Subject: [PATCH 4/7] fix: keep keys and screenshots inside the grant and report partial input as unknown Keys are sent only when the real focus, followed through open shadow roots and frames, is in the granted document and not on a sensitive field, in both engines. Once any input event reached the page, a later failure reports outcome_unknown so the agent never repeats a half-done action. Screenshot redaction is measured before and after the capture and refuses the image when the layout moved twice, a box cannot be measured or more than 64 areas need covering. Every cross-origin or unreadable frame is blacked out, and an accessibility node with no DOM record is redacted. A detach requested during an attach takes effect when the attach ends, and the debugger adapter stops when the command is cancelled. Refs #18 Co-Authored-By: Claude Opus 5.5 --- docs/browser-control-threat-model.md | 13 +- src/browser/adapters/cdp/cdp-adapter.ts | 381 ++++++++++++++------ src/browser/adapters/cdp/cdp-client.ts | 61 +++- src/browser/adapters/cdp/cdp-outline.ts | 18 +- src/browser/adapters/cdp/index.ts | 4 +- src/browser/adapters/page-driver.ts | 10 +- src/browser/chrome-api.ts | 12 +- src/browser/engine-router.ts | 22 +- src/browser/executor.ts | 40 +- src/browser/session.ts | 26 +- src/browser/snapshot.ts | 52 ++- tests/e2e/browser-control-advanced.spec.ts | 29 ++ tests/e2e/support/browser-relay.ts | 9 + tests/unit/browser/cdp-adapter.test.ts | 270 ++++++++++++-- tests/unit/browser/cdp-client.test.ts | 68 ++++ tests/unit/browser/chrome-api.test.ts | 7 + tests/unit/browser/executor.test.ts | 35 +- tests/unit/browser/session-advanced.test.ts | 30 +- tests/unit/browser/snapshot.test.ts | 67 ++++ 19 files changed, 951 insertions(+), 203 deletions(-) diff --git a/docs/browser-control-threat-model.md b/docs/browser-control-threat-model.md index 90a9b55..24fa9f4 100644 --- a/docs/browser-control-threat-model.md +++ b/docs/browser-control-threat-model.md @@ -22,7 +22,8 @@ The person's signed-in web sessions. An agent that can read and click in a tab c | Agent reaches the person's other tabs | Agents drive only tabs they opened ("owned tabs"), addressed by opaque handles (`t1`, `t2`); raw tab ids never leave the extension. `tabs` lists only owned tabs. The extension has no `tabs` permission, so other tabs' URLs and titles are invisible to it. A tab Chrome creates after the command was cancelled (Stop, revoke, profile switch) is closed, never registered. | | Navigation or redirect to an ungranted site | The URL is checked before and after every navigation and click; a page that lands outside the grant fails the command, and every later command on that tab is refused while it stays there. `navigate` also needs the page currently in the tab to be in the grant. `chrome://`, extension pages, the Chrome Web Store, `file:`, `data:`, `javascript:` and every `about:` page (`about:blank` included, since a blank tab has no granted origin) are always refused. | | Secrets read from the page | The snapshot redacts (`value=[redacted]`) any sensitive field, whatever role the page gives it: `type=password`; `autocomplete` `cc-*`, `one-time-code`, `current-password` or `new-password` (so a password revealed by a "show password" button stays redacted); a name or id containing a password, PIN, OTP, MFA or verification-code word (`userPin`, `otp_code`); payment card, CVC, expiry and IBAN hints. Hidden and `aria-hidden` nodes, hidden inputs, and frames from an origin outside the grant are left out. | -| Agent types into a secret field | `type` into any field the rule above calls sensitive is refused (`sensitive_field`), and `Backspace` never edits one. | +| Agent types into a secret field | `type` into any field the rule above calls sensitive is refused (`sensitive_field`), and no key at all (`press`, not only `Backspace`) is sent while focus is on one. The focused element is found through open shadow roots. Keys are refused (`origin_not_granted`) while focus is on a frame element, or inside a frame that is not same-origin and granted. | +| A step fails after input reached the page | A click, `type` or `press` that fails after its first input event (mouse, key, inserted text, or an injected script that may have run) is reported as `outcome_unknown`, never as a specific error, so the agent re-snapshots instead of retrying blindly. A refusal before any input keeps its specific code. Both engines follow this rule. | | Destructive or irreversible actions | `type` with `submit`, `press Enter` (which submits forms and sends chat messages), and clicks on elements named like submit, publish, delete, pay or similar wait for a per-action "Allow once" in the side panel. The card names the agent, the site and the target element, and for `type` with `submit` shows the start of the text to be sent (clipped, held only while the card is open, never stored or logged). With no side panel open the action is denied, never auto-approved. After approval the element (or the focused element, for Enter) is described again; if its role, name or sensitivity changed, the command fails with `stale_ref`. | | Stale refs act on the wrong element | Refs belong to one snapshot generation; navigation or a large DOM change bumps it and old refs fail with `stale_ref`. Clicks check that the element is visible, enabled and actually under the click point. | | Agent fights the person for the tab | Best effort: real clicks and key presses by the person in an owned tab cancel the agent's running command there, hold new ones for 5 s (`busy`) and pause control (a persisted Pause; the panel says why and the person presses Resume). Input the debugger itself dispatches is not taken for the person's (it is ignored while a debugger input runs and for 600 ms after). The watcher listens in the capture phase on the page's window; a page can still stop the event before it arrives, so this is a courtesy, not a security boundary. One command at a time per tab. | @@ -40,12 +41,12 @@ A separate build profile, Advanced (`pnpm build:advanced`, `pnpm zip:advanced`), | Threat | Control | |---|---| | The debugger used without the person knowing | Off by default. The person turns it on in the drawer ("Advanced mode") for this browser session only (`storage.session`; a Chrome restart turns it off). Chrome shows its own "is debugging this browser" bar while attached; cancelling that bar turns the opt-in off. The Gateway is told `advanced` only while the opt-in is on, so it refuses screenshot commands otherwise. | -| Attaching wider than needed | Only `chrome.debugger.attach({ tabId })` on a tab the agent owns, never a browser-level target, another tab, an extension page or a remote debugging port. It attaches when a debugger command starts and detaches when that command ends. It detaches at once on Stop, Pause, grant revoke or expiry (a timer, not the next command), disconnect, profile switch, opt-in off and tab close; on a worker restart any tab still attached from before is detached. | +| Attaching wider than needed | Only `chrome.debugger.attach({ tabId })` on a tab the agent owns, never a browser-level target, another tab, an extension page or a remote debugging port. It attaches when a debugger command starts and detaches when that command ends. It detaches at once on Stop, Pause, grant revoke or expiry (a timer, not the next command), disconnect, profile switch, opt-in off and tab close; a detach asked for while an attach is still in progress detaches as soon as that attach completes, and the command fails. On a worker restart any tab still attached from before is detached. A cancelled command (Stop, deadline) stops between debugger steps and stops waiting for a pending one. | | Arbitrary CDP from the wire | Commands never carry a CDP method or params. The adapter calls one `send` wrapper that refuses any method outside a fixed list (unit-tested): `Accessibility.getFullAXTree`/`getPartialAXTree`, `DOM.enable`/`getDocument`/`describeNode`/`getContentQuads`/`getNodeForLocation`/`focus`/`scrollIntoViewIfNeeded`, `Page.getFrameTree`/`getLayoutMetrics`/`captureScreenshot`, `Input.dispatchMouseEvent`/`dispatchKeyEvent`/`insertText`, `Overlay.enable`/`highlightNode`/`hideHighlight`. No `Runtime.evaluate`, no `Network`, `Storage`, cookies, `Fetch` or `Target`. | -| Policy bypass through the debugger | The same executor runs before the adapter: grant, agent, capability, exact origin, owned tab, host permission, approvals for risky actions. In addition the adapter asks the session before attaching and before every command (opt-in on, control active, owned tab, live grant, tab URL not restricted and in the grant), checks the main frame's origin again after reading and before every input, and checks each element's frame origin. A navigation during a command fails it (`origin_not_granted` or `stale_ref`). | -| Secrets read through the accessibility tree | The AX snapshot uses the same `[eN] role "name"` format, generations and redaction as the page script: the sensitive-field rule is shared code (`src/browser/sensitive.ts`) applied to each node's DOM attributes, so a sensitive field is `value=[redacted]` whatever role the page gives it. `type` into a sensitive field and `Backspace` on one are refused. Hidden nodes are left out. | -| Secrets in screenshots | Viewport only (`captureBeyondViewport: false`), JPEG, at most 1.5 MiB after a quality and size step-down; nothing larger is sent. Sensitive fields and every frame whose origin is not granted are painted black before encoding. The image goes only into the command result; it is never stored, logged or kept in the timeline. Screenshots never fall back to the Standard path. | -| Frames | Same-process frames whose origin is in the grant are read and driven. Cross-site (out-of-process) frames and frames from origins outside the grant are left out of the snapshot, refused for actions and blacked out in screenshots. | +| Policy bypass through the debugger | The same executor runs before the adapter: grant, agent, capability, exact origin, owned tab, host permission, approvals for risky actions. In addition the adapter asks the session before attaching and before every command (opt-in on, control active, owned tab, live grant, tab URL not restricted and in the grant; Chrome's host permission is read at attach and once per command for the tab's origin, and read again after any refusal or origin change), checks the main frame's origin again after reading and before every input, and checks each element's frame origin. A navigation during a command fails it (`origin_not_granted` or `stale_ref`). | +| Secrets read through the accessibility tree | The AX snapshot uses the same `[eN] role "name"` format, generations and redaction as the page script: the sensitive-field rule is shared code (`src/browser/sensitive.ts`) applied to each node's DOM attributes, so a sensitive field is `value=[redacted]` whatever role the page gives it. An accessibility node with no matching DOM record is treated as sensitive. `type` into a sensitive field is refused, and no key is sent while focus is on one. Focus is read from the accessibility tree of the main frame and of readable frames; if focus cannot be placed (an out-of-process frame, too many frames, a frame element) or is in a frame outside the grant, the key is refused (`origin_not_granted`). After focusing for `type`, the target must still hold focus and be non-sensitive, before the text and again before `Enter`. | +| Secrets in screenshots | Viewport only (`captureBeyondViewport: false`), JPEG, at most 1.5 MiB after a quality and size step-down; nothing larger is sent. Sensitive fields and every frame that is not same-origin with the granted main page and readable by the debugger (every cross-origin frame, granted or not, and every out-of-process frame) are painted black before encoding. Their boxes are measured before and after the capture; if the viewport or any box moved, it retries once and then refuses. A box lookup that fails (other than an element with no box), or more than 64 areas to black out, refuses the screenshot rather than send a part unredacted. The image goes only into the command result; it is never stored, logged or kept in the timeline. Screenshots never fall back to the Standard path. | +| Frames | Same-process frames whose origin is in the grant are read and driven. Cross-site (out-of-process) frames and frames from origins outside the grant are left out of the snapshot, refused for actions and keys, and blacked out in screenshots. | | Debugger busy (DevTools or another extension) | Attaching fails with `debugger_unavailable`. If the person allowed it (the "read and act the standard way" box, on by default) snapshot and `press` go the Standard way; refs then belong to that engine, and a ref from the other engine is `stale_ref`. Screenshots never fall back. | Residual risks of Advanced mode: debugger input is trusted input, so sites that accept only trusted events will accept the agent's clicks; the accessibility tree can expose text a page script would have treated as hidden when the page labels it badly; Chrome's debugging bar is the only browser-level indication beyond the extension's own tab badge. diff --git a/src/browser/adapters/cdp/cdp-adapter.ts b/src/browser/adapters/cdp/cdp-adapter.ts index 1ce578a..31fb60c 100644 --- a/src/browser/adapters/cdp/cdp-adapter.ts +++ b/src/browser/adapters/cdp/cdp-adapter.ts @@ -8,22 +8,29 @@ * - refs: `eN` handles bound to a DOM backend node id, the frame it lives in, and the tab's * snapshot generation. Navigation moves the generation; a node that no longer exists is * `element_not_found`. Refs are never shared with the Standard adapter. - * - click/type/press: trusted input (`Input.*`) after a hit test at the element's centre. - * - screenshot: the visible viewport only, sensitive fields and ungranted frames blacked out. + * - click/type/press: trusted input (`Input.*`) after a hit test at the element's centre. Keys go + * only where focus is known to be: the main frame or a granted same-process frame, never a + * sensitive field (shadow roots included). A failure once input was sent is marked + * `effect: 'maybe'`, which the executor reports as `outcome_unknown`. + * - screenshot: the visible viewport only, sensitive fields and every frame that cannot be + * redacted field by field blacked out; the boxes are measured before and after the capture and + * must agree, otherwise no image is returned. * - frames: in-process frames whose origin is in the grant are read and driven; out-of-process * (cross-site) frames are left out and their elements cannot be targeted. */ import type { PressKey } from '../../protocol'; import { urlOrigin } from '../../policy'; import type { DriverErrorCode, DriverRequest, DriverResponse, PageDriver } from '../page-driver'; -import { CdpFailure, type CdpClient } from './cdp-client'; +import { CdpFailure, type CdpClient, type CdpMethod } from './cdp-client'; import { formatOutline, indexDom, isEditableNode, isSensitiveNode, nameOf, property, roleOf, type AXNode, type DomInfo, type DomNode } from './cdp-outline'; -import { bytesToBase64, fitScreenshot, quadsToRects, type DecodedImage, type ImageCodec, type Rect } from './screenshot'; +import { bytesToBase64, fitScreenshot, quadsToRects, type DecodedImage, type ImageCodec } from './screenshot'; /** How long trusted input from the debugger is not taken for the person's own input. */ export const DISPATCH_GRACE_MS = 600; -/** Frames read per snapshot besides the main one. */ +/** Frames read per snapshot (or per focus lookup) besides the main one. */ const MAX_FRAMES = 10; +/** Most boxes a screenshot blacks out; a page with more is refused rather than sent half redacted. */ +export const MAX_REDACTIONS = 64; interface RefTarget { backend: number; @@ -48,6 +55,26 @@ interface Frames { origins: Map; } +/** The command running on a tab: its cancellation, and whether input already went out. */ +interface RunState { + signal?: AbortSignal; + effect: boolean; +} + +interface Viewport { + pageX: number; + pageY: number; + clientWidth: number; + clientHeight: number; +} + +/** What a screenshot blacks out, measured at one moment. `key` compares two measurements. */ +interface Layout { + key: string; + quads: number[][]; + width: number; +} + const KEYS: Record = { Enter: { key: 'Enter', code: 'Enter', keyCode: 13, text: '\r' }, Tab: { key: 'Tab', code: 'Tab', keyCode: 9 }, @@ -63,10 +90,20 @@ const KEYS: Record ({ ok: false, code }); /** Chrome's message for a node id that no longer resolves. */ const GONE = /No node|Could not find node|does not exist|not found/i; +/** Chrome's message for a node that is not rendered (it has no box to black out). */ +const NO_BOX = /Could not compute (content )?quads|Could not compute box model/i; + +const isFocused = (node: AXNode) => property(node, 'focused') === true; +const isDocumentNode = (node: AXNode) => { + const role = roleOf(node); + return role === 'RootWebArea' || role === 'WebArea'; +}; export interface CdpAdapterDeps { client: CdpClient; @@ -78,6 +115,8 @@ export class CdpAdapter implements PageDr private readonly refs = new Map(); private readonly dispatching = new Map(); private readonly overlay = new Set(); + /** One command per tab at a time (the executor leases the tab), so the run is kept per tab. */ + private readonly runs = new Map(); private readonly now: () => number; constructor(private readonly deps: CdpAdapterDeps) { @@ -125,80 +164,113 @@ export class CdpAdapter implements PageDr return (this.dispatching.get(tabId) ?? 0) > this.now(); } - async run(tabId: number, request: DriverRequest): Promise { + async run(tabId: number, request: DriverRequest, signal?: AbortSignal): Promise { + const state: RunState = { signal, effect: false }; + this.runs.set(tabId, state); try { await this.deps.client.attach(tabId); - switch (request.op) { - case 'snapshot': - return await this.snapshot(tabId, request.origins, request.generation, request.maxChars); - case 'describe': { - const target = await this.target(tabId, request.origins, request.generation, request.ref); - const ax = await this.axNode(tabId, target); - const dom = await this.describeDom(tabId, target); - return { ok: true, op: 'describe', role: roleOf(ax), name: nameOf(ax), sensitive: isSensitiveNode(dom), editable: isEditableNode(dom) }; + const response = await this.perform(tabId, request); + return !response.ok && state.effect ? { ...response, effect: 'maybe' } : response; + } catch (error) { + const code = error instanceof CdpFailure ? error.code : 'internal'; + // Once input went out, a failure does not mean nothing happened. + return state.effect ? { ok: false, code, effect: 'maybe' } : fail(code); + } finally { + if (this.runs.get(tabId) === state) this.runs.delete(tabId); + } + } + + private async perform(tabId: number, request: DriverRequest): Promise { + switch (request.op) { + case 'snapshot': + return this.snapshot(tabId, request.origins, request.generation, request.maxChars); + case 'describe': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + const ax = await this.axNode(tabId, target); + const dom = await this.describeDom(tabId, target); + return { ok: true, op: 'describe', role: roleOf(ax), name: nameOf(ax), sensitive: isSensitiveNode(dom), editable: isEditableNode(dom) }; + } + case 'highlight': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + await this.send(tabId, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backend }); + if (!this.overlay.has(tabId)) { + await this.send(tabId, 'DOM.enable'); + await this.send(tabId, 'Overlay.enable'); + this.overlay.add(tabId); } - case 'highlight': { - const target = await this.target(tabId, request.origins, request.generation, request.ref); - await this.deps.client.send(tabId, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backend }); - if (!this.overlay.has(tabId)) { - await this.deps.client.send(tabId, 'DOM.enable'); - await this.deps.client.send(tabId, 'Overlay.enable'); - this.overlay.add(tabId); + await this.send(tabId, 'Overlay.highlightNode', { + backendNodeId: target.backend, + highlightConfig: { showInfo: false, contentColor: { r: 124, g: 58, b: 237, a: 0.15 }, borderColor: { r: 124, g: 58, b: 237, a: 0.9 } }, + }); + return { ok: true, op: 'highlight' }; + } + case 'click': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + await this.axNode(tabId, target); + const point = await this.reachable(tabId, target); + await this.hideHighlight(tabId); + await this.input(tabId, async () => { + const base = { x: point.x, y: point.y, button: 'left', clickCount: 1 }; + await this.dispatch(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: point.x, y: point.y }); + await this.dispatch(tabId, 'Input.dispatchMouseEvent', { ...base, type: 'mousePressed', buttons: 1 }); + await this.dispatch(tabId, 'Input.dispatchMouseEvent', { ...base, type: 'mouseReleased', buttons: 0 }); + }); + return { ok: true, op: 'click' }; + } + case 'type': { + const target = await this.target(tabId, request.origins, request.generation, request.ref); + await this.axNode(tabId, target); + const dom = await this.describeDom(tabId, target); + if (isSensitiveNode(dom)) return fail('sensitive_field'); + if (!isEditableNode(dom)) return fail('element_not_interactable'); + await this.reachable(tabId, target); + await this.input(tabId, async () => { + await this.send(tabId, 'DOM.focus', { backendNodeId: target.backend }); + // A focus handler may have moved focus (to a password field, say): type only into the element itself. + await this.expectFocus(tabId, request.origins, target); + // Replace what is there, like the Standard adapter: select all, then insert (or delete). + await this.key(tabId, { key: 'a', code: 'KeyA', keyCode: 65 }, ['selectAll']); + if (request.text) await this.dispatch(tabId, 'Input.insertText', { text: request.text }); + else await this.key(tabId, { key: 'Delete', code: 'Delete', keyCode: 46 }); + if (request.submit) { + await this.expectFocus(tabId, request.origins, target); + await this.key(tabId, KEYS.Enter); } - await this.deps.client.send(tabId, 'Overlay.highlightNode', { - backendNodeId: target.backend, - highlightConfig: { showInfo: false, contentColor: { r: 124, g: 58, b: 237, a: 0.15 }, borderColor: { r: 124, g: 58, b: 237, a: 0.9 } }, - }); - return { ok: true, op: 'highlight' }; - } - case 'click': { - const target = await this.target(tabId, request.origins, request.generation, request.ref); - await this.axNode(tabId, target); - const point = await this.reachable(tabId, target); - await this.hideHighlight(tabId); - await this.input(tabId, async () => { - const base = { x: point.x, y: point.y, button: 'left', clickCount: 1 }; - await this.deps.client.send(tabId, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x: point.x, y: point.y }); - await this.deps.client.send(tabId, 'Input.dispatchMouseEvent', { ...base, type: 'mousePressed', buttons: 1 }); - await this.deps.client.send(tabId, 'Input.dispatchMouseEvent', { ...base, type: 'mouseReleased', buttons: 0 }); - }); - return { ok: true, op: 'click' }; - } - case 'type': { - const target = await this.target(tabId, request.origins, request.generation, request.ref); - await this.axNode(tabId, target); - const dom = await this.describeDom(tabId, target); - if (isSensitiveNode(dom)) return fail('sensitive_field'); - if (!isEditableNode(dom)) return fail('element_not_interactable'); - await this.reachable(tabId, target); - await this.input(tabId, async () => { - await this.deps.client.send(tabId, 'DOM.focus', { backendNodeId: target.backend }); - // Replace what is there, like the Standard adapter: select all, then insert (or delete). - await this.key(tabId, { key: 'a', code: 'KeyA', keyCode: 65 }, ['selectAll']); - if (request.text) await this.deps.client.send(tabId, 'Input.insertText', { text: request.text }); - else await this.key(tabId, { key: 'Delete', code: 'Delete', keyCode: 46 }); - if (request.submit) await this.key(tabId, KEYS.Enter); - }); - return { ok: true, op: 'type' }; - } - case 'press': { - const frames = await this.frames(tabId); - if (!request.origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); - await this.input(tabId, () => this.key(tabId, KEYS[request.key])); - return { ok: true, op: 'press' }; - } - case 'screenshot': - return await this.screenshot(tabId, request.origins); - default: - // Scrolling, waiting for text, the focused element and the input watcher stay with the page script. - return fail('internal'); + }); + return { ok: true, op: 'type' }; } - } catch (error) { - if (error instanceof CdpFailure) return fail(error.code); - return fail('internal'); + case 'press': { + const frames = await this.frames(tabId); + await this.keyTarget(tabId, request.origins, frames); + await this.input(tabId, () => this.key(tabId, KEYS[request.key])); + return { ok: true, op: 'press' }; + } + case 'screenshot': + return this.screenshot(tabId, request.origins); + default: + // Scrolling, waiting for text, the focused element and the input watcher stay with the page script. + return fail('internal'); } } + // ---------- Sending ---------- + + /** A read or a non-input step, cancelled with the command. */ + private send>(tabId: number, method: CdpMethod, params: Record = {}): Promise { + return this.deps.client.send(tabId, method, params, { signal: this.runs.get(tabId)?.signal }); + } + + /** An input step: once it passed the policy check and goes to Chrome, the page may change. */ + private dispatch(tabId: number, method: CdpMethod, params: Record): Promise { + const state = this.runs.get(tabId); + return this.deps.client.send(tabId, method, params, { + signal: state?.signal, + onDispatch: () => { + if (state) state.effect = true; + }, + }); + } + // ---------- Refs ---------- private refsFor(tabId: number, generation: number): TabRefs { @@ -223,7 +295,7 @@ export class CdpAdapter implements PageDr } private async frames(tabId: number): Promise { - const { frameTree } = await this.deps.client.send<{ frameTree: FrameTreeNode }>(tabId, 'Page.getFrameTree'); + const { frameTree } = await this.send<{ frameTree: FrameTreeNode }>(tabId, 'Page.getFrameTree'); const origins = new Map(); const visit = (node: FrameTreeNode) => { origins.set(node.frame.id, urlOrigin(node.frame.url)); @@ -234,7 +306,7 @@ export class CdpAdapter implements PageDr } private async document(tabId: number, mainFrame: string): Promise> { - const { root } = await this.deps.client.send<{ root: DomNode }>(tabId, 'DOM.getDocument', { depth: -1, pierce: true }); + const { root } = await this.send<{ root: DomNode }>(tabId, 'DOM.getDocument', { depth: -1, pierce: true }); return indexDom(root, mainFrame); } @@ -242,7 +314,7 @@ export class CdpAdapter implements PageDr private async axNode(tabId: number, target: RefTarget): Promise { let nodes: AXNode[]; try { - ({ nodes } = await this.deps.client.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getPartialAXTree', { backendNodeId: target.backend, fetchRelatives: false })); + ({ nodes } = await this.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getPartialAXTree', { backendNodeId: target.backend, fetchRelatives: false })); } catch (error) { throw this.nodeError(error); } @@ -254,7 +326,7 @@ export class CdpAdapter implements PageDr private async describeDom(tabId: number, target: RefTarget): Promise { try { - const { node } = await this.deps.client.send<{ node: DomNode }>(tabId, 'DOM.describeNode', { backendNodeId: target.backend }); + const { node } = await this.send<{ node: DomNode }>(tabId, 'DOM.describeNode', { backendNodeId: target.backend }); return indexDom(node, target.frame).get(node.backendNodeId) ?? { tag: node.nodeName.toUpperCase(), attrs: new Map(), frame: target.frame }; } catch (error) { throw this.nodeError(error); @@ -274,8 +346,8 @@ export class CdpAdapter implements PageDr private async reachable(tabId: number, target: RefTarget): Promise<{ x: number; y: number }> { let quads: number[][]; try { - await this.deps.client.send(tabId, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backend }); - ({ quads } = await this.deps.client.send<{ quads: number[][] }>(tabId, 'DOM.getContentQuads', { backendNodeId: target.backend })); + await this.send(tabId, 'DOM.scrollIntoViewIfNeeded', { backendNodeId: target.backend }); + ({ quads } = await this.send<{ quads: number[][] }>(tabId, 'DOM.getContentQuads', { backendNodeId: target.backend })); } catch (error) { throw this.nodeError(error); } @@ -283,12 +355,13 @@ export class CdpAdapter implements PageDr if (!quad) throw new CdpFailure('element_not_interactable', false); const x = ((quad[0] as number) + (quad[2] as number) + (quad[4] as number) + (quad[6] as number)) / 4; const y = ((quad[1] as number) + (quad[3] as number) + (quad[5] as number) + (quad[7] as number)) / 4; - const { cssLayoutViewport } = await this.deps.client.send<{ cssLayoutViewport: { clientWidth: number; clientHeight: number } }>(tabId, 'Page.getLayoutMetrics'); + const { cssLayoutViewport } = await this.send<{ cssLayoutViewport: { clientWidth: number; clientHeight: number } }>(tabId, 'Page.getLayoutMetrics'); if (x < 0 || y < 0 || x > cssLayoutViewport.clientWidth || y > cssLayoutViewport.clientHeight) throw new CdpFailure('element_not_interactable', false); let hit: number | undefined; try { - ({ backendNodeId: hit } = await this.deps.client.send<{ backendNodeId: number }>(tabId, 'DOM.getNodeForLocation', { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: false, ignorePointerEventsNone: false })); - } catch { + ({ backendNodeId: hit } = await this.send<{ backendNodeId: number }>(tabId, 'DOM.getNodeForLocation', { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: false, ignorePointerEventsNone: false })); + } catch (error) { + if (error instanceof CdpFailure && error.code !== 'internal') throw error; hit = undefined; } if (hit === undefined) throw new CdpFailure('element_not_interactable', false); @@ -318,6 +391,61 @@ export class CdpAdapter implements PageDr }); } + // ---------- Focus ---------- + + /** + * Where keyboard input would go now, from the `focused` state of the accessibility trees (they + * see into shadow roots, open or closed): the focused node and its frame, a frame whose document + * itself has focus, or null when no readable tree holds focus (an out-of-process frame does). + */ + private async focus(tabId: number, frames: Frames): Promise<{ frame: string; node?: AXNode } | null> { + const others = [...frames.origins.keys()].filter((id) => id !== frames.main); + let documentFocus: string | null = null; + for (const frame of [frames.main, ...others.slice(0, MAX_FRAMES)]) { + let nodes: AXNode[]; + try { + ({ nodes } = await this.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree', frame === frames.main ? {} : { frameId: frame })); + } catch (error) { + // A frame this session cannot read is skipped; if focus is there, nothing below finds it. + if (frame === frames.main || !(error instanceof CdpFailure) || error.code !== 'internal') throw error; + continue; + } + const focused = nodes.filter(isFocused); + const node = focused.find((n) => !isDocumentNode(n)); + if (node) return { frame, node }; + if (documentFocus === null && focused.length > 0) documentFocus = frame; + // The main document itself has focus: no frame holds it. + if (documentFocus === frames.main) return { frame: frames.main }; + } + if (others.length > MAX_FRAMES) return null; + return documentFocus === null ? null : { frame: documentFocus }; + } + + /** + * Checks that keys may go where focus is: a granted page, focus in the main frame or a granted + * same-process frame, not a sensitive field. Returns the focused node (undefined for a document + * with no focused element); throws the refusal otherwise. + */ + private async keyTarget(tabId: number, origins: string[], frames: Frames): Promise { + if (!origins.includes(frames.origins.get(frames.main) ?? '')) throw new CdpFailure('origin_not_granted', false); + const focus = await this.focus(tabId, frames); + if (!focus || !origins.includes(frames.origins.get(focus.frame) ?? '')) throw new CdpFailure('origin_not_granted', false); + if (!focus.node) return undefined; + const backend = focus.node.backendDOMNodeId; + const info = backend === undefined ? undefined : (await this.document(tabId, frames.main)).get(backend); + // A focused node the DOM read cannot place is treated like a secret field. + if (info === undefined || isSensitiveNode(info)) throw new CdpFailure('sensitive_field', false); + // Focus on a frame element means keys go into a document no tree here showed. + if (FRAME_TAGS.has(info.tag) || info.childFrame !== undefined) throw new CdpFailure('origin_not_granted', false); + return backend; + } + + /** Focus is on the element being typed into (and it is still allowed to receive keys). */ + private async expectFocus(tabId: number, origins: string[], target: RefTarget): Promise { + const focused = await this.keyTarget(tabId, origins, await this.frames(tabId)); + if (focused !== target.backend) throw new CdpFailure('element_not_interactable', false); + } + // ---------- Input ---------- private async input(tabId: number, send: () => Promise): Promise { @@ -331,12 +459,12 @@ export class CdpAdapter implements PageDr private async key(tabId: number, key: { key: string; code: string; keyCode: number; text?: string }, commands?: string[]): Promise { const base = { key: key.key, code: key.code, windowsVirtualKeyCode: key.keyCode, nativeVirtualKeyCode: key.keyCode }; - await this.deps.client.send(tabId, 'Input.dispatchKeyEvent', { ...base, type: key.text ? 'keyDown' : 'rawKeyDown', ...(key.text ? { text: key.text, unmodifiedText: key.text } : {}), ...(commands ? { commands } : {}) }); - await this.deps.client.send(tabId, 'Input.dispatchKeyEvent', { ...base, type: 'keyUp' }); + await this.dispatch(tabId, 'Input.dispatchKeyEvent', { ...base, type: key.text ? 'keyDown' : 'rawKeyDown', ...(key.text ? { text: key.text, unmodifiedText: key.text } : {}), ...(commands ? { commands } : {}) }); + await this.dispatch(tabId, 'Input.dispatchKeyEvent', { ...base, type: 'keyUp' }); } private async hideHighlight(tabId: number): Promise { - if (this.overlay.has(tabId)) await this.deps.client.send(tabId, 'Overlay.hideHighlight').catch(() => undefined); + if (this.overlay.has(tabId)) await this.send(tabId, 'Overlay.hideHighlight').catch(() => undefined); } // ---------- Snapshot ---------- @@ -346,12 +474,12 @@ export class CdpAdapter implements PageDr if (!origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); const dom = await this.document(tabId, frames.main); const trees = new Map(); - trees.set(frames.main, (await this.deps.client.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree')).nodes); + trees.set(frames.main, (await this.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree')).nodes); const granted = new Set([...frames.origins].filter(([, origin]) => origin !== null && origins.includes(origin)).map(([id]) => id)); for (const frame of granted) { if (frame === frames.main || trees.size > MAX_FRAMES) continue; try { - trees.set(frame, (await this.deps.client.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree', { frameId: frame })).nodes); + trees.set(frame, (await this.send<{ nodes: AXNode[] }>(tabId, 'Accessibility.getFullAXTree', { frameId: frame })).nodes); } catch (error) { // An out-of-process frame cannot be read through this tab's session: it is left out. if (error instanceof CdpFailure && error.code !== 'internal') throw error; @@ -381,21 +509,31 @@ export class CdpAdapter implements PageDr // ---------- Screenshot ---------- + /** + * Measures what to black out, captures, measures again. The two measurements must match (same + * scroll position, viewport, fields and boxes, every box found); otherwise it tries once more and + * then refuses, so an image is never returned with boxes taken from a different layout. + */ private async screenshot(tabId: number, origins: string[]): Promise { - const frames = await this.frames(tabId); - if (!origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); - const dom = await this.document(tabId, frames.main); - const { cssVisualViewport } = await this.deps.client.send<{ cssVisualViewport: { clientWidth: number; clientHeight: number } }>(tabId, 'Page.getLayoutMetrics'); - const { data } = await this.deps.client.send<{ data: string }>(tabId, 'Page.captureScreenshot', { format: 'png', fromSurface: true, captureBeyondViewport: false }); - // A navigation while capturing could have put another site in the picture. - const after = await this.frames(tabId); - if (after.main !== frames.main || !origins.includes(after.origins.get(after.main) ?? '')) return fail('origin_not_granted'); + for (let attempt = 0; attempt < 2; attempt += 1) { + const frames = await this.frames(tabId); + if (!origins.includes(frames.origins.get(frames.main) ?? '')) return fail('origin_not_granted'); + const before = await this.layout(tabId, frames, origins); + const { data } = await this.send<{ data: string }>(tabId, 'Page.captureScreenshot', { format: 'png', fromSurface: true, captureBeyondViewport: false }); + // A navigation while capturing could have put another site in the picture. + const after = await this.frames(tabId); + if (after.main !== frames.main || !origins.includes(after.origins.get(after.main) ?? '')) return fail('origin_not_granted'); + const again = await this.layout(tabId, after, origins); + if (before && again && before.key === again.key) return this.encode(data, again); + } + return fail('internal'); + } + private async encode(data: string, layout: Layout): Promise { const image = await this.deps.codec.decode(data); try { - const ratio = cssVisualViewport.clientWidth > 0 ? image.width / cssVisualViewport.clientWidth : 1; - const redact = await this.redactions(tabId, dom, after, origins, ratio, image); - const bytes = await fitScreenshot(this.deps.codec, image, redact); + const ratio = layout.width > 0 ? image.width / layout.width : 1; + const bytes = await fitScreenshot(this.deps.codec, image, quadsToRects(layout.quads, ratio, image)); if (!bytes) return fail('internal'); return { ok: true, op: 'screenshot', data: bytesToBase64(bytes) }; } finally { @@ -403,25 +541,44 @@ export class CdpAdapter implements PageDr } } - /** Boxes to black out: every sensitive field, and every frame whose origin is not in the grant. */ - private async redactions(tabId: number, dom: Map, frames: Frames, origins: string[], ratio: number, size: { width: number; height: number }): Promise { - const nodes: number[] = []; - for (const [backend, info] of dom) { - const field = info.tag === 'INPUT' || info.tag === 'TEXTAREA' || info.tag === 'SELECT' || info.attrs.has('contenteditable'); - if (field && isSensitiveNode(info)) nodes.push(backend); - else if (info.childFrame !== undefined && !origins.includes(frames.origins.get(info.childFrame) ?? '')) nodes.push(backend); - else if ((info.tag === 'IFRAME' || info.tag === 'FRAME') && info.childFrame === undefined) nodes.push(backend); - } - const rects: Rect[] = []; - for (const backendNodeId of nodes) { + /** The boxes to black out right now, or null when one of them could not be measured. */ + private async layout(tabId: number, frames: Frames, origins: string[]): Promise { + const targets = this.redactionTargets(await this.document(tabId, frames.main), frames, origins); + if (targets.length > MAX_REDACTIONS) throw new CdpFailure('internal', false, 'too many areas to black out'); + const { cssVisualViewport: view } = await this.send<{ cssVisualViewport: Viewport }>(tabId, 'Page.getLayoutMetrics'); + const quads: number[][] = []; + for (const backendNodeId of targets) { try { - const { quads } = await this.deps.client.send<{ quads: number[][] }>(tabId, 'DOM.getContentQuads', { backendNodeId }); - rects.push(...quadsToRects(quads, ratio, size)); + quads.push(...(await this.send<{ quads: number[][] }>(tabId, 'DOM.getContentQuads', { backendNodeId })).quads); } catch (error) { - // Not rendered (no box): nothing to hide. Any refusal from the guard still stops the screenshot. - if (error instanceof CdpFailure && error.code !== 'internal') throw error; + // A refusal, a timeout or a cancellation ends the screenshot. + if (!(error instanceof CdpFailure) || error.code !== 'internal') throw error; + // Not rendered: nothing to hide. Any other failure (the node went away, ...) means this measurement cannot be trusted. + if (!NO_BOX.test(error.detail)) return null; + } + } + return { key: JSON.stringify([view.pageX, view.pageY, view.clientWidth, view.clientHeight, targets, quads]), quads, width: view.clientWidth }; + } + + /** + * Every sensitive field, and every frame that cannot be redacted field by field: a frame is left + * visible only when it shows the main frame's origin, that origin is granted, and Chrome returned + * its document here (so its fields are in this list too). Cross-origin and out-of-process frames, + * granted or not, are blacked out whole. + */ + private redactionTargets(dom: Map, frames: Frames, origins: string[]): number[] { + const mainOrigin = frames.origins.get(frames.main) ?? null; + const targets: number[] = []; + for (const [backend, info] of dom) { + const field = info.tag === 'INPUT' || info.tag === 'TEXTAREA' || info.tag === 'SELECT' || info.attrs.has('contenteditable'); + if (field && isSensitiveNode(info)) { + targets.push(backend); + } else if (FRAME_TAGS.has(info.tag) || info.childFrame !== undefined) { + const origin = info.childFrame === undefined ? null : (frames.origins.get(info.childFrame) ?? null); + const readable = info.contentIndexed === true && origin !== null && origin === mainOrigin && origins.includes(origin); + if (!readable) targets.push(backend); } } - return rects; + return targets; } } diff --git a/src/browser/adapters/cdp/cdp-client.ts b/src/browser/adapters/cdp/cdp-client.ts index 6679037..e102d85 100644 --- a/src/browser/adapters/cdp/cdp-client.ts +++ b/src/browser/adapters/cdp/cdp-client.ts @@ -76,10 +76,23 @@ export class CdpFailure extends Error { } } +/** + * When the policy check runs: `attach` (the start of a command, everything looked up afresh) or + * `send` (before each command; Chrome's site permission for the origin may be reused from attach). + */ +export type CheckPhase = 'attach' | 'send'; + +export interface SendOptions { + /** The command was cancelled or reached its deadline: nothing more is sent. */ + signal?: AbortSignal; + /** Called once the policy check passed, right before the command goes to Chrome. */ + onDispatch?(): void; +} + export interface CdpClientDeps { debugger: DebuggerApi; /** Local policy for this tab right now: null when allowed, otherwise the refusal. */ - check(tabId: number): Promise; + check(tabId: number, phase: CheckPhase): Promise; /** Chrome ended a session we held (tab closed, the person pressed Cancel on Chrome's bar, ...). */ onDetached?(tabId: number, reason: string): void; /** The set of attached tabs changed (for the side panel). */ @@ -87,16 +100,31 @@ export interface CdpClientDeps { timeoutMs?: number; } -function withTimeout(promise: Promise, ms: number): Promise { +/** The failure for a command that was cancelled or ran out of time (the runner reports the real reason). */ +const aborted = () => new CdpFailure('timeout', false, 'command aborted'); + +function withTimeout(promise: Promise, ms: number, signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new CdpFailure('timeout', false, 'CDP command timed out')), ms); + const done = () => { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + }; + const onAbort = () => { + done(); + reject(aborted()); + }; + const timer = setTimeout(() => { + done(); + reject(new CdpFailure('timeout', false, 'CDP command timed out')); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); promise.then( (value) => { - clearTimeout(timer); + done(); resolve(value); }, (error: unknown) => { - clearTimeout(timer); + done(); reject(error instanceof Error ? error : new Error(String(error))); }, ); @@ -106,6 +134,8 @@ function withTimeout(promise: Promise, ms: number): Promise { export class CdpClient { private readonly attached = new Set(); private readonly pending = new Map>(); + /** Tabs asked to detach while their attach was still running. */ + private readonly detachRequested = new Set(); private readonly timeoutMs: number; constructor(private readonly deps: CdpClientDeps) { @@ -131,8 +161,9 @@ export class CdpClient { if (this.attached.has(tabId)) return; const running = this.pending.get(tabId); if (running) return running; + this.detachRequested.delete(tabId); const attaching = (async () => { - const refused = await this.deps.check(tabId); + const refused = await this.deps.check(tabId, 'attach'); if (refused) throw new CdpFailure(refused, true); try { await withTimeout(this.deps.debugger.attach({ tabId }, CDP_VERSION), this.timeoutMs); @@ -141,6 +172,11 @@ export class CdpClient { // debug, or the tab went away. throw new CdpFailure('debugger_unavailable', true, error instanceof Error ? error.message : ''); } + // Pause, Stop, revoke or opt-out asked to let go while Chrome was attaching: do it now. + if (this.detachRequested.delete(tabId)) { + await this.deps.debugger.detach({ tabId }).catch(() => undefined); + throw new CdpFailure('debugger_unavailable', true, 'detached while attaching'); + } this.attached.add(tabId); this.deps.onChange?.(); })(); @@ -153,17 +189,21 @@ export class CdpClient { } /** Sends one whitelisted command to an attached tab, after the policy check. */ - async send>(tabId: number, method: CdpMethod, params: Record = {}): Promise { + async send>(tabId: number, method: CdpMethod, params: Record = {}, options: SendOptions = {}): Promise { // The type already restricts `method`; this runtime check holds even against a cast. if (!isAllowedCdpMethod(method)) throw new CdpFailure('internal', false, `method not allowed: ${String(method)}`); + if (options.signal?.aborted) throw aborted(); if (!this.attached.has(tabId)) throw new CdpFailure('debugger_unavailable', false); - const refused = await this.deps.check(tabId); + const refused = await this.deps.check(tabId, 'send'); if (refused) { await this.detach(tabId); throw new CdpFailure(refused, false); } + if (options.signal?.aborted) throw aborted(); + if (!this.attached.has(tabId)) throw new CdpFailure('debugger_unavailable', false); + options.onDispatch?.(); try { - return (await withTimeout(this.deps.debugger.sendCommand({ tabId }, method, params), this.timeoutMs)) as T; + return (await withTimeout(this.deps.debugger.sendCommand({ tabId }, method, params), this.timeoutMs, options.signal)) as T; } catch (error) { if (error instanceof CdpFailure) throw error; // Detached meanwhile (Stop, revoke, the person cancelled Chrome's bar, the tab closed). @@ -174,6 +214,8 @@ export class CdpClient { /** Lets go of the tab. `force` also asks Chrome when this worker does not remember attaching (after a restart). */ async detach(tabId: number, force = false): Promise { + // An attach still running for the tab detaches as soon as Chrome completes it. + if (this.pending.has(tabId)) this.detachRequested.add(tabId); const held = this.attached.delete(tabId); if (!held && !force) return; await this.deps.debugger.detach({ tabId }).catch(() => undefined); @@ -181,6 +223,7 @@ export class CdpClient { } async detachAll(): Promise { + for (const tabId of this.pending.keys()) this.detachRequested.add(tabId); await Promise.all([...this.attached].map((tabId) => this.detach(tabId))); } } diff --git a/src/browser/adapters/cdp/cdp-outline.ts b/src/browser/adapters/cdp/cdp-outline.ts index a0d4f9d..0a0a569 100644 --- a/src/browser/adapters/cdp/cdp-outline.ts +++ b/src/browser/adapters/cdp/cdp-outline.ts @@ -5,7 +5,8 @@ * * The rules match the page script: values of sensitive fields (password, PIN, payment card, * one-time code; see sensitive.ts) are written as `value=[redacted]` whatever role the page gives - * them; ignored nodes (hidden, `aria-hidden`) are skipped; a frame is walked only when its origin + * them, and so is the value of any node whose DOM attributes could not be read; ignored nodes + * (hidden, `aria-hidden`) are skipped; a frame is walked only when its origin * is in the grant and Chrome could read its tree here (in-process frames). Out-of-process frames * are left out, like cross-origin frames in Standard. */ @@ -50,6 +51,8 @@ export interface DomInfo { frame: string; /** For ` `; const inner = ``; - const foreign = `

Foreign text e2e

`; + // `/foreign` writes every key it receives to `data-keys` on its body, so a test can tell none arrived. + const foreign = `

Foreign text e2e

`; + // `/shadow-frame` hides the cross-site frame in a closed shadow root: the page's + // `document.activeElement` is then the host element, never the frame. + const shadowFrame = (port: number) => ` +E2E Shadow Frame + +

Widget

+ + +`; // `/slow` has a button whose mousedown handler keeps the page busy longer than one debugger // command may wait, so the input reaches the page but its outcome is never confirmed. const slow = ` @@ -391,6 +404,7 @@ export async function startPageServer(): Promise<{ port: number; stop(): Promise if (req.url === '/frames') return void html(frames(port)); if (req.url === '/inner') return void html(inner); if (req.url === '/foreign') return void html(foreign); + if (req.url === '/shadow-frame') return void html(shadowFrame(port)); if (req.url === '/slow') return void html(slow); html(page); }); diff --git a/tests/unit/browser/cdp-adapter.test.ts b/tests/unit/browser/cdp-adapter.test.ts index 916f14b..9d595da 100644 --- a/tests/unit/browser/cdp-adapter.test.ts +++ b/tests/unit/browser/cdp-adapter.test.ts @@ -1,4 +1,4 @@ -import { CdpAdapter, DISPATCH_GRACE_MS, MAX_REDACTIONS } from '@/browser/adapters/cdp/cdp-adapter'; +import { CdpAdapter, DISPATCH_GRACE_MS, FOCUS_MAY_BE_IN_FRAME, MAX_REDACTIONS } from '@/browser/adapters/cdp/cdp-adapter'; import { CDP_METHODS, CdpClient, type DebuggerApi } from '@/browser/adapters/cdp/cdp-client'; import type { AXNode, DomNode } from '@/browser/adapters/cdp/cdp-outline'; import type { ImageCodec, Rect } from '@/browser/adapters/cdp/screenshot'; @@ -25,13 +25,40 @@ interface PageOptions { innerOutOfProcess?: boolean; /** Extra password fields in the main document. */ passwords?: number; + /** + * Where the cross-site ad (node 9) sits: an `