diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 6d99ad65dd..bf4bbac7bb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -42,6 +42,20 @@ jobs:
- name: Lint
run: pnpm eslint .
+ - name: Browser agent bundle matches its source
+ # `src-tauri/src/browser/js/agent.bundle.js` is committed so that a
+ # cargo build needs no node — `include_str!` reads it at compile time.
+ # A committed artifact is only worth anything if it is the one the
+ # source produces, and nothing else in the build would notice a stale
+ # one: the Rust side would happily embed last week's bundle.
+ run: pnpm browser:agent:check
+
+ - name: Browser agent typecheck
+ # `browser-agent/` is excluded from the app's tsconfig (its vendored
+ # half answers to Playwright's compiler settings, not ours), so the
+ # repo-wide typecheck in `pnpm build` never sees our entry point.
+ run: pnpm browser:agent:types
+
- name: Unit tests (vitest)
run: pnpm test
diff --git a/.prettierignore b/.prettierignore
index b79eb87684..bcd38a0c62 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -7,3 +7,5 @@ src-tauri/resources/opencode/models-dev.json
src-tauri/resources/codex/bundled-catalog.json
docs/
pnpm-lock.yaml
+browser-agent/vendor/
+src-tauri/src/browser/js/
diff --git a/Dockerfile b/Dockerfile
index 3a53a20365..3565ba8596 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -56,6 +56,8 @@ ENV CODEG_RUNTIME=docker
ENV CODEG_RESTART_DELAY_MS=2000
EXPOSE 3080
+# Port bridge for dev servers (CODEG_BRIDGE_PORTS; default CODEG_PORT+1..+10).
+EXPOSE 3081-3090
VOLUME /data
# Run under the built-in supervisor (PID 1) so an in-place upgrade can swap
diff --git a/browser-agent/README.md b/browser-agent/README.md
new file mode 100644
index 0000000000..92187376d1
--- /dev/null
+++ b/browser-agent/README.md
@@ -0,0 +1,86 @@
+# browser-agent
+
+The script a browser tab's **isolated world** runs when an agent needs to read
+the page. Built by esbuild into `src-tauri/src/browser/js/agent.bundle.js`,
+which is committed.
+
+```
+pnpm browser:agent # build the bundle
+pnpm browser:agent:check # is the committed bundle the one this source makes?
+pnpm browser:agent:types # typecheck (esbuild does not)
+pnpm browser:agent:probe # drive the bundle in real Chrome
+```
+
+## What it is
+
+`vendor/playwright/` is Playwright's aria tree, copied byte-for-byte at v1.63.0
+(see `vendor/playwright/VENDOR.md`). `src/index.ts` calls it in **`ai` mode** —
+the mode Playwright MCP uses — and puts `snapshot` and `elementForRef` on
+`globalThis.__codegAgent` for Rust to call through world-scoped eval.
+
+`ai` mode is why there is no second pass over the DOM here. It gives a ref to
+every element that is _visible and receives pointer events_, so a `
` with
+`cursor: pointer` and a click handler and no ARIA role is namable, and carries
+`[cursor=pointer]` to say why. An earlier plan for this package described
+writing that promotion ourselves; upstream had already made it unnecessary.
+
+## Two things it does that upstream does not
+
+**Refs are answerable across a navigation.** Playwright's ref counter lives in
+the module, so a fresh document starts again at `e1` — two pages use the same
+names for different elements. Each world here draws a random `generation` and
+reports it with every snapshot; `elementForRef` refuses a ref that quotes an
+older one. A new document destroys the world, so the next snapshot is a new
+generation and every ref an agent still holds is refused rather than resolved
+onto whatever now happens to be `e1`.
+
+A new document is not the only kind of navigation, and the other kind is the
+common one here: `pushState`, `replaceState` and hash changes leave the
+document, the world and the generation exactly as they were while the page
+becomes a different page. That is a route change in a single-page app — which
+is most of what a dev server serves — and the elements a framework keeps
+across one, the header and its buttons, are exactly the ones that would still
+resolve. So a snapshot also records the address it was taken at, and a ref is
+refused once the page has moved. The error runs in the safe direction: a
+caller told to snapshot again loses a round trip, a caller handed the wrong
+element loses the user's page.
+
+**Where that stops, and who takes over.** An address is not an identity. A
+route that goes A → B → A arrives back at a string that matches, on a page
+whose framework may have kept the DOM node and given it new meaning, and this
+world cannot see that it happened: the page's own `history.pushState` is
+invisible from an isolated world, because patching `History.prototype` here
+patches _this_ world's prototype while the page calls a different function
+object — the same isolation that keeps `__codegAgent` out of the page's reach.
+
+So the world enforces three floors it can check by looking — a new document, a
+moved address, a departed element — and `snapshot({ epoch })` mixes a host
+token into the generation an agent echoes back. Deciding _when_ refs die is
+the host's, because the host is the only party that sees the navigation.
+
+The probe measures the world's side of that, including the A → B → A case that
+the floors do not catch, and the premise underneath it: it patches
+`History.prototype.pushState` in the world, has the _page_ navigate, and
+checks that the patch never fired while the address moved anyway. It cannot
+measure the host's side, because there is no host here yet.
+
+**The tree can be capped.** `maxChars` cuts on a line boundary, so an agent
+never reads half a node, and the result says `truncated` so it knows to narrow
+the question rather than believe the page ended. A cap that lands inside the
+very first line has no boundary to use; the cap wins there and the line is cut
+where it falls.
+
+## What is not here
+
+Nothing reads this bundle yet. The Rust seam is where **authorization** lives —
+`none / read / control`, per tab and origin, with no automatic grants — and
+until that exists there must be no code path that reads a page on an agent's
+behalf. The bundle and the grant model land together, in the package that adds
+the `browser_*` tools.
+
+## Where it runs
+
+The world is separate from the page: `__codegAgent` is not reachable from page
+script, and page code cannot forge a ref or observe a snapshot. The probe
+asserts this. What the probe cannot assert is the engine — it drives Chrome,
+while the three platforms ship WKWebView, WebView2 and WebKitGTK.
diff --git a/browser-agent/probe.mjs b/browser-agent/probe.mjs
new file mode 100644
index 0000000000..58a7ff6667
--- /dev/null
+++ b/browser-agent/probe.mjs
@@ -0,0 +1,310 @@
+/**
+ * Drives the committed bundle in a real engine and prints what it produces.
+ *
+ * pnpm browser:agent:probe
+ *
+ * The unit tests cover the part of `src/index.ts` that is pure. They cannot
+ * cover the tree: jsdom reports every element as zero-sized, and `ai` mode
+ * only names elements that are visible and receive pointer events, so under
+ * jsdom the tree comes back with no refs and proves nothing. This asks a
+ * browser instead.
+ *
+ * Chrome, because it is the one engine present on all three of our
+ * development machines and it speaks CDP without a driver. It is not the
+ * engine any platform actually ships — WKWebView, WebView2 and WebKitGTK are —
+ * so a green run here says the bundle is sound, not that it is verified on a
+ * platform. Manual, not part of `pnpm test`: it needs a browser on the box.
+ */
+import { spawn } from "node:child_process"
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"
+import { tmpdir } from "node:os"
+import { join, dirname, resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), "..")
+const BUNDLE = readFileSync(
+ resolve(root, "src-tauri/src/browser/js/agent.bundle.js"),
+ "utf8"
+)
+
+const CHROME =
+ process.env.CHROME_PATH ??
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+const PORT = 9333
+
+const PAGE = `Probe
+Docs
+
+
Orders
+
+
+
Pointer but no handler
+
Handler but no pointer
+
Focusable but neither
+
+
alpha
beta
+`
+
+const dir = mkdtempSync(join(tmpdir(), "codeg-agent-probe-"))
+const pageFile = join(dir, "probe.html")
+writeFileSync(pageFile, PAGE)
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
+const chrome = spawn(
+ CHROME,
+ [
+ "--headless=new",
+ `--remote-debugging-port=${PORT}`,
+ `--user-data-dir=${join(dir, "profile")}`,
+ "--no-first-run",
+ "--disable-gpu",
+ "about:blank",
+ ],
+ { stdio: "ignore" }
+)
+
+let ws
+let failures = 0
+const check = (label, actual, expected) => {
+ const ok = JSON.stringify(actual) === JSON.stringify(expected)
+ if (!ok) failures++
+ console.log(`${ok ? "ok " : "FAIL"} ${label}: ${JSON.stringify(actual)}`)
+}
+
+try {
+ for (let i = 0; i < 100 && !ws; i++) {
+ try {
+ const list = await (
+ await fetch(`http://127.0.0.1:${PORT}/json/list`)
+ ).json()
+ const page = list.find((t) => t.type === "page")
+ if (page) ws = new WebSocket(page.webSocketDebuggerUrl)
+ } catch {
+ await sleep(100)
+ }
+ }
+ if (!ws) throw new Error(`no page target — is Chrome at ${CHROME}?`)
+ await new Promise((r) => (ws.onopen = r))
+
+ let id = 0
+ const pending = new Map()
+ ws.onmessage = (e) => {
+ const m = JSON.parse(e.data)
+ pending.get(m.id)?.(m)
+ pending.delete(m.id)
+ }
+ const send = (method, params = {}) =>
+ new Promise((r) => {
+ const i = ++id
+ pending.set(i, r)
+ ws.send(JSON.stringify({ id: i, method, params }))
+ })
+
+ await send("Page.enable")
+ await send("Page.navigate", { url: `file://${pageFile}` })
+ await sleep(700)
+
+ // An isolated world, created the way the shims create one.
+ const { result: frameTree } = await send("Page.getFrameTree")
+ const { result: world } = await send("Page.createIsolatedWorld", {
+ frameId: frameTree.frameTree.frame.id,
+ worldName: "codeg",
+ grantUniveralAccess: false,
+ })
+
+ const run = async (expression, contextId = world.executionContextId) => {
+ const { result } = await send("Runtime.evaluate", {
+ expression,
+ contextId,
+ returnByValue: true,
+ })
+ if (result.exceptionDetails)
+ throw new Error(JSON.stringify(result.exceptionDetails, null, 2))
+ return result.result.value
+ }
+
+ await run(BUNDLE)
+
+ const snap = JSON.parse(
+ await run("JSON.stringify(__codegAgent.snapshot({}))")
+ )
+ console.log("\n=== tree ===\n" + snap.tree + "\n")
+ console.log(`url=${snap.url} title=${snap.title} refs=${snap.refsCount}\n`)
+
+ // Roleless divs that behave like controls have to be namable, or an agent
+ // cannot act on the many pages that are built out of them. This is the whole
+ // reason there is no promotion pass of our own: `ai` mode refs everything
+ // visible that receives pointer events, so each of these is already named,
+ // whichever single attribute makes it interesting. Kept as three separate
+ // elements so that one of them regressing cannot hide behind another.
+ const named = (id, text) =>
+ check(
+ `a roleless div is namable — ${id}`,
+ new RegExp(`generic \\[ref=e\\d+\\][^\\n]*: ${text}`).test(snap.tree),
+ true
+ )
+ named("cursor:pointer only", "Pointer but no handler")
+ named("onclick only", "Handler but no pointer")
+ named("tabindex only", "Focusable but neither")
+
+ // …and `cursor: pointer` is additionally marked, which is how an agent tells
+ // "this looks clickable" from "this is merely visible".
+ check(
+ "cursor:pointer is reported, and only where it applies",
+ [
+ /\[cursor=pointer\][^\n]*: Pointer but no handler/.test(snap.tree),
+ /\[cursor=pointer\][^\n]*: Handler but no pointer/.test(snap.tree),
+ ],
+ [true, false]
+ )
+
+ check(
+ "a display:none subtree is left out",
+ snap.tree.includes("Invisible"),
+ false
+ )
+ check("the link keeps its href", snap.tree.includes("/url: /docs"), true)
+
+ // The page must not be able to see, call or forge the agent surface.
+ const mainWorld = await send("Runtime.evaluate", {
+ expression: "typeof globalThis.__codegAgent",
+ returnByValue: true,
+ })
+ check(
+ "the page cannot see __codegAgent",
+ mainWorld.result.result.value,
+ "undefined"
+ )
+
+ const cut = JSON.parse(
+ await run("JSON.stringify(__codegAgent.snapshot({maxChars: 40}))")
+ )
+ check("a capped tree reports the cut", cut.truncated, true)
+ check("a capped tree ends on a line boundary", cut.tree.endsWith(":"), true)
+
+ const g = JSON.stringify(snap.generation)
+ // Two refs: one to spend on the removal case, one that must stay in the page
+ // so the same-document case cannot pass for the wrong reason.
+ const kept = JSON.stringify(
+ snap.tree.match(/button "Export" \[ref=(e\d+)\]/)[1]
+ )
+ const spent = JSON.stringify(
+ snap.tree.match(/listitem \[ref=(e\d+)\]: alpha/)[1]
+ )
+
+ check(
+ "a live ref resolves",
+ await run(`!!__codegAgent.elementForRef(${g}, ${kept})`),
+ true
+ )
+ check(
+ "a ref from another document does not",
+ await run(`__codegAgent.elementForRef("other", ${kept})`),
+ null
+ )
+ check(
+ "a ref for a removed element does not",
+ await run(
+ `(() => { __codegAgent.elementForRef(${g}, ${spent}).remove();
+ return __codegAgent.elementForRef(${g}, ${spent}) })()`
+ ),
+ null
+ )
+
+ // A single-page app's route change: same document, same world, same
+ // generation, and the element is still in the page. Only the address moved.
+ // Asserting that it is still connected is the point — otherwise a `null`
+ // here would prove nothing about the address and everything about the node.
+ check(
+ "a ref does not survive a pushState, though its element does",
+ await run(
+ `(() => { const el = __codegAgent.elementForRef(${g}, ${kept});
+ history.pushState({}, "", "?routed");
+ return [el.isConnected, __codegAgent.elementForRef(${g}, ${kept})] })()`
+ ),
+ [true, null]
+ )
+ check(
+ "and a snapshot at the new address hands out refs that work again",
+ await run(
+ `(() => { const s = __codegAgent.snapshot({});
+ const m = s.tree.match(/button "Export" \\[ref=(e\\d+)\\]/);
+ return !!__codegAgent.elementForRef(s.generation, m[1]) })()`
+ ),
+ true
+ )
+
+ // The boundary of what this world can know, pinned so that it reads as
+ // known rather than as overlooked. An address is not an identity: a route
+ // that leaves and comes back arrives at a string that matches, and a
+ // framework may have kept the node and changed what it means. The world
+ // cannot see the transition — the page's own `pushState` is invisible from
+ // an isolated world — so the ref still resolves here.
+ check(
+ "an address that leaves and returns defeats the address check",
+ await run(
+ `(() => { const here = location.href;
+ const s = __codegAgent.snapshot({});
+ const m = s.tree.match(/button "Export" \\[ref=(e\\d+)\\]/);
+ history.pushState({}, "", "?elsewhere");
+ history.pushState({}, "", here);
+ return !!__codegAgent.elementForRef(s.generation, m[1]) })()`
+ ),
+ true
+ )
+
+ // …which is why the token carries whatever the host puts in it. The host
+ // does see the transition, and a ref quoting an epoch it has moved past is
+ // refused — by the host on the spot, and by this world from the next
+ // snapshot on, which is what these two assert.
+ check(
+ "a host epoch reaches the token an agent echoes",
+ await run(
+ `__codegAgent.snapshot({epoch: "nav-7"}).generation.endsWith(".nav-7")`
+ ),
+ true
+ )
+ check(
+ "and a ref from an earlier epoch dies at the next snapshot",
+ await run(
+ `(() => { const s = __codegAgent.snapshot({epoch: "nav-7"});
+ const m = s.tree.match(/button "Export" \\[ref=(e\\d+)\\]/);
+ __codegAgent.snapshot({epoch: "nav-8"});
+ return __codegAgent.elementForRef(s.generation, m[1]) })()`
+ ),
+ null
+ )
+
+ // The premise the whole design rests on, measured instead of assumed: this
+ // world cannot intercept the page's own history calls, which is why
+ // deciding when refs die has to be the host's job. Patch
+ // `History.prototype.pushState` here, then have the *page* navigate, and
+ // watch the patch not fire. Last, because it leaves the page elsewhere.
+ await run(`globalThis.__patchFired = false;
+ History.prototype.pushState = new Proxy(History.prototype.pushState, {
+ apply(t, self, args) { globalThis.__patchFired = true;
+ return Reflect.apply(t, self, args) } })`)
+ const before = await run("location.href")
+ await send("Runtime.evaluate", {
+ // No contextId: the page's own world, holding its own History.prototype.
+ expression: 'history.pushState({}, "", "?from-the-page")',
+ returnByValue: true,
+ })
+ check(
+ "a page's own pushState is invisible to a patch in this world",
+ [
+ await run("globalThis.__patchFired"),
+ (await run("location.href")) !== before,
+ ],
+ // Did not fire, yet the address did move — so the page really navigated
+ // and the patch really did not see it.
+ [false, true]
+ )
+
+ console.log(failures ? `\n${failures} failed` : "\nall checks passed")
+} finally {
+ ws?.close()
+ chrome.kill()
+}
+
+process.exit(failures ? 1 : 0)
diff --git a/browser-agent/src/index.test.ts b/browser-agent/src/index.test.ts
new file mode 100644
index 0000000000..f7f4cf5846
--- /dev/null
+++ b/browser-agent/src/index.test.ts
@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest"
+
+import { truncate } from "./index"
+
+/**
+ * The tree itself is Playwright's and is exercised against a real engine by
+ * `pnpm browser:agent:probe` — jsdom reports every box as zero-sized, so in
+ * `ai` mode, which only names elements that are visible and receive pointer
+ * events, it would produce a tree with no refs at all and prove nothing.
+ *
+ * What is worth testing here is the part that is ours and is pure.
+ */
+describe("truncate", () => {
+ const tree = `- generic [ref=e1]:\n - banner [ref=e2]:\n - link "Docs" [ref=e3]`
+
+ it("leaves a tree that fits alone", () => {
+ expect(truncate(tree, 10_000)).toEqual({ text: tree, truncated: false })
+ })
+
+ it("treats no cap, a zero cap and a negative cap as no cap", () => {
+ for (const cap of [undefined, 0, -1])
+ expect(truncate(tree, cap)).toEqual({ text: tree, truncated: false })
+ })
+
+ it("cuts on a line boundary so no node is left half-written", () => {
+ const { text, truncated } = truncate(tree, 40)
+ expect(truncated).toBe(true)
+ expect(text).toBe(`- generic [ref=e1]:\n - banner [ref=e2]:`)
+ // Every line that survived is a line the tree actually had.
+ for (const line of text.split("\n"))
+ expect(tree.split("\n")).toContain(line)
+ })
+
+ it("obeys the cap even where there is no line boundary to cut on", () => {
+ // A cap inside the first line has no boundary to fall back to. The cap is
+ // the caller's own bound, so it wins; returning nothing would read as an
+ // empty page rather than as a tree that was too long.
+ const { text, truncated } = truncate(tree, 5)
+ expect(truncated).toBe(true)
+ expect(text).toBe("- gen")
+ expect(text.length).toBe(5)
+ })
+
+ it("does not report a cut it did not make", () => {
+ // Exactly at the cap: no truncation, and the caller must not be told to
+ // ask for more.
+ expect(truncate(tree, tree.length)).toEqual({
+ text: tree,
+ truncated: false,
+ })
+ })
+})
diff --git a/browser-agent/src/index.ts b/browser-agent/src/index.ts
new file mode 100644
index 0000000000..45419f9ac4
--- /dev/null
+++ b/browser-agent/src/index.ts
@@ -0,0 +1,232 @@
+/**
+ * The agent-facing half of a browser tab, running in the isolated world.
+ *
+ * `channel.ts`'s primitive carries page state back to Rust. This bundle is the
+ * other direction: Rust evaluates `__codegAgent.(...)` in the same world
+ * and reads the JSON it returns. Nothing here is reachable from the page — the
+ * world is separate, and the page never sees `__codegAgent`.
+ *
+ * The tree itself is Playwright's, vendored under `../vendor/playwright`
+ * (see VENDOR.md). We call it in `ai` mode, which is the mode Playwright MCP
+ * uses, so the shape an agent reads here is the shape it already knows.
+ */
+
+import {
+ generateAriaTree,
+ renderAriaTreeAsJSON,
+} from "../vendor/playwright/injected/ariaSnapshot"
+import { renderAriaSnapshotAsYaml } from "../vendor/playwright/isomorphic/ariaSnapshotRenderer"
+
+/**
+ * Identifies the world this script is running in, and with it the document.
+ *
+ * Playwright hands out `e1`, `e2`, … from a counter that lives in the module,
+ * so a fresh document starts over at `e1`. Two pages therefore use the same
+ * names for different elements, and a ref an agent read before a navigation
+ * would land on whatever happens to be first in the new page — silently, and
+ * on an element the agent never saw. The generation makes that answerable:
+ * every snapshot reports the one it was taken in, and a request carrying an
+ * older one is refused instead of resolved.
+ *
+ * A page cannot influence it: the script is evaluated at document start in a
+ * world the page cannot reach, before any page script runs.
+ */
+const GENERATION = generationToken()
+
+function generationToken(): string {
+ const buf = new Uint32Array(2)
+ // `getRandomValues` is available on insecure origins too — it is
+ // `crypto.subtle` that is not — so a plain-http dev server takes this path
+ // like anything else. The fallback is there for the engine that surprises
+ // us: a predictable generation still separates one document from the next,
+ // which is all this value has to do, and the page cannot read it either way.
+ if (globalThis.crypto?.getRandomValues) globalThis.crypto.getRandomValues(buf)
+ else
+ for (let i = 0; i < buf.length; i++)
+ buf[i] = (Math.random() * 2 ** 32) >>> 0
+ return `${buf[0].toString(36)}${buf[1].toString(36)}`
+}
+
+/**
+ * The elements the last snapshot named, by ref.
+ *
+ * Replaced wholesale on every snapshot rather than accumulated: a ref only
+ * means anything against the tree it was read from, and keeping older ones
+ * around would let a stale name resolve to an element still in the document.
+ *
+ * The elements are held strongly, which is bounded — one tree's worth, dropped
+ * at the next snapshot, and the whole world dies with the document. What a
+ * WeakRef would buy is not the memory but the answer for an element that has
+ * since been removed from the page, and `isConnected` answers that exactly,
+ * without asking for a `WeakRef` the oldest WebKit we support may not have.
+ */
+let refs = new Map()
+
+/**
+ * The address the last snapshot was taken at.
+ *
+ * The generation answers for a *new document*, which is the only kind of
+ * navigation that destroys this world. It is not the only kind of navigation:
+ * `pushState`, `replaceState` and a hash change all leave the document, the
+ * world and this module exactly where they were while the page becomes a
+ * different page. That is the ordinary case for the dev servers these tabs
+ * exist to show — a route change in a single-page app — and the elements a
+ * framework keeps across one, a header's buttons and its nav, are precisely
+ * the ones still `isConnected` afterwards. Without this an agent could act on
+ * `e8` from the page it read while looking at the page it did not.
+ *
+ * Compared rather than subscribed to, because there is nothing here to
+ * subscribe to. See `epoch` on `SnapshotOptions`: this world structurally
+ * cannot observe a page-initiated history call, so a *floor* it can check by
+ * looking is worth more than a hook it cannot install.
+ *
+ * The direction of the error matters: this refuses some refs that would still
+ * be sound, because a caller told to take a new snapshot loses a round trip,
+ * while a caller handed the wrong element loses the user's page.
+ */
+let refsTakenAt = ""
+
+/**
+ * The token the last snapshot handed out, and the one a ref must quote.
+ *
+ * The world's own generation plus whatever the host attached to that snapshot
+ * (`SnapshotOptions.epoch`), so a caller echoes one opaque string back and
+ * neither side has to agree on what it is made of.
+ */
+let refsToken = ""
+
+export type SnapshotOptions = {
+ /** Cap on the rendered tree. Omitted or non-positive means no cap. */
+ maxChars?: number
+ /**
+ * An opaque token from the host, mixed into the generation this snapshot
+ * hands out and required back on every `elementForRef`.
+ *
+ * What it buys is enforcement *by the host*: the epoch rides inside the one
+ * string the caller echoes, so the host can refuse a ref the moment it knows
+ * the page moved on, by comparing against the epoch it is issuing now —
+ * without a side table mapping snapshots to navigations. This world refuses
+ * an older token only from the next snapshot onwards, because until then it
+ * has no way to learn that anything happened.
+ *
+ * It exists because there is a class of staleness this world cannot see. The
+ * page's own `history.pushState` is not observable from here: patching
+ * `History.prototype` in an isolated world patches *this world's* prototype,
+ * and the page calls a different function object — the same isolation that
+ * keeps `__codegAgent` out of the page's reach keeps the page's navigations
+ * out of ours. Comparing `location.href` catches the settled result of most
+ * of them, but an address is not an identity: a route that goes A → B → A
+ * arrives back at a string that matches, on a page whose framework may have
+ * kept the DOM node and given it new meaning.
+ *
+ * The host is the only party that can see those transitions, through the
+ * navigation it already tracks for the tab. So the decision of *when* refs
+ * die is the host's, and enforcing it is this world's. What is checked here
+ * without a host token — a new document, a moved address, a departed
+ * element — is a floor, not the contract.
+ */
+ epoch?: string
+}
+
+export type SnapshotResult = {
+ generation: string
+ url: string
+ title: string
+ viewport: { width: number; height: number; dpr: number }
+ tree: string
+ refsCount: number
+ truncated: boolean
+}
+
+/** Reads the page into the tree an agent operates on. */
+export function snapshot(options: SnapshotOptions = {}): SnapshotResult {
+ const root = document.body ?? document.documentElement
+ const next = new Map()
+ let rendered = ""
+
+ if (root) {
+ const aria = generateAriaTree(root, { mode: "ai" })
+ for (const [ref, info] of aria.info) next.set(ref, info.element)
+ const { json } = renderAriaTreeAsJSON(aria, { mode: "ai" })
+ rendered = renderAriaSnapshotAsYaml(json)
+ }
+ refs = next
+ refsTakenAt = location.href
+ // `!== undefined`, not truthiness: an empty epoch is a value the host chose
+ // and must stay distinguishable from one it never sent, or a host that
+ // happens to render an epoch as "" would silently get the untagged token and
+ // a ref issued under it would outlive the epoch change it was meant to die
+ // with.
+ refsToken =
+ options.epoch !== undefined ? `${GENERATION}.${options.epoch}` : GENERATION
+
+ const { text, truncated } = truncate(rendered, options.maxChars)
+ return {
+ generation: refsToken,
+ url: refsTakenAt,
+ title: document.title,
+ viewport: {
+ width: window.innerWidth,
+ height: window.innerHeight,
+ dpr: window.devicePixelRatio,
+ },
+ tree: text,
+ refsCount: next.size,
+ truncated,
+ }
+}
+
+/**
+ * The element a ref names, or `null` if the ref cannot be honoured.
+ *
+ * Four ways it cannot: the token it quotes is not the one the last snapshot
+ * handed out — another document, or a host that has since declared the page
+ * moved on — the address has changed since that snapshot, the last snapshot
+ * did not name this ref, or the element it named has left the page. All four
+ * are one answer to the caller, take a new snapshot, so they are one return
+ * value here.
+ */
+export function elementForRef(generation: string, ref: string): Element | null {
+ // Against the last snapshot's token, not the world's: a host that bumps its
+ // epoch between two snapshots means the earlier one's refs are no longer
+ // answerable, even though the document never changed.
+ if (!refsToken || generation !== refsToken) return null
+ if (location.href !== refsTakenAt) return null
+ const element = refs.get(ref)
+ if (!element?.isConnected) return null
+ return element
+}
+
+/**
+ * Cuts the tree to `maxChars` on a line boundary.
+ *
+ * Mid-line would hand the agent a half-written node — a ref with no role, or a
+ * role with half its name — which reads as a real entry rather than as a cut.
+ *
+ * There is one case with no line boundary to use: a cap that lands inside the
+ * very first line. The cap wins there and the line is cut where it falls,
+ * because the cap is the caller's own bound and returning nothing would read
+ * as an empty page rather than as a tree that was too long. `truncated` says
+ * which of the two happened either way.
+ */
+export function truncate(
+ text: string,
+ maxChars: number | undefined
+): { text: string; truncated: boolean } {
+ if (!maxChars || maxChars <= 0 || text.length <= maxChars)
+ return { text, truncated: false }
+ const cut = text.lastIndexOf("\n", maxChars)
+ return {
+ text: cut > 0 ? text.slice(0, cut) : text.slice(0, maxChars),
+ truncated: true,
+ }
+}
+
+declare global {
+ var __codegAgent: {
+ snapshot: typeof snapshot
+ elementForRef: typeof elementForRef
+ }
+}
+
+globalThis.__codegAgent = { snapshot, elementForRef }
diff --git a/browser-agent/tsconfig.json b/browser-agent/tsconfig.json
new file mode 100644
index 0000000000..9dd2671d3a
--- /dev/null
+++ b/browser-agent/tsconfig.json
@@ -0,0 +1,25 @@
+{
+ "//": [
+ "Separate from the repo's tsconfig.json, which excludes this directory.",
+ "The vendored Playwright sources are written against Playwright's own",
+ "compiler settings; holding them to ours (noUnusedLocals, noUnusedParameters)",
+ "would mean editing upstream code, and editing it is what makes an update a",
+ "merge instead of a copy. esbuild does not typecheck, so nothing here gates",
+ "the build — `pnpm browser:agent:check` runs this for our own entry's sake."
+ ],
+ "compilerOptions": {
+ "target": "ES2021",
+ "lib": ["ES2021", "DOM", "DOM.Iterable"],
+ "module": "ESNext",
+ "moduleResolution": "bundler",
+ "strict": true,
+ "noEmit": true,
+ "skipLibCheck": true,
+ "isolatedModules": true,
+ "types": ["vitest/globals"],
+ "paths": {
+ "@isomorphic/*": ["./vendor/playwright/isomorphic/*"]
+ }
+ },
+ "include": ["src/**/*.ts", "vendor/**/*.ts"]
+}
diff --git a/browser-agent/vendor/playwright/LICENSE b/browser-agent/vendor/playwright/LICENSE
new file mode 100644
index 0000000000..4ace03ddb6
--- /dev/null
+++ b/browser-agent/vendor/playwright/LICENSE
@@ -0,0 +1,202 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Portions Copyright (c) Microsoft Corporation.
+ Portions Copyright 2017 Google Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/browser-agent/vendor/playwright/VENDOR.md b/browser-agent/vendor/playwright/VENDOR.md
new file mode 100644
index 0000000000..49d30b4578
--- /dev/null
+++ b/browser-agent/vendor/playwright/VENDOR.md
@@ -0,0 +1,62 @@
+# Vendored from Playwright
+
+Nine files, copied byte-for-byte from
+[microsoft/playwright](https://github.com/microsoft/playwright) at tag
+**v1.63.0**, under the Apache License 2.0 (see `LICENSE`).
+
+They are the complete import closure of `injected/ariaSnapshot.ts` — the tree
+Playwright itself builds for `page.ariaSnapshot()` and Playwright MCP. We call
+it in `ai` mode, so the shape an agent reads from a codeg tab is the shape it
+already knows from Playwright MCP.
+
+| upstream path | bytes |
+| --- | --- |
+| `packages/injected/src/ariaSnapshot.ts` | 20123 |
+| `packages/injected/src/ariaSnapshotDistiller.ts` | 11941 |
+| `packages/injected/src/domUtils.ts` | 7572 |
+| `packages/injected/src/roleUtils.ts` | 60719 |
+| `packages/isomorphic/ariaSnapshot.ts` | 19236 |
+| `packages/isomorphic/ariaSnapshotRenderer.ts` | 6693 |
+| `packages/isomorphic/cssTokenizer.ts` | 26165 |
+| `packages/isomorphic/stringUtils.ts` | 8431 |
+| `packages/isomorphic/yaml.ts` | 2631 |
+
+## Why unmodified
+
+Not one line is edited, so updating is a copy and a diff rather than a merge.
+Two things make that possible:
+
+- **esbuild does not typecheck.** These files are written against Playwright's
+ tsconfig, not ours; ours would reject them. The bundler only transpiles, and
+ `browser-agent/` is excluded from the repo's `tsconfig.json` and eslint.
+- **The `@isomorphic/…` alias is resolved by the bundler**, not by an edit to
+ the import lines (`scripts/build-browser-agent.mjs`).
+
+`import type * as yamlTypes from 'yaml'` in `isomorphic/ariaSnapshot.ts` is a
+type-only import, erased before it reaches the bundle. There is no runtime
+dependency on the `yaml` package.
+
+## Updating
+
+Follow major versions only; there is no reason to track patches of a tree
+format that changes slowly.
+
+```
+git clone --depth 1 --filter=blob:none --sparse https://github.com/microsoft/playwright /tmp/pw
+cd /tmp/pw
+git sparse-checkout set packages/injected packages/isomorphic
+git fetch --depth 1 origin tag vX.Y.0 && git checkout vX.Y.0
+```
+
+Copy the nine files back over this directory, then check that the closure has
+not grown — a new `import` in any of them may pull in a tenth file:
+
+```
+pnpm browser:agent # esbuild fails loudly on an unresolved import
+pnpm browser:agent:probe # drives the rebuilt bundle in real Chrome
+```
+
+Watch for changes to `mode: 'ai'` in `injected/ariaSnapshot.ts`
+(`toInternalOptions`). That mode is the whole contract: `refs: 'interactable'`
+is what decides which elements an agent can name, and `renderCursorPointer` is
+what marks a roleless `
` that behaves like a button.
diff --git a/browser-agent/vendor/playwright/injected/ariaSnapshot.ts b/browser-agent/vendor/playwright/injected/ariaSnapshot.ts
new file mode 100644
index 0000000000..42e3a0842c
--- /dev/null
+++ b/browser-agent/vendor/playwright/injected/ariaSnapshot.ts
@@ -0,0 +1,549 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as aria from '@isomorphic/ariaSnapshot';
+import { renderAriaSnapshotAsYaml } from '@isomorphic/ariaSnapshotRenderer';
+import { normalizeWhiteSpace, truncateDataUrl } from '@isomorphic/stringUtils';
+
+import { distillAriaSnapshot } from './ariaSnapshotDistiller';
+import { computeBox, getElementComputedStyle, isElementVisible } from './domUtils';
+import * as roleUtils from './roleUtils';
+
+export type AriaSnapshot = {
+ root: aria.AriaNode;
+ info: Map;
+ refs: Map;
+ iframeRefs: string[];
+};
+
+type AriaRef = {
+ role: string;
+ name: string;
+ ref: string;
+};
+
+let lastRef = 0;
+
+export type AriaTreeOptions = {
+ mode: 'ai' | 'default' | 'codegen' | 'autoexpect';
+ refPrefix?: string;
+ doNotRenderActive?: boolean;
+ depth?: number;
+ boxes?: boolean;
+};
+
+type InternalOptions = {
+ visibility: 'aria' | 'ariaOrVisible' | 'ariaAndVisible',
+ refs: 'all' | 'interactable' | 'none',
+ refPrefix?: string,
+ includeGenericRole?: boolean,
+ renderCursorPointer?: boolean,
+ renderActive?: boolean,
+ renderBoxes?: boolean,
+};
+
+function toInternalOptions(options: AriaTreeOptions): InternalOptions {
+ const renderBoxes = options.boxes;
+ if (options.mode === 'ai') {
+ // For AI consumption.
+ return {
+ visibility: 'ariaOrVisible',
+ refs: 'interactable',
+ refPrefix: options.refPrefix,
+ includeGenericRole: true,
+ renderActive: !options.doNotRenderActive,
+ renderCursorPointer: true,
+ renderBoxes,
+ };
+ }
+ if (options.mode === 'autoexpect') {
+ // To auto-generate assertions on visible elements.
+ return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes };
+ }
+ // To match aria snapshot. In 'codegen' mode, the generated tree is the same,
+ // strings are converted to regexes when serializing to yaml.
+ return { visibility: 'aria', refs: 'none', renderBoxes };
+}
+
+export function generateAriaTree(rootElement: Element, publicOptions: AriaTreeOptions): AriaSnapshot {
+ const options = toInternalOptions(publicOptions);
+ const visited = new Set();
+ // For each node, the elements that contributed to its accessible name.
+ const nameSourceElements = new Map | undefined>();
+
+ const snapshot: AriaSnapshot = {
+ root: { role: 'fragment', name: '', children: [], props: {}, box: computeBox(rootElement), receivesPointerEvents: true },
+ info: new Map(),
+ refs: new Map(),
+ iframeRefs: [],
+ };
+ setAriaNodeElement(snapshot.root, rootElement);
+
+ const visit = (ariaNode: aria.AriaNode, node: Node, parentElementVisible: boolean) => {
+ if (visited.has(node))
+ return;
+ visited.add(node);
+
+ if (node.nodeType === Node.TEXT_NODE && node.nodeValue) {
+ if (!parentElementVisible)
+ return;
+
+ const text = node.nodeValue;
+ // should not report AAA as a child of the textarea.
+ if (ariaNode.role !== 'textbox' && text)
+ ariaNode.children.push(node.nodeValue || '');
+ return;
+ }
+
+ if (node.nodeType !== Node.ELEMENT_NODE)
+ return;
+
+ const element = node as Element;
+ const isElementVisibleForAria = !roleUtils.isElementHiddenForAria(element);
+ let visible = isElementVisibleForAria;
+ if (options.visibility === 'ariaOrVisible')
+ visible = isElementVisibleForAria || isElementVisible(element);
+ if (options.visibility === 'ariaAndVisible')
+ visible = isElementVisibleForAria && isElementVisible(element);
+
+ // Optimization: if we only consider aria visibility, we can skip child elements because
+ // they will not be visible for aria as well.
+ if (options.visibility === 'aria' && !visible)
+ return;
+
+ const ariaChildren: Element[] = [];
+ if (element.hasAttribute('aria-owns')) {
+ const ids = element.getAttribute('aria-owns')!.split(/\s+/);
+ for (const id of ids) {
+ const ownedElement = rootElement.ownerDocument.getElementById(id);
+ if (ownedElement)
+ ariaChildren.push(ownedElement);
+ }
+ }
+
+ const childAriaNode = visible ? toAriaNode(element, options, nameSourceElements) : null;
+ if (childAriaNode && element.getAttribute('aria-hidden')?.toLowerCase() === 'true')
+ childAriaNode.props['aria-hidden'] = 'true';
+ let elementInfo: { element: Element, nameFromContentRefs: string[] } | undefined;
+ if (childAriaNode) {
+ if (childAriaNode.ref) {
+ elementInfo = { element, nameFromContentRefs: [] };
+ snapshot.info.set(childAriaNode.ref, elementInfo);
+ snapshot.refs.set(element, childAriaNode.ref);
+ if (childAriaNode.role === 'iframe')
+ snapshot.iframeRefs.push(childAriaNode.ref);
+ }
+ ariaNode.children.push(childAriaNode);
+ }
+ processElement(childAriaNode || ariaNode, element, ariaChildren, visible);
+
+ // Now that the subtree is processed, every descendant that contributed to this node's
+ // accessible name has its ref assigned, so we can resolve those refs as the name's origins.
+ if (elementInfo) {
+ for (const contributor of nameSourceElements.get(childAriaNode!) || []) {
+ const ref = snapshot.refs.get(contributor);
+ if (ref && ref !== childAriaNode!.ref)
+ elementInfo.nameFromContentRefs.push(ref);
+ }
+ }
+ };
+
+ function processElement(ariaNode: aria.AriaNode, element: Element, ariaChildren: Element[], parentElementVisible: boolean) {
+ // Surround every element with spaces for the sake of concatenated text nodes.
+ const display = getElementComputedStyle(element)?.display || 'inline';
+ const treatAsBlock = (display !== 'inline' || element.nodeName === 'BR') ? ' ' : '';
+ if (treatAsBlock)
+ ariaNode.children.push(treatAsBlock);
+
+ ariaNode.children.push(roleUtils.getCSSContent(element, '::before') || '');
+ const assignedNodes = element.nodeName === 'SLOT' ? (element as HTMLSlotElement).assignedNodes() : [];
+ if (assignedNodes.length) {
+ for (const child of assignedNodes)
+ visit(ariaNode, child, parentElementVisible);
+ } else {
+ for (let child = element.firstChild; child; child = child.nextSibling) {
+ if (!(child as Element | Text).assignedSlot)
+ visit(ariaNode, child, parentElementVisible);
+ }
+ if (element.shadowRoot) {
+ for (let child = element.shadowRoot.firstChild; child; child = child.nextSibling)
+ visit(ariaNode, child, parentElementVisible);
+ }
+ }
+
+ for (const child of ariaChildren)
+ visit(ariaNode, child, parentElementVisible);
+
+ ariaNode.children.push(roleUtils.getCSSContent(element, '::after') || '');
+
+ if (treatAsBlock)
+ ariaNode.children.push(treatAsBlock);
+
+ if (ariaNode.children.length === 1 && ariaNode.name === ariaNode.children[0])
+ ariaNode.children = [];
+
+ if (ariaNode.role === 'link' && element.hasAttribute('href')) {
+ const href = element.getAttribute('href')!;
+ ariaNode.props['url'] = truncateDataUrl(href);
+ }
+
+ if (ariaNode.role === 'textbox' && element.hasAttribute('placeholder') && element.getAttribute('placeholder') !== ariaNode.name) {
+ const placeholder = element.getAttribute('placeholder')!;
+ ariaNode.props['placeholder'] = placeholder;
+ }
+ }
+
+ roleUtils.beginAriaCaches();
+ try {
+ visit(snapshot.root, rootElement, true);
+ } finally {
+ roleUtils.endAriaCaches();
+ }
+
+ distillAriaSnapshot(snapshot, publicOptions);
+ return snapshot;
+}
+
+function computeAriaRef(ariaNode: aria.AriaNode, options: InternalOptions) {
+ if (options.refs === 'none')
+ return;
+ if (options.refs === 'interactable' && (!ariaNode.box.visible || !ariaNode.receivesPointerEvents))
+ return;
+
+ const element = ariaNodeElement(ariaNode);
+ let ariaRef = (element as any)._ariaRef as AriaRef | undefined;
+ if (!ariaRef || ariaRef.role !== ariaNode.role || ariaRef.name !== ariaNode.name) {
+ ariaRef = { role: ariaNode.role, name: ariaNode.name, ref: (options.refPrefix ?? '') + 'e' + (++lastRef) };
+ (element as any)._ariaRef = ariaRef;
+ }
+ ariaNode.ref = ariaRef.ref;
+}
+
+function toAriaNode(element: Element, options: InternalOptions, nameSourceElements: Map | undefined>): aria.AriaNode | null {
+ const active = element.ownerDocument.activeElement === element && element.ownerDocument.hasFocus();
+ if (element.nodeName === 'IFRAME' || element.nodeName === 'FRAME') {
+ const ariaNode: aria.AriaNode = {
+ role: 'iframe',
+ name: '',
+ children: [],
+ props: {},
+ box: computeBox(element),
+ receivesPointerEvents: true,
+ active
+ };
+ setAriaNodeElement(ariaNode, element);
+ computeAriaRef(ariaNode, options);
+ return ariaNode;
+ }
+
+ const defaultRole = options.includeGenericRole ? 'generic' : null;
+ const role = roleUtils.getAriaRole(element) ?? defaultRole;
+ if (!role || role === 'presentation' || role === 'none')
+ return null;
+
+ const name = roleUtils.getElementAccessibleName(element, false);
+ const receivesPointerEvents = roleUtils.receivesPointerEvents(element);
+
+ const box = computeBox(element);
+ if (role === 'generic' && box.inline && element.childNodes.length === 1 && element.childNodes[0].nodeType === Node.TEXT_NODE)
+ return null;
+
+ const result: aria.AriaNode = {
+ role,
+ name: normalizeWhiteSpace(name.text),
+ children: [],
+ props: {},
+ box,
+ receivesPointerEvents,
+ active
+ };
+ setAriaNodeElement(result, element);
+ nameSourceElements.set(result, name.elements);
+ computeAriaRef(result, options);
+
+ if (roleUtils.kAriaCheckedRoles.includes(role))
+ result.checked = roleUtils.getAriaChecked(element);
+
+ if (roleUtils.kAriaDisabledRoles.includes(role))
+ result.disabled = roleUtils.getAriaDisabled(element);
+
+ if (roleUtils.kAriaExpandedRoles.includes(role))
+ result.expanded = roleUtils.getAriaExpanded(element);
+
+ if (roleUtils.kAriaInvalidRoles.includes(role)) {
+ const invalid = roleUtils.getAriaInvalid(element);
+ result.invalid = invalid === 'false' ? false : invalid === 'true' ? true : invalid;
+ }
+
+ if (roleUtils.kAriaLevelRoles.includes(role))
+ result.level = roleUtils.getAriaLevel(element);
+
+ if (roleUtils.kAriaPressedRoles.includes(role))
+ result.pressed = roleUtils.getAriaPressed(element);
+
+ if (roleUtils.kAriaSelectedRoles.includes(role))
+ result.selected = roleUtils.getAriaSelected(element);
+
+ if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
+ if (element.type !== 'checkbox' && element.type !== 'radio' && element.type !== 'file')
+ result.children = [element.value];
+ }
+
+ return result;
+}
+
+function matchesStringOrRegex(text: string, template: aria.AriaRegex | string | undefined): boolean {
+ if (!template)
+ return true;
+ if (!text)
+ return false;
+ if (typeof template === 'string')
+ return text === template;
+ return !!text.match(new RegExp(template.pattern));
+}
+
+function matchesTextValue(text: string, template: aria.AriaTextValue | undefined) {
+ if (!template?.normalized)
+ return true;
+ if (!text)
+ return false;
+ if (text === template.normalized)
+ return true;
+ // Accept pattern as value.
+ if (text === template.raw)
+ return true;
+
+ const regex = cachedRegex(template);
+ if (regex)
+ return !!text.match(regex);
+ return false;
+}
+
+const cachedRegexSymbol = Symbol('cachedRegex');
+
+function cachedRegex(template: aria.AriaTextValue): RegExp | null {
+ if ((template as any)[cachedRegexSymbol] !== undefined)
+ return (template as any)[cachedRegexSymbol];
+
+ const { raw } = template;
+ const canBeRegex = raw.startsWith('/') && raw.endsWith('/') && raw.length > 1;
+ let regex: RegExp | null;
+ try {
+ regex = canBeRegex ? new RegExp(raw.slice(1, -1)) : null;
+ } catch (e) {
+ regex = null;
+ }
+ (template as any)[cachedRegexSymbol] = regex;
+ return regex;
+}
+
+export type MatcherReceived = {
+ raw: string;
+ regex: string;
+};
+
+export function matchesExpectAriaTemplate(rootElement: Element, template: aria.AriaTemplateNode): { matches: aria.AriaNode[], received: MatcherReceived } {
+ const snapshot = generateAriaTree(rootElement, { mode: 'default' });
+ const matches = matchesNodeDeep(snapshot.root, template, false, false);
+ const { json } = renderAriaTreeAsJSON(snapshot, { mode: 'default' });
+ return {
+ matches,
+ received: {
+ raw: renderAriaSnapshotAsYaml(json),
+ regex: renderAriaSnapshotAsYaml(json, { convertStringsToRegex: true }),
+ }
+ };
+}
+
+export function getAllElementsMatchingExpectAriaTemplate(rootElement: Element, template: aria.AriaTemplateNode): Element[] {
+ const root = generateAriaTree(rootElement, { mode: 'default' }).root;
+ const matches = matchesNodeDeep(root, template, true, false);
+ return matches.map(n => ariaNodeElement(n));
+}
+
+function matchesNode(node: aria.AriaNode | string, template: aria.AriaTemplateNode, isDeepEqual: boolean): boolean {
+ if (typeof node === 'string' && template.kind === 'text')
+ return matchesTextValue(node, template.text);
+
+ if (node === null || typeof node !== 'object' || template.kind !== 'role')
+ return false;
+
+ if (template.role !== 'fragment' && template.role !== node.role)
+ return false;
+ if (template.checked !== undefined && template.checked !== node.checked)
+ return false;
+ if (template.disabled !== undefined && template.disabled !== node.disabled)
+ return false;
+ if (template.expanded !== undefined && template.expanded !== node.expanded)
+ return false;
+ if (template.invalid !== undefined && template.invalid !== node.invalid)
+ return false;
+ if (template.level !== undefined && template.level !== node.level)
+ return false;
+ if (template.pressed !== undefined && template.pressed !== node.pressed)
+ return false;
+ if (template.selected !== undefined && template.selected !== node.selected)
+ return false;
+ if (!matchesStringOrRegex(node.name, template.name))
+ return false;
+ if (!matchesTextValue(node.props.url, template.props?.url))
+ return false;
+
+ // Proceed based on the container mode.
+ if (template.containerMode === 'contain')
+ return containsList(node.children || [], template.children || []);
+ if (template.containerMode === 'equal')
+ return listEqual(node.children || [], template.children || [], false);
+ if (template.containerMode === 'deep-equal' || isDeepEqual)
+ return listEqual(node.children || [], template.children || [], true);
+ return containsList(node.children || [], template.children || []);
+}
+
+function listEqual(children: (aria.AriaNode | string)[], template: aria.AriaTemplateNode[], isDeepEqual: boolean): boolean {
+ if (template.length !== children.length)
+ return false;
+ for (let i = 0; i < template.length; ++i) {
+ if (!matchesNode(children[i], template[i], isDeepEqual))
+ return false;
+ }
+ return true;
+}
+
+function containsList(children: (aria.AriaNode | string)[], template: aria.AriaTemplateNode[]): boolean {
+ if (template.length > children.length)
+ return false;
+ const cc = children.slice();
+ const tt = template.slice();
+ for (const t of tt) {
+ let c = cc.shift();
+ while (c) {
+ if (matchesNode(c, t, false))
+ break;
+ c = cc.shift();
+ }
+ if (!c)
+ return false;
+ }
+ return true;
+}
+
+function matchesNodeDeep(root: aria.AriaNode, template: aria.AriaTemplateNode, collectAll: boolean, isDeepEqual: boolean): aria.AriaNode[] {
+ const results: aria.AriaNode[] = [];
+ const visit = (node: aria.AriaNode | string, parent: aria.AriaNode | null): boolean => {
+ if (matchesNode(node, template, isDeepEqual)) {
+ const result = typeof node === 'string' ? parent : node;
+ if (result)
+ results.push(result);
+ return !collectAll;
+ }
+ if (typeof node === 'string')
+ return false;
+ for (const child of node.children || []) {
+ if (visit(child, node))
+ return true;
+ }
+ return false;
+ };
+ visit(root, null);
+ return results;
+}
+
+export function renderAriaTreeAsJSON(ariaSnapshot: AriaSnapshot, publicOptions: AriaTreeOptions): { json: aria.AriaSnapshotJSON, iframeDepths: Record } {
+ const options = toInternalOptions(publicOptions);
+ const iframeDepths: Record = {};
+
+ const visit = (ariaNode: aria.AriaNode, depth: number, renderCursorPointer: boolean): aria.AriaNodeJSON => {
+ if (ariaNode.role === 'iframe' && ariaNode.ref)
+ iframeDepths[ariaNode.ref] = depth;
+
+ const node: aria.AriaNodeJSON = { role: ariaNode.role as aria.AriaNodeJSON['role'] };
+ if (ariaNode.name)
+ node.name = ariaNode.name;
+ if (ariaNode.checked === 'mixed' || ariaNode.checked === true)
+ node.checked = ariaNode.checked;
+ if (ariaNode.disabled)
+ node.disabled = true;
+ if (ariaNode.expanded)
+ node.expanded = true;
+ if (ariaNode.active && options.renderActive)
+ node.active = true;
+ if (ariaNode.invalid)
+ node.invalid = ariaNode.invalid;
+ if (ariaNode.level)
+ node.level = ariaNode.level;
+ if (ariaNode.pressed === 'mixed' || ariaNode.pressed === true)
+ node.pressed = ariaNode.pressed;
+ if (ariaNode.selected === true)
+ node.selected = true;
+ if (ariaNode.ref) {
+ node.ref = ariaNode.ref;
+ if (renderCursorPointer && aria.hasPointerCursor(ariaNode))
+ node.cursor = 'pointer';
+ }
+ if (options.renderBoxes) {
+ const element = ariaNodeElement(ariaNode);
+ if (element) {
+ const r = element.getBoundingClientRect();
+ node.box = { x: Math.round(r.x), y: Math.round(r.y), width: Math.round(r.width), height: Math.round(r.height) };
+ }
+ }
+ if (ariaNode.props.url !== undefined)
+ node.url = ariaNode.props.url;
+ if (ariaNode.props.placeholder !== undefined)
+ node.placeholder = ariaNode.props.placeholder;
+ if (ariaNode.props['aria-hidden'] !== undefined)
+ node.ariaHidden = true;
+
+ const singleTextChild = ariaNode.children.length === 1 && typeof ariaNode.children[0] === 'string' ? ariaNode.children[0] : undefined;
+ const isAtDepthLimit = !!publicOptions.depth && depth === publicOptions.depth;
+ if (singleTextChild !== undefined) {
+ node.text = singleTextChild;
+ } else if (!isAtDepthLimit && ariaNode.children.length) {
+ const inCursorPointer = !!ariaNode.ref && renderCursorPointer && aria.hasPointerCursor(ariaNode);
+ node.children = ariaNode.children.map(child => {
+ if (typeof child === 'string')
+ return child;
+ return visit(child, depth + 1, renderCursorPointer && !inCursorPointer);
+ });
+ }
+ return node;
+ };
+
+ const json: aria.AriaSnapshotJSON = [];
+ const nodesToRender = ariaSnapshot.root.role === 'fragment' ? ariaSnapshot.root.children : [ariaSnapshot.root];
+ for (const nodeToRender of nodesToRender) {
+ if (typeof nodeToRender === 'string')
+ json.push({ role: 'text', text: nodeToRender });
+ else
+ json.push(visit(nodeToRender, 0, !!options.renderCursorPointer));
+ }
+ return { json, iframeDepths };
+}
+
+const elementSymbol = Symbol('element');
+
+function ariaNodeElement(ariaNode: aria.AriaNode): Element {
+ return (ariaNode as any)[elementSymbol];
+}
+
+function setAriaNodeElement(ariaNode: aria.AriaNode, element: Element) {
+ (ariaNode as any)[elementSymbol] = element;
+}
+
+export function findNewElement(from: aria.AriaNode | undefined, to: aria.AriaNode): Element | undefined {
+ const node = aria.findNewNode(from, to);
+ return node ? ariaNodeElement(node) : undefined;
+}
diff --git a/browser-agent/vendor/playwright/injected/ariaSnapshotDistiller.ts b/browser-agent/vendor/playwright/injected/ariaSnapshotDistiller.ts
new file mode 100644
index 0000000000..02ab6c0f1f
--- /dev/null
+++ b/browser-agent/vendor/playwright/injected/ariaSnapshotDistiller.ts
@@ -0,0 +1,263 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { hasPointerCursor } from '@isomorphic/ariaSnapshot';
+import { normalizeWhiteSpace } from '@isomorphic/stringUtils';
+
+import type * as aria from '@isomorphic/ariaSnapshot';
+import type { AriaSnapshot, AriaTreeOptions } from './ariaSnapshot';
+
+// Distillation makes the snapshot less verbose without losing information: after the full tree is
+// built, a single traversal applies the chained plugins below, babel-style. Each plugin is a
+// visitor: `enter` runs pre-order, `exit` runs post-order after the children were traversed - and
+// possibly removed, unwrapped or inlined. Either hook can detach the node by returning 'remove'
+// (from `enter`, the subtree is then not traversed and no further hooks run for it), or replace
+// the node with its children by returning 'unwrap' (from `enter`, the hoisted children are
+// re-visited in the node's place; from `exit`, they were already traversed and are spliced in as
+// is). Plugins mutate the tree in place; `snapshot.info` and `snapshot.refs` are left intact, so
+// refs of removed nodes still resolve through the aria-ref selector engine.
+type DistillerContext = {
+ snapshot: AriaSnapshot;
+ // Depth of the current node; children of the root fragment are at depth 0.
+ depth: number;
+ // Render depth limit, plugins should not rely on anything below it being rendered.
+ maxDepth: number | undefined;
+ // The chain of ancestors of the current node, root first. Maintained by the traversal.
+ ancestors: aria.AriaNode[];
+ // Content refs of the entered nodes' accessible names that are not yet represented in the
+ // output - see `removeRedundantNames`.
+ pendingContentRefs: Set;
+};
+
+type DistillerPlugin = {
+ name: string;
+ enter?(node: aria.AriaNode, ctx: DistillerContext): 'remove' | 'unwrap' | void;
+ exit?(node: aria.AriaNode, ctx: DistillerContext): 'remove' | 'unwrap' | void;
+};
+
+export function distillAriaSnapshot(snapshot: AriaSnapshot, options: Pick) {
+ runPlugins(snapshot, options.mode === 'ai' ? aiPlugins : normalizePlugins, options);
+}
+
+function runPlugins(snapshot: AriaSnapshot, plugins: DistillerPlugin[], options: Pick) {
+ const ctx: DistillerContext = { snapshot, depth: -1, maxDepth: options.depth, ancestors: [], pendingContentRefs: new Set() };
+ const traverse = (node: aria.AriaNode, depth: number) => {
+ const children: (aria.AriaNode | string)[] = [];
+ const visitChild = (child: aria.AriaNode | string) => {
+ if (typeof child === 'string') {
+ children.push(child);
+ return;
+ }
+ ctx.depth = depth + 1;
+ for (const plugin of plugins) {
+ const result = plugin.enter?.(child, ctx);
+ if (result === 'remove')
+ return;
+ if (result === 'unwrap') {
+ child.children.forEach(visitChild);
+ return;
+ }
+ }
+ traverse(child, depth + 1);
+ ctx.depth = depth + 1;
+ for (const plugin of plugins) {
+ const result = plugin.exit?.(child, ctx);
+ if (result === 'remove')
+ return;
+ if (result === 'unwrap') {
+ children.push(...child.children);
+ return;
+ }
+ }
+ children.push(child);
+ };
+ ctx.ancestors.push(node);
+ node.children.forEach(visitChild);
+ ctx.ancestors.pop();
+ node.children = children;
+ };
+ // Hooks run on the root as well, but the root cannot be removed or unwrapped.
+ for (const plugin of plugins)
+ plugin.enter?.(snapshot.root, ctx);
+ traverse(snapshot.root, -1);
+ ctx.depth = -1;
+ for (const plugin of plugins)
+ plugin.exit?.(snapshot.root, ctx);
+}
+
+// A generic node whose only content is text - it carries no structure of its own.
+function isLeafGeneric(node: aria.AriaNode): boolean {
+ return node.role === 'generic' && node.children.every(child => typeof child === 'string');
+}
+
+// Removing the click target root would hide an actionable element from the snapshot.
+function isClickTargetRoot(node: aria.AriaNode, ctx: DistillerContext): boolean {
+ return !!node.ref && hasPointerCursor(node) && !ctx.ancestors.some(ancestor => !!ancestor.ref && hasPointerCursor(ancestor));
+}
+
+// The tree builder emits raw text tokens - text nodes, CSS content, block spacing markers - as
+// string children. Coalesce the adjacent ones, normalize whitespace and drop the empties, then
+// drop a lone text child that merely repeats the node's accessible name. Runs on `exit`, so the
+// merge sees the children in their final shape.
+const mergeStringChildren: DistillerPlugin = {
+ name: 'mergeStringChildren',
+ exit(node: aria.AriaNode) {
+ const children: (aria.AriaNode | string)[] = [];
+ const buffer: string[] = [];
+ const flush = () => {
+ if (!buffer.length)
+ return;
+ const text = normalizeWhiteSpace(buffer.join(''));
+ if (text)
+ children.push(text);
+ buffer.length = 0;
+ };
+ for (const child of node.children) {
+ if (typeof child === 'string') {
+ buffer.push(child);
+ } else {
+ flush();
+ children.push(child);
+ }
+ }
+ flush();
+ node.children = children;
+ if (node.children.length === 1 && node.children[0] === node.name)
+ node.children = [];
+ },
+};
+
+// Only unwrap a generic that encloses at most one element, logical grouping still makes sense,
+// even if it is not ref-able. The decision is made on `exit` - whether the node encloses a single
+// ref-bearing child is only known after its own descendants were unwrapped - so nested wrappers
+// collapse bottom-up. A generic emptied by the other plugins is dropped, unless it is the
+// click target root, for example an icon-only button.
+const unwrapSingleChildGenerics: DistillerPlugin = {
+ name: 'unwrapSingleChildGenerics',
+ exit(node: aria.AriaNode, ctx: DistillerContext): 'unwrap' | void {
+ if (node.role !== 'generic' || node.name || node.children.length > 1 || !node.children.every(child => typeof child !== 'string' && !!child.ref))
+ return;
+ if (!node.children.length && isClickTargetRoot(node, ctx))
+ return;
+ return 'unwrap';
+ },
+};
+
+// A decorative image - role `img` with no accessible name and no content - carries no
+// information. The decision is made on `exit` - whether the node has content is only known after
+// `mergeStringChildren` dropped the empty text tokens. A clickable image outside of any clickable
+// container is not decorative though - e.g. a bare svg icon acting as a button - and is kept.
+const removeNamelessImages: DistillerPlugin = {
+ name: 'removeNamelessImages',
+ exit(node: aria.AriaNode, ctx: DistillerContext): 'remove' | void {
+ if (node.role === 'img' && !node.name && !node.children.length && !isClickTargetRoot(node, ctx))
+ return 'remove';
+ },
+};
+
+// The node's accessible name is derived from content; when every node that contributed to it is
+// represented in the output anyway, the name would just repeat that content and is dropped.
+// Single-pass bookkeeping over the shared `pendingContentRefs` set: entering a node clears its
+// own ref - it is now represented - except for leaf generics, which only exist to supply text
+// and are dropped by `removeNameRepeatingChild` once a kept name shows it. On exit, either every
+// contributor was cleared and the name goes, or the kept name now represents its contributors,
+// so they are cleared for the benefit of the ancestors. A node removed on enter never clears its
+// ref, and an unwrapped one does - matching what remains in the tree.
+const removeRedundantNames: DistillerPlugin = {
+ name: 'removeRedundantNames',
+ enter(node: aria.AriaNode, ctx: DistillerContext) {
+ if (!node.ref)
+ return;
+ for (const ref of ctx.snapshot.info.get(node.ref)?.nameFromContentRefs || [])
+ ctx.pendingContentRefs.add(ref);
+ const beyondDepth = !!ctx.maxDepth && ctx.depth > ctx.maxDepth;
+ if (!beyondDepth && !isLeafGeneric(node))
+ ctx.pendingContentRefs.delete(node.ref);
+ },
+ exit(node: aria.AriaNode, ctx: DistillerContext) {
+ if (!node.ref)
+ return;
+ const nameFromContentRefs = ctx.snapshot.info.get(node.ref)?.nameFromContentRefs;
+ if (!nameFromContentRefs?.length)
+ return;
+ if (nameFromContentRefs.every(ref => !ctx.pendingContentRefs.has(ref))) {
+ node.name = '';
+ } else {
+ for (const ref of nameFromContentRefs)
+ ctx.pendingContentRefs.delete(ref);
+ }
+ },
+};
+
+// A generic whose whole content is a piece of text - a single text child, or just an accessible
+// name - that repeats the parent's accessible name adds no information, so it removes itself.
+// `inlineTextIntoGeneric` runs first, bubbling text up through nameless wrappers, so by the time
+// a wrapper exits its text faces the real parent - no need to look further up the ancestor chain.
+// The removed text then only survives through the names derived from it, so the node's ref is
+// marked pending again - it may have been cleared on enter, before the node's other children
+// (e.g. a decorative image) were distilled away - and `removeRedundantNames` keeps those names.
+const removeNameRepeatingChild: DistillerPlugin = {
+ name: 'removeNameRepeatingChild',
+ exit(node: aria.AriaNode, ctx: DistillerContext): 'remove' | void {
+ const parent = ctx.ancestors[ctx.ancestors.length - 1];
+ if (!parent?.name || node.role !== 'generic' || node.active || Object.keys(node.props).length)
+ return;
+ const singleTextChild = node.children.length === 1 && typeof node.children[0] === 'string' ? node.children[0] : undefined;
+ const text = node.name ? (node.children.length ? undefined : node.name) : singleTextChild;
+ if (text && text === parent.name) {
+ if (node.ref)
+ ctx.pendingContentRefs.add(node.ref);
+ return 'remove';
+ }
+ },
+};
+
+// A generic whose only child is a nameless leaf generic inlines that child's text:
+// `generic: - generic: "text"` becomes `generic: "text"`. Runs post-order, so chains collapse
+// bottom-up, and after the other plugins already removed or unwrapped the children.
+const inlineTextIntoGeneric: DistillerPlugin = {
+ name: 'inlineTextIntoGeneric',
+ exit(node: aria.AriaNode) {
+ if (node.role !== 'generic' || Object.keys(node.props).length || node.children.length !== 1)
+ return;
+ const child = node.children[0];
+ if (typeof child === 'string')
+ return;
+ if (child.role !== 'generic' || child.name || child.active || Object.keys(child.props).length)
+ return;
+ if (child.children.length === 1 && typeof child.children[0] === 'string')
+ node.children = [child.children[0]];
+ },
+};
+
+// Structural normalization applies to all modes - it defines the canonical tree shape.
+const normalizePlugins: DistillerPlugin[] = [
+ mergeStringChildren,
+ unwrapSingleChildGenerics,
+];
+
+// The ai preset compresses the snapshot on top of normalization. It runs as one traversal:
+// `removeRedundantNames` bookkeeping must observe every node the tree retains, including the
+// wrappers that `unwrapSingleChildGenerics` is about to unwrap. On exit, text is first inlined
+// into the node, so that `removeNameRepeatingChild` faces the real parent when it compares.
+const aiPlugins: DistillerPlugin[] = [
+ mergeStringChildren,
+ removeNamelessImages,
+ removeRedundantNames,
+ inlineTextIntoGeneric,
+ removeNameRepeatingChild,
+ unwrapSingleChildGenerics,
+];
diff --git a/browser-agent/vendor/playwright/injected/domUtils.ts b/browser-agent/vendor/playwright/injected/domUtils.ts
new file mode 100644
index 0000000000..c88184506a
--- /dev/null
+++ b/browser-agent/vendor/playwright/injected/domUtils.ts
@@ -0,0 +1,194 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+type GlobalOptions = {
+ browserNameForWorkarounds?: string;
+};
+let globalOptions: GlobalOptions = {};
+export function setGlobalOptions(options: GlobalOptions) {
+ globalOptions = options;
+}
+export function getGlobalOptions(): GlobalOptions {
+ return globalOptions;
+}
+
+export function isInsideScope(scope: Node, element: Element | undefined): boolean {
+ while (element) {
+ if (scope.contains(element))
+ return true;
+ element = enclosingShadowHost(element);
+ }
+ return false;
+}
+
+export function enclosingElement(node: Node) {
+ if (node.nodeType === 1 /* Node.ELEMENT_NODE */)
+ return node as Element;
+ return node.parentElement ?? undefined;
+}
+
+export function parentElementOrShadowHost(element: Element): Element | undefined {
+ if (element.parentElement)
+ return element.parentElement;
+ if (!element.parentNode)
+ return;
+ if (element.parentNode.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ && (element.parentNode as ShadowRoot).host)
+ return (element.parentNode as ShadowRoot).host;
+}
+
+export function enclosingShadowRootOrDocument(element: Element): Document | ShadowRoot | undefined {
+ let node: Node = element;
+ while (node.parentNode)
+ node = node.parentNode;
+ if (node.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ || node.nodeType === 9 /* Node.DOCUMENT_NODE */)
+ return node as Document | ShadowRoot;
+}
+
+function enclosingShadowHost(element: Element): Element | undefined {
+ while (element.parentElement)
+ element = element.parentElement;
+ return parentElementOrShadowHost(element);
+}
+
+// Assumption: if scope is provided, element must be inside scope's subtree.
+export function closestCrossShadow(element: Element | undefined, css: string, scope?: Document | Element): Element | undefined {
+ while (element) {
+ const closest = element.closest(css);
+ if (scope && closest !== scope && closest?.contains(scope))
+ return;
+ if (closest)
+ return closest;
+ element = enclosingShadowHost(element);
+ }
+}
+
+export function getElementComputedStyle(element: Element, pseudo?: string): CSSStyleDeclaration | undefined {
+ const cache = pseudo === '::before' ? cacheStyleBefore : pseudo === '::after' ? cacheStyleAfter : cacheStyle;
+ if (cache && cache.has(element))
+ return cache.get(element);
+ const style = element.ownerDocument && element.ownerDocument.defaultView ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo) : undefined;
+ cache?.set(element, style);
+ return style;
+}
+
+export function isElementStyleVisibilityVisible(element: Element, style?: CSSStyleDeclaration): boolean {
+ const cached = cacheStyleVisibility?.get(element);
+ if (cached !== undefined)
+ return cached;
+ const result = computeElementStyleVisibilityVisible(element, style);
+ cacheStyleVisibility?.set(element, result);
+ return result;
+}
+
+function computeElementStyleVisibilityVisible(element: Element, style?: CSSStyleDeclaration): boolean {
+ style = style ?? getElementComputedStyle(element);
+ if (!style)
+ return true;
+ // Element.checkVisibility checks for content-visibility and also looks at
+ // styles up the flat tree including user-agent ShadowRoots, such as the
+ // details element for example.
+ // All the browser implement it, but WebKit has a bug which prevents us from using it:
+ // https://bugs.webkit.org/show_bug.cgi?id=264733
+ // @ts-ignore
+ if (Element.prototype.checkVisibility && globalOptions.browserNameForWorkarounds !== 'webkit') {
+ if (!element.checkVisibility())
+ return false;
+ } else {
+ // Manual workaround for WebKit that does not have checkVisibility.
+ const detailsOrSummary = element.closest('details,summary');
+ if (detailsOrSummary !== element && detailsOrSummary?.nodeName === 'DETAILS' && !(detailsOrSummary as HTMLDetailsElement).open)
+ return false;
+ }
+ if (style.visibility !== 'visible')
+ return false;
+ return true;
+}
+
+export function computeBox(element: Element) {
+ // Note: this logic should be similar to waitForDisplayedAtStablePosition() to avoid surprises.
+ const style = getElementComputedStyle(element);
+ if (!style)
+ return { visible: true, inline: false };
+ const cursor = style.cursor;
+ if (style.display === 'contents') {
+ // display:contents is not rendered itself, but its child nodes are.
+ for (let child = element.firstChild; child; child = child.nextSibling) {
+ if (child.nodeType === 1 /* Node.ELEMENT_NODE */ && isElementVisible(child as Element))
+ return { visible: true, inline: false, cursor };
+ if (child.nodeType === 3 /* Node.TEXT_NODE */ && isVisibleTextNode(child as Text))
+ return { visible: true, inline: true, cursor };
+ }
+ return { visible: false, inline: false, cursor };
+ }
+ if (!isElementStyleVisibilityVisible(element, style))
+ return { cursor, visible: false, inline: false };
+ const rect = element.getBoundingClientRect();
+ return { cursor, visible: rect.width > 0 && rect.height > 0, inline: style.display === 'inline' };
+}
+
+export function isElementVisible(element: Element): boolean {
+ return computeBox(element).visible;
+}
+
+export function isVisibleTextNode(node: Text) {
+ // https://stackoverflow.com/questions/1461059/is-there-an-equivalent-to-getboundingclientrect-for-text-nodes
+ const range = node.ownerDocument.createRange();
+ range.selectNode(node);
+ const rect = range.getBoundingClientRect();
+ return rect.width > 0 && rect.height > 0;
+}
+
+export function elementSafeTagName(element: Element) {
+ const tagName = element.tagName;
+ if (typeof tagName === 'string') { // Fast path.
+ // Tag names in html documents are already uppercase. Lowercase names come from
+ // svg/mathml elements and from xml/xhtml documents, and they all start with
+ // a lowercase letter, so uppercasing can be skipped otherwise.
+ const firstCharCode = tagName.charCodeAt(0);
+ if (firstCharCode >= 97 && firstCharCode <= 122)
+ return tagName.toUpperCase();
+ return tagName;
+ }
+ // Named inputs, e.g. , will be exposed as fields on the parent