From 21476c3884f15187274070d379d4114c000580b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Mon, 17 Aug 2026 21:34:50 +0800 Subject: [PATCH 1/3] feat(record): capture Trace v3 states after pages settle Wait for DOM quiet after actions, match targets to VOM refs, and reduce drafts into deduped state-linked Trace v3 while keeping the v2 reducer. Co-authored-by: Cursor --- .../src/lib/__tests__/match-target.test.ts | 116 +++ .../src/lib/__tests__/page-settled.test.ts | 43 ++ .../lib/__tests__/record-observation.test.ts | 206 ++++++ .../lib/__tests__/trace-reducer-v2.test.ts | 75 ++ apps/extension/src/lib/describe-target.ts | 12 +- .../src/lib/format-observation-file.ts | 62 ++ apps/extension/src/lib/match-target.ts | 103 +++ apps/extension/src/lib/page-settled.ts | 92 +++ apps/extension/src/lib/record-constants.ts | 97 +++ apps/extension/src/lib/record-observation.ts | 681 ++++++++++++++++++ apps/extension/src/lib/trace-reducer-v2.ts | 301 ++++++++ 11 files changed, 1784 insertions(+), 4 deletions(-) create mode 100644 apps/extension/src/lib/__tests__/match-target.test.ts create mode 100644 apps/extension/src/lib/__tests__/page-settled.test.ts create mode 100644 apps/extension/src/lib/__tests__/record-observation.test.ts create mode 100644 apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts create mode 100644 apps/extension/src/lib/format-observation-file.ts create mode 100644 apps/extension/src/lib/match-target.ts create mode 100644 apps/extension/src/lib/page-settled.ts create mode 100644 apps/extension/src/lib/record-constants.ts create mode 100644 apps/extension/src/lib/record-observation.ts create mode 100644 apps/extension/src/lib/trace-reducer-v2.ts diff --git a/apps/extension/src/lib/__tests__/match-target.test.ts b/apps/extension/src/lib/__tests__/match-target.test.ts new file mode 100644 index 00000000..f4edddc5 --- /dev/null +++ b/apps/extension/src/lib/__tests__/match-target.test.ts @@ -0,0 +1,116 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import { describe, expect, it } from "vitest"; +import type { CapturedNode } from "@/tools/vom/capture"; +import { matchTarget } from "../match-target"; + +function node(backendNodeId: number, overrides: Partial = {}): CapturedNode { + return { + backendNodeId, + parentBackendNodeId: null, + tag: "button", + attrs: {}, + rect: { x: 10, y: 20, w: 100, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + ...overrides, + }; +} + +describe("matchTarget", () => { + const refs: RenderedRef[] = [ + { ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 }, + ]; + + it("matches a unique node by viewport coordinates", () => { + const target = matchTarget({ + geometry: { + rect: { x: 10, y: 20, w: 100, h: 30 }, + scrollX: 0, + scrollY: 0, + position: "static", + tag: "button", + }, + captured: [node(42)], + refs, + fallback: { tag: "button", name: "发布" }, + }); + expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); + }); + + it("matches a non-fixed node after the top frame scrolls", () => { + const target = matchTarget({ + geometry: { + rect: { x: 10, y: 20, w: 100, h: 30 }, + scrollX: 40, + scrollY: 300, + position: "static", + tag: "button", + }, + captured: [ + node(42, { + documentRect: { x: 50, y: 320, w: 100, h: 30 }, + localRect: { x: 10, y: 20, w: 100, h: 30 }, + rect: { x: 10, y: 20, w: 100, h: 30 }, + }), + ], + refs, + fallback: { tag: "button", name: "发布" }, + }); + + expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); + }); + + it("matches a static node by document coordinates when scroll changed after observation", () => { + const target = matchTarget({ + geometry: { + rect: { x: 10, y: 50, w: 100, h: 30 }, + scrollX: 0, + scrollY: 250, + position: "static", + tag: "button", + }, + captured: [ + node(42, { + documentRect: { x: 10, y: 300, w: 100, h: 30 }, + localRect: { x: 10, y: 200, w: 100, h: 30 }, + rect: { x: 10, y: 200, w: 100, h: 30 }, + }), + ], + refs, + fallback: { tag: "button", name: "发布" }, + }); + + expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); + }); + + it("returns unmatched when multiple candidates match", () => { + const target = matchTarget({ + geometry: { + rect: { x: 10, y: 20, w: 100, h: 30 }, + scrollX: 0, + scrollY: 0, + position: "static", + tag: "button", + }, + captured: [node(42), node(43)], + refs, + }); + expect(target.unmatched).toBe(true); + }); + + it("uses viewport coordinates for fixed elements", () => { + const target = matchTarget({ + geometry: { + rect: { x: 5, y: 5, w: 80, h: 24 }, + scrollX: 100, + scrollY: 200, + position: "fixed", + tag: "button", + }, + captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })], + refs, + }); + expect(target.ref).toBe("e1"); + }); +}); diff --git a/apps/extension/src/lib/__tests__/page-settled.test.ts b/apps/extension/src/lib/__tests__/page-settled.test.ts new file mode 100644 index 00000000..8e0b989a --- /dev/null +++ b/apps/extension/src/lib/__tests__/page-settled.test.ts @@ -0,0 +1,43 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpRunner } from "@/tools/shared"; +import { waitForPageSettled } from "../page-settled"; + +function quietCdp(): { cdp: CdpRunner; send: ReturnType } { + const send = vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })); + return { + cdp: { send: send as unknown as CdpRunner["send"] }, + send, + }; +} + +describe("waitForPageSettled", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("waits through the minimum observation floor before accepting a quiet page", async () => { + vi.useFakeTimers(); + const { cdp, send } = quietCdp(); + + const settled = waitForPageSettled(cdp, 7); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).toHaveBeenCalledTimes(3); + }); + + it("cancels before probing a superseded observation", async () => { + vi.useFakeTimers(); + let cancelled = false; + const { cdp, send } = quietCdp(); + + const settled = waitForPageSettled(cdp, 7, { cancelled: () => cancelled }); + cancelled = true; + await vi.advanceTimersByTimeAsync(60); + + await expect(settled).resolves.toBe("cancelled"); + expect(send).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/lib/__tests__/record-observation.test.ts b/apps/extension/src/lib/__tests__/record-observation.test.ts new file mode 100644 index 00000000..a978b712 --- /dev/null +++ b/apps/extension/src/lib/__tests__/record-observation.test.ts @@ -0,0 +1,206 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import type { CapturedNode } from "@/tools/vom/capture"; +import type { DraftTraceStep } from "@/transport/types"; +import { resetStateIdCounterForTests } from "../record-constants"; +import { + applyTargetMatching, + buildTraceV3, + createObservationState, + flushPendingRedirectLanding, + rememberStepOnPage, + scheduleRedirectLandingFlush, +} from "../record-observation"; +import { registerObservation } from "../trace-reducer"; + +const URL = "https://example.com/login"; + +function capturedInput(): CapturedNode { + return { + backendNodeId: 42, + parentBackendNodeId: null, + tag: "input", + attrs: {}, + rect: { x: 20, y: 40, w: 200, h: 30 }, + localRect: { x: 20, y: 40, w: 200, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }; +} + +function finalizedFillBody(value: string, redactValues: boolean): string { + const obs = createObservationState({ redactValues }); + const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]'; + const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText }); + obs.lastSettled = { + stateId, + captured: [capturedInput()], + refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], + url: URL, + vomText: rawVomText, + }; + + const draft: DraftTraceStep = { + op: "fill", + target: { unmatched: true }, + value, + geometry: { + rect: { x: 20, y: 40, w: 200, h: 30 }, + scrollX: 0, + scrollY: 0, + position: "static", + tag: "input", + }, + }; + applyTargetMatching(obs, draft, 1); + draft.postStateId = stateId; + rememberStepOnPage(obs, draft, 1); + + return ( + buildTraceV3({ + obs, + steps: [draft], + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }).states[0]?.body ?? "" + ); +} + +describe("record observation annotations", () => { + beforeEach(() => { + resetStateIdCounterForTests(); + }); + + it("omits a fill literal from finalized state bodies when values are redacted", () => { + const secret = "hunter2-private"; + const body = finalizedFillBody(secret, true); + + expect(body).toContain("step 1: fill"); + expect(body).not.toContain(secret); + }); + + it("keeps fill details in ordinary recording annotations", () => { + const value = "ordinary text"; + const body = finalizedFillBody(value, false); + + expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`); + }); +}); + +describe("applyTargetMatching without a settled observation", () => { + it("keeps the capture role/name instead of a bare unmatched target", () => { + const obs = createObservationState(); + const draft: DraftTraceStep = { + op: "hover", + target: { unmatched: true }, + captureTarget: { tag: "button", role: "button", name: "新建" }, + geometry: { + rect: { x: 900, y: 8, w: 60, h: 32 }, + scrollX: 0, + scrollY: 0, + position: "static", + tag: "button", + }, + }; + + applyTargetMatching(obs, draft, 1); + + expect(draft.preStateId).toBeUndefined(); + expect(draft.target).toEqual({ + role: "button", + name: "新建", + unmatched: true, + }); + }); +}); + +describe("applyTargetMatching while the previous action is still settling", () => { + it("does not bind a new unmatched control to the stale observation", () => { + const obs = createObservationState(); + const stateId = registerObservation(obs.stateRegistry, { + url: URL, + rawVomText: '@vom 1\n@e1 textbox "Search"', + }); + obs.lastSettled = { + stateId, + captured: [capturedInput()], + refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Search", line: 1 }], + url: URL, + vomText: '@vom 1\n@e1 textbox "Search"', + }; + obs.settles.set(0, { draftIndex: 0, cancelled: false }); + const draft: DraftTraceStep = { + op: "click", + target: { unmatched: true }, + captureTarget: { tag: "button", role: "button", name: "Confirm" }, + geometry: { + rect: { x: 400, y: 300, w: 80, h: 30 }, + scrollX: 0, + scrollY: 0, + position: "fixed", + tag: "button", + }, + }; + + applyTargetMatching(obs, draft, 2); + + expect(draft.preStateId).toBeUndefined(); + expect(draft.target).toEqual({ role: "button", name: "Confirm", unmatched: true }); + }); +}); + +describe("redirect landing coalescing", () => { + it("does not consume a newer redirect hop while reading tab metadata", async () => { + vi.useFakeTimers(); + try { + const finalUrl = "https://example.com/final"; + const intermediateUrl = "https://example.com/intermediate"; + const obs = createObservationState(); + obs.lastSettled = { + stateId: "s-final", + captured: [], + refs: [], + url: finalUrl, + vomText: "@vom 1\nFinal", + }; + const steps: DraftTraceStep[] = []; + + let resolveFirstTab!: (tab: chrome.tabs.Tab) => void; + const firstTab = new Promise((resolve) => { + resolveFirstTab = resolve; + }); + const get = vi + .fn() + .mockImplementationOnce(() => firstTab) + .mockResolvedValue({ id: 7, url: finalUrl }); + const tabsApi: ChromeTabsApi = { + get, + query: vi.fn(async () => []), + }; + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Runtime.evaluate") { + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + } + throw new Error(`unexpected CDP method ${method}`); + }); + const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] }; + + scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, intermediateUrl); + await vi.advanceTimersByTimeAsync(180); + expect(get).toHaveBeenCalledTimes(1); + + scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, finalUrl); + resolveFirstTab({ id: 7, url: intermediateUrl } as chrome.tabs.Tab); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + await flushPendingRedirectLanding(obs, steps, cdp, 7, tabsApi); + + expect(steps).toEqual([]); + expect(obs.pendingRedirect).toBeNull(); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts new file mode 100644 index 00000000..8abb8cd0 --- /dev/null +++ b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2"; +import type { DraftTraceStep } from "@/transport/types"; + +describe("trace-reducer-v2", () => { + it("drops scroll and bare character press steps", () => { + const steps: DraftTraceStep[] = [ + { + op: "scroll", + page_url: "https://example.com/", + }, + { + op: "press", + key: "a", + page_url: "https://example.com/", + }, + ]; + const trace = buildTraceV2({ + steps, + startedAt: "2026-01-01T00:00:00.000Z", + startUrl: "https://example.com/", + }); + expect(trace.steps).toHaveLength(0); + expect(trace.pages).toHaveLength(1); + }); + + it("preserves hover steps supported by protocol v2", () => { + const trace = buildTraceV2({ + steps: [ + { + op: "hover", + target: { unmatched: true }, + captureTarget: { tag: "button", role: "button", name: "结束" }, + page_url: "https://example.com/", + }, + ], + startedAt: "2026-01-01T00:00:00.000Z", + startUrl: "https://example.com/", + }); + + expect(trace.steps).toEqual([ + expect.objectContaining({ + op: "hover", + target: expect.objectContaining({ tag: "button", name: "结束" }), + }), + ]); + }); + + it("maps captureTarget to v2 target with required tag", () => { + const steps: DraftTraceStep[] = [ + { + op: "click", + target: { unmatched: true }, + captureTarget: { tag: "button", role: "button", name: "Submit" }, + page_url: "https://example.com/", + }, + ]; + const trace = buildTraceV2({ + steps, + startedAt: "2026-01-01T00:00:00.000Z", + startUrl: "https://example.com/", + }); + expect(trace.steps).toHaveLength(1); + expect(trace.steps[0]?.op).toBe("click"); + if (trace.steps[0]?.op === "click") { + expect(trace.steps[0].target.tag).toBe("button"); + expect(trace.steps[0].target.name).toBe("Submit"); + } + expect("version" in trace).toBe(false); + }); + + it("records Enter presses", () => { + expect(shouldRecordPress("Enter")).toBe(true); + }); +}); diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts index 0a1fcb22..f2414075 100644 --- a/apps/extension/src/lib/describe-target.ts +++ b/apps/extension/src/lib/describe-target.ts @@ -6,7 +6,8 @@ * `{ "tag": "div" }` / “点击div” is useless and must not be recorded. */ -export interface TargetDescriptor { +/** Content-script capture descriptor before VOM geometric matching. */ +export interface CaptureTargetDescriptor { role?: string; name?: string; tag: string; @@ -15,6 +16,9 @@ export interface TargetDescriptor { nearby_label?: string; } +/** @deprecated Capture-time alias retained until the recorder integration migrates. */ +export type TargetDescriptor = CaptureTargetDescriptor; + /** Max length for a label that is still a useful “find this on screen” hint. */ const ACTIONABLE_LABEL_MAX = 48; @@ -247,7 +251,7 @@ export function resolveHoverElement(target: Element): Element | null { * (visible name), or at least a form `name_attr` for checkbox/radio. * Recording “点击div” with no name fails this bar. */ -export function isMeaningfulClickTarget(target: TargetDescriptor): boolean { +export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean { const name = target.name?.trim(); if (name && isActionableLabel(name)) return true; if ( @@ -323,7 +327,7 @@ function nearbyLabelText(el: Element): string | undefined { return undefined; } -export function describeTarget(el: Element): TargetDescriptor { +export function describeTarget(el: Element): CaptureTargetDescriptor { const tag = el.tagName.toLowerCase(); const role = inferRole(el); const name = accessibleName(el); @@ -350,7 +354,7 @@ export function describeTarget(el: Element): TargetDescriptor { }; } -export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null { +export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null { if (!(target instanceof Element)) return null; const clickable = resolveClickableElement(target); if (!clickable) return null; diff --git a/apps/extension/src/lib/format-observation-file.ts b/apps/extension/src/lib/format-observation-file.ts new file mode 100644 index 00000000..8676553c --- /dev/null +++ b/apps/extension/src/lib/format-observation-file.ts @@ -0,0 +1,62 @@ +import type { StepV3 } from "@/transport/types"; +import { OBSERVATION_FILE_VERSION } from "./record-constants"; + +export interface ObservationAnnotation { + stepId: number; + op: StepV3["op"]; + line: number; + stateId: string; + detail?: string; +} + +function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string { + const suffix = detail ? `: ${detail}` : ""; + return ` ⟵ step ${stepId}: ${op}${suffix}`; +} + +/** Serialize a page observation file (front matter + VOM body + step annotations). */ +export function formatObservationFile(input: { + stateId: string; + url: string; + title?: string; + stepsHere: number[]; + body: string; + annotations?: ObservationAnnotation[]; +}): string { + const lines: string[] = [ + `# bsk-observation ${OBSERVATION_FILE_VERSION}`, + `state: ${input.stateId}`, + `url: ${input.url}`, + ]; + if (input.title) lines.push(`title: ${input.title}`); + if (input.stepsHere.length > 0) { + lines.push(`steps_here: [${input.stepsHere.join(", ")}]`); + } + lines.push("---"); + + const bodyLines = input.body.split("\n"); + const annotationMap = new Map(); + for (const ann of input.annotations ?? []) { + const bucket = annotationMap.get(ann.line) ?? []; + bucket.push(ann); + annotationMap.set(ann.line, bucket); + } + + for (let i = 0; i < bodyLines.length; i += 1) { + let line = bodyLines[i] ?? ""; + const anns = annotationMap.get(i); + if (anns) { + for (const ann of anns) { + line += formatAnnotation(ann); + } + } + lines.push(line); + } + + return `${lines.join("\n")}\n`; +} + +/** Hash input: VOM body **before** annotations are inserted. */ +export function observationBodyForHash(vomText: string): string { + return vomText; +} diff --git a/apps/extension/src/lib/match-target.ts b/apps/extension/src/lib/match-target.ts new file mode 100644 index 00000000..06e1435f --- /dev/null +++ b/apps/extension/src/lib/match-target.ts @@ -0,0 +1,103 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import type { CapturedNode } from "@/tools/vom/capture"; +import type { CaptureGeometry, TargetDescriptorV3 } from "@/transport/types"; +import type { CaptureTargetDescriptor } from "./describe-target"; +import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants"; + +export interface MatchTargetInput { + geometry: CaptureGeometry; + captured: CapturedNode[]; + refs: RenderedRef[]; + fallback?: CaptureTargetDescriptor; +} + +function withinTolerance(a: number, b: number): boolean { + return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX; +} + +function rectsMatch( + a: { x: number; y: number; w: number; h: number }, + b: { x: number; y: number; w: number; h: number }, +): boolean { + return ( + withinTolerance(a.x, b.x) && + withinTolerance(a.y, b.y) && + withinTolerance(a.w, b.w) && + withinTolerance(a.h, b.h) + ); +} + +function nodeViewportRect( + node: CapturedNode, +): { x: number; y: number; w: number; h: number } | null { + return node.localRect ?? node.rect; +} + +function matchingRects( + geometry: CaptureGeometry, + node: CapturedNode, +): { + target: { x: number; y: number; w: number; h: number }; + node: { x: number; y: number; w: number; h: number } | null; +} { + const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null; + const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky"; + if (!topFrame || viewportPosition || !node.documentRect) { + return { target: geometry.rect, node: nodeViewportRect(node) }; + } + return { + target: { + x: geometry.rect.x + geometry.scrollX, + y: geometry.rect.y + geometry.scrollY, + w: geometry.rect.w, + h: geometry.rect.h, + }, + node: node.documentRect, + }; +} + +function tagMatches(geometryTag: string, nodeTag: string): boolean { + return geometryTag.toLowerCase() === nodeTag.toLowerCase(); +} + +/** Best description available when the element cannot be located in the VOM. */ +export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { + if (!fallback) { + return { unmatched: true }; + } + return { + ...(fallback.role ? { role: fallback.role } : {}), + ...(fallback.name ? { name: fallback.name } : {}), + unmatched: true, + }; +} + +/** Locate the interacted element in the last settled observation by geometry. */ +export function matchTarget(input: MatchTargetInput): TargetDescriptorV3 { + const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null; + + const candidates = input.captured.filter((node) => { + if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false; + if (!tagMatches(input.geometry.tag, node.tag)) return false; + const rects = matchingRects(input.geometry, node); + if (!rects.node) return false; + return rectsMatch(rects.target, rects.node); + }); + + if (candidates.length !== 1) { + return fallbackDescriptor(input.fallback); + } + + const backendNodeId = candidates[0]!.backendNodeId; + const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId); + if (!refEntry) { + return fallbackDescriptor(input.fallback); + } + + return { + ref: refEntry.ref, + ...(refEntry.role ? { role: refEntry.role } : {}), + ...(refEntry.name ? { name: refEntry.name } : {}), + ...(refEntry.ctx ? { ctx: refEntry.ctx } : {}), + }; +} diff --git a/apps/extension/src/lib/page-settled.ts b/apps/extension/src/lib/page-settled.ts new file mode 100644 index 00000000..f95ff05b --- /dev/null +++ b/apps/extension/src/lib/page-settled.ts @@ -0,0 +1,92 @@ +// Deciding *when* a page is worth observing. +// +// A fixed delay cannot serve both a modal that toggles in one frame and a +// route change that renders for a second. Instead of guessing, ask the page: +// a document that has stopped mutating and is no longer loading is done +// reacting to whatever the user just did. + +import type { CdpRunner } from "@/tools/shared"; +import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants"; + +/** + * Installed once per document. Records only the timestamp of the last DOM + * change so each poll stays O(1) no matter how large the page is. + */ +const QUIET_PROBE = `(() => { + const scope = window; + let probe = scope.__bskRecordQuiet; + if (!probe) { + probe = { changedAt: Date.now() }; + const observer = new MutationObserver(() => { + probe.changedAt = Date.now(); + }); + observer.observe(document.documentElement, { + subtree: true, + childList: true, + attributes: true, + characterData: true, + }); + scope.__bskRecordQuiet = probe; + } + return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState }; +})()`; + +export type SettleOutcome = + /** The page stopped changing on its own. */ + | "quiet" + /** Still changing when the budget ran out; observe it as it is. */ + | "timeout" + /** A newer action took over; this observation is no longer wanted. */ + | "cancelled"; + +interface QuietProbe { + idleMs: number; + readyState: string; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise { + try { + const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", { + expression: QUIET_PROBE, + returnByValue: true, + }); + const value = reply.result?.value; + if (!value || typeof value !== "object") return null; + const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown }; + if (typeof idleMs !== "number" || typeof readyState !== "string") return null; + return { idleMs, readyState }; + } catch { + // An unreadable page is a page mid-swap; that is a reason to keep waiting, + // not a reason to give up. + return null; + } +} + +/** Wait until the page has finished reacting, or until the budget runs out. */ +export async function waitForPageSettled( + cdp: CdpRunner, + tabId: number, + options: { cancelled?: () => boolean } = {}, +): Promise { + const startedAt = Date.now(); + const floor = startedAt + SETTLE_MIN_MS; + const deadline = startedAt + SETTLE_MAX_MS; + + for (;;) { + if (options.cancelled?.()) return "cancelled"; + await sleep(SETTLE_POLL_MS); + if (options.cancelled?.()) return "cancelled"; + + const probe = await readQuietProbe(cdp, tabId); + const now = Date.now(); + if (now < floor) continue; + if (now >= deadline) return "timeout"; + if (!probe) continue; + if (probe.readyState === "loading") continue; + if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet"; + } +} diff --git a/apps/extension/src/lib/record-constants.ts b/apps/extension/src/lib/record-constants.ts new file mode 100644 index 00000000..0cbf1f32 --- /dev/null +++ b/apps/extension/src/lib/record-constants.ts @@ -0,0 +1,97 @@ +import type { FillCommit, NavigationCause } from "@/transport/types"; + +/** + * Floor before a post-action observation may be taken. A page usually has not + * started reacting in the first frames after an action, and capturing then + * would record the page as it was, not as the action left it. + */ +export const SETTLE_MIN_MS = 150; + +/** How long the DOM must stop changing before a page counts as settled. */ +export const SETTLE_QUIET_MS = 250; + +/** Upper bound on waiting for a page to settle; animations never stop. */ +export const SETTLE_MAX_MS = 2_000; + +/** How often to ask the page whether it has gone quiet. */ +export const SETTLE_POLL_MS = 60; + +/** Minimum interval between consecutive observations on the same tab. */ +export const OBSERVATION_MIN_INTERVAL_MS = 200; + +/** Delay before retrying a capture that lost its execution context. */ +export const CAPTURE_RETRY_DELAY_MS = 250; + +/** Document-coordinate matching tolerance in CSS pixels. */ +export const GEOM_MATCH_TOLERANCE_PX = 2; + +/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */ +export const DEFAULT_MAX_PAGE_TOKENS = 3000; + +/** VOM observation file format version (front matter header). */ +export const OBSERVATION_FILE_VERSION = 1; + +/** FNV-1a 64-bit offset basis. */ +const FNV_OFFSET = 0xcbf29ce484222325n; +/** FNV-1a 64-bit prime. */ +const FNV_PRIME = 0x100000001b3n; + +/** Content-hash for state deduplication (sync, non-cryptographic). */ +export function fnv1a64(text: string): string { + let hash = FNV_OFFSET; + for (let i = 0; i < text.length; i += 1) { + hash ^= BigInt(text.charCodeAt(i)); + hash = (hash * FNV_PRIME) & 0xffffffffffffffffn; + } + return hash.toString(16).padStart(16, "0"); +} + +let nextStateSerial = 0; + +/** Monotonic state id generator (`s1`, `s2`, …). */ +export function nextStateId(): string { + nextStateSerial += 1; + return `s${nextStateSerial}`; +} + +/** Test seam: reset the state id counter. */ +export function resetStateIdCounterForTests(): void { + nextStateSerial = 0; +} + +const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]); + +const TRANSITION_TO_CAUSE: Record = { + typed: "user_typed", + generated: "user_typed", + keyword: "user_typed", + keyword_generated: "user_typed", + link: "link", + form_submit: "form_submit", + reload: "reload", + auto_bookmark: "browser", + start_page: "browser", +}; + +export interface NavigationTransitionMeta { + transitionType?: string; + transitionQualifiers?: string[]; + navigationActionPending?: boolean; +} + +/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */ +export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null { + const qualifiers = meta.transitionQualifiers ?? []; + if (qualifiers.includes("forward_back")) return "history"; + if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null; + if (qualifiers.includes("from_address_bar")) return "user_typed"; + + const type = meta.transitionType ?? ""; + const mapped = TRANSITION_TO_CAUSE[type]; + if (mapped) return mapped; + + if (!type && meta.navigationActionPending === false) return "script"; + return "browser"; +} + +export const DEFAULT_FILL_COMMIT: FillCommit = "blur"; diff --git a/apps/extension/src/lib/record-observation.ts b/apps/extension/src/lib/record-observation.ts new file mode 100644 index 00000000..6a74850a --- /dev/null +++ b/apps/extension/src/lib/record-observation.ts @@ -0,0 +1,681 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file"; +import { fallbackDescriptor, matchTarget } from "@/lib/match-target"; +import { waitForPageSettled } from "@/lib/page-settled"; +import { + CAPTURE_RETRY_DELAY_MS, + DEFAULT_MAX_PAGE_TOKENS, + OBSERVATION_MIN_INTERVAL_MS, +} from "@/lib/record-constants"; +import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer"; +import { captureVomObservation } from "@/tools/capture-vom-observation"; +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import type { CapturedNode } from "@/tools/vom/capture"; +import type { DraftTraceStep, StepV3, StopReason, TraceState, TraceV3 } from "@/transport/types"; +import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer"; + +export interface LastSettledObservation { + stateId: string; + captured: CapturedNode[]; + refs: RenderedRef[]; + url: string; + title?: string; + vomText: string; +} + +/** A post-action observation that has been queued but not yet written down. */ +interface PendingSettle { + draftIndex: number; + /** Set when a newer action has already decided where this step landed. */ + cancelled: boolean; +} + +/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */ +export interface PendingRedirectLanding { + url: string; + /** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */ + generation: number; +} + +export interface RecordingObservationState { + stateRegistry: Map; + lastSettled: LastSettledObservation | null; + maxPageTokens: number; + redactValues: boolean; + lastCaptureAtMs: number; + /** + * Tail of the settle chain. Observations run one at a time so that a slow + * capture can never land after a faster one and report the pages out of the + * order the user visited them. + */ + settleQueue: Promise; + /** Queued and in-flight settles, by draft index, so newer actions can supersede them. */ + settles: Map; + stepAnnotations: Map; + /** + * OAuth / server redirects are coalesced here: intermediate hops only update + * `url`+`generation`, and one `navigate` is emitted after the page settles. + */ + pendingRedirect: PendingRedirectLanding | null; + redirectFlushQueue: Promise; +} + +export function createObservationState(options?: { + maxPageTokens?: number; + redactValues?: boolean; +}): RecordingObservationState { + return { + stateRegistry: new Map(), + lastSettled: null, + maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS, + redactValues: options?.redactValues ?? false, + lastCaptureAtMs: 0, + settleQueue: Promise.resolve(), + settles: new Map(), + stepAnnotations: new Map(), + pendingRedirect: null, + redirectFlushQueue: Promise.resolve(), + }; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function readTabMeta( + tabsApi: ChromeTabsApi, + tabId: number, +): Promise<{ url: string; title?: string }> { + try { + const tab = await tabsApi.get(tabId); + return { url: tab.url ?? "about:blank", title: tab.title }; + } catch { + return { url: "about:blank" }; + } +} + +export async function captureAndRegisterObservation( + obs: RecordingObservationState, + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + urlOverride?: string, +): Promise { + const now = Date.now(); + const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs)); + if (waitMs > 0) await sleep(waitMs); + + const { url, title } = urlOverride + ? { url: urlOverride, title: undefined } + : await readTabMeta(tabsApi, tabId); + const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title }; + const resolvedTitle = title ?? meta.title; + + const rendered = await captureVomObservation(cdp, tabId, url, { + maxTokens: obs.maxPageTokens, + redactValues: obs.redactValues, + conditionalSurfaceProbe: false, + }); + + const stateId = registerObservation(obs.stateRegistry, { + url, + title: resolvedTitle, + rawVomText: rendered.text, + truncated: rendered.truncated, + }); + + obs.lastCaptureAtMs = Date.now(); + const settled: LastSettledObservation = { + stateId, + captured: rendered.captured, + refs: rendered.refs, + url, + title: resolvedTitle, + vomText: rendered.text, + }; + obs.lastSettled = settled; + return settled; +} + +/** + * Bind a draft to the page it was performed on: origin state, geometric target + * match, and the inline annotation. All three read the *current* observation, + * which is only the page the user acted on while the draft is still fresh — + * once the action settles, `lastSettled` is the destination page instead. + */ +export function applyTargetMatching( + obs: RecordingObservationState, + draft: DraftTraceStep, + stepId: number, +): void { + if (!obs.lastSettled) { + // Initial observation can fail or race the first action. Keep whatever + // the content script captured so the exported step is still teachable. + if ("target" in draft && "captureTarget" in draft) { + draft.target = fallbackDescriptor(draft.captureTarget); + } + rememberStepAnnotation(obs, stepId, draft); + return; + } + + if ("target" in draft) { + draft.target = + "geometry" in draft && draft.geometry + ? matchTarget({ + geometry: draft.geometry, + captured: obs.lastSettled.captured, + refs: obs.lastSettled.refs, + fallback: draft.captureTarget, + }) + : // No geometry to match on, but the capture still knew what the user + // touched — keep that instead of an anonymous unmatched target. + fallbackDescriptor(draft.captureTarget); + } + + // A pending settle means `lastSettled` may still describe the page before + // the previous action. If the new control cannot be found there, attaching + // that stale state would be worse than leaving the origin unknown for the + // reducer to repair from the observations that do exist. + const originLooksStale = + obs.settles.size > 0 && (!("target" in draft) || draft.target?.unmatched === true); + if (originLooksStale) { + rememberStepAnnotation(obs, stepId, draft); + return; + } + + draft.preStateId = obs.lastSettled.stateId; + rememberStepAnnotation(obs, stepId, draft); +} + +function fillDetailForDraft( + obs: RecordingObservationState, + draft: DraftTraceStep, +): string | undefined { + if (obs.redactValues) return undefined; + if (draft.op === "fill") return JSON.stringify(draft.value); + return undefined; +} + +function refLineForDraft( + obs: RecordingObservationState, + draft: DraftTraceStep, +): number | undefined { + if (!("target" in draft) || !obs.lastSettled) return undefined; + const targetRef = draft.target?.ref; + if (!targetRef) return undefined; + return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line; +} + +function rememberStepAnnotation( + obs: RecordingObservationState, + stepId: number, + draft: DraftTraceStep, +): void { + const line = refLineForDraft(obs, draft); + if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return; + if (!draft.preStateId) return; + const ann: ObservationAnnotation = { + stepId, + op: draft.op, + line, + stateId: draft.preStateId, + detail: fillDetailForDraft(obs, draft), + }; + const bucket = obs.stepAnnotations.get(line) ?? []; + bucket.push(ann); + obs.stepAnnotations.set(line, bucket); +} + +/** Record the step under the page it was performed on, not the one it led to. */ +export function rememberStepOnPage( + obs: RecordingObservationState, + draft: DraftTraceStep, + stepId: number, +): void { + const stateId = draft.preStateId; + if (!stateId) return; + const entry = obs.stateRegistry.get(stateId); + if (entry && !entry.stepsHere.includes(stepId)) { + entry.stepsHere.push(stepId); + } +} + +/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */ +export function clearPendingRedirectLanding(obs: RecordingObservationState): void { + obs.pendingRedirect = null; +} + +function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise): void { + obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {}); +} + +/** + * Remember a redirect hop and schedule a settle-then-emit of a single navigate + * to the final URL. Later hops only bump `generation` / replace `url`. + */ +export function scheduleRedirectLandingFlush( + obs: RecordingObservationState, + steps: DraftTraceStep[], + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + url: string, + options: { cancelled?: () => boolean } = {}, +): void { + const generation = (obs.pendingRedirect?.generation ?? 0) + 1; + obs.pendingRedirect = { url, generation }; + enqueueRedirectFlush(obs, () => + coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options), + ); +} + +/** + * Drain any coalesced redirect so the next action matches against the real + * landing page. Safe to call when nothing is pending. + */ +export async function flushPendingRedirectLanding( + obs: RecordingObservationState, + steps: DraftTraceStep[], + cdp: CdpRunner | undefined, + tabId: number, + tabsApi: ChromeTabsApi, + options: { cancelled?: () => boolean } = {}, +): Promise { + if (obs.pendingRedirect && cdp) { + enqueueRedirectFlush(obs, () => + coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options), + ); + } + await obs.redirectFlushQueue; +} + +/** + * Wait until the redirect chain stops changing the document, then emit one + * `navigate` (cause `browser`) to the tab's final URL and settle its observation. + */ +async function coalesceRedirectLanding( + obs: RecordingObservationState, + steps: DraftTraceStep[], + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + options: { cancelled?: () => boolean }, +): Promise { + while (obs.pendingRedirect) { + if (options.cancelled?.()) { + obs.pendingRedirect = null; + return; + } + + const snap = obs.pendingRedirect; + const outcome = await waitForPageSettled(cdp, tabId, { + cancelled: () => + !!options.cancelled?.() || + obs.pendingRedirect === null || + obs.pendingRedirect.generation !== snap.generation, + }); + + if (options.cancelled?.()) { + obs.pendingRedirect = null; + return; + } + // A newer hop cancelled this wait — loop and settle against the latest URL. + if (outcome === "cancelled") continue; + if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) { + continue; + } + + const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url; + // `tabs.get` is asynchronous. A newer redirect hop may have arrived while + // it was in flight; only the generation that initiated this read may + // consume the pending landing. + if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) { + continue; + } + obs.pendingRedirect = null; + + if (!finalUrl || finalUrl === "about:blank") return; + if (obs.lastSettled?.url === finalUrl) return; + + const last = steps[steps.length - 1]; + if (last?.op === "navigate" && last.url === finalUrl) { + if (!last.postStateId) { + const draftIndex = steps.length - 1; + scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps); + await obs.settleQueue; + } + return; + } + + if (outcome === "timeout") { + console.debug( + `[bsk record] redirect landing still changing after settle budget; ` + + `recording navigate to ${finalUrl}`, + ); + } + + const draft: DraftTraceStep = { + op: "navigate", + url: finalUrl, + page_url: finalUrl, + cause: "browser", + preStateId: obs.lastSettled?.stateId, + }; + steps.push(draft); + const draftIndex = steps.length - 1; + const stepId = draftIndex + 1; + rememberStepOnPage(obs, draft, stepId); + scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps); + // Callers that flush before the next action need `lastSettled` to already + // be the landing page so target matching does not use the pre-redirect view. + await obs.settleQueue; + return; + } +} + +/** + * A capture that lands mid-navigation fails on a destroyed execution context. + * That is transient, so give it one more chance before the step has to fall + * back to a stale state. + */ +async function captureWithRetry( + obs: RecordingObservationState, + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + urlOverride?: string, +): Promise { + try { + return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride); + } catch (err) { + console.debug("[bsk record] observation failed, retrying once", err); + await sleep(CAPTURE_RETRY_DELAY_MS); + return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride); + } +} + +export async function settleDraftObservation( + obs: RecordingObservationState, + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + steps: DraftTraceStep[], + draftIndex: number, + pending?: PendingSettle, +): Promise { + const cancelled = () => pending?.cancelled === true; + const outcome = await waitForPageSettled(cdp, tabId, { cancelled }); + if (outcome === "cancelled") return; + if (outcome === "timeout") { + console.debug( + `[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` + + "budget; observing it as it stands", + ); + } + + const started = steps[draftIndex]; + if (!started) return; + + // The URL attached to the draft may only be an intermediate redirect hop. + // Once the page is settled, register the capture against live tab metadata. + const settled = await captureWithRetry(obs, cdp, tabId, tabsApi); + // The observation still counts as the latest view of the page even when this + // step no longer wants it — the next action will start from it. + if (cancelled()) return; + // Re-read the slot: a navigation observed while the capture ran may have + // rewritten this draft, and the observation belongs to whatever occupies the + // slot now — writing to the object we started with would strand it. + const draft = steps[draftIndex] ?? started; + draft.postStateId = settled.stateId; +} + +/** + * Recover landing states from the shape of the recording itself: wherever the + * next action was performed is, by definition, where the previous one landed. + * Only the immediately following draft counts — a later one would claim a page + * that several unobserved actions away, which is worse than admitting we saw + * no change. + */ +export function inferMissingPostStates(steps: DraftTraceStep[]): void { + for (let i = 0; i < steps.length - 1; i += 1) { + const draft = steps[i]; + const next = steps[i + 1]; + if (!draft || !next || draft.postStateId || !next.preStateId) continue; + draft.postStateId = next.preStateId; + console.debug( + `[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` + + `using where step ${i + 2} started (${next.preStateId})`, + ); + } +} + +/** + * Last chance for the closing actions of a recording. Stopping right after the + * final action is normal, and so is that action navigating away, so take one + * final look at the page. Only trailing drafts may claim it: an earlier step + * did not land on whatever the user happens to be looking at when they stop. + */ +export async function settleUnsettledDrafts( + obs: RecordingObservationState, + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + steps: DraftTraceStep[], +): Promise { + const trailing: DraftTraceStep[] = []; + for (let i = steps.length - 1; i >= 0; i -= 1) { + const draft = steps[i]; + if (!draft || draft.postStateId) break; + trailing.push(draft); + } + if (trailing.length === 0) return; + + let settled: LastSettledObservation; + try { + settled = await captureWithRetry(obs, cdp, tabId, tabsApi); + } catch (err) { + console.warn("[bsk record] final observation at stop failed", err); + return; + } + for (const draft of trailing) draft.postStateId = settled.stateId; + console.debug( + `[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` + + `(${settled.stateId} ${settled.url})`, + ); +} + +/** + * Performing a new action ends the previous one: the page the user reached for + * is, by definition, where the earlier step left them. Taking the landing from + * the newer step also keeps the trace monotonic — a capture that finished late + * would otherwise credit an earlier step with a page that only exists because + * of the newer action. + */ +function supersedeEarlierSettles( + obs: RecordingObservationState, + draftIndex: number, + steps: DraftTraceStep[], +): void { + const landing = steps[draftIndex]?.preStateId; + for (const [index, pending] of obs.settles) { + if (index >= draftIndex) continue; + pending.cancelled = true; + obs.settles.delete(index); + + const draft = steps[index]; + if (!draft || draft.postStateId) continue; + if (!landing) continue; + draft.postStateId = landing; + console.debug( + `[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` + + `${draftIndex + 1} started; landing it on ${landing}`, + ); + } +} + +export function scheduleDraftSettle( + obs: RecordingObservationState, + draftIndex: number, + stepId: number, + cdp: CdpRunner, + tabId: number, + tabsApi: ChromeTabsApi, + steps: DraftTraceStep[], +): void { + supersedeEarlierSettles(obs, draftIndex, steps); + + // Rescheduling the same step (a navigation showed up after the action) + // replaces the pending observation rather than racing it. + const superseded = obs.settles.get(draftIndex); + if (superseded) superseded.cancelled = true; + + const pending: PendingSettle = { draftIndex, cancelled: false }; + obs.settles.set(draftIndex, pending); + + enqueueSettle(obs, async () => { + if (pending.cancelled) return; + try { + await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending); + } catch (err) { + // Recoverable: stop-time repair still gives the step a landing page. + const op = steps[draftIndex]?.op ?? "?"; + console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err); + } finally { + if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex); + } + }); +} + +function enqueueSettle(obs: RecordingObservationState, task: () => Promise): void { + obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {}); +} + +export function cancelPendingSettles(obs: RecordingObservationState): void { + for (const pending of obs.settles.values()) pending.cancelled = true; + obs.settles.clear(); +} + +/** Bound the drain loop so a self-rescheduling settle cannot block stop. */ +const MAX_SETTLE_FLUSH_ROUNDS = 10; + +/** + * Drain settle work before exporting, including work queued while draining — + * a navigation observed during the last capture schedules another one, and + * the trace would drop it if stop did not wait. + */ +export async function flushPendingSettles(obs: RecordingObservationState): Promise { + for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) { + const drained = obs.settleQueue; + await drained; + if (obs.settleQueue === drained) return; + } + console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed"); +} + +function finalizeStateBodies( + entries: StateRegistryEntry[], + annotationsByState: Map, + stepIdByDraftId: Map, + idByOldId: Map, +): TraceState[] { + return entries.map((entry) => { + const id = idByOldId.get(entry.id) ?? entry.id; + const body = formatObservationFile({ + stateId: id, + url: entry.url, + title: entry.title, + stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId), + body: entry.rawVomText, + annotations: annotationsByState.get(entry.id) ?? [], + }); + return { + id, + url: entry.url, + ...(entry.title ? { title: entry.title } : {}), + body, + ...(entry.truncated ? { truncated: true } : {}), + }; + }); +} + +/** Draft ids only become step ids after collapsing and filtering. */ +function remapStepIds(draftIds: number[], stepIdByDraftId: Map): number[] { + const mapped = new Set(); + for (const draftId of draftIds) { + const stepId = stepIdByDraftId.get(draftId); + if (stepId !== undefined) mapped.add(stepId); + } + return [...mapped].sort((a, b) => a - b); +} + +function collectAnnotationsByState( + obs: RecordingObservationState, + stepIdByDraftId: Map, +): Map { + const byState = new Map(); + for (const anns of obs.stepAnnotations.values()) { + for (const ann of anns) { + const stepId = stepIdByDraftId.get(ann.stepId); + if (stepId === undefined) continue; + const bucket = byState.get(ann.stateId) ?? []; + bucket.push({ ...ann, stepId }); + byState.set(ann.stateId, bucket); + } + } + return byState; +} + +/** + * Keep only the pages the published steps point at. Redirect hops and + * mid-load captures land in the registry too, and shipping them would invite + * a reader to treat a page the flow merely passed through as a real stop. + * With no steps at all the first observation is the whole artifact, so it + * stays. + */ +function selectPublishedStates( + registry: Map, + steps: StepV3[], +): StateRegistryEntry[] { + const entries = [...registry.values()]; + if (steps.length === 0) return entries.slice(0, 1); + const referenced = new Set(); + for (const step of steps) { + referenced.add(step.state); + referenced.add(step.result.state); + } + return entries.filter((entry) => referenced.has(entry.id)); +} + +export function buildTraceV3(input: { + obs: RecordingObservationState; + steps: DraftTraceStep[]; + startedAt: string; + purpose?: string; + startUrl?: string; + stoppedBy: StopReason; + bskVersion: string; +}): TraceV3 { + const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry); + const published = selectPublishedStates(input.obs.stateRegistry, steps); + // Renumber so the shipped dictionary reads s1..sN without holes where the + // dropped captures used to be. + const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`])); + for (const step of steps) { + step.state = idByOldId.get(step.state) ?? step.state; + step.result.state = idByOldId.get(step.result.state) ?? step.result.state; + } + const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId); + const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId); + const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states); + return { + version: 3, + ...(input.purpose ? { purpose: input.purpose } : {}), + recorded_at: new Date().toISOString(), + started_at: input.startedAt, + stopped_by: input.stoppedBy, + entry: { start_url: startUrl }, + recorder: { bsk: input.bskVersion, vom: 1 }, + states, + steps, + }; +} diff --git a/apps/extension/src/lib/trace-reducer-v2.ts b/apps/extension/src/lib/trace-reducer-v2.ts new file mode 100644 index 00000000..ef57b74a --- /dev/null +++ b/apps/extension/src/lib/trace-reducer-v2.ts @@ -0,0 +1,301 @@ +import type { CaptureTargetDescriptor } from "@/lib/describe-target"; +import type { DraftTraceStep, KeyModifier } from "@/transport/types"; + +const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); +const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); + +export interface TargetDescriptorV2 { + role?: string; + name?: string; + tag: string; + name_attr?: string; + placeholder?: string; + nearby_label?: string; +} + +export interface PageRef { + id: string; + url: string; + title?: string; +} + +export interface SelectedOptionV2 { + value: string; + label?: string; +} + +export interface StepEffectV2 { + navigated_to: string; +} + +export type StepV2 = + | { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 } + | { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 } + | { op: "hover"; id: number; page: string; target: TargetDescriptorV2 } + | { + op: "fill"; + id: number; + page: string; + target: TargetDescriptorV2; + value: string; + redacted?: boolean; + effect?: StepEffectV2; + } + | { + op: "select"; + id: number; + page: string; + target: TargetDescriptorV2; + selection: SelectedOptionV2[]; + effect?: StepEffectV2; + } + | { + op: "press"; + id: number; + page: string; + key: string; + modifiers?: KeyModifier[]; + target?: TargetDescriptorV2; + effect?: StepEffectV2; + }; + +export interface TraceV2 { + recorded_at: string; + started_at?: string; + purpose?: string; + entry: { start_url: string }; + pages: PageRef[]; + steps: StepV2[]; +} + +export function shouldRecordPress( + key: string, + modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, +): boolean { + if (MODIFIER_ONLY_KEYS.has(key)) return false; + const mods = modifiers ?? []; + const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); + if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; + if (key === "Enter" || key === "Escape") return true; + if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; + return false; +} + +function shouldIncludeDraft(step: DraftTraceStep): boolean { + if (step.op === "scroll") return false; + if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false; + if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false; + return true; +} + +function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { + const out: DraftTraceStep[] = []; + for (const step of steps) { + const prev = out[out.length - 1]; + if (step.op === "navigate" && prev?.op === "navigate") { + out[out.length - 1] = step; + continue; + } + out.push(step); + } + return out; +} + +function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { + const urls: string[] = []; + if (startUrl) urls.push(startUrl); + for (const step of steps) { + if (step.op === "navigate") { + urls.push(step.url); + continue; + } + if ("page_url" in step && step.page_url) urls.push(step.page_url); + if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to); + } + const seen = new Set(); + const unique: string[] = []; + for (const url of urls) { + if (!url || seen.has(url)) continue; + seen.add(url); + unique.push(url); + } + return unique; +} + +function buildPageRegistry( + steps: DraftTraceStep[], + startUrl?: string, +): { pages: PageRef[]; urlToId: Map } { + const urls = collectUrls(steps, startUrl); + const urlToId = new Map(); + const pages = urls.map((url, index) => { + const id = `p${index + 1}`; + urlToId.set(url, id); + return { id, url }; + }); + return { pages, urlToId }; +} + +function pageIdFor( + url: string | undefined, + urlToId: Map, + fallbackUrl?: string, +): string { + if (url && urlToId.has(url)) return urlToId.get(url)!; + if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!; + return urlToId.values().next().value ?? "p1"; +} + +function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined { + if (step.op === "navigate") return step.page_url ?? step.url; + if ("page_url" in step && step.page_url) return step.page_url; + return fallbackUrl; +} + +function effectForNavigation( + navigatedTo: string | undefined, + urlToId: Map, +): StepEffectV2 | undefined { + if (!navigatedTo) return undefined; + const pageId = urlToId.get(navigatedTo); + if (!pageId) return undefined; + return { navigated_to: pageId }; +} + +function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 { + return { + tag: capture?.tag ?? "unknown", + ...(capture?.role ? { role: capture.role } : {}), + ...(capture?.name ? { name: capture.name } : {}), + ...(capture?.name_attr ? { name_attr: capture.name_attr } : {}), + ...(capture?.placeholder ? { placeholder: capture.placeholder } : {}), + ...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}), + }; +} + +function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined { + if (!("target" in step) && !("captureTarget" in step)) return undefined; + const capture = "captureTarget" in step ? step.captureTarget : undefined; + if (capture) return captureTargetToV2Target(capture); + if ("target" in step && step.target && "tag" in (step.target as object)) { + const legacy = step.target as TargetDescriptorV2 & { tag?: string }; + if (legacy.tag) return legacy; + } + return undefined; +} + +function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] { + return values.map((value, index) => ({ + value, + ...(labels?.[index] ? { label: labels[index] } : {}), + })); +} + +function toV2Step( + step: DraftTraceStep, + id: number, + urlToId: Map, + fallbackUrl?: string, +): StepV2 | null { + if (!shouldIncludeDraft(step)) return null; + + const pageUrl = pageUrlForDraft(step, fallbackUrl); + const page = pageIdFor(pageUrl, urlToId, fallbackUrl); + const effect = + "navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined; + + switch (step.op) { + case "navigate": + return { + op: "navigate", + id, + page: pageIdFor(step.url, urlToId, fallbackUrl), + to: step.url, + }; + case "click": { + const target = targetForDraft(step); + if (!target) return null; + return { op: "click", id, page, target, ...(effect ? { effect } : {}) }; + } + case "hover": { + const target = targetForDraft(step); + if (!target) return null; + return { op: "hover", id, page, target }; + } + case "fill": { + const target = targetForDraft(step); + if (!target) return null; + return { + op: "fill", + id, + page, + target, + value: step.value, + ...(step.redacted ? { redacted: true } : {}), + }; + } + case "press": + return { + op: "press", + id, + page, + key: step.key, + ...(step.target ? { target: targetForDraft(step) } : {}), + ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}), + ...(effect ? { effect } : {}), + }; + case "select": { + const target = targetForDraft(step); + if (!target) return null; + return { + op: "select", + id, + page, + target, + selection: toSelection(step.values, step.labels), + ...(effect ? { effect } : {}), + }; + } + case "scroll": + return null; + } +} + +export interface BuildTraceV2Input { + steps: DraftTraceStep[]; + startedAt: string; + startUrl?: string; + purpose?: string; +} + +export function buildTraceV2(input: BuildTraceV2Input): TraceV2 { + const collapsed = collapseNavigations(input.steps); + const startUrl = + input.startUrl ?? + collapsed.find( + (step): step is Extract => step.op === "navigate", + )?.url ?? + collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ?? + "about:blank"; + const { pages, urlToId } = buildPageRegistry(collapsed, startUrl); + const out: StepV2[] = []; + let id = 1; + let lastUrl = startUrl; + for (const draft of collapsed) { + if (draft.op === "navigate") lastUrl = draft.url; + else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to; + else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url; + const step = toV2Step(draft, id, urlToId, lastUrl); + if (!step) continue; + out.push(step); + id += 1; + } + return { + recorded_at: new Date().toISOString(), + started_at: input.startedAt, + ...(input.purpose ? { purpose: input.purpose } : {}), + entry: { start_url: startUrl }, + pages, + steps: out, + }; +} From 87cdcfe5cac51fd1ee9bdc0fd12a0454d3ebb9ef Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 13:01:48 +0800 Subject: [PATCH 2/3] fix(recording): using unified geometry --- apps/extension/src/content/record-capture.ts | 21 +- .../src/lib/__tests__/document-settle.test.ts | 101 +++ .../src/lib/__tests__/match-target.test.ts | 116 --- .../src/lib/__tests__/page-settled.test.ts | 43 -- .../lib/__tests__/record-observation.test.ts | 206 ------ .../__tests__/recording-observation.test.ts | 150 ++++ .../lib/__tests__/settle-controller.test.ts | 136 ++++ ...tep-buffer.test.ts => step-buffer.test.ts} | 16 +- .../src/lib/__tests__/target-matcher.test.ts | 126 ++++ .../lib/__tests__/trace-reducer-v2.test.ts | 219 ++++-- .../lib/__tests__/trace-reducer-v3.test.ts | 62 ++ .../src/lib/__tests__/trace-reducer.test.ts | 186 ----- apps/extension/src/lib/describe-target.ts | 5 +- .../src/lib/format-observation-file.ts | 62 -- apps/extension/src/lib/match-target.ts | 103 --- apps/extension/src/lib/page-settled.ts | 92 --- apps/extension/src/lib/record-bridge.ts | 4 +- apps/extension/src/lib/record-constants.ts | 97 --- apps/extension/src/lib/record-observation.ts | 681 ------------------ .../src/lib/recording/document-settle.ts | 134 ++++ .../src/lib/recording/draft-policy.ts | 34 + .../src/lib/recording/observation-capture.ts | 115 +++ .../src/lib/recording/observation-session.ts | 122 ++++ .../src/lib/recording/settle-controller.ts | 232 ++++++ .../src/lib/recording/state-registry.ts | 56 ++ .../step-buffer.ts} | 34 +- .../src/lib/recording/target-matcher.ts | 76 ++ .../src/lib/recording/trace-builder-v3.ts | 87 +++ .../trace-reducer-v2.ts} | 107 ++- .../src/lib/recording/trace-reducer-v3.ts | 129 ++++ .../src/lib/recording/trace-state-body.ts | 39 + apps/extension/src/lib/recording/types.ts | 83 +++ apps/extension/src/lib/trace-reducer-v2.ts | 301 -------- apps/extension/src/tools/record.ts | 22 +- apps/extension/src/transport/types.ts | 42 -- 35 files changed, 1952 insertions(+), 2087 deletions(-) create mode 100644 apps/extension/src/lib/__tests__/document-settle.test.ts delete mode 100644 apps/extension/src/lib/__tests__/match-target.test.ts delete mode 100644 apps/extension/src/lib/__tests__/page-settled.test.ts delete mode 100644 apps/extension/src/lib/__tests__/record-observation.test.ts create mode 100644 apps/extension/src/lib/__tests__/recording-observation.test.ts create mode 100644 apps/extension/src/lib/__tests__/settle-controller.test.ts rename apps/extension/src/lib/__tests__/{recording-step-buffer.test.ts => step-buffer.test.ts} (80%) create mode 100644 apps/extension/src/lib/__tests__/target-matcher.test.ts create mode 100644 apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts delete mode 100644 apps/extension/src/lib/__tests__/trace-reducer.test.ts delete mode 100644 apps/extension/src/lib/format-observation-file.ts delete mode 100644 apps/extension/src/lib/match-target.ts delete mode 100644 apps/extension/src/lib/page-settled.ts delete mode 100644 apps/extension/src/lib/record-constants.ts delete mode 100644 apps/extension/src/lib/record-observation.ts create mode 100644 apps/extension/src/lib/recording/document-settle.ts create mode 100644 apps/extension/src/lib/recording/draft-policy.ts create mode 100644 apps/extension/src/lib/recording/observation-capture.ts create mode 100644 apps/extension/src/lib/recording/observation-session.ts create mode 100644 apps/extension/src/lib/recording/settle-controller.ts create mode 100644 apps/extension/src/lib/recording/state-registry.ts rename apps/extension/src/lib/{recording-step-buffer.ts => recording/step-buffer.ts} (79%) create mode 100644 apps/extension/src/lib/recording/target-matcher.ts create mode 100644 apps/extension/src/lib/recording/trace-builder-v3.ts rename apps/extension/src/lib/{trace-reducer.ts => recording/trace-reducer-v2.ts} (59%) create mode 100644 apps/extension/src/lib/recording/trace-reducer-v3.ts create mode 100644 apps/extension/src/lib/recording/trace-state-body.ts create mode 100644 apps/extension/src/lib/recording/types.ts delete mode 100644 apps/extension/src/lib/trace-reducer-v2.ts diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index d227de66..bded922d 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,9 +1,9 @@ import { + type CaptureTargetDescriptor, describeEventTarget, describeTarget, resolveClickableElement, resolveHoverElement, - type TargetDescriptor, } from "@/lib/describe-target"; import { evaluateHoverTrigger, @@ -27,7 +27,7 @@ import { type RecordStopAck, type RecordStopMessage, } from "@/lib/record-bridge"; -import { shouldRecordPress } from "@/lib/trace-reducer"; +import { shouldRecordPress } from "@/lib/recording/draft-policy"; import { closestHoverSurfaceCandidate, collectHoverSurfaceStates, @@ -65,14 +65,14 @@ export interface RecordCaptureController { interface FillSession { element: FillableElement; - target: TargetDescriptor; + target: CaptureTargetDescriptor; baselineValue: string; lastValue: string; } interface HoverCandidate { element: Element; - target: TargetDescriptor; + target: CaptureTargetDescriptor; recordedAt: number; score: number; eligible: boolean; @@ -232,7 +232,10 @@ function collectHoverTriggerLabelText(root: Element): string { return normalizeLabelText(text); } -function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor { +function compactHoverTargetName( + el: Element, + desc: CaptureTargetDescriptor, +): CaptureTargetDescriptor { if (!desc.name) return desc; const fullText = normalizeLabelText(el.textContent ?? ""); const compactName = collectHoverTriggerLabelText(el); @@ -250,7 +253,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc return desc; } -function isWeakHoverTarget(target: TargetDescriptor): boolean { +function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean { return !target.role && !target.name && target.tag === "div"; } @@ -261,9 +264,9 @@ function looksLikeAvatarElement(el: Element): boolean { function normalizeHoverTarget( el: Element, - desc: TargetDescriptor, + desc: CaptureTargetDescriptor, decision: HoverTriggerDecision, -): TargetDescriptor { +): CaptureTargetDescriptor { if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) { return { ...desc, name: "image" }; } @@ -277,7 +280,7 @@ function normalizeHoverTarget( return desc; } -function hoverTriggerSignals(el: Element, desc: TargetDescriptor) { +function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) { if (!(el instanceof HTMLElement)) return null; const style = hoverTriggerStyle(el); return { diff --git a/apps/extension/src/lib/__tests__/document-settle.test.ts b/apps/extension/src/lib/__tests__/document-settle.test.ts new file mode 100644 index 00000000..cb0d8e26 --- /dev/null +++ b/apps/extension/src/lib/__tests__/document-settle.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpRunner } from "@/tools/shared"; +import { waitForDocumentSettled } from "../recording/document-settle"; + +function quietCdp(): { cdp: CdpRunner; send: ReturnType } { + const send = vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })); + return { + cdp: { send: send as unknown as CdpRunner["send"] }, + send, + }; +} + +describe("waitForDocumentSettled", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("waits through the minimum observation floor before accepting a quiet page", async () => { + vi.useFakeTimers(); + const { cdp, send } = quietCdp(); + + const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } }); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).toHaveBeenCalledTimes(3); + }); + + it("cancels before probing a superseded observation", async () => { + vi.useFakeTimers(); + const { cdp, send } = quietCdp(); + + const controller = new AbortController(); + const settled = waitForDocumentSettled( + cdp, + { target: { tabId: 7 } }, + { + signal: controller.signal, + }, + ); + controller.abort(); + await vi.advanceTimersByTimeAsync(60); + + await expect(settled).resolves.toBe("cancelled"); + expect(send).not.toHaveBeenCalled(); + }); + + it("probes a same-process iframe through its isolated execution context", async () => { + vi.useFakeTimers(); + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 91 }; + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + }); + const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] }; + + const settled = waitForDocumentSettled(cdp, { + target: { tabId: 7 }, + frameId: "child-frame", + }); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).toHaveBeenCalledWith( + 7, + "Page.createIsolatedWorld", + expect.objectContaining({ frameId: "child-frame" }), + ); + expect(send).toHaveBeenCalledWith( + 7, + "Runtime.evaluate", + expect.objectContaining({ contextId: 91 }), + ); + }); + + it("routes an OOPIF probe through its CDP target session", async () => { + vi.useFakeTimers(); + const send = vi.fn(); + const sendToTarget = vi.fn(async (_target, method: string) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 27 }; + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + }; + + const target = { tabId: 7, sessionId: "oopif-session" }; + const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" }); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).not.toHaveBeenCalled(); + expect(sendToTarget).toHaveBeenCalledWith( + target, + "Runtime.evaluate", + expect.objectContaining({ contextId: 27 }), + ); + }); +}); diff --git a/apps/extension/src/lib/__tests__/match-target.test.ts b/apps/extension/src/lib/__tests__/match-target.test.ts deleted file mode 100644 index f4edddc5..00000000 --- a/apps/extension/src/lib/__tests__/match-target.test.ts +++ /dev/null @@ -1,116 +0,0 @@ -import type { RenderedRef } from "@browser-skill/vom"; -import { describe, expect, it } from "vitest"; -import type { CapturedNode } from "@/tools/vom/capture"; -import { matchTarget } from "../match-target"; - -function node(backendNodeId: number, overrides: Partial = {}): CapturedNode { - return { - backendNodeId, - parentBackendNodeId: null, - tag: "button", - attrs: {}, - rect: { x: 10, y: 20, w: 100, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - ...overrides, - }; -} - -describe("matchTarget", () => { - const refs: RenderedRef[] = [ - { ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 }, - ]; - - it("matches a unique node by viewport coordinates", () => { - const target = matchTarget({ - geometry: { - rect: { x: 10, y: 20, w: 100, h: 30 }, - scrollX: 0, - scrollY: 0, - position: "static", - tag: "button", - }, - captured: [node(42)], - refs, - fallback: { tag: "button", name: "发布" }, - }); - expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); - }); - - it("matches a non-fixed node after the top frame scrolls", () => { - const target = matchTarget({ - geometry: { - rect: { x: 10, y: 20, w: 100, h: 30 }, - scrollX: 40, - scrollY: 300, - position: "static", - tag: "button", - }, - captured: [ - node(42, { - documentRect: { x: 50, y: 320, w: 100, h: 30 }, - localRect: { x: 10, y: 20, w: 100, h: 30 }, - rect: { x: 10, y: 20, w: 100, h: 30 }, - }), - ], - refs, - fallback: { tag: "button", name: "发布" }, - }); - - expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); - }); - - it("matches a static node by document coordinates when scroll changed after observation", () => { - const target = matchTarget({ - geometry: { - rect: { x: 10, y: 50, w: 100, h: 30 }, - scrollX: 0, - scrollY: 250, - position: "static", - tag: "button", - }, - captured: [ - node(42, { - documentRect: { x: 10, y: 300, w: 100, h: 30 }, - localRect: { x: 10, y: 200, w: 100, h: 30 }, - rect: { x: 10, y: 200, w: 100, h: 30 }, - }), - ], - refs, - fallback: { tag: "button", name: "发布" }, - }); - - expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); - }); - - it("returns unmatched when multiple candidates match", () => { - const target = matchTarget({ - geometry: { - rect: { x: 10, y: 20, w: 100, h: 30 }, - scrollX: 0, - scrollY: 0, - position: "static", - tag: "button", - }, - captured: [node(42), node(43)], - refs, - }); - expect(target.unmatched).toBe(true); - }); - - it("uses viewport coordinates for fixed elements", () => { - const target = matchTarget({ - geometry: { - rect: { x: 5, y: 5, w: 80, h: 24 }, - scrollX: 100, - scrollY: 200, - position: "fixed", - tag: "button", - }, - captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })], - refs, - }); - expect(target.ref).toBe("e1"); - }); -}); diff --git a/apps/extension/src/lib/__tests__/page-settled.test.ts b/apps/extension/src/lib/__tests__/page-settled.test.ts deleted file mode 100644 index 8e0b989a..00000000 --- a/apps/extension/src/lib/__tests__/page-settled.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { CdpRunner } from "@/tools/shared"; -import { waitForPageSettled } from "../page-settled"; - -function quietCdp(): { cdp: CdpRunner; send: ReturnType } { - const send = vi.fn(async () => ({ - result: { value: { idleMs: 1_000, readyState: "complete" } }, - })); - return { - cdp: { send: send as unknown as CdpRunner["send"] }, - send, - }; -} - -describe("waitForPageSettled", () => { - afterEach(() => { - vi.useRealTimers(); - }); - - it("waits through the minimum observation floor before accepting a quiet page", async () => { - vi.useFakeTimers(); - const { cdp, send } = quietCdp(); - - const settled = waitForPageSettled(cdp, 7); - await vi.advanceTimersByTimeAsync(180); - - await expect(settled).resolves.toBe("quiet"); - expect(send).toHaveBeenCalledTimes(3); - }); - - it("cancels before probing a superseded observation", async () => { - vi.useFakeTimers(); - let cancelled = false; - const { cdp, send } = quietCdp(); - - const settled = waitForPageSettled(cdp, 7, { cancelled: () => cancelled }); - cancelled = true; - await vi.advanceTimersByTimeAsync(60); - - await expect(settled).resolves.toBe("cancelled"); - expect(send).not.toHaveBeenCalled(); - }); -}); diff --git a/apps/extension/src/lib/__tests__/record-observation.test.ts b/apps/extension/src/lib/__tests__/record-observation.test.ts deleted file mode 100644 index a978b712..00000000 --- a/apps/extension/src/lib/__tests__/record-observation.test.ts +++ /dev/null @@ -1,206 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; -import type { CapturedNode } from "@/tools/vom/capture"; -import type { DraftTraceStep } from "@/transport/types"; -import { resetStateIdCounterForTests } from "../record-constants"; -import { - applyTargetMatching, - buildTraceV3, - createObservationState, - flushPendingRedirectLanding, - rememberStepOnPage, - scheduleRedirectLandingFlush, -} from "../record-observation"; -import { registerObservation } from "../trace-reducer"; - -const URL = "https://example.com/login"; - -function capturedInput(): CapturedNode { - return { - backendNodeId: 42, - parentBackendNodeId: null, - tag: "input", - attrs: {}, - rect: { x: 20, y: 40, w: 200, h: 30 }, - localRect: { x: 20, y: 40, w: 200, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }; -} - -function finalizedFillBody(value: string, redactValues: boolean): string { - const obs = createObservationState({ redactValues }); - const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]'; - const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText }); - obs.lastSettled = { - stateId, - captured: [capturedInput()], - refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], - url: URL, - vomText: rawVomText, - }; - - const draft: DraftTraceStep = { - op: "fill", - target: { unmatched: true }, - value, - geometry: { - rect: { x: 20, y: 40, w: 200, h: 30 }, - scrollX: 0, - scrollY: 0, - position: "static", - tag: "input", - }, - }; - applyTargetMatching(obs, draft, 1); - draft.postStateId = stateId; - rememberStepOnPage(obs, draft, 1); - - return ( - buildTraceV3({ - obs, - steps: [draft], - startedAt: "2026-08-12T00:00:00.000Z", - stoppedBy: "user_finish", - bskVersion: "test", - }).states[0]?.body ?? "" - ); -} - -describe("record observation annotations", () => { - beforeEach(() => { - resetStateIdCounterForTests(); - }); - - it("omits a fill literal from finalized state bodies when values are redacted", () => { - const secret = "hunter2-private"; - const body = finalizedFillBody(secret, true); - - expect(body).toContain("step 1: fill"); - expect(body).not.toContain(secret); - }); - - it("keeps fill details in ordinary recording annotations", () => { - const value = "ordinary text"; - const body = finalizedFillBody(value, false); - - expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`); - }); -}); - -describe("applyTargetMatching without a settled observation", () => { - it("keeps the capture role/name instead of a bare unmatched target", () => { - const obs = createObservationState(); - const draft: DraftTraceStep = { - op: "hover", - target: { unmatched: true }, - captureTarget: { tag: "button", role: "button", name: "新建" }, - geometry: { - rect: { x: 900, y: 8, w: 60, h: 32 }, - scrollX: 0, - scrollY: 0, - position: "static", - tag: "button", - }, - }; - - applyTargetMatching(obs, draft, 1); - - expect(draft.preStateId).toBeUndefined(); - expect(draft.target).toEqual({ - role: "button", - name: "新建", - unmatched: true, - }); - }); -}); - -describe("applyTargetMatching while the previous action is still settling", () => { - it("does not bind a new unmatched control to the stale observation", () => { - const obs = createObservationState(); - const stateId = registerObservation(obs.stateRegistry, { - url: URL, - rawVomText: '@vom 1\n@e1 textbox "Search"', - }); - obs.lastSettled = { - stateId, - captured: [capturedInput()], - refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Search", line: 1 }], - url: URL, - vomText: '@vom 1\n@e1 textbox "Search"', - }; - obs.settles.set(0, { draftIndex: 0, cancelled: false }); - const draft: DraftTraceStep = { - op: "click", - target: { unmatched: true }, - captureTarget: { tag: "button", role: "button", name: "Confirm" }, - geometry: { - rect: { x: 400, y: 300, w: 80, h: 30 }, - scrollX: 0, - scrollY: 0, - position: "fixed", - tag: "button", - }, - }; - - applyTargetMatching(obs, draft, 2); - - expect(draft.preStateId).toBeUndefined(); - expect(draft.target).toEqual({ role: "button", name: "Confirm", unmatched: true }); - }); -}); - -describe("redirect landing coalescing", () => { - it("does not consume a newer redirect hop while reading tab metadata", async () => { - vi.useFakeTimers(); - try { - const finalUrl = "https://example.com/final"; - const intermediateUrl = "https://example.com/intermediate"; - const obs = createObservationState(); - obs.lastSettled = { - stateId: "s-final", - captured: [], - refs: [], - url: finalUrl, - vomText: "@vom 1\nFinal", - }; - const steps: DraftTraceStep[] = []; - - let resolveFirstTab!: (tab: chrome.tabs.Tab) => void; - const firstTab = new Promise((resolve) => { - resolveFirstTab = resolve; - }); - const get = vi - .fn() - .mockImplementationOnce(() => firstTab) - .mockResolvedValue({ id: 7, url: finalUrl }); - const tabsApi: ChromeTabsApi = { - get, - query: vi.fn(async () => []), - }; - const send = vi.fn(async (_tabId: number, method: string) => { - if (method === "Runtime.evaluate") { - return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; - } - throw new Error(`unexpected CDP method ${method}`); - }); - const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] }; - - scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, intermediateUrl); - await vi.advanceTimersByTimeAsync(180); - expect(get).toHaveBeenCalledTimes(1); - - scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, finalUrl); - resolveFirstTab({ id: 7, url: intermediateUrl } as chrome.tabs.Tab); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(1_000); - await flushPendingRedirectLanding(obs, steps, cdp, 7, tabsApi); - - expect(steps).toEqual([]); - expect(obs.pendingRedirect).toBeNull(); - } finally { - vi.useRealTimers(); - } - }); -}); diff --git a/apps/extension/src/lib/__tests__/recording-observation.test.ts b/apps/extension/src/lib/__tests__/recording-observation.test.ts new file mode 100644 index 00000000..dd277f4b --- /dev/null +++ b/apps/extension/src/lib/__tests__/recording-observation.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from "vitest"; +import { ObservationNodeIndex } from "../recording/observation-capture"; +import { RecordingObservationSession } from "../recording/observation-session"; +import { RecordingStateRegistry } from "../recording/state-registry"; +import { buildTraceV3 } from "../recording/trace-builder-v3"; +import type { RecordingDraftStep } from "../recording/types"; + +const URL = "https://example.com/login"; + +function sessionWithInput(redactValues = false): RecordingObservationSession { + const session = new RecordingObservationSession({ redactValues }); + const state = session.registry.register({ + url: URL, + rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]', + }); + session.cursor.lastSettled = { + stateId: state.id, + rootFrameId: "root", + index: new ObservationNodeIndex({ + rootFrameId: "root", + frameDocuments: [ + { + frameId: "root", + domNodes: [ + { + backendNodeId: 42, + parentBackendNodeId: null, + frameId: "root", + tag: "input", + attrs: {}, + rect: { x: 20, y: 40, w: 200, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }, + ], + refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], + }), + url: URL, + }; + return session; +} + +function finalizedFillBody(value: string, redactValues: boolean): string { + const session = sessionWithInput(redactValues); + const draft: RecordingDraftStep = { + op: "fill", + captureTarget: { tag: "input", role: "textbox", name: "Password" }, + value, + targetHint: { + geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" }, + }, + }; + session.bindDraft(draft, 1); + draft.postStateId = draft.preStateId; + return buildTraceV3({ + registry: session.registry, + drafts: [draft], + annotations: session.annotations, + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }).states[0]!.body; +} + +describe("record observation annotations", () => { + it("omits a fill literal when values are redacted", () => { + const secret = "hunter2-private"; + const body = finalizedFillBody(secret, true); + expect(body).toContain("step 1: fill"); + expect(body).not.toContain(secret); + }); + + it("keeps ordinary fill details", () => { + expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"'); + }); + + it("encodes title and URL so line breaks cannot corrupt state metadata", () => { + const registry = new RecordingStateRegistry(); + const state = registry.register({ + url: "https://example.com/a\nb", + title: "hello\nworld", + rawVomText: "@vom 1", + }); + const trace = buildTraceV3({ + registry, + drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }], + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }); + expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"'); + expect(trace.states[0]?.body).toContain('title: "hello\\nworld"'); + }); +}); + +describe("recording state ownership", () => { + it("deduplicates within one recording and isolates ids between recordings", () => { + const first = new RecordingStateRegistry(); + const second = new RecordingStateRegistry(); + expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1"); + expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1"); + expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1"); + }); + + it("enriches metadata when a deduplicated observation becomes more complete", () => { + const registry = new RecordingStateRegistry(); + registry.register({ url: URL, rawVomText: "same" }); + const state = registry.register({ + url: URL, + title: "Login", + rawVomText: "same", + truncated: true, + }); + expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true }); + }); +}); + +describe("draft binding", () => { + it("keeps capture semantics when no observation exists", () => { + const session = new RecordingObservationSession(); + const draft: RecordingDraftStep = { + op: "hover", + captureTarget: { tag: "button", role: "button", name: "新建" }, + }; + session.bindDraft(draft, 1); + expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true }); + expect(draft.preStateId).toBeUndefined(); + }); + + it("does not bind an unmatched new action to a stale observation", () => { + const session = sessionWithInput(); + const draft: RecordingDraftStep = { + op: "click", + captureTarget: { tag: "button", role: "button", name: "Confirm" }, + targetHint: { + geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" }, + }, + }; + session.bindDraft(draft, 2, true); + expect(draft.preStateId).toBeUndefined(); + expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({ + role: "button", + name: "Confirm", + unmatched: true, + }); + }); +}); diff --git a/apps/extension/src/lib/__tests__/settle-controller.test.ts b/apps/extension/src/lib/__tests__/settle-controller.test.ts new file mode 100644 index 00000000..b72e2437 --- /dev/null +++ b/apps/extension/src/lib/__tests__/settle-controller.test.ts @@ -0,0 +1,136 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; +import { RecordingObservationSession } from "../recording/observation-session"; +import { SettleController } from "../recording/settle-controller"; +import type { RecordingDraftStep } from "../recording/types"; + +const OBSERVATION: RegisteredObservation = { + stateId: "s-next", + rootFrameId: "root", + index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: [], refs: [] }), + url: "https://example.com/next", +}; + +describe("SettleController", () => { + afterEach(() => vi.useRealTimers()); + + it("aborts superseded capture work and lands the prior action on the next origin", async () => { + vi.useFakeTimers(); + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })) as unknown as CdpRunner["send"], + }; + const tabsApi: ChromeTabsApi = { + get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab), + query: vi.fn(async () => []), + }; + const session = new RecordingObservationSession(); + let firstSignal: AbortSignal | undefined; + vi.spyOn(session, "capture") + .mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => { + firstSignal = signal; + return new Promise((_resolve, reject) => { + signal?.addEventListener( + "abort", + () => reject(new DOMException("observation aborted", "AbortError")), + { once: true }, + ); + }); + }) + .mockResolvedValueOnce(OBSERVATION); + + const drafts: RecordingDraftStep[] = [ + { op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" }, + ]; + const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 }); + controller.schedule(drafts, 0); + await vi.advanceTimersByTimeAsync(180); + + drafts.push({ + op: "click", + captureTarget: { tag: "button", name: "Second" }, + preStateId: "s-next", + }); + controller.schedule(drafts, 1); + expect(firstSignal?.aborted).toBe(true); + expect(drafts[0]?.postStateId).toBe("s-next"); + + await vi.advanceTimersByTimeAsync(500); + await controller.flush(); + expect(drafts[1]?.postStateId).toBe("s-next"); + }); + + it("does not consume a newer redirect while reading the prior landing URL", async () => { + vi.useFakeTimers(); + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })) as unknown as CdpRunner["send"], + }; + let resolveFirstTab!: (tab: chrome.tabs.Tab) => void; + const firstTab = new Promise((resolve) => { + resolveFirstTab = resolve; + }); + const tabsApi: ChromeTabsApi = { + get: vi + .fn() + .mockImplementationOnce(() => firstTab) + .mockResolvedValue({ id: 7, url: OBSERVATION.url }), + query: vi.fn(async () => []), + }; + const session = new RecordingObservationSession(); + session.cursor.lastSettled = OBSERVATION; + const drafts: RecordingDraftStep[] = []; + const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 }); + + controller.scheduleRedirect(drafts, "https://example.com/intermediate"); + await vi.advanceTimersByTimeAsync(180); + expect(tabsApi.get).toHaveBeenCalledTimes(1); + + controller.scheduleRedirect(drafts, OBSERVATION.url); + resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab); + await vi.advanceTimersByTimeAsync(500); + await controller.flushRedirects(); + + expect(drafts).toEqual([]); + expect(controller.hasPending).toBe(false); + }); + + it("settles an action in its own OOPIF document scope", async () => { + vi.useFakeTimers(); + const send = vi.fn(); + const sendToTarget = vi.fn(async (_target, method: string) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 39 }; + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + }; + const tabsApi: ChromeTabsApi = { + get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab), + query: vi.fn(async () => []), + }; + const session = new RecordingObservationSession(); + vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION); + const drafts: RecordingDraftStep[] = [ + { op: "click", captureTarget: { tag: "button", name: "Inside frame" } }, + ]; + const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 }); + const target = { tabId: 7, sessionId: "oopif-session" }; + + controller.schedule(drafts, 0, { target, frameId: "oopif-frame" }); + await vi.advanceTimersByTimeAsync(180); + await controller.flush(); + + expect(send).not.toHaveBeenCalled(); + expect(sendToTarget).toHaveBeenCalledWith( + target, + "Runtime.evaluate", + expect.objectContaining({ contextId: 39 }), + ); + expect(drafts[0]?.postStateId).toBe("s-next"); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts b/apps/extension/src/lib/__tests__/step-buffer.test.ts similarity index 80% rename from apps/extension/src/lib/__tests__/recording-step-buffer.test.ts rename to apps/extension/src/lib/__tests__/step-buffer.test.ts index 396253a5..ca212b8b 100644 --- a/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts +++ b/apps/extension/src/lib/__tests__/step-buffer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer"; +import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer"; describe("recording-step-buffer", () => { it("stores semantic click without summary", () => { @@ -12,7 +12,7 @@ describe("recording-step-buffer", () => { expect(buffer.steps).toEqual([ { op: "click", - target: { tag: "button", role: "button", name: "发布" }, + captureTarget: { tag: "button", role: "button", name: "发布" }, }, ]); expect(buffer.pendingNavigation).toBe(true); @@ -23,7 +23,7 @@ describe("recording-step-buffer", () => { steps: [ { op: "click" as const, - target: { tag: "button", role: "button", name: "发布" }, + captureTarget: { tag: "button", role: "button", name: "发布" }, }, ], currentUrl: "https://example.com/a", @@ -34,8 +34,8 @@ describe("recording-step-buffer", () => { expect(buffer.steps).toEqual([ { op: "click", - target: { tag: "button", role: "button", name: "发布" }, - navigated_to: "https://example.com/b", + captureTarget: { tag: "button", role: "button", name: "发布" }, + navigatedTo: "https://example.com/b", }, ]); expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation"); @@ -46,7 +46,7 @@ describe("recording-step-buffer", () => { steps: [ { op: "select" as const, - target: { tag: "select", role: "combobox", name: "分类" }, + captureTarget: { tag: "select", role: "combobox", name: "分类" }, values: ["tech"], }, ], @@ -58,9 +58,9 @@ describe("recording-step-buffer", () => { expect(buffer.steps).toEqual([ { op: "select", - target: { tag: "select", role: "combobox", name: "分类" }, + captureTarget: { tag: "select", role: "combobox", name: "分类" }, values: ["tech"], - navigated_to: "https://example.com/list?cat=tech", + navigatedTo: "https://example.com/list?cat=tech", }, ]); }); diff --git a/apps/extension/src/lib/__tests__/target-matcher.test.ts b/apps/extension/src/lib/__tests__/target-matcher.test.ts new file mode 100644 index 00000000..b7081181 --- /dev/null +++ b/apps/extension/src/lib/__tests__/target-matcher.test.ts @@ -0,0 +1,126 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import { describe, expect, it } from "vitest"; +import type { CapturedNode } from "@/tools/vom/capture"; +import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; +import { matchObservationTarget } from "../recording/target-matcher"; + +function node(backendNodeId: number, frameId: string, x = 10): CapturedNode { + return { + backendNodeId, + parentBackendNodeId: null, + frameId, + tag: "button", + attrs: {}, + rect: { x, y: 20, w: 100, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }; +} + +function observation( + documents: Array<{ frameId: string; domNodes: CapturedNode[] }>, + refs: RenderedRef[], +): RegisteredObservation { + return { + stateId: "s1", + rootFrameId: "root", + index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: documents, refs }), + url: "https://example.com", + }; +} + +describe("matchObservationTarget", () => { + it("matches a unique node using canonical top-level viewport geometry", () => { + const target = matchObservationTarget({ + observation: observation( + [{ frameId: "root", domNodes: [node(42, "root")] }], + [{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + }); + expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); + }); + + it("uses frame id with backend node id so sibling frames cannot collide", () => { + const target = matchObservationTarget({ + observation: observation( + [ + { frameId: "left", domNodes: [node(42, "left")] }, + { frameId: "right", domNodes: [node(42, "right")] }, + ], + [ + { ref: "e1", backendNodeId: 42, frameId: "left", line: 1 }, + { ref: "e2", backendNodeId: 42, frameId: "right", line: 2 }, + ], + ), + hint: { + frameId: "right", + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }); + expect(target.ref).toBe("e2"); + }); + + it("restricts a missing frame hint to the root frame", () => { + const target = matchObservationTarget({ + observation: observation( + [ + { frameId: "root", domNodes: [] }, + { frameId: "child", domNodes: [node(42, "child")] }, + ], + [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + fallback: { tag: "button", name: "发布" }, + }); + expect(target).toEqual({ name: "发布", unmatched: true }); + }); + + it("returns unmatched for ambiguous geometry", () => { + const target = matchObservationTarget({ + observation: observation( + [{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }], + [ + { ref: "e1", backendNodeId: 42, line: 1 }, + { ref: "e2", backendNodeId: 43, line: 2 }, + ], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + }); + expect(target.unmatched).toBe(true); + }); + + it("uses semantics when geometry is unavailable without crossing frame boundaries", () => { + const target = matchObservationTarget({ + observation: observation( + [ + { frameId: "root", domNodes: [node(41, "root")] }, + { frameId: "child", domNodes: [node(42, "child")] }, + ], + [ + { ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 }, + { ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 }, + ], + ), + hint: { frameId: "child" }, + fallback: { tag: "button", role: "button", name: "保存" }, + }); + expect(target.ref).toBe("e2"); + }); + + it("uses semantics to disambiguate equal geometry in one frame", () => { + const target = matchObservationTarget({ + observation: observation( + [{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }], + [ + { ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 }, + { ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 }, + ], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + fallback: { tag: "button", role: "button", name: "取消" }, + }); + expect(target.ref).toBe("e2"); + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts index 8abb8cd0..2b7fb1d7 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts @@ -1,75 +1,196 @@ import { describe, expect, it } from "vitest"; -import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2"; -import type { DraftTraceStep } from "@/transport/types"; +import type { RecordingDraftStep } from "@/lib/recording/types"; +import { shouldRecordPress } from "../recording/draft-policy"; +import { buildTraceV2 } from "../recording/trace-reducer-v2"; -describe("trace-reducer-v2", () => { - it("drops scroll and bare character press steps", () => { - const steps: DraftTraceStep[] = [ +function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) { + return buildTraceV2({ + steps, + startedAt: "2026-01-01T00:00:00.000Z", + ...(startUrl ? { startUrl } : {}), + }); +} + +describe("shouldRecordPress", () => { + it("keeps Enter and Escape", () => { + expect(shouldRecordPress("Enter")).toBe(true); + expect(shouldRecordPress("Escape")).toBe(true); + }); + + it("drops modifiers, clipboard shortcuts, and bare typing", () => { + expect(shouldRecordPress("Meta")).toBe(false); + expect(shouldRecordPress("c", ["meta"])).toBe(false); + expect(shouldRecordPress("a", ["ctrl"])).toBe(false); + expect(shouldRecordPress("x")).toBe(false); + expect(shouldRecordPress("中")).toBe(false); + }); +}); + +describe("reduceTraceSteps", () => { + it("builds steps with pages dictionary and page id references", () => { + const drafts: RecordingDraftStep[] = [ + { + op: "navigate", + url: "https://example.com/search?q=hello&utm_source=x", + pageUrl: "https://example.com/search?q=hello&utm_source=x", + }, { - op: "scroll", - page_url: "https://example.com/", + op: "fill", + captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, + value: "browser skill", + pageUrl: "https://example.com/search?q=hello&utm_source=x", + }, + { + op: "press", + key: "Enter", + captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, + navigatedTo: "https://example.com/results/42", + pageUrl: "https://example.com/search?q=hello&utm_source=x", + }, + { + op: "click", + captureTarget: { tag: "button", role: "button", name: "发布" }, + navigatedTo: "https://example.com/p/99", + pageUrl: "https://example.com/results/42", }, { op: "press", key: "a", - page_url: "https://example.com/", + pageUrl: "https://example.com/p/99", }, ]; - const trace = buildTraceV2({ - steps, - startedAt: "2026-01-01T00:00:00.000Z", - startUrl: "https://example.com/", + + const { pages, steps } = reduceTraceSteps( + drafts, + "https://example.com/search?q=hello&utm_source=x", + ); + expect(JSON.stringify(steps)).not.toContain("parameters"); + expect(JSON.stringify(steps)).not.toContain("intent"); + expect(JSON.stringify(steps)).not.toContain("summary"); + expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]); + expect(pages.map((p) => p.url)).toEqual([ + "https://example.com/search?q=hello&utm_source=x", + "https://example.com/results/42", + "https://example.com/p/99", + ]); + + expect(steps[0]).toMatchObject({ + id: 1, + op: "navigate", + page: "p1", + to: "https://example.com/search?q=hello&utm_source=x", + }); + + expect(steps[1]).toMatchObject({ + op: "fill", + page: "p1", + value: "browser skill", + }); + + expect(steps[2]).toMatchObject({ + op: "press", + key: "Enter", + page: "p1", + effect: { navigated_to: "p2" }, + }); + + expect(steps[3]).toMatchObject({ + op: "click", + page: "p2", + effect: { navigated_to: "p3" }, }); - expect(trace.steps).toHaveLength(0); - expect(trace.pages).toHaveLength(1); }); - it("preserves hover steps supported by protocol v2", () => { - const trace = buildTraceV2({ - steps: [ + it("collapses consecutive navigations", () => { + const { steps } = reduceTraceSteps([ + { op: "navigate", url: "https://a.example/redirect1" }, + { op: "navigate", url: "https://a.example/final" }, + ]); + expect(steps).toHaveLength(1); + expect(steps[0]).toMatchObject({ + op: "navigate", + to: "https://a.example/final", + }); + }); + + it("keeps committed empty fill steps", () => { + const { steps } = reduceTraceSteps( + [ { - op: "hover", - target: { unmatched: true }, - captureTarget: { tag: "button", role: "button", name: "结束" }, - page_url: "https://example.com/", + op: "fill", + captureTarget: { tag: "input", role: "textbox", name: "Search query" }, + value: "", + pageUrl: "https://example.com/search", }, ], - startedAt: "2026-01-01T00:00:00.000Z", - startUrl: "https://example.com/", - }); + "https://example.com/search", + ); - expect(trace.steps).toEqual([ + expect(steps).toEqual([ expect.objectContaining({ - op: "hover", - target: expect.objectContaining({ tag: "button", name: "结束" }), + op: "fill", + value: "", }), ]); }); - it("maps captureTarget to v2 target with required tag", () => { - const steps: DraftTraceStep[] = [ - { - op: "click", - target: { unmatched: true }, - captureTarget: { tag: "button", role: "button", name: "Submit" }, - page_url: "https://example.com/", - }, - ]; - const trace = buildTraceV2({ - steps, - startedAt: "2026-01-01T00:00:00.000Z", - startUrl: "https://example.com/", + it("maps select navigatedTo onto effect.navigatedTo (page id)", () => { + const { pages, steps } = reduceTraceSteps( + [ + { + op: "select", + captureTarget: { tag: "select", role: "combobox", name: "分类" }, + values: ["tech"], + labels: ["技术"], + navigatedTo: "https://example.com/list?cat=tech", + pageUrl: "https://example.com/list", + }, + ], + "https://example.com/list", + ); + expect(pages.map((p) => p.url)).toEqual([ + "https://example.com/list", + "https://example.com/list?cat=tech", + ]); + expect(steps[0]).toMatchObject({ + op: "select", + page: "p1", + selection: [{ value: "tech", label: "技术" }], + effect: { navigated_to: "p2" }, }); - expect(trace.steps).toHaveLength(1); - expect(trace.steps[0]?.op).toBe("click"); - if (trace.steps[0]?.op === "click") { - expect(trace.steps[0].target.tag).toBe("button"); - expect(trace.steps[0].target.name).toBe("Submit"); - } - expect("version" in trace).toBe(false); }); - it("records Enter presses", () => { - expect(shouldRecordPress("Enter")).toBe(true); + it("keeps hover steps before menu clicks", () => { + const { steps } = reduceTraceSteps( + [ + { + op: "hover", + captureTarget: { tag: "span", role: "button", name: "Account" }, + pageUrl: "https://example.com/app", + }, + { + op: "click", + captureTarget: { tag: "a", role: "link", name: "Profile" }, + pageUrl: "https://example.com/app", + }, + ], + "https://example.com/app", + ); + expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); + expect(steps[0]).toMatchObject({ + op: "hover", + page: "p1", + target: { name: "Account" }, + }); + }); + + it("resolveTraceStartUrl prefers explicit start URL", () => { + expect( + buildTraceV2({ + steps: [{ op: "navigate", url: "https://example.com/other" }], + startedAt: "2026-01-01T00:00:00.000Z", + startUrl: "https://example.com/start", + }).entry.start_url, + ).toBe("https://example.com/start"); }); }); diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts new file mode 100644 index 00000000..93434d14 --- /dev/null +++ b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types"; +import { RecordingStateRegistry } from "../recording/state-registry"; +import { buildTraceV3 } from "../recording/trace-builder-v3"; +import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3"; +import type { RecordingDraftStep } from "../recording/types"; + +describe("trace reducer v3", () => { + it("collapses redirect hops while retaining draft-to-step identity", () => { + const drafts: RecordingDraftStep[] = [ + { op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" }, + { + op: "navigate", + url: "https://example.com/final", + transitionQualifiers: ["server_redirect"], + preStateId: "s2", + postStateId: "s3", + }, + ]; + const before = structuredClone(drafts); + const reduced = reduceTraceStepsV3(drafts); + + expect(drafts).toEqual(before); + expect(reduced.steps).toEqual([ + expect.objectContaining({ + id: 1, + op: "navigate", + state: "s1", + to: "https://example.com/final", + result: { state: "s3" }, + }), + ]); + expect(reduced.stepIdByDraftId.get(1)).toBe(1); + expect(reduced.stepIdByDraftId.get(2)).toBe(1); + }); + + it("builds the wire model from protocol constants", () => { + const registry = new RecordingStateRegistry(); + const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" }); + const trace = buildTraceV3({ + registry, + drafts: [ + { + op: "click", + captureTarget: { tag: "button", role: "button", name: "Save" }, + preStateId: state.id, + postStateId: state.id, + }, + ], + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }); + + expect(trace.version).toBe(TRACE_VERSION_V3); + expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION); + expect(trace.steps[0]).toMatchObject({ + op: "click", + target: { role: "button", name: "Save", unmatched: true }, + }); + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer.test.ts deleted file mode 100644 index f5c6db51..00000000 --- a/apps/extension/src/lib/__tests__/trace-reducer.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { DraftTraceStep } from "@/transport/types"; -import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer"; - -describe("shouldRecordPress", () => { - it("keeps Enter and Escape", () => { - expect(shouldRecordPress("Enter")).toBe(true); - expect(shouldRecordPress("Escape")).toBe(true); - }); - - it("drops modifiers, clipboard shortcuts, and bare typing", () => { - expect(shouldRecordPress("Meta")).toBe(false); - expect(shouldRecordPress("c", ["meta"])).toBe(false); - expect(shouldRecordPress("a", ["ctrl"])).toBe(false); - expect(shouldRecordPress("x")).toBe(false); - expect(shouldRecordPress("中")).toBe(false); - }); -}); - -describe("reduceTraceSteps", () => { - it("builds steps with pages dictionary and page id references", () => { - const drafts: DraftTraceStep[] = [ - { - op: "navigate", - url: "https://example.com/search?q=hello&utm_source=x", - page_url: "https://example.com/search?q=hello&utm_source=x", - }, - { - op: "fill", - target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, - value: "browser skill", - page_url: "https://example.com/search?q=hello&utm_source=x", - }, - { - op: "press", - key: "Enter", - target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, - navigated_to: "https://example.com/results/42", - page_url: "https://example.com/search?q=hello&utm_source=x", - }, - { - op: "click", - target: { tag: "button", role: "button", name: "发布" }, - navigated_to: "https://example.com/p/99", - page_url: "https://example.com/results/42", - }, - { - op: "press", - key: "a", - page_url: "https://example.com/p/99", - }, - ]; - - const { pages, steps } = reduceTraceSteps( - drafts, - "https://example.com/search?q=hello&utm_source=x", - ); - expect(JSON.stringify(steps)).not.toContain("parameters"); - expect(JSON.stringify(steps)).not.toContain("intent"); - expect(JSON.stringify(steps)).not.toContain("summary"); - expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]); - expect(pages.map((p) => p.url)).toEqual([ - "https://example.com/search?q=hello&utm_source=x", - "https://example.com/results/42", - "https://example.com/p/99", - ]); - - expect(steps[0]).toMatchObject({ - id: 1, - op: "navigate", - page: "p1", - to: "https://example.com/search?q=hello&utm_source=x", - }); - - expect(steps[1]).toMatchObject({ - op: "fill", - page: "p1", - value: "browser skill", - }); - - expect(steps[2]).toMatchObject({ - op: "press", - key: "Enter", - page: "p1", - effect: { navigated_to: "p2" }, - }); - - expect(steps[3]).toMatchObject({ - op: "click", - page: "p2", - effect: { navigated_to: "p3" }, - }); - }); - - it("collapses consecutive navigations", () => { - const { steps } = reduceTraceSteps([ - { op: "navigate", url: "https://a.example/redirect1" }, - { op: "navigate", url: "https://a.example/final" }, - ]); - expect(steps).toHaveLength(1); - expect(steps[0]).toMatchObject({ - op: "navigate", - to: "https://a.example/final", - }); - }); - - it("keeps committed empty fill steps", () => { - const { steps } = reduceTraceSteps( - [ - { - op: "fill", - target: { tag: "input", role: "textbox", name: "Search query" }, - value: "", - page_url: "https://example.com/search", - }, - ], - "https://example.com/search", - ); - - expect(steps).toEqual([ - expect.objectContaining({ - op: "fill", - value: "", - }), - ]); - }); - - it("maps select navigated_to onto effect.navigated_to (page id)", () => { - const { pages, steps } = reduceTraceSteps( - [ - { - op: "select", - target: { tag: "select", role: "combobox", name: "分类" }, - values: ["tech"], - labels: ["技术"], - navigated_to: "https://example.com/list?cat=tech", - page_url: "https://example.com/list", - }, - ], - "https://example.com/list", - ); - expect(pages.map((p) => p.url)).toEqual([ - "https://example.com/list", - "https://example.com/list?cat=tech", - ]); - expect(steps[0]).toMatchObject({ - op: "select", - page: "p1", - selection: [{ value: "tech", label: "技术" }], - effect: { navigated_to: "p2" }, - }); - }); - - it("keeps hover steps before menu clicks", () => { - const { steps } = reduceTraceSteps( - [ - { - op: "hover", - target: { tag: "span", role: "button", name: "Account" }, - page_url: "https://example.com/app", - }, - { - op: "click", - target: { tag: "a", role: "link", name: "Profile" }, - page_url: "https://example.com/app", - }, - ], - "https://example.com/app", - ); - expect(steps.map((s) => s.op)).toEqual(["hover", "click"]); - expect(steps[0]).toMatchObject({ - op: "hover", - page: "p1", - target: { name: "Account" }, - }); - }); - - it("resolveTraceStartUrl prefers explicit start URL", () => { - expect( - resolveTraceStartUrl( - [{ op: "navigate", url: "https://example.com/other" }], - "https://example.com/start", - ), - ).toBe("https://example.com/start"); - }); -}); diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts index f2414075..c090b0a6 100644 --- a/apps/extension/src/lib/describe-target.ts +++ b/apps/extension/src/lib/describe-target.ts @@ -1,5 +1,5 @@ /** - * Build a semantic TargetDescriptor for an interacted element. + * Build a semantic capture descriptor for an interacted element. * * Trace steps are an LLM *textbook*: each click must say what to look for * on screen (usually a short visible name). Tag-only noise like @@ -16,9 +16,6 @@ export interface CaptureTargetDescriptor { nearby_label?: string; } -/** @deprecated Capture-time alias retained until the recorder integration migrates. */ -export type TargetDescriptor = CaptureTargetDescriptor; - /** Max length for a label that is still a useful “find this on screen” hint. */ const ACTIONABLE_LABEL_MAX = 48; diff --git a/apps/extension/src/lib/format-observation-file.ts b/apps/extension/src/lib/format-observation-file.ts deleted file mode 100644 index 8676553c..00000000 --- a/apps/extension/src/lib/format-observation-file.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { StepV3 } from "@/transport/types"; -import { OBSERVATION_FILE_VERSION } from "./record-constants"; - -export interface ObservationAnnotation { - stepId: number; - op: StepV3["op"]; - line: number; - stateId: string; - detail?: string; -} - -function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string { - const suffix = detail ? `: ${detail}` : ""; - return ` ⟵ step ${stepId}: ${op}${suffix}`; -} - -/** Serialize a page observation file (front matter + VOM body + step annotations). */ -export function formatObservationFile(input: { - stateId: string; - url: string; - title?: string; - stepsHere: number[]; - body: string; - annotations?: ObservationAnnotation[]; -}): string { - const lines: string[] = [ - `# bsk-observation ${OBSERVATION_FILE_VERSION}`, - `state: ${input.stateId}`, - `url: ${input.url}`, - ]; - if (input.title) lines.push(`title: ${input.title}`); - if (input.stepsHere.length > 0) { - lines.push(`steps_here: [${input.stepsHere.join(", ")}]`); - } - lines.push("---"); - - const bodyLines = input.body.split("\n"); - const annotationMap = new Map(); - for (const ann of input.annotations ?? []) { - const bucket = annotationMap.get(ann.line) ?? []; - bucket.push(ann); - annotationMap.set(ann.line, bucket); - } - - for (let i = 0; i < bodyLines.length; i += 1) { - let line = bodyLines[i] ?? ""; - const anns = annotationMap.get(i); - if (anns) { - for (const ann of anns) { - line += formatAnnotation(ann); - } - } - lines.push(line); - } - - return `${lines.join("\n")}\n`; -} - -/** Hash input: VOM body **before** annotations are inserted. */ -export function observationBodyForHash(vomText: string): string { - return vomText; -} diff --git a/apps/extension/src/lib/match-target.ts b/apps/extension/src/lib/match-target.ts deleted file mode 100644 index 06e1435f..00000000 --- a/apps/extension/src/lib/match-target.ts +++ /dev/null @@ -1,103 +0,0 @@ -import type { RenderedRef } from "@browser-skill/vom"; -import type { CapturedNode } from "@/tools/vom/capture"; -import type { CaptureGeometry, TargetDescriptorV3 } from "@/transport/types"; -import type { CaptureTargetDescriptor } from "./describe-target"; -import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants"; - -export interface MatchTargetInput { - geometry: CaptureGeometry; - captured: CapturedNode[]; - refs: RenderedRef[]; - fallback?: CaptureTargetDescriptor; -} - -function withinTolerance(a: number, b: number): boolean { - return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX; -} - -function rectsMatch( - a: { x: number; y: number; w: number; h: number }, - b: { x: number; y: number; w: number; h: number }, -): boolean { - return ( - withinTolerance(a.x, b.x) && - withinTolerance(a.y, b.y) && - withinTolerance(a.w, b.w) && - withinTolerance(a.h, b.h) - ); -} - -function nodeViewportRect( - node: CapturedNode, -): { x: number; y: number; w: number; h: number } | null { - return node.localRect ?? node.rect; -} - -function matchingRects( - geometry: CaptureGeometry, - node: CapturedNode, -): { - target: { x: number; y: number; w: number; h: number }; - node: { x: number; y: number; w: number; h: number } | null; -} { - const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null; - const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky"; - if (!topFrame || viewportPosition || !node.documentRect) { - return { target: geometry.rect, node: nodeViewportRect(node) }; - } - return { - target: { - x: geometry.rect.x + geometry.scrollX, - y: geometry.rect.y + geometry.scrollY, - w: geometry.rect.w, - h: geometry.rect.h, - }, - node: node.documentRect, - }; -} - -function tagMatches(geometryTag: string, nodeTag: string): boolean { - return geometryTag.toLowerCase() === nodeTag.toLowerCase(); -} - -/** Best description available when the element cannot be located in the VOM. */ -export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { - if (!fallback) { - return { unmatched: true }; - } - return { - ...(fallback.role ? { role: fallback.role } : {}), - ...(fallback.name ? { name: fallback.name } : {}), - unmatched: true, - }; -} - -/** Locate the interacted element in the last settled observation by geometry. */ -export function matchTarget(input: MatchTargetInput): TargetDescriptorV3 { - const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null; - - const candidates = input.captured.filter((node) => { - if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false; - if (!tagMatches(input.geometry.tag, node.tag)) return false; - const rects = matchingRects(input.geometry, node); - if (!rects.node) return false; - return rectsMatch(rects.target, rects.node); - }); - - if (candidates.length !== 1) { - return fallbackDescriptor(input.fallback); - } - - const backendNodeId = candidates[0]!.backendNodeId; - const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId); - if (!refEntry) { - return fallbackDescriptor(input.fallback); - } - - return { - ref: refEntry.ref, - ...(refEntry.role ? { role: refEntry.role } : {}), - ...(refEntry.name ? { name: refEntry.name } : {}), - ...(refEntry.ctx ? { ctx: refEntry.ctx } : {}), - }; -} diff --git a/apps/extension/src/lib/page-settled.ts b/apps/extension/src/lib/page-settled.ts deleted file mode 100644 index f95ff05b..00000000 --- a/apps/extension/src/lib/page-settled.ts +++ /dev/null @@ -1,92 +0,0 @@ -// Deciding *when* a page is worth observing. -// -// A fixed delay cannot serve both a modal that toggles in one frame and a -// route change that renders for a second. Instead of guessing, ask the page: -// a document that has stopped mutating and is no longer loading is done -// reacting to whatever the user just did. - -import type { CdpRunner } from "@/tools/shared"; -import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants"; - -/** - * Installed once per document. Records only the timestamp of the last DOM - * change so each poll stays O(1) no matter how large the page is. - */ -const QUIET_PROBE = `(() => { - const scope = window; - let probe = scope.__bskRecordQuiet; - if (!probe) { - probe = { changedAt: Date.now() }; - const observer = new MutationObserver(() => { - probe.changedAt = Date.now(); - }); - observer.observe(document.documentElement, { - subtree: true, - childList: true, - attributes: true, - characterData: true, - }); - scope.__bskRecordQuiet = probe; - } - return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState }; -})()`; - -export type SettleOutcome = - /** The page stopped changing on its own. */ - | "quiet" - /** Still changing when the budget ran out; observe it as it is. */ - | "timeout" - /** A newer action took over; this observation is no longer wanted. */ - | "cancelled"; - -interface QuietProbe { - idleMs: number; - readyState: string; -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise { - try { - const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", { - expression: QUIET_PROBE, - returnByValue: true, - }); - const value = reply.result?.value; - if (!value || typeof value !== "object") return null; - const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown }; - if (typeof idleMs !== "number" || typeof readyState !== "string") return null; - return { idleMs, readyState }; - } catch { - // An unreadable page is a page mid-swap; that is a reason to keep waiting, - // not a reason to give up. - return null; - } -} - -/** Wait until the page has finished reacting, or until the budget runs out. */ -export async function waitForPageSettled( - cdp: CdpRunner, - tabId: number, - options: { cancelled?: () => boolean } = {}, -): Promise { - const startedAt = Date.now(); - const floor = startedAt + SETTLE_MIN_MS; - const deadline = startedAt + SETTLE_MAX_MS; - - for (;;) { - if (options.cancelled?.()) return "cancelled"; - await sleep(SETTLE_POLL_MS); - if (options.cancelled?.()) return "cancelled"; - - const probe = await readQuietProbe(cdp, tabId); - const now = Date.now(); - if (now < floor) continue; - if (now >= deadline) return "timeout"; - if (!probe) continue; - if (probe.readyState === "loading") continue; - if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet"; - } -} diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts index c4259e38..09399ba5 100644 --- a/apps/extension/src/lib/record-bridge.ts +++ b/apps/extension/src/lib/record-bridge.ts @@ -3,7 +3,7 @@ * service worker and a tab's content script. */ -import type { TargetDescriptor } from "./describe-target"; +import type { CaptureTargetDescriptor } from "./describe-target"; export const RECORD_START = "bsk-record-start"; export const RECORD_STEP = "bsk-record-step"; @@ -42,7 +42,7 @@ export interface RecordStartMessage { export interface RecordStepPayload { op: "click" | "hover" | "fill" | "press" | "select" | "navigate"; - target?: TargetDescriptor; + target?: CaptureTargetDescriptor; value?: string; key?: string; modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">; diff --git a/apps/extension/src/lib/record-constants.ts b/apps/extension/src/lib/record-constants.ts deleted file mode 100644 index 0cbf1f32..00000000 --- a/apps/extension/src/lib/record-constants.ts +++ /dev/null @@ -1,97 +0,0 @@ -import type { FillCommit, NavigationCause } from "@/transport/types"; - -/** - * Floor before a post-action observation may be taken. A page usually has not - * started reacting in the first frames after an action, and capturing then - * would record the page as it was, not as the action left it. - */ -export const SETTLE_MIN_MS = 150; - -/** How long the DOM must stop changing before a page counts as settled. */ -export const SETTLE_QUIET_MS = 250; - -/** Upper bound on waiting for a page to settle; animations never stop. */ -export const SETTLE_MAX_MS = 2_000; - -/** How often to ask the page whether it has gone quiet. */ -export const SETTLE_POLL_MS = 60; - -/** Minimum interval between consecutive observations on the same tab. */ -export const OBSERVATION_MIN_INTERVAL_MS = 200; - -/** Delay before retrying a capture that lost its execution context. */ -export const CAPTURE_RETRY_DELAY_MS = 250; - -/** Document-coordinate matching tolerance in CSS pixels. */ -export const GEOM_MATCH_TOLERANCE_PX = 2; - -/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */ -export const DEFAULT_MAX_PAGE_TOKENS = 3000; - -/** VOM observation file format version (front matter header). */ -export const OBSERVATION_FILE_VERSION = 1; - -/** FNV-1a 64-bit offset basis. */ -const FNV_OFFSET = 0xcbf29ce484222325n; -/** FNV-1a 64-bit prime. */ -const FNV_PRIME = 0x100000001b3n; - -/** Content-hash for state deduplication (sync, non-cryptographic). */ -export function fnv1a64(text: string): string { - let hash = FNV_OFFSET; - for (let i = 0; i < text.length; i += 1) { - hash ^= BigInt(text.charCodeAt(i)); - hash = (hash * FNV_PRIME) & 0xffffffffffffffffn; - } - return hash.toString(16).padStart(16, "0"); -} - -let nextStateSerial = 0; - -/** Monotonic state id generator (`s1`, `s2`, …). */ -export function nextStateId(): string { - nextStateSerial += 1; - return `s${nextStateSerial}`; -} - -/** Test seam: reset the state id counter. */ -export function resetStateIdCounterForTests(): void { - nextStateSerial = 0; -} - -const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]); - -const TRANSITION_TO_CAUSE: Record = { - typed: "user_typed", - generated: "user_typed", - keyword: "user_typed", - keyword_generated: "user_typed", - link: "link", - form_submit: "form_submit", - reload: "reload", - auto_bookmark: "browser", - start_page: "browser", -}; - -export interface NavigationTransitionMeta { - transitionType?: string; - transitionQualifiers?: string[]; - navigationActionPending?: boolean; -} - -/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */ -export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null { - const qualifiers = meta.transitionQualifiers ?? []; - if (qualifiers.includes("forward_back")) return "history"; - if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null; - if (qualifiers.includes("from_address_bar")) return "user_typed"; - - const type = meta.transitionType ?? ""; - const mapped = TRANSITION_TO_CAUSE[type]; - if (mapped) return mapped; - - if (!type && meta.navigationActionPending === false) return "script"; - return "browser"; -} - -export const DEFAULT_FILL_COMMIT: FillCommit = "blur"; diff --git a/apps/extension/src/lib/record-observation.ts b/apps/extension/src/lib/record-observation.ts deleted file mode 100644 index 6a74850a..00000000 --- a/apps/extension/src/lib/record-observation.ts +++ /dev/null @@ -1,681 +0,0 @@ -import type { RenderedRef } from "@browser-skill/vom"; -import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file"; -import { fallbackDescriptor, matchTarget } from "@/lib/match-target"; -import { waitForPageSettled } from "@/lib/page-settled"; -import { - CAPTURE_RETRY_DELAY_MS, - DEFAULT_MAX_PAGE_TOKENS, - OBSERVATION_MIN_INTERVAL_MS, -} from "@/lib/record-constants"; -import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer"; -import { captureVomObservation } from "@/tools/capture-vom-observation"; -import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; -import type { CapturedNode } from "@/tools/vom/capture"; -import type { DraftTraceStep, StepV3, StopReason, TraceState, TraceV3 } from "@/transport/types"; -import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer"; - -export interface LastSettledObservation { - stateId: string; - captured: CapturedNode[]; - refs: RenderedRef[]; - url: string; - title?: string; - vomText: string; -} - -/** A post-action observation that has been queued but not yet written down. */ -interface PendingSettle { - draftIndex: number; - /** Set when a newer action has already decided where this step landed. */ - cancelled: boolean; -} - -/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */ -export interface PendingRedirectLanding { - url: string; - /** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */ - generation: number; -} - -export interface RecordingObservationState { - stateRegistry: Map; - lastSettled: LastSettledObservation | null; - maxPageTokens: number; - redactValues: boolean; - lastCaptureAtMs: number; - /** - * Tail of the settle chain. Observations run one at a time so that a slow - * capture can never land after a faster one and report the pages out of the - * order the user visited them. - */ - settleQueue: Promise; - /** Queued and in-flight settles, by draft index, so newer actions can supersede them. */ - settles: Map; - stepAnnotations: Map; - /** - * OAuth / server redirects are coalesced here: intermediate hops only update - * `url`+`generation`, and one `navigate` is emitted after the page settles. - */ - pendingRedirect: PendingRedirectLanding | null; - redirectFlushQueue: Promise; -} - -export function createObservationState(options?: { - maxPageTokens?: number; - redactValues?: boolean; -}): RecordingObservationState { - return { - stateRegistry: new Map(), - lastSettled: null, - maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS, - redactValues: options?.redactValues ?? false, - lastCaptureAtMs: 0, - settleQueue: Promise.resolve(), - settles: new Map(), - stepAnnotations: new Map(), - pendingRedirect: null, - redirectFlushQueue: Promise.resolve(), - }; -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function readTabMeta( - tabsApi: ChromeTabsApi, - tabId: number, -): Promise<{ url: string; title?: string }> { - try { - const tab = await tabsApi.get(tabId); - return { url: tab.url ?? "about:blank", title: tab.title }; - } catch { - return { url: "about:blank" }; - } -} - -export async function captureAndRegisterObservation( - obs: RecordingObservationState, - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - urlOverride?: string, -): Promise { - const now = Date.now(); - const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs)); - if (waitMs > 0) await sleep(waitMs); - - const { url, title } = urlOverride - ? { url: urlOverride, title: undefined } - : await readTabMeta(tabsApi, tabId); - const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title }; - const resolvedTitle = title ?? meta.title; - - const rendered = await captureVomObservation(cdp, tabId, url, { - maxTokens: obs.maxPageTokens, - redactValues: obs.redactValues, - conditionalSurfaceProbe: false, - }); - - const stateId = registerObservation(obs.stateRegistry, { - url, - title: resolvedTitle, - rawVomText: rendered.text, - truncated: rendered.truncated, - }); - - obs.lastCaptureAtMs = Date.now(); - const settled: LastSettledObservation = { - stateId, - captured: rendered.captured, - refs: rendered.refs, - url, - title: resolvedTitle, - vomText: rendered.text, - }; - obs.lastSettled = settled; - return settled; -} - -/** - * Bind a draft to the page it was performed on: origin state, geometric target - * match, and the inline annotation. All three read the *current* observation, - * which is only the page the user acted on while the draft is still fresh — - * once the action settles, `lastSettled` is the destination page instead. - */ -export function applyTargetMatching( - obs: RecordingObservationState, - draft: DraftTraceStep, - stepId: number, -): void { - if (!obs.lastSettled) { - // Initial observation can fail or race the first action. Keep whatever - // the content script captured so the exported step is still teachable. - if ("target" in draft && "captureTarget" in draft) { - draft.target = fallbackDescriptor(draft.captureTarget); - } - rememberStepAnnotation(obs, stepId, draft); - return; - } - - if ("target" in draft) { - draft.target = - "geometry" in draft && draft.geometry - ? matchTarget({ - geometry: draft.geometry, - captured: obs.lastSettled.captured, - refs: obs.lastSettled.refs, - fallback: draft.captureTarget, - }) - : // No geometry to match on, but the capture still knew what the user - // touched — keep that instead of an anonymous unmatched target. - fallbackDescriptor(draft.captureTarget); - } - - // A pending settle means `lastSettled` may still describe the page before - // the previous action. If the new control cannot be found there, attaching - // that stale state would be worse than leaving the origin unknown for the - // reducer to repair from the observations that do exist. - const originLooksStale = - obs.settles.size > 0 && (!("target" in draft) || draft.target?.unmatched === true); - if (originLooksStale) { - rememberStepAnnotation(obs, stepId, draft); - return; - } - - draft.preStateId = obs.lastSettled.stateId; - rememberStepAnnotation(obs, stepId, draft); -} - -function fillDetailForDraft( - obs: RecordingObservationState, - draft: DraftTraceStep, -): string | undefined { - if (obs.redactValues) return undefined; - if (draft.op === "fill") return JSON.stringify(draft.value); - return undefined; -} - -function refLineForDraft( - obs: RecordingObservationState, - draft: DraftTraceStep, -): number | undefined { - if (!("target" in draft) || !obs.lastSettled) return undefined; - const targetRef = draft.target?.ref; - if (!targetRef) return undefined; - return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line; -} - -function rememberStepAnnotation( - obs: RecordingObservationState, - stepId: number, - draft: DraftTraceStep, -): void { - const line = refLineForDraft(obs, draft); - if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return; - if (!draft.preStateId) return; - const ann: ObservationAnnotation = { - stepId, - op: draft.op, - line, - stateId: draft.preStateId, - detail: fillDetailForDraft(obs, draft), - }; - const bucket = obs.stepAnnotations.get(line) ?? []; - bucket.push(ann); - obs.stepAnnotations.set(line, bucket); -} - -/** Record the step under the page it was performed on, not the one it led to. */ -export function rememberStepOnPage( - obs: RecordingObservationState, - draft: DraftTraceStep, - stepId: number, -): void { - const stateId = draft.preStateId; - if (!stateId) return; - const entry = obs.stateRegistry.get(stateId); - if (entry && !entry.stepsHere.includes(stepId)) { - entry.stepsHere.push(stepId); - } -} - -/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */ -export function clearPendingRedirectLanding(obs: RecordingObservationState): void { - obs.pendingRedirect = null; -} - -function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise): void { - obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {}); -} - -/** - * Remember a redirect hop and schedule a settle-then-emit of a single navigate - * to the final URL. Later hops only bump `generation` / replace `url`. - */ -export function scheduleRedirectLandingFlush( - obs: RecordingObservationState, - steps: DraftTraceStep[], - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - url: string, - options: { cancelled?: () => boolean } = {}, -): void { - const generation = (obs.pendingRedirect?.generation ?? 0) + 1; - obs.pendingRedirect = { url, generation }; - enqueueRedirectFlush(obs, () => - coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options), - ); -} - -/** - * Drain any coalesced redirect so the next action matches against the real - * landing page. Safe to call when nothing is pending. - */ -export async function flushPendingRedirectLanding( - obs: RecordingObservationState, - steps: DraftTraceStep[], - cdp: CdpRunner | undefined, - tabId: number, - tabsApi: ChromeTabsApi, - options: { cancelled?: () => boolean } = {}, -): Promise { - if (obs.pendingRedirect && cdp) { - enqueueRedirectFlush(obs, () => - coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options), - ); - } - await obs.redirectFlushQueue; -} - -/** - * Wait until the redirect chain stops changing the document, then emit one - * `navigate` (cause `browser`) to the tab's final URL and settle its observation. - */ -async function coalesceRedirectLanding( - obs: RecordingObservationState, - steps: DraftTraceStep[], - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - options: { cancelled?: () => boolean }, -): Promise { - while (obs.pendingRedirect) { - if (options.cancelled?.()) { - obs.pendingRedirect = null; - return; - } - - const snap = obs.pendingRedirect; - const outcome = await waitForPageSettled(cdp, tabId, { - cancelled: () => - !!options.cancelled?.() || - obs.pendingRedirect === null || - obs.pendingRedirect.generation !== snap.generation, - }); - - if (options.cancelled?.()) { - obs.pendingRedirect = null; - return; - } - // A newer hop cancelled this wait — loop and settle against the latest URL. - if (outcome === "cancelled") continue; - if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) { - continue; - } - - const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url; - // `tabs.get` is asynchronous. A newer redirect hop may have arrived while - // it was in flight; only the generation that initiated this read may - // consume the pending landing. - if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) { - continue; - } - obs.pendingRedirect = null; - - if (!finalUrl || finalUrl === "about:blank") return; - if (obs.lastSettled?.url === finalUrl) return; - - const last = steps[steps.length - 1]; - if (last?.op === "navigate" && last.url === finalUrl) { - if (!last.postStateId) { - const draftIndex = steps.length - 1; - scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps); - await obs.settleQueue; - } - return; - } - - if (outcome === "timeout") { - console.debug( - `[bsk record] redirect landing still changing after settle budget; ` + - `recording navigate to ${finalUrl}`, - ); - } - - const draft: DraftTraceStep = { - op: "navigate", - url: finalUrl, - page_url: finalUrl, - cause: "browser", - preStateId: obs.lastSettled?.stateId, - }; - steps.push(draft); - const draftIndex = steps.length - 1; - const stepId = draftIndex + 1; - rememberStepOnPage(obs, draft, stepId); - scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps); - // Callers that flush before the next action need `lastSettled` to already - // be the landing page so target matching does not use the pre-redirect view. - await obs.settleQueue; - return; - } -} - -/** - * A capture that lands mid-navigation fails on a destroyed execution context. - * That is transient, so give it one more chance before the step has to fall - * back to a stale state. - */ -async function captureWithRetry( - obs: RecordingObservationState, - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - urlOverride?: string, -): Promise { - try { - return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride); - } catch (err) { - console.debug("[bsk record] observation failed, retrying once", err); - await sleep(CAPTURE_RETRY_DELAY_MS); - return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride); - } -} - -export async function settleDraftObservation( - obs: RecordingObservationState, - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - steps: DraftTraceStep[], - draftIndex: number, - pending?: PendingSettle, -): Promise { - const cancelled = () => pending?.cancelled === true; - const outcome = await waitForPageSettled(cdp, tabId, { cancelled }); - if (outcome === "cancelled") return; - if (outcome === "timeout") { - console.debug( - `[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` + - "budget; observing it as it stands", - ); - } - - const started = steps[draftIndex]; - if (!started) return; - - // The URL attached to the draft may only be an intermediate redirect hop. - // Once the page is settled, register the capture against live tab metadata. - const settled = await captureWithRetry(obs, cdp, tabId, tabsApi); - // The observation still counts as the latest view of the page even when this - // step no longer wants it — the next action will start from it. - if (cancelled()) return; - // Re-read the slot: a navigation observed while the capture ran may have - // rewritten this draft, and the observation belongs to whatever occupies the - // slot now — writing to the object we started with would strand it. - const draft = steps[draftIndex] ?? started; - draft.postStateId = settled.stateId; -} - -/** - * Recover landing states from the shape of the recording itself: wherever the - * next action was performed is, by definition, where the previous one landed. - * Only the immediately following draft counts — a later one would claim a page - * that several unobserved actions away, which is worse than admitting we saw - * no change. - */ -export function inferMissingPostStates(steps: DraftTraceStep[]): void { - for (let i = 0; i < steps.length - 1; i += 1) { - const draft = steps[i]; - const next = steps[i + 1]; - if (!draft || !next || draft.postStateId || !next.preStateId) continue; - draft.postStateId = next.preStateId; - console.debug( - `[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` + - `using where step ${i + 2} started (${next.preStateId})`, - ); - } -} - -/** - * Last chance for the closing actions of a recording. Stopping right after the - * final action is normal, and so is that action navigating away, so take one - * final look at the page. Only trailing drafts may claim it: an earlier step - * did not land on whatever the user happens to be looking at when they stop. - */ -export async function settleUnsettledDrafts( - obs: RecordingObservationState, - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - steps: DraftTraceStep[], -): Promise { - const trailing: DraftTraceStep[] = []; - for (let i = steps.length - 1; i >= 0; i -= 1) { - const draft = steps[i]; - if (!draft || draft.postStateId) break; - trailing.push(draft); - } - if (trailing.length === 0) return; - - let settled: LastSettledObservation; - try { - settled = await captureWithRetry(obs, cdp, tabId, tabsApi); - } catch (err) { - console.warn("[bsk record] final observation at stop failed", err); - return; - } - for (const draft of trailing) draft.postStateId = settled.stateId; - console.debug( - `[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` + - `(${settled.stateId} ${settled.url})`, - ); -} - -/** - * Performing a new action ends the previous one: the page the user reached for - * is, by definition, where the earlier step left them. Taking the landing from - * the newer step also keeps the trace monotonic — a capture that finished late - * would otherwise credit an earlier step with a page that only exists because - * of the newer action. - */ -function supersedeEarlierSettles( - obs: RecordingObservationState, - draftIndex: number, - steps: DraftTraceStep[], -): void { - const landing = steps[draftIndex]?.preStateId; - for (const [index, pending] of obs.settles) { - if (index >= draftIndex) continue; - pending.cancelled = true; - obs.settles.delete(index); - - const draft = steps[index]; - if (!draft || draft.postStateId) continue; - if (!landing) continue; - draft.postStateId = landing; - console.debug( - `[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` + - `${draftIndex + 1} started; landing it on ${landing}`, - ); - } -} - -export function scheduleDraftSettle( - obs: RecordingObservationState, - draftIndex: number, - stepId: number, - cdp: CdpRunner, - tabId: number, - tabsApi: ChromeTabsApi, - steps: DraftTraceStep[], -): void { - supersedeEarlierSettles(obs, draftIndex, steps); - - // Rescheduling the same step (a navigation showed up after the action) - // replaces the pending observation rather than racing it. - const superseded = obs.settles.get(draftIndex); - if (superseded) superseded.cancelled = true; - - const pending: PendingSettle = { draftIndex, cancelled: false }; - obs.settles.set(draftIndex, pending); - - enqueueSettle(obs, async () => { - if (pending.cancelled) return; - try { - await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending); - } catch (err) { - // Recoverable: stop-time repair still gives the step a landing page. - const op = steps[draftIndex]?.op ?? "?"; - console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err); - } finally { - if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex); - } - }); -} - -function enqueueSettle(obs: RecordingObservationState, task: () => Promise): void { - obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {}); -} - -export function cancelPendingSettles(obs: RecordingObservationState): void { - for (const pending of obs.settles.values()) pending.cancelled = true; - obs.settles.clear(); -} - -/** Bound the drain loop so a self-rescheduling settle cannot block stop. */ -const MAX_SETTLE_FLUSH_ROUNDS = 10; - -/** - * Drain settle work before exporting, including work queued while draining — - * a navigation observed during the last capture schedules another one, and - * the trace would drop it if stop did not wait. - */ -export async function flushPendingSettles(obs: RecordingObservationState): Promise { - for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) { - const drained = obs.settleQueue; - await drained; - if (obs.settleQueue === drained) return; - } - console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed"); -} - -function finalizeStateBodies( - entries: StateRegistryEntry[], - annotationsByState: Map, - stepIdByDraftId: Map, - idByOldId: Map, -): TraceState[] { - return entries.map((entry) => { - const id = idByOldId.get(entry.id) ?? entry.id; - const body = formatObservationFile({ - stateId: id, - url: entry.url, - title: entry.title, - stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId), - body: entry.rawVomText, - annotations: annotationsByState.get(entry.id) ?? [], - }); - return { - id, - url: entry.url, - ...(entry.title ? { title: entry.title } : {}), - body, - ...(entry.truncated ? { truncated: true } : {}), - }; - }); -} - -/** Draft ids only become step ids after collapsing and filtering. */ -function remapStepIds(draftIds: number[], stepIdByDraftId: Map): number[] { - const mapped = new Set(); - for (const draftId of draftIds) { - const stepId = stepIdByDraftId.get(draftId); - if (stepId !== undefined) mapped.add(stepId); - } - return [...mapped].sort((a, b) => a - b); -} - -function collectAnnotationsByState( - obs: RecordingObservationState, - stepIdByDraftId: Map, -): Map { - const byState = new Map(); - for (const anns of obs.stepAnnotations.values()) { - for (const ann of anns) { - const stepId = stepIdByDraftId.get(ann.stepId); - if (stepId === undefined) continue; - const bucket = byState.get(ann.stateId) ?? []; - bucket.push({ ...ann, stepId }); - byState.set(ann.stateId, bucket); - } - } - return byState; -} - -/** - * Keep only the pages the published steps point at. Redirect hops and - * mid-load captures land in the registry too, and shipping them would invite - * a reader to treat a page the flow merely passed through as a real stop. - * With no steps at all the first observation is the whole artifact, so it - * stays. - */ -function selectPublishedStates( - registry: Map, - steps: StepV3[], -): StateRegistryEntry[] { - const entries = [...registry.values()]; - if (steps.length === 0) return entries.slice(0, 1); - const referenced = new Set(); - for (const step of steps) { - referenced.add(step.state); - referenced.add(step.result.state); - } - return entries.filter((entry) => referenced.has(entry.id)); -} - -export function buildTraceV3(input: { - obs: RecordingObservationState; - steps: DraftTraceStep[]; - startedAt: string; - purpose?: string; - startUrl?: string; - stoppedBy: StopReason; - bskVersion: string; -}): TraceV3 { - const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry); - const published = selectPublishedStates(input.obs.stateRegistry, steps); - // Renumber so the shipped dictionary reads s1..sN without holes where the - // dropped captures used to be. - const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`])); - for (const step of steps) { - step.state = idByOldId.get(step.state) ?? step.state; - step.result.state = idByOldId.get(step.result.state) ?? step.result.state; - } - const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId); - const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId); - const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states); - return { - version: 3, - ...(input.purpose ? { purpose: input.purpose } : {}), - recorded_at: new Date().toISOString(), - started_at: input.startedAt, - stopped_by: input.stoppedBy, - entry: { start_url: startUrl }, - recorder: { bsk: input.bskVersion, vom: 1 }, - states, - steps, - }; -} diff --git a/apps/extension/src/lib/recording/document-settle.ts b/apps/extension/src/lib/recording/document-settle.ts new file mode 100644 index 00000000..d3b89c7f --- /dev/null +++ b/apps/extension/src/lib/recording/document-settle.ts @@ -0,0 +1,134 @@ +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { type CdpRunner, sendToCdpTarget } from "@/tools/shared"; + +const SETTLE_MIN_MS = 150; +const SETTLE_QUIET_MS = 250; +const SETTLE_MAX_MS = 2_000; +const SETTLE_POLL_MS = 60; +const SETTLE_WORLD_NAME = "__bsk_record_settle__"; + +const QUIET_PROBE = `(() => { + const scope = window; + let probe = scope.__bskRecordQuiet; + if (!probe) { + probe = { changedAt: Date.now() }; + const observer = new MutationObserver(() => { + probe.changedAt = Date.now(); + }); + observer.observe(document.documentElement, { + subtree: true, + childList: true, + attributes: true, + characterData: true, + }); + scope.__bskRecordQuiet = probe; + } + return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState }; +})()`; + +export interface DocumentSettleScope { + target: CdpTarget; + /** Omit only for the target's root document. */ + frameId?: string; +} + +type SettleOutcome = "quiet" | "timeout" | "cancelled"; + +interface QuietProbe { + idleMs: number; + readyState: string; +} + +interface ProbeContext { + executionContextId?: number; +} + +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +async function executionContextForScope( + cdp: CdpRunner, + scope: DocumentSettleScope, +): Promise { + if (!scope.frameId) return undefined; + const result = await sendToCdpTarget<{ executionContextId?: number }>( + cdp, + scope.target, + "Page.createIsolatedWorld", + { + frameId: scope.frameId, + worldName: SETTLE_WORLD_NAME, + grantUniveralAccess: false, + }, + ); + return result.executionContextId; +} + +async function readQuietProbe( + cdp: CdpRunner, + scope: DocumentSettleScope, + context: ProbeContext, +): Promise { + try { + if (scope.frameId && context.executionContextId === undefined) { + context.executionContextId = await executionContextForScope(cdp, scope); + if (context.executionContextId === undefined) return null; + } + const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>( + cdp, + scope.target, + "Runtime.evaluate", + { + expression: QUIET_PROBE, + returnByValue: true, + ...(context.executionContextId !== undefined + ? { contextId: context.executionContextId } + : {}), + }, + ); + const value = reply.result?.value; + if (!value || typeof value !== "object") return null; + const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown }; + if (typeof idleMs !== "number" || typeof readyState !== "string") return null; + return { idleMs, readyState }; + } catch { + context.executionContextId = undefined; + return null; + } +} + +export async function waitForDocumentSettled( + cdp: CdpRunner, + scope: DocumentSettleScope, + options: { signal?: AbortSignal } = {}, +): Promise { + const startedAt = Date.now(); + const floor = startedAt + SETTLE_MIN_MS; + const deadline = startedAt + SETTLE_MAX_MS; + const context: ProbeContext = {}; + + for (;;) { + if (options.signal?.aborted) return "cancelled"; + await sleep(SETTLE_POLL_MS, options.signal); + if (options.signal?.aborted) return "cancelled"; + + const probe = await readQuietProbe(cdp, scope, context); + const now = Date.now(); + if (now < floor) continue; + if (now >= deadline) return "timeout"; + if (!probe || probe.readyState === "loading") continue; + if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet"; + } +} diff --git a/apps/extension/src/lib/recording/draft-policy.ts b/apps/extension/src/lib/recording/draft-policy.ts new file mode 100644 index 00000000..94723b13 --- /dev/null +++ b/apps/extension/src/lib/recording/draft-policy.ts @@ -0,0 +1,34 @@ +import type { RecordingDraftStep } from "./types"; + +const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); +const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); + +export function shouldRecordPress( + key: string, + modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, +): boolean { + if (MODIFIER_ONLY_KEYS.has(key)) return false; + const mods = modifiers ?? []; + const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); + if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; + if (key === "Enter" || key === "Escape") return true; + if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; + return false; +} + +export function shouldIncludeDraft(step: RecordingDraftStep): boolean { + return step.op !== "press" || shouldRecordPress(step.key, step.modifiers); +} + +export function resolveDraftStartUrl( + drafts: RecordingDraftStep[], + explicitStartUrl?: string, + fallbackUrl?: string, +): string { + if (explicitStartUrl) return explicitStartUrl; + const navigation = drafts.find( + (step): step is Extract => step.op === "navigate", + ); + if (navigation) return navigation.url; + return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank"; +} diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts new file mode 100644 index 00000000..8006f74d --- /dev/null +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -0,0 +1,115 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import { captureVomObservation } from "@/tools/capture-vom-observation"; +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import type { CapturedNode } from "@/tools/vom/capture"; + +export interface IndexedObservationNode { + frameId: string; + node: CapturedNode; + ref?: RenderedRef; +} + +export interface CapturedRecordingObservation { + rootFrameId: string; + index: ObservationNodeIndex; + url: string; + title?: string; + vomText: string; + truncated: boolean; +} + +export interface RegisteredObservation { + stateId: string; + rootFrameId: string; + index: ObservationNodeIndex; + url: string; +} + +function nodeKey(frameId: string, backendNodeId: number): string { + return `${frameId}:${backendNodeId}`; +} + +function frameTagKey(frameId: string, tag: string): string { + return `${frameId}:${tag.toLowerCase()}`; +} + +export class ObservationNodeIndex { + readonly #nodesByFrameTag = new Map(); + readonly #refById = new Map(); + readonly #refsByFrame = new Map(); + + constructor(input: { + rootFrameId: string; + frameDocuments: Array<{ frameId: string; domNodes: CapturedNode[] }>; + refs: RenderedRef[]; + }) { + const refByNode = new Map(); + for (const ref of input.refs) { + const frameId = ref.frameId ?? input.rootFrameId; + refByNode.set(nodeKey(frameId, ref.backendNodeId), ref); + this.#refById.set(ref.ref, ref); + const frameRefs = this.#refsByFrame.get(frameId) ?? []; + frameRefs.push(ref); + this.#refsByFrame.set(frameId, frameRefs); + } + for (const document of input.frameDocuments) { + for (const node of document.domNodes) { + const frameId = node.frameId ?? document.frameId; + const entry = { frameId, node, ref: refByNode.get(nodeKey(frameId, node.backendNodeId)) }; + const key = frameTagKey(frameId, node.tag); + const bucket = this.#nodesByFrameTag.get(key) ?? []; + bucket.push(entry); + this.#nodesByFrameTag.set(key, bucket); + } + } + } + + candidates(frameId: string, tag: string): readonly IndexedObservationNode[] { + return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? []; + } + + ref(refId: string): RenderedRef | undefined { + return this.#refById.get(refId); + } + + refs(frameId: string): readonly RenderedRef[] { + return this.#refsByFrame.get(frameId) ?? []; + } +} + +async function readTabMeta( + tabsApi: ChromeTabsApi, + tabId: number, +): Promise<{ url: string; title?: string }> { + try { + const tab = await tabsApi.get(tabId); + return { url: tab.url ?? "about:blank", title: tab.title }; + } catch { + return { url: "about:blank" }; + } +} + +export async function captureRecordingObservation(input: { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + tabId: number; + maxTokens: number; + redactValues: boolean; + signal?: AbortSignal; +}): Promise { + const { url, title } = await readTabMeta(input.tabsApi, input.tabId); + const captured = await captureVomObservation(input.cdp, input.tabId, url, { + maxTokens: input.maxTokens, + redactValues: input.redactValues, + conditionalSurfaceProbe: false, + signal: input.signal, + }); + return { + rootFrameId: captured.rootFrameId, + index: new ObservationNodeIndex(captured), + url, + title, + vomText: captured.text, + truncated: captured.truncated, + }; +} diff --git a/apps/extension/src/lib/recording/observation-session.ts b/apps/extension/src/lib/recording/observation-session.ts new file mode 100644 index 00000000..9cfd43de --- /dev/null +++ b/apps/extension/src/lib/recording/observation-session.ts @@ -0,0 +1,122 @@ +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture"; +import { RecordingStateRegistry } from "./state-registry"; +import { matchObservationTarget, unmatchedTarget } from "./target-matcher"; +import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types"; + +const DEFAULT_MAX_PAGE_TOKENS = 3_000; +const MIN_CAPTURE_INTERVAL_MS = 200; + +export interface TabObservationCursor { + lastSettled: RegisteredObservation | null; + lastCaptureAt: number; +} + +function abortableDelay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft { + return draft.op !== "navigate" && draft.op !== "scroll"; +} + +export class RecordingObservationSession { + readonly registry: RecordingStateRegistry; + readonly cursor: TabObservationCursor; + readonly annotations: StepAnnotation[]; + readonly #maxTokens: number; + readonly #redactValues: boolean; + + constructor( + options: { + registry?: RecordingStateRegistry; + cursor?: TabObservationCursor; + annotations?: StepAnnotation[]; + maxTokens?: number; + redactValues?: boolean; + } = {}, + ) { + this.registry = options.registry ?? new RecordingStateRegistry(); + this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 }; + this.annotations = options.annotations ?? []; + this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS; + this.#redactValues = options.redactValues ?? false; + } + + async capture( + cdp: CdpRunner, + tabsApi: ChromeTabsApi, + tabId: number, + signal?: AbortSignal, + ): Promise { + const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt)); + if (waitMs > 0) await abortableDelay(waitMs, signal); + if (signal?.aborted) throw new DOMException("observation aborted", "AbortError"); + const captured = await captureRecordingObservation({ + cdp, + tabsApi, + tabId, + maxTokens: this.#maxTokens, + redactValues: this.#redactValues, + signal, + }); + const state = this.registry.register({ + url: captured.url, + title: captured.title, + rawVomText: captured.vomText, + truncated: captured.truncated, + }); + const observation: RegisteredObservation = { + stateId: state.id, + rootFrameId: captured.rootFrameId, + index: captured.index, + url: captured.url, + }; + this.cursor.lastSettled = observation; + this.cursor.lastCaptureAt = Date.now(); + return observation; + } + + bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void { + const observation = this.cursor.lastSettled; + if (isTargeted(draft)) { + draft.matchedTarget = observation + ? matchObservationTarget({ + observation, + hint: draft.targetHint, + fallback: draft.captureTarget, + }) + : unmatchedTarget(draft.captureTarget); + } + if (!observation) return; + + const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true; + if (previousActionPending && unmatched) return; + draft.preStateId = observation.stateId; + this.registry.markStep(observation.stateId, draftId); + + if (!isTargeted(draft) || !draft.matchedTarget?.ref) return; + const ref = observation.index.ref(draft.matchedTarget.ref); + if (!ref) return; + this.annotations.push({ + draftId, + op: draft.op, + line: ref.line, + stateId: observation.stateId, + ...(draft.op === "fill" && !this.#redactValues + ? { detail: JSON.stringify(draft.value) } + : {}), + }); + } +} diff --git a/apps/extension/src/lib/recording/settle-controller.ts b/apps/extension/src/lib/recording/settle-controller.ts new file mode 100644 index 00000000..c63f46fa --- /dev/null +++ b/apps/extension/src/lib/recording/settle-controller.ts @@ -0,0 +1,232 @@ +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle"; +import { RecordingObservationSession } from "./observation-session"; +import type { RecordingDraftStep } from "./types"; + +interface PendingSettle { + abort: AbortController; + scope: DocumentSettleScope; +} + +interface PendingRedirect { + url: string; + abort: AbortController; +} + +const CAPTURE_RETRY_DELAY_MS = 250; + +function delay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +function isAbortError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { name?: string }).name === "AbortError" + ); +} + +export function inferMissingPostStates(drafts: RecordingDraftStep[]): void { + for (let index = 0; index < drafts.length - 1; index += 1) { + const draft = drafts[index]; + const next = drafts[index + 1]; + if (draft && next && !draft.postStateId && next.preStateId) { + draft.postStateId = next.preStateId; + } + } +} + +export class SettleController { + readonly #session: RecordingObservationSession; + readonly #cdp: CdpRunner; + readonly #tabsApi: ChromeTabsApi; + readonly #tabId: number; + readonly #rootScope: DocumentSettleScope; + readonly #pending = new Map(); + #queue = Promise.resolve(); + #pendingRedirect: PendingRedirect | null = null; + #redirectQueue = Promise.resolve(); + + constructor(input: { + session: RecordingObservationSession; + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + tabId: number; + }) { + this.#session = input.session; + this.#cdp = input.cdp; + this.#tabsApi = input.tabsApi; + this.#tabId = input.tabId; + this.#rootScope = { target: { tabId: input.tabId } }; + } + + get hasPending(): boolean { + return this.#pending.size > 0 || this.#pendingRedirect !== null; + } + + async #captureWithRetry(signal?: AbortSignal) { + try { + return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal); + } catch (error) { + if (signal?.aborted) throw error; + await delay(CAPTURE_RETRY_DELAY_MS, signal); + return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal); + } + } + + schedule( + drafts: RecordingDraftStep[], + draftIndex: number, + scope: DocumentSettleScope = this.#rootScope, + ): void { + if (scope.target.tabId !== this.#tabId) { + throw new Error( + `settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`, + ); + } + const landing = drafts[draftIndex]?.preStateId; + for (const [index, pending] of this.#pending) { + if (index >= draftIndex) continue; + pending.abort.abort(); + this.#pending.delete(index); + if (landing && drafts[index] && !drafts[index].postStateId) { + drafts[index].postStateId = landing; + } + } + + this.#pending.get(draftIndex)?.abort.abort(); + const pending = { abort: new AbortController(), scope }; + this.#pending.set(draftIndex, pending); + const task = async () => { + if (pending.abort.signal.aborted) return; + try { + const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, { + signal: pending.abort.signal, + }); + if (outcome === "cancelled") return; + const observation = await this.#captureWithRetry(pending.abort.signal); + if (!pending.abort.signal.aborted && drafts[draftIndex]) { + drafts[draftIndex].postStateId = observation.stateId; + } + } catch (error) { + if (!isAbortError(error)) { + console.warn( + `[bsk record] post-action observation failed for step ${draftIndex + 1}`, + error, + ); + } + } finally { + if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex); + } + }; + this.#queue = this.#queue.then(task, task).catch(() => {}); + } + + cancel(): void { + for (const pending of this.#pending.values()) pending.abort.abort(); + this.#pending.clear(); + this.clearRedirect(); + } + + async flush(): Promise { + for (let round = 0; round < 10; round += 1) { + const current = this.#queue; + await current; + if (current === this.#queue) return; + } + console.warn("[bsk record] settle queue kept growing; using completed observations"); + } + + async settleTrailing(drafts: RecordingDraftStep[]): Promise { + const trailing: RecordingDraftStep[] = []; + for (let index = drafts.length - 1; index >= 0; index -= 1) { + const draft = drafts[index]; + if (!draft || draft.postStateId) break; + trailing.push(draft); + } + if (trailing.length === 0) return; + try { + const observation = await this.#captureWithRetry(); + for (const draft of trailing) draft.postStateId = observation.stateId; + } catch (error) { + console.warn("[bsk record] final observation at stop failed", error); + } + } + + clearRedirect(): void { + this.#pendingRedirect?.abort.abort(); + this.#pendingRedirect = null; + } + + scheduleRedirect(drafts: RecordingDraftStep[], url: string): void { + this.#pendingRedirect?.abort.abort(); + const pending: PendingRedirect = { url, abort: new AbortController() }; + this.#pendingRedirect = pending; + const task = () => this.#settleRedirect(drafts, pending); + this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {}); + } + + async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise { + const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, { + signal: pending.abort.signal, + }); + if (outcome === "cancelled" || this.#pendingRedirect !== pending) return; + + let finalUrl = pending.url; + try { + finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl; + } catch { + // The navigation event URL remains the best available destination. + } + if (this.#pendingRedirect !== pending) return; + this.#pendingRedirect = null; + if ( + !finalUrl || + finalUrl === "about:blank" || + this.#session.cursor.lastSettled?.url === finalUrl + ) { + return; + } + + const last = drafts[drafts.length - 1]; + if (last?.op === "navigate" && last.url === finalUrl) { + if (!last.postStateId) this.schedule(drafts, drafts.length - 1); + await this.flush(); + return; + } + + const draft: RecordingDraftStep = { + op: "navigate", + url: finalUrl, + pageUrl: finalUrl, + cause: "browser", + preStateId: this.#session.cursor.lastSettled?.stateId, + }; + drafts.push(draft); + const draftIndex = drafts.length - 1; + if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1); + this.schedule(drafts, draftIndex); + await this.flush(); + } + + async flushRedirects(): Promise { + for (let round = 0; round < 10; round += 1) { + const current = this.#redirectQueue; + await current; + if (current === this.#redirectQueue) return; + } + console.warn("[bsk record] redirect queue kept growing; using the final completed landing"); + } +} diff --git a/apps/extension/src/lib/recording/state-registry.ts b/apps/extension/src/lib/recording/state-registry.ts new file mode 100644 index 00000000..fe5f0eab --- /dev/null +++ b/apps/extension/src/lib/recording/state-registry.ts @@ -0,0 +1,56 @@ +export interface RecordedStateEntry { + id: string; + url: string; + title?: string; + rawVomText: string; + truncated: boolean; + stepsHere: number[]; +} + +function stateIdentity(url: string, body: string): string { + return `${url}\0${body}`; +} + +export class RecordingStateRegistry { + readonly #entriesById = new Map(); + readonly #idByIdentity = new Map(); + #nextId = 1; + + register(input: { + url: string; + title?: string; + rawVomText: string; + truncated?: boolean; + }): RecordedStateEntry { + const identity = stateIdentity(input.url, input.rawVomText); + const existingId = this.#idByIdentity.get(identity); + if (existingId) { + const existing = this.#entriesById.get(existingId)!; + if (!existing.title && input.title) existing.title = input.title; + if (input.truncated) existing.truncated = true; + return existing; + } + + const entry: RecordedStateEntry = { + id: `s${this.#nextId}`, + url: input.url, + ...(input.title ? { title: input.title } : {}), + rawVomText: input.rawVomText, + truncated: input.truncated ?? false, + stepsHere: [], + }; + this.#nextId += 1; + this.#entriesById.set(entry.id, entry); + this.#idByIdentity.set(identity, entry.id); + return entry; + } + + values(): RecordedStateEntry[] { + return [...this.#entriesById.values()]; + } + + markStep(stateId: string, draftId: number): void { + const entry = this.#entriesById.get(stateId); + if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId); + } +} diff --git a/apps/extension/src/lib/recording-step-buffer.ts b/apps/extension/src/lib/recording/step-buffer.ts similarity index 79% rename from apps/extension/src/lib/recording-step-buffer.ts rename to apps/extension/src/lib/recording/step-buffer.ts index c1c5086b..a2732e1d 100644 --- a/apps/extension/src/lib/recording-step-buffer.ts +++ b/apps/extension/src/lib/recording/step-buffer.ts @@ -1,8 +1,8 @@ -import type { DraftTraceStep } from "@/transport/types"; -import type { RecordStepPayload } from "./record-bridge"; +import type { RecordStepPayload } from "../record-bridge"; +import type { RecordingDraftStep } from "./types"; export interface RecordingStepBuffer { - steps: DraftTraceStep[]; + steps: RecordingDraftStep[]; currentUrl?: string; pendingNavigation: boolean; pendingNavigationDeadline?: number; @@ -10,33 +10,33 @@ export interface RecordingStepBuffer { const NAVIGATION_TRIGGER_WINDOW_MS = 3_000; -function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { +function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null { const pageUrl = payload.page_url; switch (payload.op) { case "click": return payload.target ? { op: "click", - target: payload.target, - ...(pageUrl ? { page_url: pageUrl } : {}), + captureTarget: payload.target, + ...(pageUrl ? { pageUrl } : {}), } : null; case "hover": return payload.target ? { op: "hover", - target: payload.target, - ...(pageUrl ? { page_url: pageUrl } : {}), + captureTarget: payload.target, + ...(pageUrl ? { pageUrl } : {}), } : null; case "fill": return payload.target ? { op: "fill", - target: payload.target, + captureTarget: payload.target, value: payload.value ?? "", ...(payload.redacted ? { redacted: true } : {}), - ...(pageUrl ? { page_url: pageUrl } : {}), + ...(pageUrl ? { pageUrl } : {}), } : null; case "press": @@ -44,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { ? { op: "press", key: payload.key, - ...(payload.target ? { target: payload.target } : {}), + ...(payload.target ? { captureTarget: payload.target } : {}), ...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}), - ...(pageUrl ? { page_url: pageUrl } : {}), + ...(pageUrl ? { pageUrl } : {}), } : null; case "select": return payload.target && payload.values ? { op: "select", - target: payload.target, + captureTarget: payload.target, values: payload.values, ...(payload.labels?.length ? { labels: payload.labels } : {}), - ...(pageUrl ? { page_url: pageUrl } : {}), + ...(pageUrl ? { pageUrl } : {}), } : null; case "navigate": @@ -69,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b const step = buffer.steps[i]; if (!step) continue; if (step.op === "click" || step.op === "press" || step.op === "select") { - buffer.steps[i] = { ...step, navigated_to: url }; + buffer.steps[i] = { ...step, navigatedTo: url }; return true; } break; @@ -96,7 +96,7 @@ export function observeRecordedNavigation( buffer.steps.push({ op: "navigate", url, - page_url: url, + pageUrl: url, }); } return; @@ -107,7 +107,7 @@ export function observeRecordedNavigation( buffer.steps.push({ op: "navigate", url, - page_url: url, + pageUrl: url, }); } diff --git a/apps/extension/src/lib/recording/target-matcher.ts b/apps/extension/src/lib/recording/target-matcher.ts new file mode 100644 index 00000000..0df1abee --- /dev/null +++ b/apps/extension/src/lib/recording/target-matcher.ts @@ -0,0 +1,76 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import type { TargetDescriptorV3 } from "@/transport/types"; +import type { CaptureTargetDescriptor } from "../describe-target"; +import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture"; +import type { TargetGeometry, TargetMatchHint } from "./types"; + +const MATCH_TOLERANCE_PX = 2; + +function close(a: number, b: number): boolean { + return Math.abs(a - b) <= MATCH_TOLERANCE_PX; +} + +function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean { + return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h); +} + +function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean { + return candidate.node.rect !== null && rectMatches(geometry.rect, candidate.node.rect); +} + +export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { + return { + ...(fallback?.role ? { role: fallback.role } : {}), + ...(fallback?.name ? { name: fallback.name } : {}), + unmatched: true, + }; +} + +function normalized(value: string | undefined): string { + return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? ""; +} + +function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean { + const role = normalized(fallback?.role); + const name = normalized(fallback?.name); + if (!role && !name) return false; + if (role && normalized(ref.role) !== role) return false; + if (name && normalized(ref.name) !== name) return false; + return true; +} + +function descriptor(ref: RenderedRef): TargetDescriptorV3 { + return { + ref: ref.ref, + ...(ref.role ? { role: ref.role } : {}), + ...(ref.name ? { name: ref.name } : {}), + ...(ref.ctx ? { ctx: ref.ctx } : {}), + }; +} + +export function matchObservationTarget(input: { + observation: RegisteredObservation; + hint?: TargetMatchHint; + fallback?: CaptureTargetDescriptor; +}): TargetDescriptorV3 { + const frameId = input.hint?.frameId ?? input.observation.rootFrameId; + const geometry = input.hint?.geometry; + if (geometry) { + const matches = input.observation.index + .candidates(frameId, geometry.tag) + .filter((candidate) => candidate.ref && candidateMatches(candidate, geometry)); + if (matches.length === 1) return descriptor(matches[0]!.ref!); + const semanticMatches = matches.filter( + (candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback), + ); + if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!); + return unmatchedTarget(input.fallback); + } + + const semanticMatches = input.observation.index + .refs(frameId) + .filter((ref) => matchesSemantics(ref, input.fallback)); + return semanticMatches.length === 1 + ? descriptor(semanticMatches[0]!) + : unmatchedTarget(input.fallback); +} diff --git a/apps/extension/src/lib/recording/trace-builder-v3.ts b/apps/extension/src/lib/recording/trace-builder-v3.ts new file mode 100644 index 00000000..5798c222 --- /dev/null +++ b/apps/extension/src/lib/recording/trace-builder-v3.ts @@ -0,0 +1,87 @@ +import { + type StepV3, + type StopReason, + TRACE_VERSION_V3, + type TraceStateV3, + type TraceV3, + VOM_FORMAT_VERSION, +} from "@/transport/types"; +import { resolveDraftStartUrl } from "./draft-policy"; +import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry"; +import { reduceTraceStepsV3 } from "./trace-reducer-v3"; +import { formatTraceStateBody } from "./trace-state-body"; +import type { RecordingDraftStep, StepAnnotation } from "./types"; + +function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] { + const entries = registry.values(); + if (steps.length === 0) return entries.slice(0, 1); + const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state])); + return entries.filter((entry) => referenced.has(entry.id)); +} + +function remapDraftIds(draftIds: number[], stepIdByDraftId: Map): number[] { + return [ + ...new Set( + draftIds.flatMap((id) => { + const stepId = stepIdByDraftId.get(id); + return stepId === undefined ? [] : [stepId]; + }), + ), + ].sort((a, b) => a - b); +} + +export function buildTraceV3(input: { + registry: RecordingStateRegistry; + drafts: RecordingDraftStep[]; + annotations?: StepAnnotation[]; + startedAt: string; + purpose?: string; + startUrl?: string; + stoppedBy: StopReason; + bskVersion: string; +}): TraceV3 { + const reduced = reduceTraceStepsV3(input.drafts); + const entries = publishedEntries(input.registry, reduced.steps); + const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`])); + const annotationsByState = new Map(); + for (const annotation of input.annotations ?? []) { + const bucket = annotationsByState.get(annotation.stateId) ?? []; + bucket.push(annotation); + annotationsByState.set(annotation.stateId, bucket); + } + const steps = reduced.steps.map((step) => ({ + ...step, + state: publishedId.get(step.state) ?? step.state, + result: { state: publishedId.get(step.result.state) ?? step.result.state }, + })); + const states: TraceStateV3[] = entries.map((entry) => { + const id = publishedId.get(entry.id) ?? entry.id; + return { + id, + url: entry.url, + ...(entry.title ? { title: entry.title } : {}), + body: formatTraceStateBody({ + stateId: id, + url: entry.url, + title: entry.title, + stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId), + vomText: entry.rawVomText, + annotations: annotationsByState.get(entry.id) ?? [], + stepIdByDraftId: reduced.stepIdByDraftId, + }), + ...(entry.truncated ? { truncated: true } : {}), + }; + }); + + return { + version: TRACE_VERSION_V3, + ...(input.purpose ? { purpose: input.purpose } : {}), + recorded_at: new Date().toISOString(), + started_at: input.startedAt, + stopped_by: input.stoppedBy, + entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) }, + recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION }, + states, + steps, + }; +} diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/recording/trace-reducer-v2.ts similarity index 59% rename from apps/extension/src/lib/trace-reducer.ts rename to apps/extension/src/lib/recording/trace-reducer-v2.ts index b58b7939..35399389 100644 --- a/apps/extension/src/lib/trace-reducer.ts +++ b/apps/extension/src/lib/recording/trace-reducer-v2.ts @@ -1,30 +1,10 @@ -import type { DraftTraceStep, PageRefV2, SelectedOptionV2, StepV2 } from "@/transport/types"; - -const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); -const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); - -export function shouldRecordPress( - key: string, - modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, -): boolean { - if (MODIFIER_ONLY_KEYS.has(key)) return false; - const mods = modifiers ?? []; - const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); - if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; - if (key === "Enter" || key === "Escape") return true; - // Drop bare character typing — FillSession already records the value. - if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; - return false; -} - -function shouldIncludeDraft(step: DraftTraceStep): boolean { - if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false; - return true; -} +import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types"; +import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy"; +import type { RecordingDraftStep } from "./types"; /** Collapse consecutive navigations to the last hop. */ -function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { - const out: DraftTraceStep[] = []; +function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] { + const out: RecordingDraftStep[] = []; for (const step of steps) { const prev = out[out.length - 1]; if (step.op === "navigate" && prev?.op === "navigate") { @@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { return out; } -function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { +function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] { const urls: string[] = []; if (startUrl) urls.push(startUrl); for (const step of steps) { @@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { urls.push(step.url); continue; } - if ("page_url" in step && step.page_url) urls.push(step.page_url); - if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to); + if (step.pageUrl) urls.push(step.pageUrl); + if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo); } const seen = new Set(); const unique: string[] = []; @@ -58,7 +38,7 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { } function buildPageRegistry( - steps: DraftTraceStep[], + steps: RecordingDraftStep[], startUrl?: string, ): { pages: PageRefV2[]; urlToId: Map } { const urls = collectUrls(steps, startUrl); @@ -81,9 +61,9 @@ function pageIdFor( return urlToId.values().next().value ?? "p1"; } -function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined { - if (step.op === "navigate") return step.page_url ?? step.url; - if ("page_url" in step && step.page_url) return step.page_url; +function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined { + if (step.op === "navigate") return step.pageUrl ?? step.url; + if (step.pageUrl) return step.pageUrl; return fallbackUrl; } @@ -110,7 +90,7 @@ function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] { } function toV2Step( - step: DraftTraceStep, + step: RecordingDraftStep, id: number, urlToId: Map, fallbackUrl?: string, @@ -129,28 +109,31 @@ function toV2Step( to: step.url, }; case "click": + if (!step.captureTarget) return null; return withEffect( { op: "click", id, page, - target: step.target, + target: step.captureTarget, }, - effectForNavigation(step.navigated_to, urlToId), + effectForNavigation(step.navigatedTo, urlToId), ); case "hover": + if (!step.captureTarget) return null; return { op: "hover", id, page, - target: step.target, + target: step.captureTarget, }; case "fill": + if (!step.captureTarget) return null; return { op: "fill", id, page, - target: step.target, + target: step.captureTarget, value: step.value, ...(step.redacted ? { redacted: true } : {}), }; @@ -161,26 +144,29 @@ function toV2Step( id, page, key: step.key, - ...(step.target ? { target: step.target } : {}), + ...(step.captureTarget ? { target: step.captureTarget } : {}), ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}), }, - effectForNavigation(step.navigated_to, urlToId), + effectForNavigation(step.navigatedTo, urlToId), ); case "select": + if (!step.captureTarget) return null; return withEffect( { op: "select", id, page, - target: step.target, + target: step.captureTarget, selection: toSelection(step.values, step.labels), }, - effectForNavigation(step.navigated_to, urlToId), + effectForNavigation(step.navigatedTo, urlToId), ); + case "scroll": + return null; } } -export interface ReducedTrace { +interface ReducedTrace { pages: PageRefV2[]; steps: StepV2[]; } @@ -189,7 +175,7 @@ export interface ReducedTrace { * Compile capture drafts into record-only trace v2 steps. * Variable inputs are NOT classified here — executing agents infer that at run time. */ -export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace { +function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace { const collapsed = collapseNavigations(steps); const { pages, urlToId } = buildPageRegistry(collapsed, startUrl); const out: StepV2[] = []; @@ -197,8 +183,8 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re let lastUrl = startUrl; for (const draft of collapsed) { if (draft.op === "navigate") lastUrl = draft.url; - else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to; - else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url; + else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo; + else if (draft.pageUrl) lastUrl = draft.pageUrl; const step = toV2Step(draft, id, urlToId, lastUrl); if (!step) continue; out.push(step); @@ -207,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re return { pages, steps: out }; } -export function resolveTraceStartUrl( - drafts: DraftTraceStep[], +function resolveTraceStartUrl( + drafts: RecordingDraftStep[], startUrl?: string, pages?: PageRefV2[], ): string { - if (startUrl) return startUrl; - const navigate = drafts.find((step): step is Extract => { - return step.op === "navigate"; - }); - if (navigate) return navigate.url; - for (const draft of drafts) { - if ("page_url" in draft && draft.page_url) return draft.page_url; - } - return pages?.[0]?.url ?? "about:blank"; + return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url); +} + +export function buildTraceV2(input: { + steps: RecordingDraftStep[]; + startedAt: string; + startUrl?: string; + purpose?: string; +}): TraceV2 { + const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl); + return { + recorded_at: new Date().toISOString(), + started_at: input.startedAt, + ...(input.purpose ? { purpose: input.purpose } : {}), + entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) }, + pages, + steps, + }; } diff --git a/apps/extension/src/lib/recording/trace-reducer-v3.ts b/apps/extension/src/lib/recording/trace-reducer-v3.ts new file mode 100644 index 00000000..901429bc --- /dev/null +++ b/apps/extension/src/lib/recording/trace-reducer-v3.ts @@ -0,0 +1,129 @@ +import type { NavigationCause, StepV3 } from "@/transport/types"; +import { shouldIncludeDraft } from "./draft-policy"; +import { unmatchedTarget } from "./target-matcher"; +import type { RecordingDraftStep } from "./types"; + +interface CollapsedDraft { + draft: RecordingDraftStep; + draftIds: number[]; +} + +const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]); +const TRANSITION_CAUSES: Record = { + typed: "user_typed", + generated: "user_typed", + keyword: "user_typed", + keyword_generated: "user_typed", + link: "link", + form_submit: "form_submit", + reload: "reload", + auto_bookmark: "browser", + start_page: "browser", +}; + +function isRedirect(step: Extract): boolean { + return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier)); +} + +function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] { + const output: CollapsedDraft[] = []; + steps.forEach((step, index) => { + const previous = output[output.length - 1]; + if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) { + previous.draft = { + ...previous.draft, + url: step.url, + postStateId: step.postStateId ?? previous.draft.postStateId, + }; + previous.draftIds.push(index + 1); + return; + } + output.push({ draft: { ...step }, draftIds: [index + 1] }); + }); + return output; +} + +function navigationCause(step: Extract): NavigationCause { + if (step.cause) return step.cause; + const qualifiers = step.transitionQualifiers ?? []; + if (qualifiers.includes("forward_back")) return "history"; + if (qualifiers.includes("from_address_bar")) return "user_typed"; + return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser"; +} + +function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> { + return values.map((value, index) => ({ + value, + ...(labels?.[index] ? { label: labels[index] } : {}), + })); +} + +function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null { + if (!shouldIncludeDraft(draft)) return null; + const state = draft.preStateId ?? draft.postStateId; + const resultState = draft.postStateId ?? draft.preStateId; + if (!state || !resultState) return null; + const common = { id, state, result: { state: resultState } }; + + switch (draft.op) { + case "navigate": + return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) }; + case "click": + return { + op: "click", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + }; + case "hover": + return { + op: "hover", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + }; + case "fill": + return { + op: "fill", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + value: draft.value, + commit: draft.commit ?? "blur", + ...(draft.redacted ? { redacted: true } : {}), + }; + case "press": + return { + op: "press", + ...common, + key: draft.key, + ...(draft.captureTarget || draft.matchedTarget + ? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) } + : {}), + ...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}), + }; + case "select": + return { + op: "select", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + selection: selection(draft.values, draft.labels), + }; + case "scroll": + return { op: "scroll", ...common }; + } +} + +export interface ReducedTraceV3 { + steps: StepV3[]; + stepIdByDraftId: Map; +} + +export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 { + const output: StepV3[] = []; + const stepIdByDraftId = new Map(); + for (const { draft, draftIds } of collapseRedirects(steps)) { + const step = reduceDraft(draft, output.length + 1); + if (!step) continue; + output.push(step); + for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id); + } + return { steps: output, stepIdByDraftId }; +} diff --git a/apps/extension/src/lib/recording/trace-state-body.ts b/apps/extension/src/lib/recording/trace-state-body.ts new file mode 100644 index 00000000..fc87349b --- /dev/null +++ b/apps/extension/src/lib/recording/trace-state-body.ts @@ -0,0 +1,39 @@ +import type { StepAnnotation } from "./types"; + +function annotationText(annotation: StepAnnotation, stepId: number): string { + return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`; +} + +export function formatTraceStateBody(input: { + stateId: string; + url: string; + title?: string; + stepIds: number[]; + vomText: string; + annotations: StepAnnotation[]; + stepIdByDraftId: Map; +}): string { + const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`]; + lines.push(`url: ${JSON.stringify(input.url)}`); + if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`); + if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`); + lines.push("---"); + + const byLine = new Map>(); + for (const annotation of input.annotations) { + const stepId = input.stepIdByDraftId.get(annotation.draftId); + if (stepId === undefined) continue; + const bucket = byLine.get(annotation.line) ?? []; + bucket.push({ annotation, stepId }); + byLine.set(annotation.line, bucket); + } + + input.vomText.split("\n").forEach((bodyLine, lineIndex) => { + let line = bodyLine; + for (const item of byLine.get(lineIndex) ?? []) { + line += annotationText(item.annotation, item.stepId); + } + lines.push(line); + }); + return `${lines.join("\n")}\n`; +} diff --git a/apps/extension/src/lib/recording/types.ts b/apps/extension/src/lib/recording/types.ts new file mode 100644 index 00000000..96df3fee --- /dev/null +++ b/apps/extension/src/lib/recording/types.ts @@ -0,0 +1,83 @@ +import type { CaptureTargetDescriptor } from "@/lib/describe-target"; +import type { + FillCommit, + KeyModifier, + NavigationCause, + StepV3, + TargetDescriptorV3, +} from "@/transport/types"; + +export interface TargetGeometry { + /** Top-level viewport-relative CSS pixels, as defined by the geometry module. */ + rect: { x: number; y: number; w: number; h: number }; + tag: string; +} + +export interface TargetMatchHint { + geometry?: TargetGeometry; + /** Missing means the current top frame, never an unrestricted frame search. */ + frameId?: string; +} + +export interface StepAnnotation { + draftId: number; + op: StepV3["op"]; + line: number; + stateId: string; + detail?: string; +} + +interface DraftStateLink { + pageUrl?: string; + preStateId?: string; + postStateId?: string; +} + +interface DraftTarget { + captureTarget?: CaptureTargetDescriptor; + targetHint?: TargetMatchHint; + matchedTarget?: TargetDescriptorV3; +} + +interface DraftNavigationEffect { + navigatedTo?: string; +} + +export type RecordingDraftStep = + | ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect) + | ({ op: "hover" } & DraftStateLink & DraftTarget) + | ({ + op: "fill"; + value: string; + commit?: FillCommit; + redacted?: boolean; + } & DraftStateLink & + DraftTarget & + DraftNavigationEffect) + | ({ + op: "press"; + key: string; + modifiers?: KeyModifier[]; + } & DraftStateLink & + DraftTarget & + DraftNavigationEffect) + | ({ + op: "select"; + values: string[]; + labels?: string[]; + } & DraftStateLink & + DraftTarget & + DraftNavigationEffect) + | ({ op: "scroll" } & DraftStateLink) + | ({ + op: "navigate"; + url: string; + cause?: NavigationCause; + transitionType?: string; + transitionQualifiers?: string[]; + } & DraftStateLink); + +export type TargetedRecordingDraft = Extract< + RecordingDraftStep, + { op: "click" | "hover" | "fill" | "press" | "select" } +>; diff --git a/apps/extension/src/lib/trace-reducer-v2.ts b/apps/extension/src/lib/trace-reducer-v2.ts deleted file mode 100644 index ef57b74a..00000000 --- a/apps/extension/src/lib/trace-reducer-v2.ts +++ /dev/null @@ -1,301 +0,0 @@ -import type { CaptureTargetDescriptor } from "@/lib/describe-target"; -import type { DraftTraceStep, KeyModifier } from "@/transport/types"; - -const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); -const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); - -export interface TargetDescriptorV2 { - role?: string; - name?: string; - tag: string; - name_attr?: string; - placeholder?: string; - nearby_label?: string; -} - -export interface PageRef { - id: string; - url: string; - title?: string; -} - -export interface SelectedOptionV2 { - value: string; - label?: string; -} - -export interface StepEffectV2 { - navigated_to: string; -} - -export type StepV2 = - | { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 } - | { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 } - | { op: "hover"; id: number; page: string; target: TargetDescriptorV2 } - | { - op: "fill"; - id: number; - page: string; - target: TargetDescriptorV2; - value: string; - redacted?: boolean; - effect?: StepEffectV2; - } - | { - op: "select"; - id: number; - page: string; - target: TargetDescriptorV2; - selection: SelectedOptionV2[]; - effect?: StepEffectV2; - } - | { - op: "press"; - id: number; - page: string; - key: string; - modifiers?: KeyModifier[]; - target?: TargetDescriptorV2; - effect?: StepEffectV2; - }; - -export interface TraceV2 { - recorded_at: string; - started_at?: string; - purpose?: string; - entry: { start_url: string }; - pages: PageRef[]; - steps: StepV2[]; -} - -export function shouldRecordPress( - key: string, - modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, -): boolean { - if (MODIFIER_ONLY_KEYS.has(key)) return false; - const mods = modifiers ?? []; - const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); - if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; - if (key === "Enter" || key === "Escape") return true; - if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; - return false; -} - -function shouldIncludeDraft(step: DraftTraceStep): boolean { - if (step.op === "scroll") return false; - if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false; - if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false; - return true; -} - -function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { - const out: DraftTraceStep[] = []; - for (const step of steps) { - const prev = out[out.length - 1]; - if (step.op === "navigate" && prev?.op === "navigate") { - out[out.length - 1] = step; - continue; - } - out.push(step); - } - return out; -} - -function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { - const urls: string[] = []; - if (startUrl) urls.push(startUrl); - for (const step of steps) { - if (step.op === "navigate") { - urls.push(step.url); - continue; - } - if ("page_url" in step && step.page_url) urls.push(step.page_url); - if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to); - } - const seen = new Set(); - const unique: string[] = []; - for (const url of urls) { - if (!url || seen.has(url)) continue; - seen.add(url); - unique.push(url); - } - return unique; -} - -function buildPageRegistry( - steps: DraftTraceStep[], - startUrl?: string, -): { pages: PageRef[]; urlToId: Map } { - const urls = collectUrls(steps, startUrl); - const urlToId = new Map(); - const pages = urls.map((url, index) => { - const id = `p${index + 1}`; - urlToId.set(url, id); - return { id, url }; - }); - return { pages, urlToId }; -} - -function pageIdFor( - url: string | undefined, - urlToId: Map, - fallbackUrl?: string, -): string { - if (url && urlToId.has(url)) return urlToId.get(url)!; - if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!; - return urlToId.values().next().value ?? "p1"; -} - -function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined { - if (step.op === "navigate") return step.page_url ?? step.url; - if ("page_url" in step && step.page_url) return step.page_url; - return fallbackUrl; -} - -function effectForNavigation( - navigatedTo: string | undefined, - urlToId: Map, -): StepEffectV2 | undefined { - if (!navigatedTo) return undefined; - const pageId = urlToId.get(navigatedTo); - if (!pageId) return undefined; - return { navigated_to: pageId }; -} - -function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 { - return { - tag: capture?.tag ?? "unknown", - ...(capture?.role ? { role: capture.role } : {}), - ...(capture?.name ? { name: capture.name } : {}), - ...(capture?.name_attr ? { name_attr: capture.name_attr } : {}), - ...(capture?.placeholder ? { placeholder: capture.placeholder } : {}), - ...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}), - }; -} - -function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined { - if (!("target" in step) && !("captureTarget" in step)) return undefined; - const capture = "captureTarget" in step ? step.captureTarget : undefined; - if (capture) return captureTargetToV2Target(capture); - if ("target" in step && step.target && "tag" in (step.target as object)) { - const legacy = step.target as TargetDescriptorV2 & { tag?: string }; - if (legacy.tag) return legacy; - } - return undefined; -} - -function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] { - return values.map((value, index) => ({ - value, - ...(labels?.[index] ? { label: labels[index] } : {}), - })); -} - -function toV2Step( - step: DraftTraceStep, - id: number, - urlToId: Map, - fallbackUrl?: string, -): StepV2 | null { - if (!shouldIncludeDraft(step)) return null; - - const pageUrl = pageUrlForDraft(step, fallbackUrl); - const page = pageIdFor(pageUrl, urlToId, fallbackUrl); - const effect = - "navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined; - - switch (step.op) { - case "navigate": - return { - op: "navigate", - id, - page: pageIdFor(step.url, urlToId, fallbackUrl), - to: step.url, - }; - case "click": { - const target = targetForDraft(step); - if (!target) return null; - return { op: "click", id, page, target, ...(effect ? { effect } : {}) }; - } - case "hover": { - const target = targetForDraft(step); - if (!target) return null; - return { op: "hover", id, page, target }; - } - case "fill": { - const target = targetForDraft(step); - if (!target) return null; - return { - op: "fill", - id, - page, - target, - value: step.value, - ...(step.redacted ? { redacted: true } : {}), - }; - } - case "press": - return { - op: "press", - id, - page, - key: step.key, - ...(step.target ? { target: targetForDraft(step) } : {}), - ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}), - ...(effect ? { effect } : {}), - }; - case "select": { - const target = targetForDraft(step); - if (!target) return null; - return { - op: "select", - id, - page, - target, - selection: toSelection(step.values, step.labels), - ...(effect ? { effect } : {}), - }; - } - case "scroll": - return null; - } -} - -export interface BuildTraceV2Input { - steps: DraftTraceStep[]; - startedAt: string; - startUrl?: string; - purpose?: string; -} - -export function buildTraceV2(input: BuildTraceV2Input): TraceV2 { - const collapsed = collapseNavigations(input.steps); - const startUrl = - input.startUrl ?? - collapsed.find( - (step): step is Extract => step.op === "navigate", - )?.url ?? - collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ?? - "about:blank"; - const { pages, urlToId } = buildPageRegistry(collapsed, startUrl); - const out: StepV2[] = []; - let id = 1; - let lastUrl = startUrl; - for (const draft of collapsed) { - if (draft.op === "navigate") lastUrl = draft.url; - else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to; - else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url; - const step = toV2Step(draft, id, urlToId, lastUrl); - if (!step) continue; - out.push(step); - id += 1; - } - return { - recorded_at: new Date().toISOString(), - started_at: input.startedAt, - ...(input.purpose ? { purpose: input.purpose } : {}), - entry: { start_url: startUrl }, - pages, - steps: out, - }; -} diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index be869ef7..294c083a 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -17,11 +17,11 @@ import { type RecordStartMessage, type RecordStopMessage, } from "@/lib/record-bridge"; -import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer"; -import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer"; +import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer"; +import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2"; +import type { RecordingDraftStep } from "@/lib/recording/types"; import type { SessionManager } from "@/session-manager/manager"; import type { - DraftTraceStep, RecordAwaitParams, RecordAwaitResult, RecordStartParams, @@ -47,7 +47,7 @@ interface ActiveRecording { agentWindowId: number; startUrl?: string; purpose?: string; - steps: DraftTraceStep[]; + steps: RecordingDraftStep[]; startedAt: string; startedAtMs: number; finishPromise: Promise; @@ -155,16 +155,12 @@ async function sendRecordStartWithAck( } function buildTrace(recording: ActiveRecording): TraceV2 { - const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl); - const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages); - return { + return buildTraceV2({ + steps: recording.steps, + startedAt: recording.startedAt, + ...(recording.startUrl ? { startUrl: recording.startUrl } : {}), ...(recording.purpose ? { purpose: recording.purpose } : {}), - recorded_at: new Date().toISOString(), - started_at: recording.startedAt, - entry: { start_url: startUrl }, - pages, - steps, - }; + }); } export interface RecordDeps { diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index c72a44c1..8db006f8 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -744,48 +744,6 @@ export interface StepCommonV2 { effect?: StepEffectV2; } -/** Capture/buffer draft before v2 reduction. */ -export type DraftTraceStep = - | { - op: "click"; - target: TargetDescriptorV2; - navigated_to?: string; - page_url?: string; - } - | { - op: "hover"; - target: TargetDescriptorV2; - page_url?: string; - } - | { - op: "fill"; - target: TargetDescriptorV2; - value: string; - redacted?: boolean; - page_url?: string; - } - | { - op: "press"; - key: string; - target?: TargetDescriptorV2; - modifiers?: KeyModifier[]; - navigated_to?: string; - page_url?: string; - } - | { - op: "select"; - target: TargetDescriptorV2; - values: string[]; - labels?: string[]; - navigated_to?: string; - page_url?: string; - } - | { - op: "navigate"; - url: string; - page_url?: string; - }; - /** Exported record-only step (trace v2). */ export type StepV2 = | ({ op: "navigate" } & StepCommonV2 & { to: string }) From 6300eda0aac0f5e1818eec2bd8e5af7724c296c5 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 11:45:51 +0800 Subject: [PATCH 3/3] fix(record): rebase and dto policy --- .../__tests__/recording-observation.test.ts | 19 ++----- .../lib/__tests__/settle-controller.test.ts | 2 +- .../src/lib/__tests__/target-matcher.test.ts | 49 ++++++++++--------- .../src/lib/recording/observation-capture.ts | 35 ++++++------- .../src/lib/recording/target-matcher.ts | 2 +- 5 files changed, 50 insertions(+), 57 deletions(-) diff --git a/apps/extension/src/lib/__tests__/recording-observation.test.ts b/apps/extension/src/lib/__tests__/recording-observation.test.ts index dd277f4b..18b4959e 100644 --- a/apps/extension/src/lib/__tests__/recording-observation.test.ts +++ b/apps/extension/src/lib/__tests__/recording-observation.test.ts @@ -18,22 +18,13 @@ function sessionWithInput(redactValues = false): RecordingObservationSession { rootFrameId: "root", index: new ObservationNodeIndex({ rootFrameId: "root", - frameDocuments: [ + matchNodes: [ { frameId: "root", - domNodes: [ - { - backendNodeId: 42, - parentBackendNodeId: null, - frameId: "root", - tag: "input", - attrs: {}, - rect: { x: 20, y: 40, w: 200, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }, - ], + backendNodeId: 42, + tag: "input", + rect: { x: 20, y: 40, w: 200, h: 30 }, + localRect: { x: 20, y: 40, w: 200, h: 30 }, }, ], refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], diff --git a/apps/extension/src/lib/__tests__/settle-controller.test.ts b/apps/extension/src/lib/__tests__/settle-controller.test.ts index b72e2437..5ec5043a 100644 --- a/apps/extension/src/lib/__tests__/settle-controller.test.ts +++ b/apps/extension/src/lib/__tests__/settle-controller.test.ts @@ -8,7 +8,7 @@ import type { RecordingDraftStep } from "../recording/types"; const OBSERVATION: RegisteredObservation = { stateId: "s-next", rootFrameId: "root", - index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: [], refs: [] }), + index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }), url: "https://example.com/next", }; diff --git a/apps/extension/src/lib/__tests__/target-matcher.test.ts b/apps/extension/src/lib/__tests__/target-matcher.test.ts index b7081181..d3efae2a 100644 --- a/apps/extension/src/lib/__tests__/target-matcher.test.ts +++ b/apps/extension/src/lib/__tests__/target-matcher.test.ts @@ -1,40 +1,50 @@ import type { RenderedRef } from "@browser-skill/vom"; import { describe, expect, it } from "vitest"; -import type { CapturedNode } from "@/tools/vom/capture"; +import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation"; import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; import { matchObservationTarget } from "../recording/target-matcher"; -function node(backendNodeId: number, frameId: string, x = 10): CapturedNode { +function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode { return { backendNodeId, - parentBackendNodeId: null, frameId, tag: "button", - attrs: {}, rect: { x, y: 20, w: 100, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", + localRect: { x, y: 20, w: 100, h: 30 }, }; } function observation( - documents: Array<{ frameId: string; domNodes: CapturedNode[] }>, + matchNodes: CaptureVomMatchNode[], refs: RenderedRef[], ): RegisteredObservation { return { stateId: "s1", rootFrameId: "root", - index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: documents, refs }), + index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }), url: "https://example.com", }; } describe("matchObservationTarget", () => { + it("indexes the safe geometry DTO without discarding frame-local geometry", () => { + const geometry = node(42, "child"); + geometry.localRect = { x: 5, y: 6, w: 100, h: 30 }; + const ref: RenderedRef = { + ref: "e1", + backendNodeId: 42, + frameId: "child", + line: 1, + }; + const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0]; + + expect(candidate).toEqual({ frameId: "child", geometry, ref }); + }); + it("matches a unique node using canonical top-level viewport geometry", () => { const target = matchObservationTarget({ observation: observation( - [{ frameId: "root", domNodes: [node(42, "root")] }], + [node(42, "root")], [{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }], ), hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, @@ -45,10 +55,7 @@ describe("matchObservationTarget", () => { it("uses frame id with backend node id so sibling frames cannot collide", () => { const target = matchObservationTarget({ observation: observation( - [ - { frameId: "left", domNodes: [node(42, "left")] }, - { frameId: "right", domNodes: [node(42, "right")] }, - ], + [node(42, "left"), node(42, "right")], [ { ref: "e1", backendNodeId: 42, frameId: "left", line: 1 }, { ref: "e2", backendNodeId: 42, frameId: "right", line: 2 }, @@ -65,10 +72,7 @@ describe("matchObservationTarget", () => { it("restricts a missing frame hint to the root frame", () => { const target = matchObservationTarget({ observation: observation( - [ - { frameId: "root", domNodes: [] }, - { frameId: "child", domNodes: [node(42, "child")] }, - ], + [node(42, "child")], [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], ), hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, @@ -80,7 +84,7 @@ describe("matchObservationTarget", () => { it("returns unmatched for ambiguous geometry", () => { const target = matchObservationTarget({ observation: observation( - [{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }], + [node(42, "root"), node(43, "root")], [ { ref: "e1", backendNodeId: 42, line: 1 }, { ref: "e2", backendNodeId: 43, line: 2 }, @@ -94,10 +98,7 @@ describe("matchObservationTarget", () => { it("uses semantics when geometry is unavailable without crossing frame boundaries", () => { const target = matchObservationTarget({ observation: observation( - [ - { frameId: "root", domNodes: [node(41, "root")] }, - { frameId: "child", domNodes: [node(42, "child")] }, - ], + [node(41, "root"), node(42, "child")], [ { ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 }, { ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 }, @@ -112,7 +113,7 @@ describe("matchObservationTarget", () => { it("uses semantics to disambiguate equal geometry in one frame", () => { const target = matchObservationTarget({ observation: observation( - [{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }], + [node(42, "root"), node(43, "root")], [ { ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 }, { ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 }, diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts index 8006f74d..9c1f4591 100644 --- a/apps/extension/src/lib/recording/observation-capture.ts +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -1,11 +1,14 @@ import type { RenderedRef } from "@browser-skill/vom"; -import { captureVomObservation } from "@/tools/capture-vom-observation"; +import { + type CaptureVomMatchNode, + type CaptureVomObservationResult, + captureVomObservation, +} from "@/tools/capture-vom-observation"; import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; -import type { CapturedNode } from "@/tools/vom/capture"; export interface IndexedObservationNode { frameId: string; - node: CapturedNode; + geometry: CaptureVomMatchNode; ref?: RenderedRef; } @@ -38,11 +41,7 @@ export class ObservationNodeIndex { readonly #refById = new Map(); readonly #refsByFrame = new Map(); - constructor(input: { - rootFrameId: string; - frameDocuments: Array<{ frameId: string; domNodes: CapturedNode[] }>; - refs: RenderedRef[]; - }) { + constructor(input: Pick) { const refByNode = new Map(); for (const ref of input.refs) { const frameId = ref.frameId ?? input.rootFrameId; @@ -52,15 +51,17 @@ export class ObservationNodeIndex { frameRefs.push(ref); this.#refsByFrame.set(frameId, frameRefs); } - for (const document of input.frameDocuments) { - for (const node of document.domNodes) { - const frameId = node.frameId ?? document.frameId; - const entry = { frameId, node, ref: refByNode.get(nodeKey(frameId, node.backendNodeId)) }; - const key = frameTagKey(frameId, node.tag); - const bucket = this.#nodesByFrameTag.get(key) ?? []; - bucket.push(entry); - this.#nodesByFrameTag.set(key, bucket); - } + for (const geometry of input.matchNodes) { + const { frameId } = geometry; + const entry = { + frameId, + geometry, + ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)), + }; + const key = frameTagKey(frameId, geometry.tag); + const bucket = this.#nodesByFrameTag.get(key) ?? []; + bucket.push(entry); + this.#nodesByFrameTag.set(key, bucket); } } diff --git a/apps/extension/src/lib/recording/target-matcher.ts b/apps/extension/src/lib/recording/target-matcher.ts index 0df1abee..5cf21aac 100644 --- a/apps/extension/src/lib/recording/target-matcher.ts +++ b/apps/extension/src/lib/recording/target-matcher.ts @@ -15,7 +15,7 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool } function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean { - return candidate.node.rect !== null && rectMatches(geometry.rect, candidate.node.rect); + return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect); } export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {