diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index d227de66..bded922d 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -1,9 +1,9 @@ import { + type CaptureTargetDescriptor, describeEventTarget, describeTarget, resolveClickableElement, resolveHoverElement, - type TargetDescriptor, } from "@/lib/describe-target"; import { evaluateHoverTrigger, @@ -27,7 +27,7 @@ import { type RecordStopAck, type RecordStopMessage, } from "@/lib/record-bridge"; -import { shouldRecordPress } from "@/lib/trace-reducer"; +import { shouldRecordPress } from "@/lib/recording/draft-policy"; import { closestHoverSurfaceCandidate, collectHoverSurfaceStates, @@ -65,14 +65,14 @@ export interface RecordCaptureController { interface FillSession { element: FillableElement; - target: TargetDescriptor; + target: CaptureTargetDescriptor; baselineValue: string; lastValue: string; } interface HoverCandidate { element: Element; - target: TargetDescriptor; + target: CaptureTargetDescriptor; recordedAt: number; score: number; eligible: boolean; @@ -232,7 +232,10 @@ function collectHoverTriggerLabelText(root: Element): string { return normalizeLabelText(text); } -function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor { +function compactHoverTargetName( + el: Element, + desc: CaptureTargetDescriptor, +): CaptureTargetDescriptor { if (!desc.name) return desc; const fullText = normalizeLabelText(el.textContent ?? ""); const compactName = collectHoverTriggerLabelText(el); @@ -250,7 +253,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc return desc; } -function isWeakHoverTarget(target: TargetDescriptor): boolean { +function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean { return !target.role && !target.name && target.tag === "div"; } @@ -261,9 +264,9 @@ function looksLikeAvatarElement(el: Element): boolean { function normalizeHoverTarget( el: Element, - desc: TargetDescriptor, + desc: CaptureTargetDescriptor, decision: HoverTriggerDecision, -): TargetDescriptor { +): CaptureTargetDescriptor { if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) { return { ...desc, name: "image" }; } @@ -277,7 +280,7 @@ function normalizeHoverTarget( return desc; } -function hoverTriggerSignals(el: Element, desc: TargetDescriptor) { +function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) { if (!(el instanceof HTMLElement)) return null; const style = hoverTriggerStyle(el); return { diff --git a/apps/extension/src/lib/__tests__/document-settle.test.ts b/apps/extension/src/lib/__tests__/document-settle.test.ts new file mode 100644 index 00000000..cb0d8e26 --- /dev/null +++ b/apps/extension/src/lib/__tests__/document-settle.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpRunner } from "@/tools/shared"; +import { waitForDocumentSettled } from "../recording/document-settle"; + +function quietCdp(): { cdp: CdpRunner; send: ReturnType } { + const send = vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })); + return { + cdp: { send: send as unknown as CdpRunner["send"] }, + send, + }; +} + +describe("waitForDocumentSettled", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("waits through the minimum observation floor before accepting a quiet page", async () => { + vi.useFakeTimers(); + const { cdp, send } = quietCdp(); + + const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } }); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).toHaveBeenCalledTimes(3); + }); + + it("cancels before probing a superseded observation", async () => { + vi.useFakeTimers(); + const { cdp, send } = quietCdp(); + + const controller = new AbortController(); + const settled = waitForDocumentSettled( + cdp, + { target: { tabId: 7 } }, + { + signal: controller.signal, + }, + ); + controller.abort(); + await vi.advanceTimersByTimeAsync(60); + + await expect(settled).resolves.toBe("cancelled"); + expect(send).not.toHaveBeenCalled(); + }); + + it("probes a same-process iframe through its isolated execution context", async () => { + vi.useFakeTimers(); + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 91 }; + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + }); + const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] }; + + const settled = waitForDocumentSettled(cdp, { + target: { tabId: 7 }, + frameId: "child-frame", + }); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).toHaveBeenCalledWith( + 7, + "Page.createIsolatedWorld", + expect.objectContaining({ frameId: "child-frame" }), + ); + expect(send).toHaveBeenCalledWith( + 7, + "Runtime.evaluate", + expect.objectContaining({ contextId: 91 }), + ); + }); + + it("routes an OOPIF probe through its CDP target session", async () => { + vi.useFakeTimers(); + const send = vi.fn(); + const sendToTarget = vi.fn(async (_target, method: string) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 27 }; + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + }; + + const target = { tabId: 7, sessionId: "oopif-session" }; + const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" }); + await vi.advanceTimersByTimeAsync(180); + + await expect(settled).resolves.toBe("quiet"); + expect(send).not.toHaveBeenCalled(); + expect(sendToTarget).toHaveBeenCalledWith( + target, + "Runtime.evaluate", + expect.objectContaining({ contextId: 27 }), + ); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-observation.test.ts b/apps/extension/src/lib/__tests__/recording-observation.test.ts new file mode 100644 index 00000000..18b4959e --- /dev/null +++ b/apps/extension/src/lib/__tests__/recording-observation.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import { ObservationNodeIndex } from "../recording/observation-capture"; +import { RecordingObservationSession } from "../recording/observation-session"; +import { RecordingStateRegistry } from "../recording/state-registry"; +import { buildTraceV3 } from "../recording/trace-builder-v3"; +import type { RecordingDraftStep } from "../recording/types"; + +const URL = "https://example.com/login"; + +function sessionWithInput(redactValues = false): RecordingObservationSession { + const session = new RecordingObservationSession({ redactValues }); + const state = session.registry.register({ + url: URL, + rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]', + }); + session.cursor.lastSettled = { + stateId: state.id, + rootFrameId: "root", + index: new ObservationNodeIndex({ + rootFrameId: "root", + matchNodes: [ + { + frameId: "root", + backendNodeId: 42, + tag: "input", + rect: { x: 20, y: 40, w: 200, h: 30 }, + localRect: { x: 20, y: 40, w: 200, h: 30 }, + }, + ], + refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], + }), + url: URL, + }; + return session; +} + +function finalizedFillBody(value: string, redactValues: boolean): string { + const session = sessionWithInput(redactValues); + const draft: RecordingDraftStep = { + op: "fill", + captureTarget: { tag: "input", role: "textbox", name: "Password" }, + value, + targetHint: { + geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" }, + }, + }; + session.bindDraft(draft, 1); + draft.postStateId = draft.preStateId; + return buildTraceV3({ + registry: session.registry, + drafts: [draft], + annotations: session.annotations, + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }).states[0]!.body; +} + +describe("record observation annotations", () => { + it("omits a fill literal when values are redacted", () => { + const secret = "hunter2-private"; + const body = finalizedFillBody(secret, true); + expect(body).toContain("step 1: fill"); + expect(body).not.toContain(secret); + }); + + it("keeps ordinary fill details", () => { + expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"'); + }); + + it("encodes title and URL so line breaks cannot corrupt state metadata", () => { + const registry = new RecordingStateRegistry(); + const state = registry.register({ + url: "https://example.com/a\nb", + title: "hello\nworld", + rawVomText: "@vom 1", + }); + const trace = buildTraceV3({ + registry, + drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }], + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }); + expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"'); + expect(trace.states[0]?.body).toContain('title: "hello\\nworld"'); + }); +}); + +describe("recording state ownership", () => { + it("deduplicates within one recording and isolates ids between recordings", () => { + const first = new RecordingStateRegistry(); + const second = new RecordingStateRegistry(); + expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1"); + expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1"); + expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1"); + }); + + it("enriches metadata when a deduplicated observation becomes more complete", () => { + const registry = new RecordingStateRegistry(); + registry.register({ url: URL, rawVomText: "same" }); + const state = registry.register({ + url: URL, + title: "Login", + rawVomText: "same", + truncated: true, + }); + expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true }); + }); +}); + +describe("draft binding", () => { + it("keeps capture semantics when no observation exists", () => { + const session = new RecordingObservationSession(); + const draft: RecordingDraftStep = { + op: "hover", + captureTarget: { tag: "button", role: "button", name: "新建" }, + }; + session.bindDraft(draft, 1); + expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true }); + expect(draft.preStateId).toBeUndefined(); + }); + + it("does not bind an unmatched new action to a stale observation", () => { + const session = sessionWithInput(); + const draft: RecordingDraftStep = { + op: "click", + captureTarget: { tag: "button", role: "button", name: "Confirm" }, + targetHint: { + geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" }, + }, + }; + session.bindDraft(draft, 2, true); + expect(draft.preStateId).toBeUndefined(); + expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({ + role: "button", + name: "Confirm", + unmatched: true, + }); + }); +}); diff --git a/apps/extension/src/lib/__tests__/settle-controller.test.ts b/apps/extension/src/lib/__tests__/settle-controller.test.ts new file mode 100644 index 00000000..5ec5043a --- /dev/null +++ b/apps/extension/src/lib/__tests__/settle-controller.test.ts @@ -0,0 +1,136 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; +import { RecordingObservationSession } from "../recording/observation-session"; +import { SettleController } from "../recording/settle-controller"; +import type { RecordingDraftStep } from "../recording/types"; + +const OBSERVATION: RegisteredObservation = { + stateId: "s-next", + rootFrameId: "root", + index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }), + url: "https://example.com/next", +}; + +describe("SettleController", () => { + afterEach(() => vi.useRealTimers()); + + it("aborts superseded capture work and lands the prior action on the next origin", async () => { + vi.useFakeTimers(); + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })) as unknown as CdpRunner["send"], + }; + const tabsApi: ChromeTabsApi = { + get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab), + query: vi.fn(async () => []), + }; + const session = new RecordingObservationSession(); + let firstSignal: AbortSignal | undefined; + vi.spyOn(session, "capture") + .mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => { + firstSignal = signal; + return new Promise((_resolve, reject) => { + signal?.addEventListener( + "abort", + () => reject(new DOMException("observation aborted", "AbortError")), + { once: true }, + ); + }); + }) + .mockResolvedValueOnce(OBSERVATION); + + const drafts: RecordingDraftStep[] = [ + { op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" }, + ]; + const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 }); + controller.schedule(drafts, 0); + await vi.advanceTimersByTimeAsync(180); + + drafts.push({ + op: "click", + captureTarget: { tag: "button", name: "Second" }, + preStateId: "s-next", + }); + controller.schedule(drafts, 1); + expect(firstSignal?.aborted).toBe(true); + expect(drafts[0]?.postStateId).toBe("s-next"); + + await vi.advanceTimersByTimeAsync(500); + await controller.flush(); + expect(drafts[1]?.postStateId).toBe("s-next"); + }); + + it("does not consume a newer redirect while reading the prior landing URL", async () => { + vi.useFakeTimers(); + const cdp: CdpRunner = { + send: vi.fn(async () => ({ + result: { value: { idleMs: 1_000, readyState: "complete" } }, + })) as unknown as CdpRunner["send"], + }; + let resolveFirstTab!: (tab: chrome.tabs.Tab) => void; + const firstTab = new Promise((resolve) => { + resolveFirstTab = resolve; + }); + const tabsApi: ChromeTabsApi = { + get: vi + .fn() + .mockImplementationOnce(() => firstTab) + .mockResolvedValue({ id: 7, url: OBSERVATION.url }), + query: vi.fn(async () => []), + }; + const session = new RecordingObservationSession(); + session.cursor.lastSettled = OBSERVATION; + const drafts: RecordingDraftStep[] = []; + const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 }); + + controller.scheduleRedirect(drafts, "https://example.com/intermediate"); + await vi.advanceTimersByTimeAsync(180); + expect(tabsApi.get).toHaveBeenCalledTimes(1); + + controller.scheduleRedirect(drafts, OBSERVATION.url); + resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab); + await vi.advanceTimersByTimeAsync(500); + await controller.flushRedirects(); + + expect(drafts).toEqual([]); + expect(controller.hasPending).toBe(false); + }); + + it("settles an action in its own OOPIF document scope", async () => { + vi.useFakeTimers(); + const send = vi.fn(); + const sendToTarget = vi.fn(async (_target, method: string) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 39 }; + return { result: { value: { idleMs: 1_000, readyState: "complete" } } }; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + }; + const tabsApi: ChromeTabsApi = { + get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab), + query: vi.fn(async () => []), + }; + const session = new RecordingObservationSession(); + vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION); + const drafts: RecordingDraftStep[] = [ + { op: "click", captureTarget: { tag: "button", name: "Inside frame" } }, + ]; + const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 }); + const target = { tabId: 7, sessionId: "oopif-session" }; + + controller.schedule(drafts, 0, { target, frameId: "oopif-frame" }); + await vi.advanceTimersByTimeAsync(180); + await controller.flush(); + + expect(send).not.toHaveBeenCalled(); + expect(sendToTarget).toHaveBeenCalledWith( + target, + "Runtime.evaluate", + expect.objectContaining({ contextId: 39 }), + ); + expect(drafts[0]?.postStateId).toBe("s-next"); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts b/apps/extension/src/lib/__tests__/step-buffer.test.ts similarity index 80% rename from apps/extension/src/lib/__tests__/recording-step-buffer.test.ts rename to apps/extension/src/lib/__tests__/step-buffer.test.ts index 396253a5..ca212b8b 100644 --- a/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts +++ b/apps/extension/src/lib/__tests__/step-buffer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer"; +import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer"; describe("recording-step-buffer", () => { it("stores semantic click without summary", () => { @@ -12,7 +12,7 @@ describe("recording-step-buffer", () => { expect(buffer.steps).toEqual([ { op: "click", - target: { tag: "button", role: "button", name: "发布" }, + captureTarget: { tag: "button", role: "button", name: "发布" }, }, ]); expect(buffer.pendingNavigation).toBe(true); @@ -23,7 +23,7 @@ describe("recording-step-buffer", () => { steps: [ { op: "click" as const, - target: { tag: "button", role: "button", name: "发布" }, + captureTarget: { tag: "button", role: "button", name: "发布" }, }, ], currentUrl: "https://example.com/a", @@ -34,8 +34,8 @@ describe("recording-step-buffer", () => { expect(buffer.steps).toEqual([ { op: "click", - target: { tag: "button", role: "button", name: "发布" }, - navigated_to: "https://example.com/b", + captureTarget: { tag: "button", role: "button", name: "发布" }, + navigatedTo: "https://example.com/b", }, ]); expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation"); @@ -46,7 +46,7 @@ describe("recording-step-buffer", () => { steps: [ { op: "select" as const, - target: { tag: "select", role: "combobox", name: "分类" }, + captureTarget: { tag: "select", role: "combobox", name: "分类" }, values: ["tech"], }, ], @@ -58,9 +58,9 @@ describe("recording-step-buffer", () => { expect(buffer.steps).toEqual([ { op: "select", - target: { tag: "select", role: "combobox", name: "分类" }, + captureTarget: { tag: "select", role: "combobox", name: "分类" }, values: ["tech"], - navigated_to: "https://example.com/list?cat=tech", + navigatedTo: "https://example.com/list?cat=tech", }, ]); }); diff --git a/apps/extension/src/lib/__tests__/target-matcher.test.ts b/apps/extension/src/lib/__tests__/target-matcher.test.ts new file mode 100644 index 00000000..d3efae2a --- /dev/null +++ b/apps/extension/src/lib/__tests__/target-matcher.test.ts @@ -0,0 +1,127 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import { describe, expect, it } from "vitest"; +import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation"; +import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; +import { matchObservationTarget } from "../recording/target-matcher"; + +function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode { + return { + backendNodeId, + frameId, + tag: "button", + rect: { x, y: 20, w: 100, h: 30 }, + localRect: { x, y: 20, w: 100, h: 30 }, + }; +} + +function observation( + matchNodes: CaptureVomMatchNode[], + refs: RenderedRef[], +): RegisteredObservation { + return { + stateId: "s1", + rootFrameId: "root", + index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }), + url: "https://example.com", + }; +} + +describe("matchObservationTarget", () => { + it("indexes the safe geometry DTO without discarding frame-local geometry", () => { + const geometry = node(42, "child"); + geometry.localRect = { x: 5, y: 6, w: 100, h: 30 }; + const ref: RenderedRef = { + ref: "e1", + backendNodeId: 42, + frameId: "child", + line: 1, + }; + const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0]; + + expect(candidate).toEqual({ frameId: "child", geometry, ref }); + }); + + it("matches a unique node using canonical top-level viewport geometry", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "root")], + [{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + }); + expect(target).toEqual({ ref: "e1", role: "button", name: "发布" }); + }); + + it("uses frame id with backend node id so sibling frames cannot collide", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "left"), node(42, "right")], + [ + { ref: "e1", backendNodeId: 42, frameId: "left", line: 1 }, + { ref: "e2", backendNodeId: 42, frameId: "right", line: 2 }, + ], + ), + hint: { + frameId: "right", + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }); + expect(target.ref).toBe("e2"); + }); + + it("restricts a missing frame hint to the root frame", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "child")], + [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + fallback: { tag: "button", name: "发布" }, + }); + expect(target).toEqual({ name: "发布", unmatched: true }); + }); + + it("returns unmatched for ambiguous geometry", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "root"), node(43, "root")], + [ + { ref: "e1", backendNodeId: 42, line: 1 }, + { ref: "e2", backendNodeId: 43, line: 2 }, + ], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + }); + expect(target.unmatched).toBe(true); + }); + + it("uses semantics when geometry is unavailable without crossing frame boundaries", () => { + const target = matchObservationTarget({ + observation: observation( + [node(41, "root"), node(42, "child")], + [ + { ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 }, + { ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 }, + ], + ), + hint: { frameId: "child" }, + fallback: { tag: "button", role: "button", name: "保存" }, + }); + expect(target.ref).toBe("e2"); + }); + + it("uses semantics to disambiguate equal geometry in one frame", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "root"), node(43, "root")], + [ + { ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 }, + { ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 }, + ], + ), + hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, + fallback: { tag: "button", role: "button", name: "取消" }, + }); + expect(target.ref).toBe("e2"); + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts similarity index 67% rename from apps/extension/src/lib/__tests__/trace-reducer.test.ts rename to apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts index f5c6db51..2b7fb1d7 100644 --- a/apps/extension/src/lib/__tests__/trace-reducer.test.ts +++ b/apps/extension/src/lib/__tests__/trace-reducer-v2.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it } from "vitest"; -import type { DraftTraceStep } from "@/transport/types"; -import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer"; +import type { RecordingDraftStep } from "@/lib/recording/types"; +import { shouldRecordPress } from "../recording/draft-policy"; +import { buildTraceV2 } from "../recording/trace-reducer-v2"; + +function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) { + return buildTraceV2({ + steps, + startedAt: "2026-01-01T00:00:00.000Z", + ...(startUrl ? { startUrl } : {}), + }); +} describe("shouldRecordPress", () => { it("keeps Enter and Escape", () => { @@ -19,35 +28,35 @@ describe("shouldRecordPress", () => { describe("reduceTraceSteps", () => { it("builds steps with pages dictionary and page id references", () => { - const drafts: DraftTraceStep[] = [ + const drafts: RecordingDraftStep[] = [ { op: "navigate", url: "https://example.com/search?q=hello&utm_source=x", - page_url: "https://example.com/search?q=hello&utm_source=x", + pageUrl: "https://example.com/search?q=hello&utm_source=x", }, { op: "fill", - target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, + captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, value: "browser skill", - page_url: "https://example.com/search?q=hello&utm_source=x", + pageUrl: "https://example.com/search?q=hello&utm_source=x", }, { op: "press", key: "Enter", - target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, - navigated_to: "https://example.com/results/42", - page_url: "https://example.com/search?q=hello&utm_source=x", + captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, + navigatedTo: "https://example.com/results/42", + pageUrl: "https://example.com/search?q=hello&utm_source=x", }, { op: "click", - target: { tag: "button", role: "button", name: "发布" }, - navigated_to: "https://example.com/p/99", - page_url: "https://example.com/results/42", + captureTarget: { tag: "button", role: "button", name: "发布" }, + navigatedTo: "https://example.com/p/99", + pageUrl: "https://example.com/results/42", }, { op: "press", key: "a", - page_url: "https://example.com/p/99", + pageUrl: "https://example.com/p/99", }, ]; @@ -109,9 +118,9 @@ describe("reduceTraceSteps", () => { [ { op: "fill", - target: { tag: "input", role: "textbox", name: "Search query" }, + captureTarget: { tag: "input", role: "textbox", name: "Search query" }, value: "", - page_url: "https://example.com/search", + pageUrl: "https://example.com/search", }, ], "https://example.com/search", @@ -125,16 +134,16 @@ describe("reduceTraceSteps", () => { ]); }); - it("maps select navigated_to onto effect.navigated_to (page id)", () => { + it("maps select navigatedTo onto effect.navigatedTo (page id)", () => { const { pages, steps } = reduceTraceSteps( [ { op: "select", - target: { tag: "select", role: "combobox", name: "分类" }, + captureTarget: { tag: "select", role: "combobox", name: "分类" }, values: ["tech"], labels: ["技术"], - navigated_to: "https://example.com/list?cat=tech", - page_url: "https://example.com/list", + navigatedTo: "https://example.com/list?cat=tech", + pageUrl: "https://example.com/list", }, ], "https://example.com/list", @@ -156,13 +165,13 @@ describe("reduceTraceSteps", () => { [ { op: "hover", - target: { tag: "span", role: "button", name: "Account" }, - page_url: "https://example.com/app", + captureTarget: { tag: "span", role: "button", name: "Account" }, + pageUrl: "https://example.com/app", }, { op: "click", - target: { tag: "a", role: "link", name: "Profile" }, - page_url: "https://example.com/app", + captureTarget: { tag: "a", role: "link", name: "Profile" }, + pageUrl: "https://example.com/app", }, ], "https://example.com/app", @@ -177,10 +186,11 @@ describe("reduceTraceSteps", () => { it("resolveTraceStartUrl prefers explicit start URL", () => { expect( - resolveTraceStartUrl( - [{ op: "navigate", url: "https://example.com/other" }], - "https://example.com/start", - ), + buildTraceV2({ + steps: [{ op: "navigate", url: "https://example.com/other" }], + startedAt: "2026-01-01T00:00:00.000Z", + startUrl: "https://example.com/start", + }).entry.start_url, ).toBe("https://example.com/start"); }); }); diff --git a/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts new file mode 100644 index 00000000..93434d14 --- /dev/null +++ b/apps/extension/src/lib/__tests__/trace-reducer-v3.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types"; +import { RecordingStateRegistry } from "../recording/state-registry"; +import { buildTraceV3 } from "../recording/trace-builder-v3"; +import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3"; +import type { RecordingDraftStep } from "../recording/types"; + +describe("trace reducer v3", () => { + it("collapses redirect hops while retaining draft-to-step identity", () => { + const drafts: RecordingDraftStep[] = [ + { op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" }, + { + op: "navigate", + url: "https://example.com/final", + transitionQualifiers: ["server_redirect"], + preStateId: "s2", + postStateId: "s3", + }, + ]; + const before = structuredClone(drafts); + const reduced = reduceTraceStepsV3(drafts); + + expect(drafts).toEqual(before); + expect(reduced.steps).toEqual([ + expect.objectContaining({ + id: 1, + op: "navigate", + state: "s1", + to: "https://example.com/final", + result: { state: "s3" }, + }), + ]); + expect(reduced.stepIdByDraftId.get(1)).toBe(1); + expect(reduced.stepIdByDraftId.get(2)).toBe(1); + }); + + it("builds the wire model from protocol constants", () => { + const registry = new RecordingStateRegistry(); + const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" }); + const trace = buildTraceV3({ + registry, + drafts: [ + { + op: "click", + captureTarget: { tag: "button", role: "button", name: "Save" }, + preStateId: state.id, + postStateId: state.id, + }, + ], + startedAt: "2026-08-12T00:00:00.000Z", + stoppedBy: "user_finish", + bskVersion: "test", + }); + + expect(trace.version).toBe(TRACE_VERSION_V3); + expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION); + expect(trace.steps[0]).toMatchObject({ + op: "click", + target: { role: "button", name: "Save", unmatched: true }, + }); + }); +}); diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts index 0a1fcb22..c090b0a6 100644 --- a/apps/extension/src/lib/describe-target.ts +++ b/apps/extension/src/lib/describe-target.ts @@ -1,12 +1,13 @@ /** - * Build a semantic TargetDescriptor for an interacted element. + * Build a semantic capture descriptor for an interacted element. * * Trace steps are an LLM *textbook*: each click must say what to look for * on screen (usually a short visible name). Tag-only noise like * `{ "tag": "div" }` / “点击div” is useless and must not be recorded. */ -export interface TargetDescriptor { +/** Content-script capture descriptor before VOM geometric matching. */ +export interface CaptureTargetDescriptor { role?: string; name?: string; tag: string; @@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null { * (visible name), or at least a form `name_attr` for checkbox/radio. * Recording “点击div” with no name fails this bar. */ -export function isMeaningfulClickTarget(target: TargetDescriptor): boolean { +export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean { const name = target.name?.trim(); if (name && isActionableLabel(name)) return true; if ( @@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined { return undefined; } -export function describeTarget(el: Element): TargetDescriptor { +export function describeTarget(el: Element): CaptureTargetDescriptor { const tag = el.tagName.toLowerCase(); const role = inferRole(el); const name = accessibleName(el); @@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor { }; } -export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null { +export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null { if (!(target instanceof Element)) return null; const clickable = resolveClickableElement(target); if (!clickable) return null; diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts index c4259e38..09399ba5 100644 --- a/apps/extension/src/lib/record-bridge.ts +++ b/apps/extension/src/lib/record-bridge.ts @@ -3,7 +3,7 @@ * service worker and a tab's content script. */ -import type { TargetDescriptor } from "./describe-target"; +import type { CaptureTargetDescriptor } from "./describe-target"; export const RECORD_START = "bsk-record-start"; export const RECORD_STEP = "bsk-record-step"; @@ -42,7 +42,7 @@ export interface RecordStartMessage { export interface RecordStepPayload { op: "click" | "hover" | "fill" | "press" | "select" | "navigate"; - target?: TargetDescriptor; + target?: CaptureTargetDescriptor; value?: string; key?: string; modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">; diff --git a/apps/extension/src/lib/recording/document-settle.ts b/apps/extension/src/lib/recording/document-settle.ts new file mode 100644 index 00000000..d3b89c7f --- /dev/null +++ b/apps/extension/src/lib/recording/document-settle.ts @@ -0,0 +1,134 @@ +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { type CdpRunner, sendToCdpTarget } from "@/tools/shared"; + +const SETTLE_MIN_MS = 150; +const SETTLE_QUIET_MS = 250; +const SETTLE_MAX_MS = 2_000; +const SETTLE_POLL_MS = 60; +const SETTLE_WORLD_NAME = "__bsk_record_settle__"; + +const QUIET_PROBE = `(() => { + const scope = window; + let probe = scope.__bskRecordQuiet; + if (!probe) { + probe = { changedAt: Date.now() }; + const observer = new MutationObserver(() => { + probe.changedAt = Date.now(); + }); + observer.observe(document.documentElement, { + subtree: true, + childList: true, + attributes: true, + characterData: true, + }); + scope.__bskRecordQuiet = probe; + } + return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState }; +})()`; + +export interface DocumentSettleScope { + target: CdpTarget; + /** Omit only for the target's root document. */ + frameId?: string; +} + +type SettleOutcome = "quiet" | "timeout" | "cancelled"; + +interface QuietProbe { + idleMs: number; + readyState: string; +} + +interface ProbeContext { + executionContextId?: number; +} + +function sleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +async function executionContextForScope( + cdp: CdpRunner, + scope: DocumentSettleScope, +): Promise { + if (!scope.frameId) return undefined; + const result = await sendToCdpTarget<{ executionContextId?: number }>( + cdp, + scope.target, + "Page.createIsolatedWorld", + { + frameId: scope.frameId, + worldName: SETTLE_WORLD_NAME, + grantUniveralAccess: false, + }, + ); + return result.executionContextId; +} + +async function readQuietProbe( + cdp: CdpRunner, + scope: DocumentSettleScope, + context: ProbeContext, +): Promise { + try { + if (scope.frameId && context.executionContextId === undefined) { + context.executionContextId = await executionContextForScope(cdp, scope); + if (context.executionContextId === undefined) return null; + } + const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>( + cdp, + scope.target, + "Runtime.evaluate", + { + expression: QUIET_PROBE, + returnByValue: true, + ...(context.executionContextId !== undefined + ? { contextId: context.executionContextId } + : {}), + }, + ); + const value = reply.result?.value; + if (!value || typeof value !== "object") return null; + const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown }; + if (typeof idleMs !== "number" || typeof readyState !== "string") return null; + return { idleMs, readyState }; + } catch { + context.executionContextId = undefined; + return null; + } +} + +export async function waitForDocumentSettled( + cdp: CdpRunner, + scope: DocumentSettleScope, + options: { signal?: AbortSignal } = {}, +): Promise { + const startedAt = Date.now(); + const floor = startedAt + SETTLE_MIN_MS; + const deadline = startedAt + SETTLE_MAX_MS; + const context: ProbeContext = {}; + + for (;;) { + if (options.signal?.aborted) return "cancelled"; + await sleep(SETTLE_POLL_MS, options.signal); + if (options.signal?.aborted) return "cancelled"; + + const probe = await readQuietProbe(cdp, scope, context); + const now = Date.now(); + if (now < floor) continue; + if (now >= deadline) return "timeout"; + if (!probe || probe.readyState === "loading") continue; + if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet"; + } +} diff --git a/apps/extension/src/lib/recording/draft-policy.ts b/apps/extension/src/lib/recording/draft-policy.ts new file mode 100644 index 00000000..94723b13 --- /dev/null +++ b/apps/extension/src/lib/recording/draft-policy.ts @@ -0,0 +1,34 @@ +import type { RecordingDraftStep } from "./types"; + +const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); +const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); + +export function shouldRecordPress( + key: string, + modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, +): boolean { + if (MODIFIER_ONLY_KEYS.has(key)) return false; + const mods = modifiers ?? []; + const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); + if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; + if (key === "Enter" || key === "Escape") return true; + if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; + return false; +} + +export function shouldIncludeDraft(step: RecordingDraftStep): boolean { + return step.op !== "press" || shouldRecordPress(step.key, step.modifiers); +} + +export function resolveDraftStartUrl( + drafts: RecordingDraftStep[], + explicitStartUrl?: string, + fallbackUrl?: string, +): string { + if (explicitStartUrl) return explicitStartUrl; + const navigation = drafts.find( + (step): step is Extract => step.op === "navigate", + ); + if (navigation) return navigation.url; + return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank"; +} diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts new file mode 100644 index 00000000..9c1f4591 --- /dev/null +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -0,0 +1,116 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import { + type CaptureVomMatchNode, + type CaptureVomObservationResult, + captureVomObservation, +} from "@/tools/capture-vom-observation"; +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; + +export interface IndexedObservationNode { + frameId: string; + geometry: CaptureVomMatchNode; + ref?: RenderedRef; +} + +export interface CapturedRecordingObservation { + rootFrameId: string; + index: ObservationNodeIndex; + url: string; + title?: string; + vomText: string; + truncated: boolean; +} + +export interface RegisteredObservation { + stateId: string; + rootFrameId: string; + index: ObservationNodeIndex; + url: string; +} + +function nodeKey(frameId: string, backendNodeId: number): string { + return `${frameId}:${backendNodeId}`; +} + +function frameTagKey(frameId: string, tag: string): string { + return `${frameId}:${tag.toLowerCase()}`; +} + +export class ObservationNodeIndex { + readonly #nodesByFrameTag = new Map(); + readonly #refById = new Map(); + readonly #refsByFrame = new Map(); + + constructor(input: Pick) { + const refByNode = new Map(); + for (const ref of input.refs) { + const frameId = ref.frameId ?? input.rootFrameId; + refByNode.set(nodeKey(frameId, ref.backendNodeId), ref); + this.#refById.set(ref.ref, ref); + const frameRefs = this.#refsByFrame.get(frameId) ?? []; + frameRefs.push(ref); + this.#refsByFrame.set(frameId, frameRefs); + } + for (const geometry of input.matchNodes) { + const { frameId } = geometry; + const entry = { + frameId, + geometry, + ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)), + }; + const key = frameTagKey(frameId, geometry.tag); + const bucket = this.#nodesByFrameTag.get(key) ?? []; + bucket.push(entry); + this.#nodesByFrameTag.set(key, bucket); + } + } + + candidates(frameId: string, tag: string): readonly IndexedObservationNode[] { + return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? []; + } + + ref(refId: string): RenderedRef | undefined { + return this.#refById.get(refId); + } + + refs(frameId: string): readonly RenderedRef[] { + return this.#refsByFrame.get(frameId) ?? []; + } +} + +async function readTabMeta( + tabsApi: ChromeTabsApi, + tabId: number, +): Promise<{ url: string; title?: string }> { + try { + const tab = await tabsApi.get(tabId); + return { url: tab.url ?? "about:blank", title: tab.title }; + } catch { + return { url: "about:blank" }; + } +} + +export async function captureRecordingObservation(input: { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + tabId: number; + maxTokens: number; + redactValues: boolean; + signal?: AbortSignal; +}): Promise { + const { url, title } = await readTabMeta(input.tabsApi, input.tabId); + const captured = await captureVomObservation(input.cdp, input.tabId, url, { + maxTokens: input.maxTokens, + redactValues: input.redactValues, + conditionalSurfaceProbe: false, + signal: input.signal, + }); + return { + rootFrameId: captured.rootFrameId, + index: new ObservationNodeIndex(captured), + url, + title, + vomText: captured.text, + truncated: captured.truncated, + }; +} diff --git a/apps/extension/src/lib/recording/observation-session.ts b/apps/extension/src/lib/recording/observation-session.ts new file mode 100644 index 00000000..9cfd43de --- /dev/null +++ b/apps/extension/src/lib/recording/observation-session.ts @@ -0,0 +1,122 @@ +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture"; +import { RecordingStateRegistry } from "./state-registry"; +import { matchObservationTarget, unmatchedTarget } from "./target-matcher"; +import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types"; + +const DEFAULT_MAX_PAGE_TOKENS = 3_000; +const MIN_CAPTURE_INTERVAL_MS = 200; + +export interface TabObservationCursor { + lastSettled: RegisteredObservation | null; + lastCaptureAt: number; +} + +function abortableDelay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft { + return draft.op !== "navigate" && draft.op !== "scroll"; +} + +export class RecordingObservationSession { + readonly registry: RecordingStateRegistry; + readonly cursor: TabObservationCursor; + readonly annotations: StepAnnotation[]; + readonly #maxTokens: number; + readonly #redactValues: boolean; + + constructor( + options: { + registry?: RecordingStateRegistry; + cursor?: TabObservationCursor; + annotations?: StepAnnotation[]; + maxTokens?: number; + redactValues?: boolean; + } = {}, + ) { + this.registry = options.registry ?? new RecordingStateRegistry(); + this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 }; + this.annotations = options.annotations ?? []; + this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS; + this.#redactValues = options.redactValues ?? false; + } + + async capture( + cdp: CdpRunner, + tabsApi: ChromeTabsApi, + tabId: number, + signal?: AbortSignal, + ): Promise { + const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt)); + if (waitMs > 0) await abortableDelay(waitMs, signal); + if (signal?.aborted) throw new DOMException("observation aborted", "AbortError"); + const captured = await captureRecordingObservation({ + cdp, + tabsApi, + tabId, + maxTokens: this.#maxTokens, + redactValues: this.#redactValues, + signal, + }); + const state = this.registry.register({ + url: captured.url, + title: captured.title, + rawVomText: captured.vomText, + truncated: captured.truncated, + }); + const observation: RegisteredObservation = { + stateId: state.id, + rootFrameId: captured.rootFrameId, + index: captured.index, + url: captured.url, + }; + this.cursor.lastSettled = observation; + this.cursor.lastCaptureAt = Date.now(); + return observation; + } + + bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void { + const observation = this.cursor.lastSettled; + if (isTargeted(draft)) { + draft.matchedTarget = observation + ? matchObservationTarget({ + observation, + hint: draft.targetHint, + fallback: draft.captureTarget, + }) + : unmatchedTarget(draft.captureTarget); + } + if (!observation) return; + + const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true; + if (previousActionPending && unmatched) return; + draft.preStateId = observation.stateId; + this.registry.markStep(observation.stateId, draftId); + + if (!isTargeted(draft) || !draft.matchedTarget?.ref) return; + const ref = observation.index.ref(draft.matchedTarget.ref); + if (!ref) return; + this.annotations.push({ + draftId, + op: draft.op, + line: ref.line, + stateId: observation.stateId, + ...(draft.op === "fill" && !this.#redactValues + ? { detail: JSON.stringify(draft.value) } + : {}), + }); + } +} diff --git a/apps/extension/src/lib/recording/settle-controller.ts b/apps/extension/src/lib/recording/settle-controller.ts new file mode 100644 index 00000000..c63f46fa --- /dev/null +++ b/apps/extension/src/lib/recording/settle-controller.ts @@ -0,0 +1,232 @@ +import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; +import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle"; +import { RecordingObservationSession } from "./observation-session"; +import type { RecordingDraftStep } from "./types"; + +interface PendingSettle { + abort: AbortController; + scope: DocumentSettleScope; +} + +interface PendingRedirect { + url: string; + abort: AbortController; +} + +const CAPTURE_RETRY_DELAY_MS = 250; + +function delay(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) return resolve(); + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +function isAbortError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { name?: string }).name === "AbortError" + ); +} + +export function inferMissingPostStates(drafts: RecordingDraftStep[]): void { + for (let index = 0; index < drafts.length - 1; index += 1) { + const draft = drafts[index]; + const next = drafts[index + 1]; + if (draft && next && !draft.postStateId && next.preStateId) { + draft.postStateId = next.preStateId; + } + } +} + +export class SettleController { + readonly #session: RecordingObservationSession; + readonly #cdp: CdpRunner; + readonly #tabsApi: ChromeTabsApi; + readonly #tabId: number; + readonly #rootScope: DocumentSettleScope; + readonly #pending = new Map(); + #queue = Promise.resolve(); + #pendingRedirect: PendingRedirect | null = null; + #redirectQueue = Promise.resolve(); + + constructor(input: { + session: RecordingObservationSession; + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + tabId: number; + }) { + this.#session = input.session; + this.#cdp = input.cdp; + this.#tabsApi = input.tabsApi; + this.#tabId = input.tabId; + this.#rootScope = { target: { tabId: input.tabId } }; + } + + get hasPending(): boolean { + return this.#pending.size > 0 || this.#pendingRedirect !== null; + } + + async #captureWithRetry(signal?: AbortSignal) { + try { + return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal); + } catch (error) { + if (signal?.aborted) throw error; + await delay(CAPTURE_RETRY_DELAY_MS, signal); + return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal); + } + } + + schedule( + drafts: RecordingDraftStep[], + draftIndex: number, + scope: DocumentSettleScope = this.#rootScope, + ): void { + if (scope.target.tabId !== this.#tabId) { + throw new Error( + `settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`, + ); + } + const landing = drafts[draftIndex]?.preStateId; + for (const [index, pending] of this.#pending) { + if (index >= draftIndex) continue; + pending.abort.abort(); + this.#pending.delete(index); + if (landing && drafts[index] && !drafts[index].postStateId) { + drafts[index].postStateId = landing; + } + } + + this.#pending.get(draftIndex)?.abort.abort(); + const pending = { abort: new AbortController(), scope }; + this.#pending.set(draftIndex, pending); + const task = async () => { + if (pending.abort.signal.aborted) return; + try { + const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, { + signal: pending.abort.signal, + }); + if (outcome === "cancelled") return; + const observation = await this.#captureWithRetry(pending.abort.signal); + if (!pending.abort.signal.aborted && drafts[draftIndex]) { + drafts[draftIndex].postStateId = observation.stateId; + } + } catch (error) { + if (!isAbortError(error)) { + console.warn( + `[bsk record] post-action observation failed for step ${draftIndex + 1}`, + error, + ); + } + } finally { + if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex); + } + }; + this.#queue = this.#queue.then(task, task).catch(() => {}); + } + + cancel(): void { + for (const pending of this.#pending.values()) pending.abort.abort(); + this.#pending.clear(); + this.clearRedirect(); + } + + async flush(): Promise { + for (let round = 0; round < 10; round += 1) { + const current = this.#queue; + await current; + if (current === this.#queue) return; + } + console.warn("[bsk record] settle queue kept growing; using completed observations"); + } + + async settleTrailing(drafts: RecordingDraftStep[]): Promise { + const trailing: RecordingDraftStep[] = []; + for (let index = drafts.length - 1; index >= 0; index -= 1) { + const draft = drafts[index]; + if (!draft || draft.postStateId) break; + trailing.push(draft); + } + if (trailing.length === 0) return; + try { + const observation = await this.#captureWithRetry(); + for (const draft of trailing) draft.postStateId = observation.stateId; + } catch (error) { + console.warn("[bsk record] final observation at stop failed", error); + } + } + + clearRedirect(): void { + this.#pendingRedirect?.abort.abort(); + this.#pendingRedirect = null; + } + + scheduleRedirect(drafts: RecordingDraftStep[], url: string): void { + this.#pendingRedirect?.abort.abort(); + const pending: PendingRedirect = { url, abort: new AbortController() }; + this.#pendingRedirect = pending; + const task = () => this.#settleRedirect(drafts, pending); + this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {}); + } + + async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise { + const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, { + signal: pending.abort.signal, + }); + if (outcome === "cancelled" || this.#pendingRedirect !== pending) return; + + let finalUrl = pending.url; + try { + finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl; + } catch { + // The navigation event URL remains the best available destination. + } + if (this.#pendingRedirect !== pending) return; + this.#pendingRedirect = null; + if ( + !finalUrl || + finalUrl === "about:blank" || + this.#session.cursor.lastSettled?.url === finalUrl + ) { + return; + } + + const last = drafts[drafts.length - 1]; + if (last?.op === "navigate" && last.url === finalUrl) { + if (!last.postStateId) this.schedule(drafts, drafts.length - 1); + await this.flush(); + return; + } + + const draft: RecordingDraftStep = { + op: "navigate", + url: finalUrl, + pageUrl: finalUrl, + cause: "browser", + preStateId: this.#session.cursor.lastSettled?.stateId, + }; + drafts.push(draft); + const draftIndex = drafts.length - 1; + if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1); + this.schedule(drafts, draftIndex); + await this.flush(); + } + + async flushRedirects(): Promise { + for (let round = 0; round < 10; round += 1) { + const current = this.#redirectQueue; + await current; + if (current === this.#redirectQueue) return; + } + console.warn("[bsk record] redirect queue kept growing; using the final completed landing"); + } +} diff --git a/apps/extension/src/lib/recording/state-registry.ts b/apps/extension/src/lib/recording/state-registry.ts new file mode 100644 index 00000000..fe5f0eab --- /dev/null +++ b/apps/extension/src/lib/recording/state-registry.ts @@ -0,0 +1,56 @@ +export interface RecordedStateEntry { + id: string; + url: string; + title?: string; + rawVomText: string; + truncated: boolean; + stepsHere: number[]; +} + +function stateIdentity(url: string, body: string): string { + return `${url}\0${body}`; +} + +export class RecordingStateRegistry { + readonly #entriesById = new Map(); + readonly #idByIdentity = new Map(); + #nextId = 1; + + register(input: { + url: string; + title?: string; + rawVomText: string; + truncated?: boolean; + }): RecordedStateEntry { + const identity = stateIdentity(input.url, input.rawVomText); + const existingId = this.#idByIdentity.get(identity); + if (existingId) { + const existing = this.#entriesById.get(existingId)!; + if (!existing.title && input.title) existing.title = input.title; + if (input.truncated) existing.truncated = true; + return existing; + } + + const entry: RecordedStateEntry = { + id: `s${this.#nextId}`, + url: input.url, + ...(input.title ? { title: input.title } : {}), + rawVomText: input.rawVomText, + truncated: input.truncated ?? false, + stepsHere: [], + }; + this.#nextId += 1; + this.#entriesById.set(entry.id, entry); + this.#idByIdentity.set(identity, entry.id); + return entry; + } + + values(): RecordedStateEntry[] { + return [...this.#entriesById.values()]; + } + + markStep(stateId: string, draftId: number): void { + const entry = this.#entriesById.get(stateId); + if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId); + } +} diff --git a/apps/extension/src/lib/recording-step-buffer.ts b/apps/extension/src/lib/recording/step-buffer.ts similarity index 79% rename from apps/extension/src/lib/recording-step-buffer.ts rename to apps/extension/src/lib/recording/step-buffer.ts index c1c5086b..a2732e1d 100644 --- a/apps/extension/src/lib/recording-step-buffer.ts +++ b/apps/extension/src/lib/recording/step-buffer.ts @@ -1,8 +1,8 @@ -import type { DraftTraceStep } from "@/transport/types"; -import type { RecordStepPayload } from "./record-bridge"; +import type { RecordStepPayload } from "../record-bridge"; +import type { RecordingDraftStep } from "./types"; export interface RecordingStepBuffer { - steps: DraftTraceStep[]; + steps: RecordingDraftStep[]; currentUrl?: string; pendingNavigation: boolean; pendingNavigationDeadline?: number; @@ -10,33 +10,33 @@ export interface RecordingStepBuffer { const NAVIGATION_TRIGGER_WINDOW_MS = 3_000; -function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { +function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null { const pageUrl = payload.page_url; switch (payload.op) { case "click": return payload.target ? { op: "click", - target: payload.target, - ...(pageUrl ? { page_url: pageUrl } : {}), + captureTarget: payload.target, + ...(pageUrl ? { pageUrl } : {}), } : null; case "hover": return payload.target ? { op: "hover", - target: payload.target, - ...(pageUrl ? { page_url: pageUrl } : {}), + captureTarget: payload.target, + ...(pageUrl ? { pageUrl } : {}), } : null; case "fill": return payload.target ? { op: "fill", - target: payload.target, + captureTarget: payload.target, value: payload.value ?? "", ...(payload.redacted ? { redacted: true } : {}), - ...(pageUrl ? { page_url: pageUrl } : {}), + ...(pageUrl ? { pageUrl } : {}), } : null; case "press": @@ -44,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { ? { op: "press", key: payload.key, - ...(payload.target ? { target: payload.target } : {}), + ...(payload.target ? { captureTarget: payload.target } : {}), ...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}), - ...(pageUrl ? { page_url: pageUrl } : {}), + ...(pageUrl ? { pageUrl } : {}), } : null; case "select": return payload.target && payload.values ? { op: "select", - target: payload.target, + captureTarget: payload.target, values: payload.values, ...(payload.labels?.length ? { labels: payload.labels } : {}), - ...(pageUrl ? { page_url: pageUrl } : {}), + ...(pageUrl ? { pageUrl } : {}), } : null; case "navigate": @@ -69,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b const step = buffer.steps[i]; if (!step) continue; if (step.op === "click" || step.op === "press" || step.op === "select") { - buffer.steps[i] = { ...step, navigated_to: url }; + buffer.steps[i] = { ...step, navigatedTo: url }; return true; } break; @@ -96,7 +96,7 @@ export function observeRecordedNavigation( buffer.steps.push({ op: "navigate", url, - page_url: url, + pageUrl: url, }); } return; @@ -107,7 +107,7 @@ export function observeRecordedNavigation( buffer.steps.push({ op: "navigate", url, - page_url: url, + pageUrl: url, }); } diff --git a/apps/extension/src/lib/recording/target-matcher.ts b/apps/extension/src/lib/recording/target-matcher.ts new file mode 100644 index 00000000..5cf21aac --- /dev/null +++ b/apps/extension/src/lib/recording/target-matcher.ts @@ -0,0 +1,76 @@ +import type { RenderedRef } from "@browser-skill/vom"; +import type { TargetDescriptorV3 } from "@/transport/types"; +import type { CaptureTargetDescriptor } from "../describe-target"; +import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture"; +import type { TargetGeometry, TargetMatchHint } from "./types"; + +const MATCH_TOLERANCE_PX = 2; + +function close(a: number, b: number): boolean { + return Math.abs(a - b) <= MATCH_TOLERANCE_PX; +} + +function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean { + return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h); +} + +function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean { + return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect); +} + +export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { + return { + ...(fallback?.role ? { role: fallback.role } : {}), + ...(fallback?.name ? { name: fallback.name } : {}), + unmatched: true, + }; +} + +function normalized(value: string | undefined): string { + return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? ""; +} + +function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean { + const role = normalized(fallback?.role); + const name = normalized(fallback?.name); + if (!role && !name) return false; + if (role && normalized(ref.role) !== role) return false; + if (name && normalized(ref.name) !== name) return false; + return true; +} + +function descriptor(ref: RenderedRef): TargetDescriptorV3 { + return { + ref: ref.ref, + ...(ref.role ? { role: ref.role } : {}), + ...(ref.name ? { name: ref.name } : {}), + ...(ref.ctx ? { ctx: ref.ctx } : {}), + }; +} + +export function matchObservationTarget(input: { + observation: RegisteredObservation; + hint?: TargetMatchHint; + fallback?: CaptureTargetDescriptor; +}): TargetDescriptorV3 { + const frameId = input.hint?.frameId ?? input.observation.rootFrameId; + const geometry = input.hint?.geometry; + if (geometry) { + const matches = input.observation.index + .candidates(frameId, geometry.tag) + .filter((candidate) => candidate.ref && candidateMatches(candidate, geometry)); + if (matches.length === 1) return descriptor(matches[0]!.ref!); + const semanticMatches = matches.filter( + (candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback), + ); + if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!); + return unmatchedTarget(input.fallback); + } + + const semanticMatches = input.observation.index + .refs(frameId) + .filter((ref) => matchesSemantics(ref, input.fallback)); + return semanticMatches.length === 1 + ? descriptor(semanticMatches[0]!) + : unmatchedTarget(input.fallback); +} diff --git a/apps/extension/src/lib/recording/trace-builder-v3.ts b/apps/extension/src/lib/recording/trace-builder-v3.ts new file mode 100644 index 00000000..5798c222 --- /dev/null +++ b/apps/extension/src/lib/recording/trace-builder-v3.ts @@ -0,0 +1,87 @@ +import { + type StepV3, + type StopReason, + TRACE_VERSION_V3, + type TraceStateV3, + type TraceV3, + VOM_FORMAT_VERSION, +} from "@/transport/types"; +import { resolveDraftStartUrl } from "./draft-policy"; +import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry"; +import { reduceTraceStepsV3 } from "./trace-reducer-v3"; +import { formatTraceStateBody } from "./trace-state-body"; +import type { RecordingDraftStep, StepAnnotation } from "./types"; + +function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] { + const entries = registry.values(); + if (steps.length === 0) return entries.slice(0, 1); + const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state])); + return entries.filter((entry) => referenced.has(entry.id)); +} + +function remapDraftIds(draftIds: number[], stepIdByDraftId: Map): number[] { + return [ + ...new Set( + draftIds.flatMap((id) => { + const stepId = stepIdByDraftId.get(id); + return stepId === undefined ? [] : [stepId]; + }), + ), + ].sort((a, b) => a - b); +} + +export function buildTraceV3(input: { + registry: RecordingStateRegistry; + drafts: RecordingDraftStep[]; + annotations?: StepAnnotation[]; + startedAt: string; + purpose?: string; + startUrl?: string; + stoppedBy: StopReason; + bskVersion: string; +}): TraceV3 { + const reduced = reduceTraceStepsV3(input.drafts); + const entries = publishedEntries(input.registry, reduced.steps); + const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`])); + const annotationsByState = new Map(); + for (const annotation of input.annotations ?? []) { + const bucket = annotationsByState.get(annotation.stateId) ?? []; + bucket.push(annotation); + annotationsByState.set(annotation.stateId, bucket); + } + const steps = reduced.steps.map((step) => ({ + ...step, + state: publishedId.get(step.state) ?? step.state, + result: { state: publishedId.get(step.result.state) ?? step.result.state }, + })); + const states: TraceStateV3[] = entries.map((entry) => { + const id = publishedId.get(entry.id) ?? entry.id; + return { + id, + url: entry.url, + ...(entry.title ? { title: entry.title } : {}), + body: formatTraceStateBody({ + stateId: id, + url: entry.url, + title: entry.title, + stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId), + vomText: entry.rawVomText, + annotations: annotationsByState.get(entry.id) ?? [], + stepIdByDraftId: reduced.stepIdByDraftId, + }), + ...(entry.truncated ? { truncated: true } : {}), + }; + }); + + return { + version: TRACE_VERSION_V3, + ...(input.purpose ? { purpose: input.purpose } : {}), + recorded_at: new Date().toISOString(), + started_at: input.startedAt, + stopped_by: input.stoppedBy, + entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) }, + recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION }, + states, + steps, + }; +} diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/recording/trace-reducer-v2.ts similarity index 59% rename from apps/extension/src/lib/trace-reducer.ts rename to apps/extension/src/lib/recording/trace-reducer-v2.ts index b58b7939..35399389 100644 --- a/apps/extension/src/lib/trace-reducer.ts +++ b/apps/extension/src/lib/recording/trace-reducer-v2.ts @@ -1,30 +1,10 @@ -import type { DraftTraceStep, PageRefV2, SelectedOptionV2, StepV2 } from "@/transport/types"; - -const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); -const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); - -export function shouldRecordPress( - key: string, - modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, -): boolean { - if (MODIFIER_ONLY_KEYS.has(key)) return false; - const mods = modifiers ?? []; - const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); - if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; - if (key === "Enter" || key === "Escape") return true; - // Drop bare character typing — FillSession already records the value. - if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; - return false; -} - -function shouldIncludeDraft(step: DraftTraceStep): boolean { - if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false; - return true; -} +import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types"; +import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy"; +import type { RecordingDraftStep } from "./types"; /** Collapse consecutive navigations to the last hop. */ -function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { - const out: DraftTraceStep[] = []; +function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] { + const out: RecordingDraftStep[] = []; for (const step of steps) { const prev = out[out.length - 1]; if (step.op === "navigate" && prev?.op === "navigate") { @@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { return out; } -function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { +function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] { const urls: string[] = []; if (startUrl) urls.push(startUrl); for (const step of steps) { @@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { urls.push(step.url); continue; } - if ("page_url" in step && step.page_url) urls.push(step.page_url); - if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to); + if (step.pageUrl) urls.push(step.pageUrl); + if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo); } const seen = new Set(); const unique: string[] = []; @@ -58,7 +38,7 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { } function buildPageRegistry( - steps: DraftTraceStep[], + steps: RecordingDraftStep[], startUrl?: string, ): { pages: PageRefV2[]; urlToId: Map } { const urls = collectUrls(steps, startUrl); @@ -81,9 +61,9 @@ function pageIdFor( return urlToId.values().next().value ?? "p1"; } -function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined { - if (step.op === "navigate") return step.page_url ?? step.url; - if ("page_url" in step && step.page_url) return step.page_url; +function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined { + if (step.op === "navigate") return step.pageUrl ?? step.url; + if (step.pageUrl) return step.pageUrl; return fallbackUrl; } @@ -110,7 +90,7 @@ function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] { } function toV2Step( - step: DraftTraceStep, + step: RecordingDraftStep, id: number, urlToId: Map, fallbackUrl?: string, @@ -129,28 +109,31 @@ function toV2Step( to: step.url, }; case "click": + if (!step.captureTarget) return null; return withEffect( { op: "click", id, page, - target: step.target, + target: step.captureTarget, }, - effectForNavigation(step.navigated_to, urlToId), + effectForNavigation(step.navigatedTo, urlToId), ); case "hover": + if (!step.captureTarget) return null; return { op: "hover", id, page, - target: step.target, + target: step.captureTarget, }; case "fill": + if (!step.captureTarget) return null; return { op: "fill", id, page, - target: step.target, + target: step.captureTarget, value: step.value, ...(step.redacted ? { redacted: true } : {}), }; @@ -161,26 +144,29 @@ function toV2Step( id, page, key: step.key, - ...(step.target ? { target: step.target } : {}), + ...(step.captureTarget ? { target: step.captureTarget } : {}), ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}), }, - effectForNavigation(step.navigated_to, urlToId), + effectForNavigation(step.navigatedTo, urlToId), ); case "select": + if (!step.captureTarget) return null; return withEffect( { op: "select", id, page, - target: step.target, + target: step.captureTarget, selection: toSelection(step.values, step.labels), }, - effectForNavigation(step.navigated_to, urlToId), + effectForNavigation(step.navigatedTo, urlToId), ); + case "scroll": + return null; } } -export interface ReducedTrace { +interface ReducedTrace { pages: PageRefV2[]; steps: StepV2[]; } @@ -189,7 +175,7 @@ export interface ReducedTrace { * Compile capture drafts into record-only trace v2 steps. * Variable inputs are NOT classified here — executing agents infer that at run time. */ -export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace { +function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace { const collapsed = collapseNavigations(steps); const { pages, urlToId } = buildPageRegistry(collapsed, startUrl); const out: StepV2[] = []; @@ -197,8 +183,8 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re let lastUrl = startUrl; for (const draft of collapsed) { if (draft.op === "navigate") lastUrl = draft.url; - else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to; - else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url; + else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo; + else if (draft.pageUrl) lastUrl = draft.pageUrl; const step = toV2Step(draft, id, urlToId, lastUrl); if (!step) continue; out.push(step); @@ -207,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re return { pages, steps: out }; } -export function resolveTraceStartUrl( - drafts: DraftTraceStep[], +function resolveTraceStartUrl( + drafts: RecordingDraftStep[], startUrl?: string, pages?: PageRefV2[], ): string { - if (startUrl) return startUrl; - const navigate = drafts.find((step): step is Extract => { - return step.op === "navigate"; - }); - if (navigate) return navigate.url; - for (const draft of drafts) { - if ("page_url" in draft && draft.page_url) return draft.page_url; - } - return pages?.[0]?.url ?? "about:blank"; + return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url); +} + +export function buildTraceV2(input: { + steps: RecordingDraftStep[]; + startedAt: string; + startUrl?: string; + purpose?: string; +}): TraceV2 { + const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl); + return { + recorded_at: new Date().toISOString(), + started_at: input.startedAt, + ...(input.purpose ? { purpose: input.purpose } : {}), + entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) }, + pages, + steps, + }; } diff --git a/apps/extension/src/lib/recording/trace-reducer-v3.ts b/apps/extension/src/lib/recording/trace-reducer-v3.ts new file mode 100644 index 00000000..901429bc --- /dev/null +++ b/apps/extension/src/lib/recording/trace-reducer-v3.ts @@ -0,0 +1,129 @@ +import type { NavigationCause, StepV3 } from "@/transport/types"; +import { shouldIncludeDraft } from "./draft-policy"; +import { unmatchedTarget } from "./target-matcher"; +import type { RecordingDraftStep } from "./types"; + +interface CollapsedDraft { + draft: RecordingDraftStep; + draftIds: number[]; +} + +const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]); +const TRANSITION_CAUSES: Record = { + typed: "user_typed", + generated: "user_typed", + keyword: "user_typed", + keyword_generated: "user_typed", + link: "link", + form_submit: "form_submit", + reload: "reload", + auto_bookmark: "browser", + start_page: "browser", +}; + +function isRedirect(step: Extract): boolean { + return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier)); +} + +function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] { + const output: CollapsedDraft[] = []; + steps.forEach((step, index) => { + const previous = output[output.length - 1]; + if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) { + previous.draft = { + ...previous.draft, + url: step.url, + postStateId: step.postStateId ?? previous.draft.postStateId, + }; + previous.draftIds.push(index + 1); + return; + } + output.push({ draft: { ...step }, draftIds: [index + 1] }); + }); + return output; +} + +function navigationCause(step: Extract): NavigationCause { + if (step.cause) return step.cause; + const qualifiers = step.transitionQualifiers ?? []; + if (qualifiers.includes("forward_back")) return "history"; + if (qualifiers.includes("from_address_bar")) return "user_typed"; + return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser"; +} + +function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> { + return values.map((value, index) => ({ + value, + ...(labels?.[index] ? { label: labels[index] } : {}), + })); +} + +function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null { + if (!shouldIncludeDraft(draft)) return null; + const state = draft.preStateId ?? draft.postStateId; + const resultState = draft.postStateId ?? draft.preStateId; + if (!state || !resultState) return null; + const common = { id, state, result: { state: resultState } }; + + switch (draft.op) { + case "navigate": + return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) }; + case "click": + return { + op: "click", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + }; + case "hover": + return { + op: "hover", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + }; + case "fill": + return { + op: "fill", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + value: draft.value, + commit: draft.commit ?? "blur", + ...(draft.redacted ? { redacted: true } : {}), + }; + case "press": + return { + op: "press", + ...common, + key: draft.key, + ...(draft.captureTarget || draft.matchedTarget + ? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) } + : {}), + ...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}), + }; + case "select": + return { + op: "select", + ...common, + target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget), + selection: selection(draft.values, draft.labels), + }; + case "scroll": + return { op: "scroll", ...common }; + } +} + +export interface ReducedTraceV3 { + steps: StepV3[]; + stepIdByDraftId: Map; +} + +export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 { + const output: StepV3[] = []; + const stepIdByDraftId = new Map(); + for (const { draft, draftIds } of collapseRedirects(steps)) { + const step = reduceDraft(draft, output.length + 1); + if (!step) continue; + output.push(step); + for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id); + } + return { steps: output, stepIdByDraftId }; +} diff --git a/apps/extension/src/lib/recording/trace-state-body.ts b/apps/extension/src/lib/recording/trace-state-body.ts new file mode 100644 index 00000000..fc87349b --- /dev/null +++ b/apps/extension/src/lib/recording/trace-state-body.ts @@ -0,0 +1,39 @@ +import type { StepAnnotation } from "./types"; + +function annotationText(annotation: StepAnnotation, stepId: number): string { + return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`; +} + +export function formatTraceStateBody(input: { + stateId: string; + url: string; + title?: string; + stepIds: number[]; + vomText: string; + annotations: StepAnnotation[]; + stepIdByDraftId: Map; +}): string { + const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`]; + lines.push(`url: ${JSON.stringify(input.url)}`); + if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`); + if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`); + lines.push("---"); + + const byLine = new Map>(); + for (const annotation of input.annotations) { + const stepId = input.stepIdByDraftId.get(annotation.draftId); + if (stepId === undefined) continue; + const bucket = byLine.get(annotation.line) ?? []; + bucket.push({ annotation, stepId }); + byLine.set(annotation.line, bucket); + } + + input.vomText.split("\n").forEach((bodyLine, lineIndex) => { + let line = bodyLine; + for (const item of byLine.get(lineIndex) ?? []) { + line += annotationText(item.annotation, item.stepId); + } + lines.push(line); + }); + return `${lines.join("\n")}\n`; +} diff --git a/apps/extension/src/lib/recording/types.ts b/apps/extension/src/lib/recording/types.ts new file mode 100644 index 00000000..96df3fee --- /dev/null +++ b/apps/extension/src/lib/recording/types.ts @@ -0,0 +1,83 @@ +import type { CaptureTargetDescriptor } from "@/lib/describe-target"; +import type { + FillCommit, + KeyModifier, + NavigationCause, + StepV3, + TargetDescriptorV3, +} from "@/transport/types"; + +export interface TargetGeometry { + /** Top-level viewport-relative CSS pixels, as defined by the geometry module. */ + rect: { x: number; y: number; w: number; h: number }; + tag: string; +} + +export interface TargetMatchHint { + geometry?: TargetGeometry; + /** Missing means the current top frame, never an unrestricted frame search. */ + frameId?: string; +} + +export interface StepAnnotation { + draftId: number; + op: StepV3["op"]; + line: number; + stateId: string; + detail?: string; +} + +interface DraftStateLink { + pageUrl?: string; + preStateId?: string; + postStateId?: string; +} + +interface DraftTarget { + captureTarget?: CaptureTargetDescriptor; + targetHint?: TargetMatchHint; + matchedTarget?: TargetDescriptorV3; +} + +interface DraftNavigationEffect { + navigatedTo?: string; +} + +export type RecordingDraftStep = + | ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect) + | ({ op: "hover" } & DraftStateLink & DraftTarget) + | ({ + op: "fill"; + value: string; + commit?: FillCommit; + redacted?: boolean; + } & DraftStateLink & + DraftTarget & + DraftNavigationEffect) + | ({ + op: "press"; + key: string; + modifiers?: KeyModifier[]; + } & DraftStateLink & + DraftTarget & + DraftNavigationEffect) + | ({ + op: "select"; + values: string[]; + labels?: string[]; + } & DraftStateLink & + DraftTarget & + DraftNavigationEffect) + | ({ op: "scroll" } & DraftStateLink) + | ({ + op: "navigate"; + url: string; + cause?: NavigationCause; + transitionType?: string; + transitionQualifiers?: string[]; + } & DraftStateLink); + +export type TargetedRecordingDraft = Extract< + RecordingDraftStep, + { op: "click" | "hover" | "fill" | "press" | "select" } +>; diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index be869ef7..294c083a 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -17,11 +17,11 @@ import { type RecordStartMessage, type RecordStopMessage, } from "@/lib/record-bridge"; -import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer"; -import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer"; +import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer"; +import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2"; +import type { RecordingDraftStep } from "@/lib/recording/types"; import type { SessionManager } from "@/session-manager/manager"; import type { - DraftTraceStep, RecordAwaitParams, RecordAwaitResult, RecordStartParams, @@ -47,7 +47,7 @@ interface ActiveRecording { agentWindowId: number; startUrl?: string; purpose?: string; - steps: DraftTraceStep[]; + steps: RecordingDraftStep[]; startedAt: string; startedAtMs: number; finishPromise: Promise; @@ -155,16 +155,12 @@ async function sendRecordStartWithAck( } function buildTrace(recording: ActiveRecording): TraceV2 { - const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl); - const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages); - return { + return buildTraceV2({ + steps: recording.steps, + startedAt: recording.startedAt, + ...(recording.startUrl ? { startUrl: recording.startUrl } : {}), ...(recording.purpose ? { purpose: recording.purpose } : {}), - recorded_at: new Date().toISOString(), - started_at: recording.startedAt, - entry: { start_url: startUrl }, - pages, - steps, - }; + }); } export interface RecordDeps { diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index c72a44c1..8db006f8 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -744,48 +744,6 @@ export interface StepCommonV2 { effect?: StepEffectV2; } -/** Capture/buffer draft before v2 reduction. */ -export type DraftTraceStep = - | { - op: "click"; - target: TargetDescriptorV2; - navigated_to?: string; - page_url?: string; - } - | { - op: "hover"; - target: TargetDescriptorV2; - page_url?: string; - } - | { - op: "fill"; - target: TargetDescriptorV2; - value: string; - redacted?: boolean; - page_url?: string; - } - | { - op: "press"; - key: string; - target?: TargetDescriptorV2; - modifiers?: KeyModifier[]; - navigated_to?: string; - page_url?: string; - } - | { - op: "select"; - target: TargetDescriptorV2; - values: string[]; - labels?: string[]; - navigated_to?: string; - page_url?: string; - } - | { - op: "navigate"; - url: string; - page_url?: string; - }; - /** Exported record-only step (trace v2). */ export type StepV2 = | ({ op: "navigate" } & StepCommonV2 & { to: string })