diff --git a/apps/extension/src/tools/__tests__/scroll.test.ts b/apps/extension/src/tools/__tests__/scroll.test.ts new file mode 100644 index 00000000..6ed86e20 --- /dev/null +++ b/apps/extension/src/tools/__tests__/scroll.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it, vi } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import type { CdpRunner } from "@/tools/shared"; +import { handleScrollTo } from "../scroll"; + +function fakeAgentWindow(ids: number[]) { + let i = 0; + return { + create: vi.fn(async () => { + const id = ids[i++]; + if (id === undefined) throw new Error("ran out of fake ids"); + return id; + }), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }; +} + +function makeFakeCdp(handlers: Record unknown>) { + const sent: Array<{ tabId: number; method: string; params?: object }> = []; + const sendImpl = async (tabId: number, method: string, params?: object) => { + sent.push({ tabId, method, params }); + const handler = handlers[method]; + if (!handler && method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (!handler) throw new Error(`unexpected CDP call ${method}`); + return handler(params); + }; + const cdp: CdpRunner = { + send: vi.fn(sendImpl) as unknown as CdpRunner["send"], + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + return { cdp, tabsApi, sent }; +} + +describe("handleScrollTo", () => { + it("scrolls a ref into view and returns its visible top-viewport bounds", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + }); + + const result = await handleScrollTo( + manager, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result).toMatchObject({ + tab_id: 4, + used_ref: "e3", + x: 10, + y: 20, + width: 100, + height: 40, + }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.scrollIntoViewIfNeeded", + "DOM.getContentQuads", + "Page.getLayoutMetrics", + ]); + }); + + it("scrolls OOPIF refs through their parent frame and CDP session", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getBoxModel": () => ({ + model: { content: [204, 306, 604, 306, 604, 506, 204, 506] }, + }), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const targetCalls: Array<{ sessionId?: string; method: string }> = []; + fake.cdp.sendToTarget = vi.fn(async (target, method) => { + targetCalls.push({ sessionId: target.sessionId, method }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }; + } + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const result = await handleScrollTo( + manager, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result).toMatchObject({ x: 224, y: 346, width: 200, height: 80 }); + expect(targetCalls).toEqual([ + { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, + { sessionId: "child-session", method: "DOM.getContentQuads" }, + { sessionId: "child-session", method: "Page.getLayoutMetrics" }, + ]); + }); + + it("does not issue CDP calls after an early cancellation", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await manager.start("aa11"); + const abort = new AbortController(); + abort.abort(); + const fake = makeFakeCdp({}); + + const result = await handleScrollTo( + manager, + { session_id: "aa11", selector: "#target" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + + expect(result).toMatchObject({ code: "cancelled" }); + expect(fake.cdp.send).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 25b5f3ac..8afdd0f0 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -26,6 +26,7 @@ import type { ResponseFrame, RpcError, ScreenshotParams, + ScrollToParams, SelectParams, SnapshotParams, WaitForNavigationParams, @@ -52,6 +53,7 @@ import { handleSnapshot, } from "./observation"; import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record"; +import { handleScrollTo } from "./scroll"; import { handleSessionStart, handleSessionStop, @@ -471,6 +473,17 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.scroll_to": + return this.withHoverReleaseForRequest( + req.params as ScrollToParams, + () => + handleScrollTo( + this.sessions, + req.params as ScrollToParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.fill": return this.withHoverReleaseForRequest( req.params as FillParams, @@ -718,6 +731,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.scroll_to": case "tool.fill": case "tool.press": case "tool.select": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabdf..3ea1373a 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise { * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ -async function resolveBackendNode( +export async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, diff --git a/apps/extension/src/tools/scroll.ts b/apps/extension/src/tools/scroll.ts new file mode 100644 index 00000000..523f758d --- /dev/null +++ b/apps/extension/src/tools/scroll.ts @@ -0,0 +1,69 @@ +import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import type { SessionManager } from "@/session-manager/manager"; +import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { resolveNodeGeometry } from "./frame-geometry"; +import { resolveBackendNode } from "./interaction"; +import { + type CdpRunner, + type ChromeTabsApi, + chromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, +} from "./shared"; + +export interface ScrollToDeps { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + signal?: AbortSignal; +} + +let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; +function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { + if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; + return defaultDeps; +} + +export async function handleScrollTo( + manager: SessionManager, + params: ScrollToParams, + deps: ScrollToDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "scroll-to"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "scroll-to"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "scroll-to"); + if (isRpcError(node)) return node; + if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" }; + + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const geometry = await resolveNodeGeometry( + deps.cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: true }, + ); + if (isRpcError(geometry)) return geometry; + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + x: geometry.topBounds.x, + y: geometry.topBounds.y, + width: geometry.topBounds.width, + height: geometry.topBounds.height, + }); +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index d53825a1..73384ccc 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -444,6 +444,25 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface ScrollToParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface ScrollToResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + width: number; + height: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e1cffb92..ee68f3f8 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -202,6 +202,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | | `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | +| `bsk scroll-to ` | Scroll an element and its frame owners into the visible viewport | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |