diff --git a/apps/extension/src/tools/__tests__/wheel.test.ts b/apps/extension/src/tools/__tests__/wheel.test.ts new file mode 100644 index 00000000..03372a22 --- /dev/null +++ b/apps/extension/src/tools/__tests__/wheel.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it, vi } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import type { CdpRunner } from "@/tools/shared"; +import { handleWheel } from "../wheel"; + +function fakeAgentWindow(ids: number[]) { + let index = 0; + return { + create: vi.fn(async () => { + const id = ids[index++]; + if (id === undefined) throw new Error("ran out of fake ids"); + return id; + }), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }; +} + +function makeFakeCdp(handlers: Record unknown> = {}) { + const sent: Array<{ tabId: number; method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (tabId: number, method: string, params?: object) => { + sent.push({ tabId, method, params }); + const handler = handlers[method]; + if (handler) return handler(params); + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + } + if (method === "Input.dispatchMouseEvent") return {}; + throw new Error(`unexpected CDP call ${method}`); + }) as unknown as CdpRunner["send"], + trackSessionTab: vi.fn(), + }; + const tabsApi = { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; + return { cdp, tabsApi, sent }; +} + +describe("handleWheel", () => { + it("dispatches a native mouseWheel event at the viewport centre", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await manager.start("aa11"); + const fake = makeFakeCdp(); + const bypassOverlay = vi.fn(async () => {}); + + const result = await handleWheel( + manager, + { + session_id: "aa11", + delta_y: 600, + delta_x: -20, + modifiers: ["ctrl", "shift"], + }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, bypassOverlay }, + ); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result).toMatchObject({ + tab_id: 4, + x: 500, + y: 400, + delta_x: -20, + delta_y: 600, + }); + expect(fake.sent).toEqual([ + { tabId: 4, method: "Page.getLayoutMetrics", params: {} }, + { + tabId: 4, + method: "Input.dispatchMouseEvent", + params: { type: "mouseMoved", x: 500, y: 400, modifiers: 10 }, + }, + { + tabId: 4, + method: "Input.dispatchMouseEvent", + params: { + type: "mouseWheel", + x: 500, + y: 400, + deltaX: -20, + deltaY: 600, + modifiers: 10, + }, + }, + ]); + expect(bypassOverlay.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + }); + + it("uses an element's visible action point for targeted wheel input", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await manager.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), + }); + + const result = await handleWheel( + manager, + { session_id: "aa11", ref: "@e3", delta_y: 120 }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result).toMatchObject({ used_ref: "e3", x: 60, y: 40, delta_y: 120 }); + expect(fake.sent.at(-1)).toMatchObject({ + method: "Input.dispatchMouseEvent", + params: { type: "mouseWheel", x: 60, y: 40, deltaY: 120 }, + }); + }); + + it("rejects a zero-distance wheel without issuing CDP calls", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await manager.start("aa11"); + const fake = makeFakeCdp(); + + const result = await handleWheel( + manager, + { session_id: "aa11", delta_x: 0, delta_y: 0 }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(result).toMatchObject({ code: "invalid_params" }); + expect(fake.cdp.send).not.toHaveBeenCalled(); + }); + + it("does not issue CDP calls after an early cancellation", async () => { + const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await manager.start("aa11"); + const fake = makeFakeCdp(); + const abort = new AbortController(); + abort.abort(); + + const result = await handleWheel( + manager, + { session_id: "aa11", delta_y: 120 }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + + expect(result).toMatchObject({ code: "cancelled" }); + expect(fake.cdp.send).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 25b5f3ac..9183134f 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -29,6 +29,7 @@ import type { SelectParams, SnapshotParams, WaitForNavigationParams, + WheelParams, } from "@/transport/types"; import { isRequestFrame } from "@/transport/types"; import { handleConsole } from "./console"; @@ -75,6 +76,7 @@ import { type TabSelectParams, } from "./tabs"; import { handleWaitForNavigation } from "./waits"; +import { handleWheel } from "./wheel"; import { handleWindowResize, type WindowResizeParams } from "./window"; type DispatcherCdpRunner = CdpRunner & @@ -471,6 +473,20 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.wheel": + return this.withHoverReassert( + req.params as WheelParams, + () => + handleWheel( + this.sessions, + req.params as WheelParams, + this.cdp + ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal, bypassOverlay } + : undefined, + ), + { releaseAfter: true }, + signal, + ); case "tool.fill": return this.withHoverReleaseForRequest( req.params as FillParams, @@ -718,6 +734,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.wheel": case "tool.fill": case "tool.press": case "tool.select": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabdf..3ea1373a 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise { * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ -async function resolveBackendNode( +export async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, diff --git a/apps/extension/src/tools/wheel.ts b/apps/extension/src/tools/wheel.ts new file mode 100644 index 00000000..7466e6a3 --- /dev/null +++ b/apps/extension/src/tools/wheel.ts @@ -0,0 +1,180 @@ +import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; +import type { SessionContext, SessionManager } from "@/session-manager/manager"; +import type { RpcError, WheelParams, WheelResult } from "@/transport/types"; +import { attachDialogs, markDialogCursor } from "./dialogs"; +import { resolveNodeGeometry } from "./frame-geometry"; +import { modifiersBitfield, resolveBackendNode } from "./interaction"; +import { + type CdpRunner, + type ChromeTabsApi, + chromeTabsApi, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, +} from "./shared"; + +export interface WheelDeps { + cdp: CdpRunner; + tabsApi: ChromeTabsApi; + signal?: AbortSignal; + /** Temporarily disable the Agent Window overlay's input blocker. */ + bypassOverlay?: (tabId: number, enabled: boolean) => Promise; +} + +interface WheelPoint { + x: number; + y: number; + usedRef?: string; + usedSelector?: string; +} + +let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null; +function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } { + if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi }; + return defaultDeps; +} + +export async function handleWheel( + manager: SessionManager, + params: WheelParams, + deps: WheelDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "wheel"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const deltaX = params.delta_x ?? 0; + const deltaY = params.delta_y; + if (!Number.isFinite(deltaX) || !Number.isFinite(deltaY)) { + return { code: "invalid_params", message: "wheel deltas must be finite numbers" }; + } + if (deltaX === 0 && deltaY === 0) { + return { code: "invalid_params", message: "at least one wheel delta must be non-zero" }; + } + if (deps.signal?.aborted) return cancelled(); + + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "wheel"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + + const point = await resolveWheelPoint(deps.cdp, ctx, target, params); + if (isRpcError(point)) return point; + if (deps.signal?.aborted) return cancelled(); + + let bypassEnabled = false; + if (deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, true); + bypassEnabled = true; + } catch (error) { + console.debug("[bsk wheel] overlay bypass enable failed", error); + } + } + + try { + if (deps.signal?.aborted) return cancelled(); + const modifiers = modifiersBitfield(params.modifiers); + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + modifiers, + }); + if (deps.signal?.aborted) return cancelled(); + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseWheel", + x: point.x, + y: point.y, + deltaX, + deltaY, + modifiers, + }); + } catch (error) { + return { + code: "cdp_failed", + message: error instanceof Error ? error.message : String(error), + }; + } finally { + if (bypassEnabled && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (error) { + console.debug("[bsk wheel] overlay bypass disable failed", error); + } + } + } + + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: point.usedRef, + used_selector: point.usedSelector, + x: point.x, + y: point.y, + delta_x: deltaX, + delta_y: deltaY, + }); +} + +async function resolveWheelPoint( + cdp: CdpRunner, + ctx: SessionContext, + target: { tabId: number }, + params: WheelParams, +): Promise { + const hasRef = typeof params.ref === "string" && params.ref.length > 0; + const hasSelector = typeof params.selector === "string" && params.selector.length > 0; + if (hasRef || hasSelector) { + const node = await resolveBackendNode(cdp, ctx, target, params, "wheel"); + if (isRpcError(node)) return node; + const geometry = await resolveNodeGeometry( + cdp, + target.tabId, + { + target: node.cdpTarget, + backendNodeId: node.backendNodeId, + ...(node.frameId ? { frameId: node.frameId } : {}), + }, + { scrollIntoView: true }, + ); + if (isRpcError(geometry)) return geometry; + return { + x: geometry.actionPoint.x, + y: geometry.actionPoint.y, + usedRef: node.usedRef, + usedSelector: node.usedSelector, + }; + } + + try { + const metrics = await cdp.send<{ + cssLayoutViewport?: { clientWidth?: number; clientHeight?: number }; + layoutViewport?: { clientWidth?: number; clientHeight?: number }; + }>(target.tabId, "Page.getLayoutMetrics", {}); + const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; + const width = viewport.clientWidth; + const height = viewport.clientHeight; + if ( + typeof width !== "number" || + typeof height !== "number" || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return { code: "cdp_failed", message: "Page.getLayoutMetrics returned no viewport size" }; + } + return { x: width / 2, y: height / 2 }; + } catch (error) { + return { + code: "cdp_failed", + message: error instanceof Error ? error.message : String(error), + }; + } +} + +function cancelled(): RpcError { + return { code: "cancelled", message: "wheel aborted" }; +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index d53825a1..065a338f 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -444,6 +444,28 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface WheelParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + delta_x?: number; + delta_y: number; + modifiers?: KeyModifier[]; + timeout_ms?: number; +} + +export interface WheelResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + x: number; + y: number; + delta_x: number; + delta_y: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e1cffb92..eea194ee 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -202,6 +202,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | | `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | +| `bsk wheel --delta-y [ref-or-selector]` | Dispatch a real mouse-wheel event at the viewport centre or an element (`--delta-x`, `--modifiers`) | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |