diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index b78877ad..7a8d83c3 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { + handleBlur, handleClick, handleFill, + handleFocus, handleHover, handlePress, handleSelect, @@ -540,6 +542,156 @@ describe("handleHover", () => { }); }); +describe("handleFocus and handleBlur", () => { + it("focuses a ref and verifies the deep active element", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + "DOM.focus": () => ({}), + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ result: { value: { focused: true } } }), + }); + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.scrollIntoViewIfNeeded", + "DOM.focus", + "DOM.resolveNode", + "Runtime.callFunctionOn", + ]); + }); + + it("focuses an OOPIF ref in its CDP session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { + tabId: 4, + frameId: "child-frame", + cdpSessionId: "child-session", + }); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => ({}), + }); + fake.cdp.getFrameGraph = vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child-frame", + parentFrameId: "main", + ownerBackendNodeId: 99, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })); + const targetCalls: Array<{ sessionId?: string; method: string }> = []; + fake.cdp.sendToTarget = vi.fn(async (target, method) => { + targetCalls.push({ sessionId: target.sessionId, method }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.focus") return {}; + if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } }; + if (method === "Runtime.callFunctionOn") return { result: { value: { focused: true } } }; + throw new Error(`unexpected child CDP call ${method}`); + }) as CdpRunner["sendToTarget"]; + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.focused).toBe(true); + expect(targetCalls).toEqual([ + { sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" }, + { sessionId: "child-session", method: "DOM.focus" }, + { sessionId: "child-session", method: "DOM.resolveNode" }, + { sessionId: "child-session", method: "Runtime.callFunctionOn" }, + ]); + }); + + it("does not focus after cancellation during scrolling", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const abort = new AbortController(); + const fake = makeFakeCdp({ + "DOM.scrollIntoViewIfNeeded": () => { + abort.abort(); + return {}; + }, + }); + + const res = await handleFocus( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal }, + ); + + expect(res).toMatchObject({ code: "cancelled" }); + expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false); + }); + + it("blurs a ref and returns its previous focus state", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ + result: { value: { ok: true, was_focused: true, focused: false } }, + }), + }); + + const res = await handleBlur( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ + tab_id: 4, + used_ref: "e3", + was_focused: true, + focused: false, + }); + expect(fake.sent.map((call) => call.method)).toEqual([ + "DOM.resolveNode", + "Runtime.callFunctionOn", + ]); + }); + + it("rejects a target that does not implement blur", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e3", 1234, { tabId: 4 }); + const fake = makeFakeCdp({ + "DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }), + "Runtime.callFunctionOn": () => ({ + result: { value: { ok: false, was_focused: false, focused: false } }, + }), + }); + + const res = await handleBlur( + sm, + { session_id: "aa11", ref: "@e3" }, + { cdp: fake.cdp, tabsApi: fake.tabsApi }, + ); + + expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ }); + }); +}); + describe("handleFill", () => { it("returns not_found for unknown ref", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 25b5f3ac..ea87f4e8 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -2,11 +2,13 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { Transport } from "@/transport/transport"; import type { + BlurParams, ClickParams, ConsoleParams, EmulateParams, EvaluateParams, FillParams, + FocusParams, GetHtmlParams, HoverParams, HoverResult, @@ -35,7 +37,15 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; +import { + handleBlur, + handleClick, + handleFill, + handleFocus, + handleHover, + handlePress, + handleSelect, +} from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -471,6 +481,28 @@ export class ToolDispatcher { ); return this.rememberHover((req.params as HoverParams).session_id, result); } + case "tool.focus": + return this.withHoverReleaseForRequest( + req.params as FocusParams, + () => + handleFocus( + this.sessions, + req.params as FocusParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); + case "tool.blur": + return this.withHoverReleaseForRequest( + req.params as BlurParams, + () => + handleBlur( + this.sessions, + req.params as BlurParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ), + signal, + ); case "tool.fill": return this.withHoverReleaseForRequest( req.params as FillParams, @@ -718,6 +750,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.reload": case "tool.click": case "tool.hover": + case "tool.focus": + case "tool.blur": case "tool.fill": case "tool.press": case "tool.select": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabdf..83068990 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -1,5 +1,4 @@ -// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and -// `tool.select`. +// DOM interaction tools — click, hover, focus/blur, fill, press, and select. // // All interaction tools: // 1. Resolve target tab (sandbox: must be inside Agent Window). @@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { + BlurParams, + BlurResult, ClickParams, ClickResult, FillParams, FillResult, + FocusParams, + FocusResult, HoverParams, HoverResult, KeyModifier, @@ -215,6 +218,157 @@ async function resolveBackendNode( } } +type FocusCheck = { focused: boolean }; +type BlurMutation = + | { ok: true; was_focused: boolean; focused: boolean } + | { ok: false; was_focused: boolean; focused: boolean }; + +const DEEP_FOCUS_CHECK = `function() { + const deepActiveElement = (root) => { + let active = root && root.activeElement; + while (active && active.shadowRoot && active.shadowRoot.activeElement) { + active = active.shadowRoot.activeElement; + } + return active; + }; + return { focused: deepActiveElement(this.ownerDocument) === this }; +}`; + +const BLUR_TARGET = `function() { + const deepActiveElement = (root) => { + let active = root && root.activeElement; + while (active && active.shadowRoot && active.shadowRoot.activeElement) { + active = active.shadowRoot.activeElement; + } + return active; + }; + const wasFocused = deepActiveElement(this.ownerDocument) === this; + if (typeof this.blur !== 'function') { + return { ok: false, was_focused: wasFocused, focused: wasFocused }; + } + this.blur(); + return { + ok: true, + was_focused: wasFocused, + focused: deepActiveElement(this.ownerDocument) === this, + }; +}`; + +// --------------------------------------------------------------------------- +// tool.focus / tool.blur +// --------------------------------------------------------------------------- + +export async function handleFocus( + manager: SessionManager, + params: FocusParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "focus"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "focus"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "focus"); + if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollElementAndFramesIntoView( + deps.cdp, + target.tabId, + node.cdpTarget, + node.backendNodeId, + node.frameId, + ); + if (scrollErr) return scrollErr; + const abortedBeforeFocus = throwIfAborted(deps.signal); + if (abortedBeforeFocus) return abortedBeforeFocus; + + try { + await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId }); + const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); + if (isRpcError(objectIdOrErr)) return objectIdOrErr; + const evaluated = await nodeCdp.send<{ result?: { value?: FocusCheck } }>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId: objectIdOrErr, + functionDeclaration: DEEP_FOCUS_CHECK, + returnByValue: true, + }, + ); + if (evaluated.result?.value?.focused !== true) { + return { code: "invalid_params", message: "target element did not become focused" }; + } + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + focused: true, + }); + } catch (err) { + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } +} + +export async function handleBlur( + manager: SessionManager, + params: BlurParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "blur"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + const aborted = throwIfAborted(deps.signal); + if (aborted) return aborted; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "blur"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "blur"); + if (isRpcError(node)) return node; + const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget); + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const abortedBeforeBlur = throwIfAborted(deps.signal); + if (abortedBeforeBlur) return abortedBeforeBlur; + + try { + const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId); + if (isRpcError(objectIdOrErr)) return objectIdOrErr; + const evaluated = await nodeCdp.send<{ result?: { value?: BlurMutation } }>( + target.tabId, + "Runtime.callFunctionOn", + { + objectId: objectIdOrErr, + functionDeclaration: BLUR_TARGET, + returnByValue: true, + }, + ); + const mutation = evaluated.result?.value; + if (!mutation?.ok) { + return { code: "invalid_params", message: "target element does not support blur()" }; + } + if (mutation.focused) { + return { code: "cdp_failed", message: "target element remained focused after blur()" }; + } + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: node.usedRef, + used_selector: node.usedSelector, + was_focused: mutation.was_focused, + focused: false, + }); + } catch (err) { + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } +} + // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index d53825a1..5162e22f 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -444,6 +444,39 @@ export interface HoverResult { dialogs?: JavaScriptDialogInfo[]; } +export interface FocusParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface FocusResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + focused: boolean; + dialogs?: JavaScriptDialogInfo[]; +} + +export interface BlurParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; +} + +export interface BlurResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + was_focused: boolean; + focused: boolean; + dialogs?: JavaScriptDialogInfo[]; +} + export interface FillParams { session_id: string; value: string; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index e1cffb92..e6c47924 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -70,6 +70,8 @@ bsk navigate --session bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces bsk snapshot --session → static aria tree fallback when VOM is insufficient bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking +bsk focus @e4 --session → explicitly enter focus-driven UI states +bsk blur @e4 --session → explicitly leave focus-driven UI states bsk click @e4 --session → or bsk fill, bsk select, bsk press bsk observe --session → again after navigation / DOM change ``` @@ -202,6 +204,8 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff |---------|---------| | `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | | `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | +| `bsk focus ` | Focus an element and verify it became the deep active element | +| `bsk blur ` | Remove focus from an element and report whether it was focused | | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option(s) by `value` (repeat `--value` for multi-select) | | `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |