Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions packages/cli/src/lib/driver/commands/mouse.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { z } from "zod";

import { updateCursorOverlayPosition } from "../cursor-overlay.js";
import type { DriverPage, DriverSessionManager } from "../session-manager.js";
import type { DriverCommandHandlers } from "./types.js";

const ButtonSchema = z.enum(["left", "right", "middle"]).optional();
Expand All @@ -17,6 +19,7 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, x, y);
await page.click(x, y, {
...(button === undefined ? {} : { button }),
...(clickCount === undefined ? {} : { clickCount }),
Expand All @@ -34,6 +37,7 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, x, y);
await page.hover(x, y);
return { hovered: true };
},
Expand All @@ -50,6 +54,7 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, x, y);
await page.scroll(x, y, deltaX, deltaY);
return { scrolled: true };
},
Expand All @@ -69,15 +74,33 @@ export const mouseHandlers: DriverCommandHandlers = {
.parse(params);
assertXPathUnavailable(returnXPath);
const page = await manager.activePage();
await positionCursorOverlay(manager, page, fromX, fromY);
await page.dragAndDrop(fromX, fromY, toX, toY, {
...(button === undefined ? {} : { button }),
...(delay === undefined ? {} : { delay }),
...(steps === undefined ? {} : { steps }),
});
await positionCursorOverlay(manager, page, toX, toY);
return { dragged: true };
},
};

async function positionCursorOverlay(
manager: DriverSessionManager,
page: DriverPage,
x: number,
y: number,
): Promise<void> {
if (!manager.isCursorOverlayEnabled(page)) return;
// The overlay is visual-only. A navigation can destroy its execution
// context, but that must not prevent or invalidate the real mouse action.
try {
await page.evaluate(updateCursorOverlayPosition, { x, y });
} catch {
// Best-effort parity with V3's cursor updates.
}
}

function assertXPathUnavailable(returnXPath: boolean | undefined): void {
if (returnXPath) {
throw new Error("Coordinate XPath lookup is not exposed by Stagehand V4");
Expand Down
10 changes: 8 additions & 2 deletions packages/cli/src/lib/driver/commands/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import { promises as fs } from "node:fs";

import { z } from "zod";

import { CURSOR_OVERLAY_SCRIPT } from "../cursor-overlay.js";
import type { DriverCommandHandlers } from "./types.js";
import { unavailableCursorOverlay } from "./unavailable.js";

export const runtimeHandlers: DriverCommandHandlers = {
async screenshot(manager, params) {
Expand Down Expand Up @@ -89,7 +89,13 @@ export const runtimeHandlers: DriverCommandHandlers = {
return { waited: true };
},

cursor: unavailableCursorOverlay,
async cursor(manager) {
const page = await manager.activePage();
await page.addInitScript(CURSOR_OVERLAY_SCRIPT);
Comment thread
shrey150 marked this conversation as resolved.
await page.evaluate(CURSOR_OVERLAY_SCRIPT);
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
manager.markCursorOverlayEnabled(page);
return { cursor: "enabled" };
},
};

function parseTimeoutMs(value: string | undefined): number {
Expand Down
9 changes: 0 additions & 9 deletions packages/cli/src/lib/driver/commands/unavailable.ts

This file was deleted.

80 changes: 80 additions & 0 deletions packages/cli/src/lib/driver/cursor-overlay.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
export const CURSOR_OVERLAY_SCRIPT = `(() => {
if (globalThis !== globalThis.top) return;

const cursorId = "__browse_cursor_overlay__";
const ensureCursor = () => {
const existing = document.getElementById(cursorId);
if (existing instanceof HTMLDivElement) return existing;

const root = document.documentElement || document.body;
if (!root) return null;

const cursor = document.createElement("div");
cursor.id = cursorId;
cursor.setAttribute("aria-hidden", "true");
Object.assign(cursor.style, {
contain: "layout style paint",
height: "24px",
left: "0px",
mixBlendMode: "normal",
pointerEvents: "none",
position: "fixed",
top: "0px",
userSelect: "none",
width: "16px",
willChange: "left,top",
zIndex: "2147483647",
});
cursor.innerHTML =
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="24" viewBox="0 0 16 24"><path d="M1 0 L1 22 L6 14 L15 14 Z" fill="black" stroke="white" stroke-width="0.7"/></svg>';
root.appendChild(cursor);
return cursor;
};

const moveCursor = (x, y) => {
const cursor = ensureCursor();
if (!cursor) return;
cursor.style.left = Math.max(0, x) + "px";
cursor.style.top = Math.max(0, y) + "px";
};

const installCursor = () => {
if (ensureCursor()) return;
if (globalThis.__browseCursorOverlayDomReadyListenerInstalled__) return;

document.addEventListener(
"DOMContentLoaded",
() => {
globalThis.__browseCursorOverlayDomReadyListenerInstalled__ = false;
ensureCursor();
},
{ once: true },
);
globalThis.__browseCursorOverlayDomReadyListenerInstalled__ = true;
};

globalThis.__browseMoveCursorOverlay__ = moveCursor;
installCursor();
if (!globalThis.__browseCursorOverlayListenerInstalled__) {
document.addEventListener(
"mousemove",
Comment thread
shrey150 marked this conversation as resolved.
(event) => {
moveCursor(event.clientX, event.clientY);
},
{ capture: true },
);
globalThis.__browseCursorOverlayListenerInstalled__ = true;
}
})()`;

export function updateCursorOverlayPosition(position: {
x: number;
y: number;
}): void {
const moveCursor = (
globalThis as typeof globalThis & {
__browseMoveCursorOverlay__?: (x: number, y: number) => void;
}
).__browseMoveCursorOverlay__;
moveCursor?.(position.x, position.y);
}
10 changes: 10 additions & 0 deletions packages/cli/src/lib/driver/session-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ export class DriverSessionManager {
private browserbaseIdentityValue: BrowserbaseIdentity = {};
private consecutiveInitFailures = 0;
private context: DriverContext | null = null;
private cursorOverlayPageIds = new Set<string>();
private lastForwardedEnvSignature: string | null = null;
private pendingEnv: ForwardedEnv | undefined;
private initFailure: InitFailure | null = null;
Expand Down Expand Up @@ -207,6 +208,7 @@ export class DriverSessionManager {
this.stagehand = null;
this.browser = null;
this.context = null;
this.cursorOverlayPageIds.clear();
this.browserbaseIdentityValue = {};
this.initFailure = null;
this.consecutiveInitFailures = 0;
Expand All @@ -223,6 +225,14 @@ export class DriverSessionManager {
return resolveCachedSelector(selector, this.refMaps);
}

markCursorOverlayEnabled(page: DriverPage): void {
this.cursorOverlayPageIds.add(page.pageId);
}

isCursorOverlayEnabled(page: DriverPage): boolean {
return this.cursorOverlayPageIds.has(page.pageId);
}

setRefMaps(refMaps: RefMaps): void {
this.refMaps = refMaps;
}
Expand Down
123 changes: 123 additions & 0 deletions packages/cli/tests/cursor-overlay.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import { createContext, runInContext } from "node:vm";

import { describe, expect, it, vi } from "vitest";

import { CURSOR_OVERLAY_SCRIPT } from "../src/lib/driver/cursor-overlay.js";

describe("cursor overlay", () => {
it("installs after DOMContentLoaded when the document root is not ready", () => {
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
const harness = createCursorHarness({ ready: false });

harness.install();

expect(harness.elements.size).toBe(0);
expect(harness.listeners.has("DOMContentLoaded")).toBe(true);

harness.makeDocumentReady();
harness.listeners.get("DOMContentLoaded")!();

expect(harness.cursor()).toBeInstanceOf(FakeDiv);
});

it("installs a click-through cursor once in an already-ready document", () => {
const harness = createCursorHarness();

harness.install();

expect(harness.cursor()?.style).toMatchObject({
left: "0px",
pointerEvents: "none",
position: "fixed",
top: "0px",
zIndex: "2147483647",
});
expect(harness.listeners.has("DOMContentLoaded")).toBe(false);
expect(harness.listeners.has("mousemove")).toBe(true);

harness.install();

expect(harness.document.createElement).toHaveBeenCalledOnce();
expect(harness.document.addEventListener).toHaveBeenCalledOnce();
expect(harness.elements.size).toBe(1);
});

it("moves and clamps the cursor from top-document mouse events", () => {
const harness = createCursorHarness();
harness.install();

const mousemove = harness.listeners.get("mousemove")!;
mousemove({ clientX: -25, clientY: 80 });
expect(harness.cursor()?.style).toMatchObject({
left: "0px",
top: "80px",
});

mousemove({ clientX: 140, clientY: -10 });
expect(harness.cursor()?.style).toMatchObject({
left: "140px",
top: "0px",
});
});

it("does not install inside a child frame", () => {
const harness = createCursorHarness({ topFrame: false });

harness.install();

expect(harness.elements.size).toBe(0);
expect(harness.document.createElement).not.toHaveBeenCalled();
expect(harness.document.addEventListener).not.toHaveBeenCalled();
});
});

class FakeDiv {
id = "";
innerHTML = "";
style: Record<string, string> = {};

setAttribute(): void {}
}

type CursorEvent = { clientX: number; clientY: number };
type CursorListener = (event?: CursorEvent) => void;

function createCursorHarness(
options: { ready?: boolean; topFrame?: boolean } = {},
) {
const elements = new Map<string, FakeDiv>();
const listeners = new Map<string, CursorListener>();
const root = {
appendChild(element: FakeDiv) {
elements.set(element.id, element);
},
};
let documentElement: typeof root | null =
options.ready === false ? null : root;
const document = {
addEventListener: vi.fn((name: string, listener: CursorListener) => {
listeners.set(name, listener);
}),
body: null,
createElement: vi.fn(() => new FakeDiv()),
get documentElement() {
return documentElement;
},
getElementById: vi.fn((id: string) => elements.get(id) ?? null),
};
const context = createContext({ document, HTMLDivElement: FakeDiv });
runInContext(
`globalThis.top = ${options.topFrame === false ? "{}" : "globalThis"}`,
context,
);

return {
cursor: () => elements.get("__browse_cursor_overlay__"),
document,
elements,
install: () => runInContext(CURSOR_OVERLAY_SCRIPT, context),
listeners,
makeDocumentReady: () => {
documentElement = root;
},
};
}
Loading
Loading