diff --git a/packages/evals/browseCliPaths.ts b/packages/evals/browseCliPaths.ts index fe2ca89e3b..d6e4fbc561 100644 --- a/packages/evals/browseCliPaths.ts +++ b/packages/evals/browseCliPaths.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import { fileURLToPath } from "node:url"; import path from "node:path"; @@ -10,3 +11,8 @@ export const BROWSE_CLI_BUILD_ARTIFACTS = [ ]; export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json"); export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md"); + +export function createBrowseCliSessionName(): string { + const random = randomBytes(8).toString("hex"); + return `eval-${process.pid}-${random}`; +} diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index 4e3d7371f3..61ff7fc43d 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -14,8 +14,13 @@ import type { ToolStartInput, ToolStartResult, } from "../contracts/tool.js"; -import { BROWSE_CLI_BUILD_ARTIFACTS, BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; +import { + BROWSE_CLI_BUILD_ARTIFACTS, + BROWSE_CLI_ENTRYPOINT, + createBrowseCliSessionName, +} from "../../browseCliPaths.js"; import { getRepoRootDir } from "../../runtimePaths.js"; +import { EvalsError } from "../../errors.js"; const execFileAsync = promisify(execFile); @@ -79,11 +84,11 @@ function buildSelectorQuery(selector: string): string { `; } -type BrowseCliPagesResult = { - pages: Array<{ +type BrowseCliTabsResult = { + tabs: Array<{ index: number; url: string; - targetId: string; + targetId?: string; }>; }; @@ -93,7 +98,7 @@ class BrowseCliRuntime { async runJson(args: string[]): Promise { const { stdout, stderr } = await execFileAsync( process.execPath, - [resolveBrowseCliEntrypoint(), "--json", "--session", this.session, ...args], + [resolveBrowseCliEntrypoint(), ...args, "--session", this.session], { cwd: getRepoRootDir(), env: process.env, @@ -208,7 +213,7 @@ class BrowseCliPageHandle implements CorePageHandle { args.push("--wait", opts.waitUntil); } if (typeof opts?.timeoutMs === "number") { - args.push("-t", String(opts.timeoutMs)); + args.push("--timeout", String(opts.timeoutMs)); } const result = await this.runCommandAfterSelecting<{ url: string }>(args); this.cachedUrl = result.url; @@ -280,15 +285,15 @@ class BrowseCliPageHandle implements CorePageHandle { type?: "png" | "jpeg"; quality?: number; }): Promise { - const args = ["screenshot"]; + const args = ["screenshot", "--base64"]; if (opts?.fullPage) { - args.push("-f"); + args.push("--full-page"); } if (opts?.type) { - args.push("-t", opts.type); + args.push("--type", opts.type); } if (typeof opts?.quality === "number") { - args.push("-q", String(opts.quality)); + args.push("--quality", String(opts.quality)); } const result = await this.runCommandAfterSelecting<{ base64: string }>(args); @@ -310,9 +315,9 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "selector", spec.selector, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), - "-s", + "--state", spec.state ?? "visible", ]); return; @@ -324,7 +329,7 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "load", spec.state, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), ]); return; @@ -372,7 +377,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("click(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["click_xy", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(targetOrX), String(y)]); return; } @@ -387,7 +392,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["click", this.refSelector(target.value)]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support click target kind "${target.kind}" yet`); @@ -399,7 +404,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("hover(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["hover", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(targetOrX), String(y)]); return; } @@ -409,11 +414,11 @@ class BrowseCliPageHandle implements CorePageHandle { switch (target.kind) { case "selector": { const point = await this.resolveHoverPoint(target.value); - await this.runCommandAfterSelecting(["hover", String(point.x), String(point.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(point.x), String(point.y)]); return; } case "coords": - await this.runCommandAfterSelecting(["hover", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support hover target kind "${target.kind}" yet`); @@ -422,6 +427,7 @@ class BrowseCliPageHandle implements CorePageHandle { async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise { await this.runCommandAfterSelecting([ + "mouse", "scroll", String(x), String(y), @@ -450,7 +456,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["type", text]); return; case "selector": - await this.runCommandAfterSelecting(["fill", target.value, text, "--no-press-enter"]); + await this.runCommandAfterSelecting(["fill", target.value, text]); return; default: throw new Error(`browse_cli does not support type target kind "${target.kind}" yet`); @@ -485,7 +491,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["press", key]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); await this.runCommandAfterSelecting(["press", key]); return; default: @@ -519,7 +525,7 @@ class BrowseCliPageHandle implements CorePageHandle { } } -class BrowseCliSession implements CoreSession { +export class BrowseCliSession implements CoreSession { readonly runtime: BrowseCliRuntime; private readonly handles = new Map(); private activePageId: string | null = null; @@ -541,9 +547,14 @@ class BrowseCliSession implements CoreSession { return handle; } - private async fetchPages(): Promise { - const result = await this.runtime.runJson(["pages"]); - const pages = result.pages ?? []; + private async fetchPages(): Promise> { + const result = await this.runtime.runJson(["tab", "list"]); + const pages = (result.tabs ?? []).map((tab) => { + if (!tab.targetId) { + throw new EvalsError(`browse tab list returned no targetId for tab index ${tab.index}`); + } + return { ...tab, targetId: tab.targetId }; + }); for (const page of pages) { this.wrap(page); @@ -583,12 +594,11 @@ class BrowseCliSession implements CoreSession { } async newPage(url?: string): Promise { - const args = ["newpage"]; + const args = ["tab", "new"]; if (url) { args.push(url); } const result = await this.runtime.runJson<{ - created: boolean; url: string; targetId: string; }>(args); @@ -604,7 +614,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_switch", String(page.index)]); + await this.runtime.runJson(["tab", "switch", page.targetId]); this.activePageId = pageId; } @@ -615,10 +625,14 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_close", String(page.index)]); + const result = await this.runtime.runJson<{ selectedTargetId?: string }>([ + "tab", + "close", + page.targetId, + ]); this.handles.delete(pageId); - const remaining = await this.fetchPages(); - this.activePageId = remaining[0]?.targetId ?? null; + this.activePageId = result.selectedTargetId ?? null; + await this.fetchPages(); } async close(): Promise { @@ -655,10 +669,6 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo return "launch"; } -function createSessionName(): string { - return `evals-browse-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - export class BrowseCliTool implements CoreTool { readonly id = "browse_cli"; readonly surface = "cli"; @@ -687,11 +697,17 @@ export class BrowseCliTool implements CoreTool { ); } - const session = new BrowseCliSession(createSessionName()); - await session.runtime.runJson([ - "env", - input.environment === "BROWSERBASE" ? "remote" : "local", - ]); + const session = new BrowseCliSession(createBrowseCliSessionName()); + try { + await session.runtime.runJson([ + "open", + "about:blank", + input.environment === "BROWSERBASE" ? "--remote" : "--local", + ]); + } catch (error) { + await session.close().catch(() => {}); + throw error; + } return { session, diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index d0e7feb152..9df2063dfe 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -11,6 +11,7 @@ import { BROWSE_CLI_ENTRYPOINT, BROWSE_CLI_PACKAGE_JSON, BROWSE_SKILL_SOURCE, + createBrowseCliSessionName, } from "../browseCliPaths.js"; import { AGENT_RUN_TOOL_NAME, @@ -277,7 +278,7 @@ export async function prepareBrowseCliHarnessAdapter( ); } - const session = createBrowseSessionName(); + const session = createBrowseCliSessionName(); const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-browse-")); const wrapperPath = path.join(cwd, "browse"); await installBrowseSkill(cwd); @@ -726,10 +727,6 @@ function clip(value: string, maxLength: number): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; } -function createBrowseSessionName(): string { - return `evals-claude-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - async function runBrowseCommand( wrapperPath: string, args: string[], diff --git a/packages/evals/package.json b/packages/evals/package.json index 3b6c143ead..0e26ace598 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -40,7 +40,7 @@ "@opentelemetry/sdk-trace-node": "^2.9.0", "ai": "^5.0.133", "braintrust": "^3.21.0", - "browse": "0.9.5", + "browse": "workspace:*", "dotenv": "^17.3.1", "langsmith": "^0.5.16", "openai": "^4.104.0", diff --git a/packages/evals/tests/browseCliPaths.test.ts b/packages/evals/tests/browseCliPaths.test.ts new file mode 100644 index 0000000000..1c635951e1 --- /dev/null +++ b/packages/evals/tests/browseCliPaths.test.ts @@ -0,0 +1,22 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createBrowseCliSessionName } from "../browseCliPaths.js"; + +describe("Browse CLI session names", () => { + afterEach(() => vi.restoreAllMocks()); + + it("includes the PID and stays short with socket-safe characters", () => { + const name = createBrowseCliSessionName(); + + expect(name).toMatch(new RegExp(`^eval-${process.pid}-[A-Za-z0-9_-]+$`)); + expect(Buffer.byteLength(name)).toBeLessThanOrEqual(32); + }); + + it("keeps repeated calls distinct at the same timestamp without Math.random entropy", () => { + vi.spyOn(Date, "now").mockReturnValue(1_000); + vi.spyOn(Math, "random").mockReturnValue(0.5); + + const names = Array.from({ length: 1_000 }, () => createBrowseCliSessionName()); + + expect(new Set(names).size).toBe(names.length); + }); +}); diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts new file mode 100644 index 0000000000..beca0f31e2 --- /dev/null +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi } from "vitest"; +import { BrowseCliSession } from "../../core/tools/browse_cli.js"; +import { EvalsError } from "../../errors.js"; + +describe("workspace Browse adapter", () => { + it("preserves page identity when tabs are reordered and addresses commands by target ID", async () => { + const session = new BrowseCliSession("eval-session"); + let tabs = [ + { index: 0, targetId: "tab-a", url: "about:blank" }, + { index: 1, targetId: "tab-b", url: "https://example.com" }, + ]; + const runJson = vi.spyOn(session.runtime, "runJson").mockImplementation(async (args) => { + if (args[0] === "tab" && args[1] === "list") return { tabs }; + if (args[0] === "tab" && args[1] === "close") { + tabs = tabs.filter((tab) => tab.targetId !== args[2]); + return { selectedTargetId: tabs[0]?.targetId }; + } + return {}; + }); + + const [, secondPage] = await session.listPages(); + tabs = [ + { index: 0, targetId: "tab-b", url: "https://example.com/updated" }, + { index: 1, targetId: "tab-a", url: "about:blank" }, + ]; + const reorderedPages = await session.listPages(); + expect(reorderedPages[0]).toBe(secondPage); + expect(secondPage.url()).toBe("https://example.com/updated"); + + await session.selectPage(secondPage.id); + expect(runJson).toHaveBeenLastCalledWith(["tab", "switch", "tab-b"]); + await session.closePage(secondPage.id); + expect(runJson).toHaveBeenCalledWith(["tab", "close", "tab-b"]); + await expect(session.listPages()).resolves.toEqual([expect.objectContaining({ id: "tab-a" })]); + }); + + it.each([ + { active: "tab-b", closed: "tab-b", selected: "tab-c" }, + { active: "tab-c", closed: "tab-b", selected: "tab-c" }, + { active: "tab-c", closed: "tab-c", selected: "tab-b" }, + ])( + "tracks the CLI-selected tab $selected after closing $closed with $active active", + async ({ active, closed, selected }) => { + const session = new BrowseCliSession("eval-session"); + let tabs = ["tab-a", "tab-b", "tab-c"].map((targetId, index) => ({ + index, + targetId, + url: "about:blank", + })); + const runJson = vi.spyOn(session.runtime, "runJson").mockImplementation(async (args) => { + if (args[0] === "tab" && args[1] === "list") return { tabs }; + if (args[0] === "tab" && args[1] === "close") { + tabs = tabs.filter((tab) => tab.targetId !== args[2]); + return { selectedTargetId: selected }; + } + return {}; + }); + + await session.selectPage(active); + await session.closePage(closed); + expect((await session.activePage()).id).toBe(selected); + + // A command targeting a different surviving tab must first switch the CLI to it. + await session.selectIfNeeded("tab-a"); + expect(runJson).toHaveBeenLastCalledWith(["tab", "switch", "tab-a"]); + }, + ); + + it("rejects tab results that cannot identify a page reliably", async () => { + const session = new BrowseCliSession("eval-session"); + vi.spyOn(session.runtime, "runJson").mockResolvedValue({ + tabs: [{ index: 0, url: "about:blank" }], + }); + + await expect(session.listPages()).rejects.toBeInstanceOf(EvalsError); + await expect(session.listPages()).rejects.toThrow( + "browse tab list returned no targetId for tab index 0", + ); + }); +}); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5acd0d939a..c214dd8729 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -110,11 +110,14 @@ describe("claude code tool adapter resolution", () => { expect(getBrowseCliAllowedTools()).toEqual(["Skill", "Bash"]); }); - it("exposes browse cli metadata for Braintrust rows", () => { + it("exposes workspace browse cli metadata for Braintrust rows", async () => { + const { version } = JSON.parse( + await fsp.readFile(new URL("../../../cli/package.json", import.meta.url), "utf8"), + ); expect(getBrowseCliToolMetadata()).toMatchObject({ toolCommand: "browse", - browseCliVersion: "0.9.5", - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliVersion: version, + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), }); }); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index 4a0cab5e5b..2d7f73c21c 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -259,7 +259,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), harnessConfig: { harness: "claude_code", model: "anthropic/claude-sonnet-4-20250514", @@ -317,7 +317,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), harnessConfig: { harness: "codex", model: "openai/gpt-5.4-mini", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c769a88398..4e74460909 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -596,8 +596,8 @@ importers: specifier: ^3.21.0 version: 3.28.0(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3) browse: - specifier: 0.9.5 - version: 0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1) + specifier: workspace:* + version: link:../cli dotenv: specifier: ^17.3.1 version: 17.4.2