diff --git a/packages/evals/README.md b/packages/evals/README.md index 784ee1b975..765ff3bb46 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -70,6 +70,7 @@ Use `Esc` to abort an in-flight run without exiting the REPL. | `-c, --concurrency ` | Max parallel sessions | | `-m, --model ` | Override the model matrix | | `--api` | Run via the Stagehand API instead of the SDK | +| `--verbose` | Stream detailed logs while the eval runs | | `--harness ` | Which agent harness drives the bench task | | `-l, --limit ` / `-s, --sample ` / `-f, --filter key=value` | Suite shaping for benchmark targets | | `--preview` | Print the resolved plan and exit — no browser, no LLM calls | diff --git a/packages/evals/browseCliPaths.ts b/packages/evals/browseCliPaths.ts index fe2ca89e3b..817b1bbb04 100644 --- a/packages/evals/browseCliPaths.ts +++ b/packages/evals/browseCliPaths.ts @@ -10,3 +10,9 @@ export const BROWSE_CLI_BUILD_ARTIFACTS = [ ]; export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json"); export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md"); + +export function createBrowseCliSessionName(): string { + const timestamp = Date.now().toString(36); + const random = Math.random().toString(36).slice(2, 6); + return `eval-${process.pid}-${timestamp}-${random}`; +} diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index 4e3d7371f3..83460c7419 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -14,7 +14,11 @@ import type { ToolStartInput, ToolStartResult, } from "../contracts/tool.js"; -import { BROWSE_CLI_BUILD_ARTIFACTS, BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; +import { + BROWSE_CLI_BUILD_ARTIFACTS, + BROWSE_CLI_ENTRYPOINT, + createBrowseCliSessionName, +} from "../../browseCliPaths.js"; import { getRepoRootDir } from "../../runtimePaths.js"; const execFileAsync = promisify(execFile); @@ -79,25 +83,38 @@ function buildSelectorQuery(selector: string): string { `; } -type BrowseCliPagesResult = { - pages: Array<{ +type BrowseCliTabsResult = { + tabs: Array<{ index: number; url: string; - targetId: string; + targetId?: string; }>; }; +export function buildBrowseCliProcessArgs( + entrypoint: string, + session: string, + args: string[], +): string[] { + return [entrypoint, ...args, "--session", session]; +} + +export function browseCliStartupArgs(environment: ToolStartInput["environment"]): string[] { + return ["open", "about:blank", environment === "BROWSERBASE" ? "--remote" : "--local"]; +} + class BrowseCliRuntime { constructor(private readonly session: string) {} - async runJson(args: string[]): Promise { + async runJson(args: string[], timeoutMs?: number): Promise { const { stdout, stderr } = await execFileAsync( process.execPath, - [resolveBrowseCliEntrypoint(), "--json", "--session", this.session, ...args], + buildBrowseCliProcessArgs(resolveBrowseCliEntrypoint(), this.session, args), { cwd: getRepoRootDir(), env: process.env, maxBuffer: 10 * 1024 * 1024, + ...(timeoutMs === undefined ? {} : { timeout: timeoutMs }), }, ); @@ -162,7 +179,7 @@ class BrowseCliLocatorHandle implements CoreLocatorHandle { } } -class BrowseCliPageHandle implements CorePageHandle { +export class BrowseCliPageHandle implements CorePageHandle { constructor( private readonly session: BrowseCliSession, readonly id: string, @@ -208,7 +225,7 @@ class BrowseCliPageHandle implements CorePageHandle { args.push("--wait", opts.waitUntil); } if (typeof opts?.timeoutMs === "number") { - args.push("-t", String(opts.timeoutMs)); + args.push("--timeout", String(opts.timeoutMs)); } const result = await this.runCommandAfterSelecting<{ url: string }>(args); this.cachedUrl = result.url; @@ -280,15 +297,15 @@ class BrowseCliPageHandle implements CorePageHandle { type?: "png" | "jpeg"; quality?: number; }): Promise { - const args = ["screenshot"]; + const args = ["screenshot", "--base64"]; if (opts?.fullPage) { - args.push("-f"); + args.push("--full-page"); } if (opts?.type) { - args.push("-t", opts.type); + args.push("--type", opts.type); } if (typeof opts?.quality === "number") { - args.push("-q", String(opts.quality)); + args.push("--quality", String(opts.quality)); } const result = await this.runCommandAfterSelecting<{ base64: string }>(args); @@ -310,9 +327,9 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "selector", spec.selector, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), - "-s", + "--state", spec.state ?? "visible", ]); return; @@ -324,7 +341,7 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "load", spec.state, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), ]); return; @@ -372,7 +389,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("click(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["click_xy", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(targetOrX), String(y)]); return; } @@ -387,7 +404,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["click", this.refSelector(target.value)]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support click target kind "${target.kind}" yet`); @@ -399,7 +416,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("hover(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["hover", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(targetOrX), String(y)]); return; } @@ -409,11 +426,11 @@ class BrowseCliPageHandle implements CorePageHandle { switch (target.kind) { case "selector": { const point = await this.resolveHoverPoint(target.value); - await this.runCommandAfterSelecting(["hover", String(point.x), String(point.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(point.x), String(point.y)]); return; } case "coords": - await this.runCommandAfterSelecting(["hover", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support hover target kind "${target.kind}" yet`); @@ -422,6 +439,7 @@ class BrowseCliPageHandle implements CorePageHandle { async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise { await this.runCommandAfterSelecting([ + "mouse", "scroll", String(x), String(y), @@ -450,7 +468,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["type", text]); return; case "selector": - await this.runCommandAfterSelecting(["fill", target.value, text, "--no-press-enter"]); + await this.runCommandAfterSelecting(["fill", target.value, text]); return; default: throw new Error(`browse_cli does not support type target kind "${target.kind}" yet`); @@ -485,7 +503,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["press", key]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); await this.runCommandAfterSelecting(["press", key]); return; default: @@ -519,7 +537,7 @@ class BrowseCliPageHandle implements CorePageHandle { } } -class BrowseCliSession implements CoreSession { +export class BrowseCliSession implements CoreSession { readonly runtime: BrowseCliRuntime; private readonly handles = new Map(); private activePageId: string | null = null; @@ -541,9 +559,14 @@ class BrowseCliSession implements CoreSession { return handle; } - private async fetchPages(): Promise { - const result = await this.runtime.runJson(["pages"]); - const pages = result.pages ?? []; + private async fetchPages(): Promise> { + const result = await this.runtime.runJson(["tab", "list"]); + const pages = (result.tabs ?? []).map((tab) => { + if (!tab.targetId) { + throw new Error(`browse tab list returned no targetId for tab index ${tab.index}`); + } + return { ...tab, targetId: tab.targetId }; + }); for (const page of pages) { this.wrap(page); @@ -583,12 +606,11 @@ class BrowseCliSession implements CoreSession { } async newPage(url?: string): Promise { - const args = ["newpage"]; + const args = ["tab", "new"]; if (url) { args.push(url); } const result = await this.runtime.runJson<{ - created: boolean; url: string; targetId: string; }>(args); @@ -604,7 +626,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_switch", String(page.index)]); + await this.runtime.runJson(["tab", "switch", page.targetId]); this.activePageId = pageId; } @@ -615,7 +637,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_close", String(page.index)]); + await this.runtime.runJson(["tab", "close", page.targetId]); this.handles.delete(pageId); const remaining = await this.fetchPages(); this.activePageId = remaining[0]?.targetId ?? null; @@ -624,12 +646,7 @@ class BrowseCliSession implements CoreSession { async close(): Promise { if (this.closed) return; this.closed = true; - - try { - await this.runtime.runJson(["stop", "--force"]); - } catch { - // best-effort only - } + await this.runtime.runJson(["stop", "--force"], 5_000); } async getArtifacts(): Promise { @@ -655,10 +672,6 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo return "launch"; } -function createSessionName(): string { - return `evals-browse-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - export class BrowseCliTool implements CoreTool { readonly id = "browse_cli"; readonly surface = "cli"; @@ -687,11 +700,13 @@ export class BrowseCliTool implements CoreTool { ); } - const session = new BrowseCliSession(createSessionName()); - await session.runtime.runJson([ - "env", - input.environment === "BROWSERBASE" ? "remote" : "local", - ]); + const session = new BrowseCliSession(createBrowseCliSessionName()); + try { + await session.runtime.runJson(browseCliStartupArgs(input.environment)); + } catch (error) { + await session.close().catch(() => {}); + throw error; + } return { session, diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index 7541f6dc52..f0c675fccd 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -13,6 +13,7 @@ import { prepareCodexToolAdapter, type PreparedCodexToolAdapter } from "./codexT import { buildExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; import type { DiscoveredTask, TaskResult } from "./types.js"; import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js"; +import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js"; export interface BenchHarnessStartInput { task: DiscoveredTask; @@ -152,6 +153,11 @@ export const claudeCodeHarness: BenchHarness = { // the adapter and the carrier. const carrierV3 = buildVerifierCarrierV3(logger); let toolAdapter: PreparedClaudeCodeToolAdapter | undefined; + const cleanup = onceAsync(async () => { + await toolAdapter?.cleanup(); + await carrierV3.close().catch(() => {}); + }); + const unregisterCleanup = registerActiveRunCleanup(cleanup); try { toolAdapter = await prepareClaudeCodeToolAdapter({ toolSurface: row.config.toolSurface, @@ -173,11 +179,11 @@ export const claudeCodeHarness: BenchHarness = { }, }); } finally { - await toolAdapter?.cleanup(); - // Deregister the never-init()-ed carrier (instance registry, event - // store, logger binding) so long matrix runs don't accumulate one - // V3 object graph per task. - await carrierV3.close().catch(() => {}); + try { + await cleanup(); + } finally { + unregisterCleanup(); + } } }, async start(): Promise { @@ -201,6 +207,11 @@ export const codexHarness: BenchHarness = { // the adapter and the carrier. const carrierV3 = buildVerifierCarrierV3(logger); let toolAdapter: PreparedCodexToolAdapter | undefined; + const cleanup = onceAsync(async () => { + await toolAdapter?.cleanup(); + await carrierV3.close().catch(() => {}); + }); + const unregisterCleanup = registerActiveRunCleanup(cleanup); try { toolAdapter = await prepareCodexToolAdapter({ toolSurface: row.config.toolSurface, @@ -222,11 +233,11 @@ export const codexHarness: BenchHarness = { }, }); } finally { - await toolAdapter?.cleanup(); - // Deregister the never-init()-ed carrier (instance registry, event - // store, logger binding) so long matrix runs don't accumulate one - // V3 object graph per task. - await carrierV3.close().catch(() => {}); + try { + await cleanup(); + } finally { + unregisterCleanup(); + } } }, async start(): Promise { diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index 0c3e06bcfb..fd28c51cb8 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -11,6 +11,7 @@ import { BROWSE_CLI_ENTRYPOINT, BROWSE_CLI_PACKAGE_JSON, BROWSE_SKILL_SOURCE, + createBrowseCliSessionName, } from "../browseCliPaths.js"; import { AGENT_RUN_TOOL_NAME, @@ -309,57 +310,87 @@ export async function prepareBrowseCliHarnessAdapter( ); } - const session = createBrowseSessionName(); + const session = createBrowseCliSessionName(); const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-browse-")); - const wrapperPath = path.join(cwd, "browse"); - await installBrowseSkill(cwd); - input.logger.log({ - category: input.logCategory, - message: `Installed browse skill at ${path.join(cwd, ".claude", "skills", "browse", "SKILL.md")}`, - level: 1, - }); - const env = { - ...process.env, - BROWSE_SESSION: session, - PATH: `${cwd}${path.delimiter}${process.env.PATH ?? ""}`, - } as Record; + try { + const wrapperPath = path.join(cwd, "browse"); + await installBrowseSkill(cwd); + input.logger.log({ + category: input.logCategory, + message: `Installed browse skill at ${path.join(cwd, ".claude", "skills", "browse", "SKILL.md")}`, + level: 1, + }); + const env = { + ...process.env, + BROWSE_SESSION: session, + PATH: `${cwd}${path.delimiter}${process.env.PATH ?? ""}`, + } as Record; + + const modeFlag = input.environment === "BROWSERBASE" ? "--remote" : "--local"; + await fsp.writeFile( + wrapperPath, + buildBrowseCliWrapperScript({ + entrypoint: BROWSE_CLI_ENTRYPOINT, + modeFlag, + nodePath: process.execPath, + session, + }), + { mode: 0o755 }, + ); - const modeFlag = input.environment === "BROWSERBASE" ? "--remote" : "--local"; - await fsp.writeFile( - wrapperPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - // The mode flag (--local/--remote) selects the environment when the daemon - // is first started and must be explicit so a set BROWSERBASE_API_KEY does - // not silently auto-select remote. It is only accepted by the driver - // commands, so skip it for the few subcommands that reject it (stop, - // status). The session name is safe on every command. - "cmd=${1:-}", - "mode=()", - 'if [[ "$cmd" != "stop" && "$cmd" != "status" ]]; then', - ` mode=(${JSON.stringify(modeFlag)})`, - "fi", - `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(BROWSE_CLI_ENTRYPOINT)} "$@" "\${mode[@]+\${mode[@]}}" --session ${JSON.stringify(session)}`, - "", - ].join("\n"), - { mode: 0o755 }, - ); + return { + toolSurface: "browse_cli", + startupProfile: input.startupProfile, + cwd, + env, + promptInstructions: buildBrowseCliPromptInstructions(input.plan), + metadata: getBrowseCliToolMetadata(), + cleanup: async () => { + try { + await runBrowseCommand(wrapperPath, ["stop", "--force"], input.logger, env, cwd, 5_000); + } catch (error) { + input.logger.warn({ + category: input.logCategory, + message: `browse_cli cleanup failed: ${error instanceof Error ? error.message : String(error)}`, + level: 0, + }); + } finally { + await fsp.rm(cwd, { recursive: true, force: true }); + } + }, + }; + } catch (error) { + await fsp.rm(cwd, { recursive: true, force: true }); + throw error; + } +} - return { - toolSurface: "browse_cli", - startupProfile: input.startupProfile, - cwd, - env, - promptInstructions: buildBrowseCliPromptInstructions(input.plan), - metadata: getBrowseCliToolMetadata(), - cleanup: async () => { - await runBrowseCommand(wrapperPath, ["stop", "--force"], input.logger, env, cwd).catch( - (): undefined => undefined, - ); - await fsp.rm(cwd, { recursive: true, force: true }); - }, - }; +export function buildBrowseCliWrapperScript({ + entrypoint, + modeFlag, + nodePath, + session, +}: { + entrypoint: string; + modeFlag: "--local" | "--remote"; + nodePath: string; + session: string; +}): string { + return [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "cmd=${1:-}", + 'case "$cmd" in', + " back|click|cursor|doctor|eval|fill|forward|get|highlight|is|mouse|network|open|press|reload|screenshot|select|snapshot|status|stop|tab|type|upload|viewport|wait) ;;", + ' *) echo "The browse eval harness only supports browser driver commands (received: ${cmd:-})." >&2; exit 64 ;;', + "esac", + "mode=()", + 'if [[ "$cmd" != "stop" && "$cmd" != "status" ]]; then', + ` mode=(${JSON.stringify(modeFlag)})`, + "fi", + `exec ${JSON.stringify(nodePath)} ${JSON.stringify(entrypoint)} "$@" "\${mode[@]+\${mode[@]}}" --session ${JSON.stringify(session)}`, + "", + ].join("\n"); } /** @@ -758,16 +789,13 @@ function clip(value: string, maxLength: number): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; } -function createBrowseSessionName(): string { - return `evals-claude-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - async function runBrowseCommand( wrapperPath: string, args: string[], logger: EvalLogger, env: Record, cwd: string, + timeoutMs: number, ): Promise { const { spawn } = await import("node:child_process"); await new Promise((resolve, reject) => { @@ -777,6 +805,19 @@ async function runBrowseCommand( stdio: ["ignore", "pipe", "pipe"], }); let stderr = ""; + let timedOut = false; + let settled = false; + const finish = (error?: Error): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + if (error) reject(error); + else resolve(); + }; + const timeout = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk: string) => { logger.log({ category: "browse_cli", message: chunk, level: 1 }); @@ -786,13 +827,19 @@ async function runBrowseCommand( stderr += chunk; logger.log({ category: "browse_cli", message: chunk, level: 1 }); }); - child.on("error", reject); + child.on("error", (error) => finish(error)); child.on("close", (code) => { + if (timedOut) { + finish( + new EvalsError(`browse_cli command timed out after ${timeoutMs}ms (${args.join(" ")})`), + ); + return; + } if (code === 0) { - resolve(); + finish(); return; } - reject(new EvalsError(`browse_cli command failed (${args.join(" ")}): ${stderr.trim()}`)); + finish(new EvalsError(`browse_cli command failed (${args.join(" ")}): ${stderr.trim()}`)); }); }); } diff --git a/packages/evals/package.json b/packages/evals/package.json index 40f321dd8c..b99abf09f5 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -25,7 +25,7 @@ "@browserbasehq/stagehand": "workspace:*", "@openai/codex-sdk": "0.125.0", "ai": "^5.0.133", - "browse": "0.9.5", + "browse": "workspace:*", "dotenv": "^17.3.1", "openai": "^4.104.0", "playwright": ">=1.55.1 <1.57.0", diff --git a/packages/evals/tests/cli.test.ts b/packages/evals/tests/cli.test.ts index 763d27247e..a8b321c15b 100644 --- a/packages/evals/tests/cli.test.ts +++ b/packages/evals/tests/cli.test.ts @@ -247,6 +247,12 @@ describe("CLI entrypoint", () => { expect(() => JSON.parse(stdout)).toThrow(); }, 20_000); + it("accepts --verbose for a one-shot run", async () => { + const { stdout, code } = await runCli(["run", "act", "--dry-run", "--verbose"]); + expect(code).toBe(0); + expect(JSON.parse(stdout).runOptions.verbose).toBe(true); + }); + it("rejects --preview combined with --dry-run", async () => { const { stdout, stderr, code } = await runCli(["run", "act", "--dry-run", "--preview"]); expect(code).toBe(1); diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts new file mode 100644 index 0000000000..f23b2f958c --- /dev/null +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + BrowseCliPageHandle, + BrowseCliSession, + browseCliStartupArgs, + buildBrowseCliProcessArgs, +} from "../../core/tools/browse_cli.js"; +import { createBrowseCliSessionName } from "../../browseCliPaths.js"; +import { getSocketPath } from "../../../cli/src/lib/driver/daemon/paths.js"; + +describe("browse_cli V4 adapter", () => { + it("places session selection on each V4 command and uses mode-specific startup commands", () => { + expect(buildBrowseCliProcessArgs("/tmp/browse", "eval-session", ["tab", "list"])).toEqual([ + "/tmp/browse", + "tab", + "list", + "--session", + "eval-session", + ]); + expect(browseCliStartupArgs("LOCAL")).toEqual(["open", "about:blank", "--local"]); + expect(browseCliStartupArgs("BROWSERBASE")).toEqual(["open", "about:blank", "--remote"]); + }); + + it("keeps eval session names short enough for temporary Unix socket paths", () => { + const sessionName = createBrowseCliSessionName(); + + expect(sessionName).toMatch(/^eval-\d+-[a-z0-9]+-[a-z0-9]{4}$/u); + expect(sessionName.length).toBeLessThanOrEqual(32); + if (process.platform !== "win32") { + expect(Buffer.byteLength(getSocketPath(sessionName))).toBeLessThanOrEqual(103); + } + }); + + it("maps page capabilities onto the V4 command tree and flags", async () => { + const runJson = vi.fn(async (args: string[]) => { + if (args[0] === "open") return { url: args[1] }; + if (args[0] === "screenshot") { + return { base64: Buffer.from("screenshot").toString("base64") }; + } + if (args[0] === "get" && args[1] === "box") return { x: 12, y: 34 }; + return {}; + }); + const session = { + runtime: { runJson }, + selectIfNeeded: vi.fn(), + } as unknown as BrowseCliSession; + const page = new BrowseCliPageHandle(session, "page-1"); + + await page.goto("https://example.com", { + timeoutMs: 1_234, + waitUntil: "networkidle", + }); + await expect(page.screenshot({ fullPage: true, quality: 80, type: "jpeg" })).resolves.toEqual( + Buffer.from("screenshot"), + ); + await page.wait({ + kind: "selector", + selector: "#ready", + state: "attached", + timeoutMs: 2_345, + }); + await page.click(10, 20); + await page.hover({ kind: "selector", value: "#hover-target" }); + await page.scroll(30, 40, 0, 500); + await page.type({ kind: "selector", value: "#text-input" }, "hello"); + await page.press({ kind: "coords", x: 50, y: 60 }, "Enter"); + + expect(runJson.mock.calls.map(([args]) => args)).toEqual([ + ["open", "https://example.com", "--wait", "networkidle", "--timeout", "1234"], + ["screenshot", "--base64", "--full-page", "--type", "jpeg", "--quality", "80"], + ["wait", "selector", "#ready", "--timeout", "2345", "--state", "attached"], + ["mouse", "click", "10", "20"], + ["get", "box", "#hover-target"], + ["mouse", "hover", "12", "34"], + ["mouse", "scroll", "30", "40", "0", "500"], + ["fill", "#text-input", "hello"], + ["mouse", "click", "50", "60"], + ["press", "Enter"], + ]); + }); + + it("reads V4 tab-list results and addresses tabs by stable targetId", async () => { + const session = new BrowseCliSession("eval-session"); + const runJson = vi.spyOn(session.runtime, "runJson").mockImplementation(async (args) => { + if (args[0] === "tab" && args[1] === "list") { + return { + tabs: [ + { index: 0, targetId: "tab-a", url: "about:blank" }, + { index: 1, targetId: "tab-b", url: "https://example.com" }, + ], + }; + } + return {}; + }); + + await expect(session.listPages()).resolves.toEqual([ + expect.objectContaining({ id: "tab-a" }), + expect.objectContaining({ id: "tab-b" }), + ]); + await session.selectPage("tab-b"); + + expect(runJson).toHaveBeenLastCalledWith(["tab", "switch", "tab-b"]); + }); + + it("rejects V4 tab results without a stable targetId", async () => { + const session = new BrowseCliSession("eval-session"); + vi.spyOn(session.runtime, "runJson").mockResolvedValue({ + tabs: [{ index: 0, url: "about:blank" }], + }); + + await expect(session.listPages()).rejects.toThrow( + "browse tab list returned no targetId for tab index 0", + ); + }); +}); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index f7c937030d..6a5ba6a3c6 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -1,8 +1,11 @@ +import { execFile } from "node:child_process"; import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { + buildBrowseCliWrapperScript, getBrowseCliAllowedTools, getBrowseCliToolMetadata, insertAfterFrontmatter, @@ -17,6 +20,9 @@ import { resolveCodexToolSurface, } from "../../framework/codexToolAdapter.js"; import type { CdpEventMessage } from "../../core/tools/cdp_code.js"; +import { BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; + +const execFileAsync = promisify(execFile); describe("claude code tool adapter resolution", () => { afterEach(() => { @@ -124,8 +130,8 @@ describe("claude code tool adapter resolution", () => { it("exposes browse cli metadata for Braintrust rows", () => { expect(getBrowseCliToolMetadata()).toMatchObject({ toolCommand: "browse", - browseCliVersion: "0.9.5", - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliVersion: "0.9.6", + browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, }); }); @@ -252,6 +258,46 @@ describe("claude code tool adapter resolution", () => { } }); + it("routes only driver commands through the pinned browse wrapper", async () => { + const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-wrapper-test-")); + const entrypoint = path.join(cwd, "capture.mjs"); + const wrapper = path.join(cwd, "browse"); + try { + await fsp.writeFile( + entrypoint, + "process.stdout.write(JSON.stringify(process.argv.slice(2)));\n", + ); + await fsp.writeFile( + wrapper, + buildBrowseCliWrapperScript({ + entrypoint, + modeFlag: "--local", + nodePath: process.execPath, + session: "eval-session", + }), + { mode: 0o755 }, + ); + + const open = await execFileAsync(wrapper, ["open", "about:blank"]); + expect(JSON.parse(open.stdout)).toEqual([ + "open", + "about:blank", + "--local", + "--session", + "eval-session", + ]); + + const stop = await execFileAsync(wrapper, ["stop", "--force"]); + expect(JSON.parse(stop.stdout)).toEqual(["stop", "--force", "--session", "eval-session"]); + + await expect(execFileAsync(wrapper, ["cloud", "projects", "list"])).rejects.toMatchObject({ + code: 64, + }); + } finally { + await fsp.rm(cwd, { recursive: true, force: true }); + } + }); + it("keeps abandoned CDP event waits from becoming unhandled rejections", async () => { const listeners = new Set<(event: CdpEventMessage) => void>(); const connection = { diff --git a/packages/evals/tests/tui/parse.test.ts b/packages/evals/tests/tui/parse.test.ts index c4fd7459e4..cc88c0de33 100644 --- a/packages/evals/tests/tui/parse.test.ts +++ b/packages/evals/tests/tui/parse.test.ts @@ -17,6 +17,11 @@ describe("resolveRunOptions", () => { expect(resolved.verbose).toBe(true); }); + it("lets --verbose override the config default", () => { + const resolved = resolveRunOptions(parseRunArgs(["act", "--verbose"]), { verbose: false }, {}); + expect(resolved.verbose).toBe(true); + }); + it("defaults to the stagehand bench harness", () => { const resolved = resolveRunOptions({}, {}, {}); expect(resolved.harness).toBe("stagehand"); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index e21e980232..9a595a7fdc 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -5,6 +5,7 @@ import { deriveCategoryFilter, runCommand, } from "../../tui/commands/run.js"; +import { BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; const runEvalsMock = vi.hoisted(() => vi.fn(async () => ({ @@ -218,7 +219,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, harnessConfig: { harness: "claude_code", model: "anthropic/claude-sonnet-4-20250514", @@ -276,7 +277,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, harnessConfig: { harness: "codex", model: "openai/gpt-5.4-mini", diff --git a/packages/evals/tui/commands/help.ts b/packages/evals/tui/commands/help.ts index eb233a185e..0b6745f0a0 100644 --- a/packages/evals/tui/commands/help.ts +++ b/packages/evals/tui/commands/help.ts @@ -63,6 +63,7 @@ export function printRunHelp(): void { row(`${cyan("-e, --env")} ${dim("")}`, `Environment: ${gray("local | browserbase")}`), row(`${cyan("-m, --model")} ${dim("")}`, "Model override"), row(cyan("--api"), "Use Stagehand API mode"), + row(cyan("--verbose"), "Stream detailed logs instead of the animated progress view"), "", ` ${bold("Core options:")}`, "", diff --git a/packages/evals/tui/commands/parse.ts b/packages/evals/tui/commands/parse.ts index 74988f0234..bba9167344 100644 --- a/packages/evals/tui/commands/parse.ts +++ b/packages/evals/tui/commands/parse.ts @@ -34,6 +34,7 @@ export interface RunFlags { filter?: Array<[string, string]>; dryRun?: boolean; preview?: boolean; + verbose?: boolean; /** * Rubric success mode for the verifier — outcome | process | both. * outcome (default): binary EvaluationResult.outcomeSuccess. @@ -88,7 +89,7 @@ const SUPPORTED_BENCHMARKS = new Set([ "odysseysbench", ]); -const BOOLEAN_FLAGS = new Set(["api", "dry-run", "preview"]); +const BOOLEAN_FLAGS = new Set(["api", "dry-run", "preview", "verbose"]); const VALUE_FLAGS = new Set([ "trials", "concurrency", @@ -178,6 +179,7 @@ export function parseRunArgs(tokens: string[]): RunFlags { if (name === "api") flags.api = true; else if (name === "dry-run") flags.dryRun = true; else if (name === "preview") flags.preview = true; + else if (name === "verbose") flags.verbose = true; i++; continue; } @@ -393,7 +395,7 @@ export function resolveRunOptions( envOverrides, dryRun: flags.dryRun ?? false, preview: flags.preview ?? false, - verbose: defaults.verbose ?? false, + verbose: flags.verbose ?? defaults.verbose ?? false, }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4db873a417..b9353e2faa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -331,8 +331,8 @@ importers: specifier: ^5.0.133 version: 5.0.220(zod@4.4.3) browse: - specifier: 0.9.5 - version: 0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1) + specifier: workspace:* + version: link:../cli dotenv: specifier: ^17.3.1 version: 17.4.2 @@ -998,22 +998,6 @@ packages: '@browserbasehq/sdk@2.16.0': resolution: {integrity: sha512-mPAuLRU9jWR7o0KJi9+gQnOBDUSIkoKbbFv4HjrA+80qWVcFacrNPlZmf4mguQnfZ0oP2t5c3ws6yuFyAX9vpA==} - '@browserbasehq/stagehand@3.6.0': - resolution: {integrity: sha512-wQ3Mv5b2xy64SntWNUbvgd7liWPbQRvGa8KEb1LorwheTkpUZFkt3BpZE8ovMY/lIleV9DfYjbvqghbeOOla4w==} - engines: {node: ^20.19.0 || >=22.12.0} - peerDependencies: - patchright-core: ^1.55.2 - playwright-core: ^1.55.1 - puppeteer-core: ^24.43.0 - zod: ^3.25.76 || ^4.2.0 - peerDependenciesMeta: - patchright-core: - optional: true - playwright-core: - optional: true - puppeteer-core: - optional: true - '@browserbasehq/stagehand@3.7.1': resolution: {integrity: sha512-vAuYSZWIhh3d76BxwppNVE3dB0ztEBLBi85G6TWulZNiebdWptNoANOMuprOB/cw5dE+80b/ZZQo4G33Pc9i6w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3461,11 +3445,6 @@ packages: peerDependencies: zod: ^3.25.34 - browse@0.9.5: - resolution: {integrity: sha512-5icxS7urivYaSaAfFxj+vSWU2zNPol8aBIU3S+3m1ervovG3qnQ0nfHEG9BhV2YXdAAACo5jT+sA0xYkugwFjA==} - engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true - buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} @@ -8205,47 +8184,6 @@ snapshots: transitivePeerDependencies: - encoding - '@browserbasehq/stagehand@3.6.0(playwright-core@1.56.1)(zod@4.4.3)': - dependencies: - '@ai-sdk/provider': 2.0.3 - '@anthropic-ai/sdk': 0.39.0 - '@browserbasehq/sdk': 2.16.0 - '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0) - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - ai: 5.0.220(zod@4.4.3) - devtools-protocol: 0.0.1642743 - fetch-cookie: 3.2.0 - openai: 4.104.0(ws@8.21.0(bufferutil@4.1.0))(zod@4.4.3) - pino: 9.14.0 - pino-pretty: 13.1.3 - uuid: 11.1.1 - ws: 8.21.0(bufferutil@4.1.0) - zod: 4.4.3 - zod-to-json-schema: 3.25.2(zod@4.4.3) - optionalDependencies: - '@ai-sdk/amazon-bedrock': 3.0.111(zod@4.4.3) - '@ai-sdk/anthropic': 2.0.91(zod@4.4.3) - '@ai-sdk/azure': 2.0.120(zod@4.4.3) - '@ai-sdk/cerebras': 1.0.51(zod@4.4.3) - '@ai-sdk/deepseek': 1.0.48(zod@4.4.3) - '@ai-sdk/google': 2.0.85(zod@4.4.3) - '@ai-sdk/google-vertex': 3.0.158(zod@4.4.3) - '@ai-sdk/groq': 2.0.45(zod@4.4.3) - '@ai-sdk/mistral': 2.0.40(zod@4.4.3) - '@ai-sdk/openai': 2.0.115(zod@4.4.3) - '@ai-sdk/perplexity': 2.0.36(zod@4.4.3) - '@ai-sdk/togetherai': 1.0.49(zod@4.4.3) - '@ai-sdk/xai': 2.0.82(zod@4.4.3) - bufferutil: 4.1.0 - chrome-launcher: 1.2.1 - ollama-ai-provider-v2: 1.5.5(zod@4.4.3) - playwright-core: 1.56.1 - transitivePeerDependencies: - - '@cfworker/json-schema' - - encoding - - supports-color - - utf-8-validate - '@browserbasehq/stagehand@3.7.1(playwright-core@1.56.1)(zod@4.4.3)': dependencies: '@ai-sdk/provider': 2.0.3 @@ -11018,36 +10956,6 @@ snapshots: - '@aws-sdk/credential-provider-web-identity' - supports-color - browse@0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1): - dependencies: - '@browserbasehq/sdk': 2.16.0 - '@browserbasehq/stagehand': 3.6.0(playwright-core@1.56.1)(zod@4.4.3) - '@oclif/core': 4.13.0 - '@vercel/detect-agent': 1.2.3 - archiver: 7.0.1 - deepmerge: 4.3.1 - dotenv: 16.6.1 - fastest-levenshtein: 1.0.16 - http-status-codes: 2.3.0 - ignore: 7.0.5 - node-html-markdown: 1.3.0 - semver: 7.8.5 - tsx: 4.23.1 - ws: 8.21.0(bufferutil@4.1.0) - zod: 4.4.3 - transitivePeerDependencies: - - '@cfworker/json-schema' - - bare-abort-controller - - bare-buffer - - bufferutil - - encoding - - patchright-core - - playwright-core - - puppeteer-core - - react-native-b4a - - supports-color - - utf-8-validate - buffer-crc32@0.2.13: {} buffer-crc32@1.0.0: {}