From a9ea36d7d9b5e90cfad6443ef56f1fe3b1d1fbb7 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 17 Sep 2026 17:19:44 +0000 Subject: [PATCH 1/4] fix(evals): use workspace Browse with compatible CLI commands --- packages/evals/browseCliPaths.ts | 6 ++ packages/evals/core/tools/browse_cli.ts | 85 +++++++++++-------- .../evals/framework/claudeCodeToolAdapter.ts | 7 +- packages/evals/package.json | 2 +- .../tests/core/browse-cli-adapter.test.ts | 45 ++++++++++ .../framework/claudeCodeToolAdapter.test.ts | 9 +- packages/evals/tests/tui/run.test.ts | 4 +- pnpm-lock.yaml | 4 +- 8 files changed, 112 insertions(+), 50 deletions(-) create mode 100644 packages/evals/tests/core/browse-cli-adapter.test.ts diff --git a/packages/evals/browseCliPaths.ts b/packages/evals/browseCliPaths.ts index fe2ca89e3b..817b1bbb04 100644 --- a/packages/evals/browseCliPaths.ts +++ b/packages/evals/browseCliPaths.ts @@ -10,3 +10,9 @@ export const BROWSE_CLI_BUILD_ARTIFACTS = [ ]; export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json"); export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md"); + +export function createBrowseCliSessionName(): string { + const timestamp = Date.now().toString(36); + const random = Math.random().toString(36).slice(2, 6); + return `eval-${process.pid}-${timestamp}-${random}`; +} diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index 4e3d7371f3..38a7a9ba4b 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -14,7 +14,11 @@ import type { ToolStartInput, ToolStartResult, } from "../contracts/tool.js"; -import { BROWSE_CLI_BUILD_ARTIFACTS, BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; +import { + BROWSE_CLI_BUILD_ARTIFACTS, + BROWSE_CLI_ENTRYPOINT, + createBrowseCliSessionName, +} from "../../browseCliPaths.js"; import { getRepoRootDir } from "../../runtimePaths.js"; const execFileAsync = promisify(execFile); @@ -79,11 +83,11 @@ function buildSelectorQuery(selector: string): string { `; } -type BrowseCliPagesResult = { - pages: Array<{ +type BrowseCliTabsResult = { + tabs: Array<{ index: number; url: string; - targetId: string; + targetId?: string; }>; }; @@ -93,7 +97,7 @@ class BrowseCliRuntime { async runJson(args: string[]): Promise { const { stdout, stderr } = await execFileAsync( process.execPath, - [resolveBrowseCliEntrypoint(), "--json", "--session", this.session, ...args], + [resolveBrowseCliEntrypoint(), ...args, "--session", this.session], { cwd: getRepoRootDir(), env: process.env, @@ -208,7 +212,7 @@ class BrowseCliPageHandle implements CorePageHandle { args.push("--wait", opts.waitUntil); } if (typeof opts?.timeoutMs === "number") { - args.push("-t", String(opts.timeoutMs)); + args.push("--timeout", String(opts.timeoutMs)); } const result = await this.runCommandAfterSelecting<{ url: string }>(args); this.cachedUrl = result.url; @@ -280,15 +284,15 @@ class BrowseCliPageHandle implements CorePageHandle { type?: "png" | "jpeg"; quality?: number; }): Promise { - const args = ["screenshot"]; + const args = ["screenshot", "--base64"]; if (opts?.fullPage) { - args.push("-f"); + args.push("--full-page"); } if (opts?.type) { - args.push("-t", opts.type); + args.push("--type", opts.type); } if (typeof opts?.quality === "number") { - args.push("-q", String(opts.quality)); + args.push("--quality", String(opts.quality)); } const result = await this.runCommandAfterSelecting<{ base64: string }>(args); @@ -310,9 +314,9 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "selector", spec.selector, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), - "-s", + "--state", spec.state ?? "visible", ]); return; @@ -324,7 +328,7 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "load", spec.state, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), ]); return; @@ -372,7 +376,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("click(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["click_xy", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(targetOrX), String(y)]); return; } @@ -387,7 +391,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["click", this.refSelector(target.value)]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support click target kind "${target.kind}" yet`); @@ -399,7 +403,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("hover(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["hover", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(targetOrX), String(y)]); return; } @@ -409,11 +413,11 @@ class BrowseCliPageHandle implements CorePageHandle { switch (target.kind) { case "selector": { const point = await this.resolveHoverPoint(target.value); - await this.runCommandAfterSelecting(["hover", String(point.x), String(point.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(point.x), String(point.y)]); return; } case "coords": - await this.runCommandAfterSelecting(["hover", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support hover target kind "${target.kind}" yet`); @@ -422,6 +426,7 @@ class BrowseCliPageHandle implements CorePageHandle { async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise { await this.runCommandAfterSelecting([ + "mouse", "scroll", String(x), String(y), @@ -450,7 +455,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["type", text]); return; case "selector": - await this.runCommandAfterSelecting(["fill", target.value, text, "--no-press-enter"]); + await this.runCommandAfterSelecting(["fill", target.value, text]); return; default: throw new Error(`browse_cli does not support type target kind "${target.kind}" yet`); @@ -485,7 +490,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["press", key]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); await this.runCommandAfterSelecting(["press", key]); return; default: @@ -519,7 +524,7 @@ class BrowseCliPageHandle implements CorePageHandle { } } -class BrowseCliSession implements CoreSession { +export class BrowseCliSession implements CoreSession { readonly runtime: BrowseCliRuntime; private readonly handles = new Map(); private activePageId: string | null = null; @@ -541,9 +546,14 @@ class BrowseCliSession implements CoreSession { return handle; } - private async fetchPages(): Promise { - const result = await this.runtime.runJson(["pages"]); - const pages = result.pages ?? []; + private async fetchPages(): Promise> { + const result = await this.runtime.runJson(["tab", "list"]); + const pages = (result.tabs ?? []).map((tab) => { + if (!tab.targetId) { + throw new Error(`browse tab list returned no targetId for tab index ${tab.index}`); + } + return { ...tab, targetId: tab.targetId }; + }); for (const page of pages) { this.wrap(page); @@ -583,12 +593,11 @@ class BrowseCliSession implements CoreSession { } async newPage(url?: string): Promise { - const args = ["newpage"]; + const args = ["tab", "new"]; if (url) { args.push(url); } const result = await this.runtime.runJson<{ - created: boolean; url: string; targetId: string; }>(args); @@ -604,7 +613,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_switch", String(page.index)]); + await this.runtime.runJson(["tab", "switch", page.targetId]); this.activePageId = pageId; } @@ -615,7 +624,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_close", String(page.index)]); + await this.runtime.runJson(["tab", "close", page.targetId]); this.handles.delete(pageId); const remaining = await this.fetchPages(); this.activePageId = remaining[0]?.targetId ?? null; @@ -655,10 +664,6 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo return "launch"; } -function createSessionName(): string { - return `evals-browse-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - export class BrowseCliTool implements CoreTool { readonly id = "browse_cli"; readonly surface = "cli"; @@ -687,11 +692,17 @@ export class BrowseCliTool implements CoreTool { ); } - const session = new BrowseCliSession(createSessionName()); - await session.runtime.runJson([ - "env", - input.environment === "BROWSERBASE" ? "remote" : "local", - ]); + const session = new BrowseCliSession(createBrowseCliSessionName()); + try { + await session.runtime.runJson([ + "open", + "about:blank", + input.environment === "BROWSERBASE" ? "--remote" : "--local", + ]); + } catch (error) { + await session.close().catch(() => {}); + throw error; + } return { session, diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index d0e7feb152..9df2063dfe 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -11,6 +11,7 @@ import { BROWSE_CLI_ENTRYPOINT, BROWSE_CLI_PACKAGE_JSON, BROWSE_SKILL_SOURCE, + createBrowseCliSessionName, } from "../browseCliPaths.js"; import { AGENT_RUN_TOOL_NAME, @@ -277,7 +278,7 @@ export async function prepareBrowseCliHarnessAdapter( ); } - const session = createBrowseSessionName(); + const session = createBrowseCliSessionName(); const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-browse-")); const wrapperPath = path.join(cwd, "browse"); await installBrowseSkill(cwd); @@ -726,10 +727,6 @@ function clip(value: string, maxLength: number): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; } -function createBrowseSessionName(): string { - return `evals-claude-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - async function runBrowseCommand( wrapperPath: string, args: string[], diff --git a/packages/evals/package.json b/packages/evals/package.json index 3b6c143ead..0e26ace598 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -40,7 +40,7 @@ "@opentelemetry/sdk-trace-node": "^2.9.0", "ai": "^5.0.133", "braintrust": "^3.21.0", - "browse": "0.9.5", + "browse": "workspace:*", "dotenv": "^17.3.1", "langsmith": "^0.5.16", "openai": "^4.104.0", diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts new file mode 100644 index 0000000000..0ac198ae5b --- /dev/null +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from "vitest"; +import { BrowseCliSession } from "../../core/tools/browse_cli.js"; + +describe("workspace Browse adapter", () => { + it("preserves page identity when tabs are reordered and addresses commands by target ID", async () => { + const session = new BrowseCliSession("eval-session"); + let tabs = [ + { index: 0, targetId: "tab-a", url: "about:blank" }, + { index: 1, targetId: "tab-b", url: "https://example.com" }, + ]; + const runJson = vi.spyOn(session.runtime, "runJson").mockImplementation(async (args) => { + if (args[0] === "tab" && args[1] === "list") return { tabs }; + if (args[0] === "tab" && args[1] === "close") { + tabs = tabs.filter((tab) => tab.targetId !== args[2]); + } + return {}; + }); + + const [, secondPage] = await session.listPages(); + tabs = [ + { index: 0, targetId: "tab-b", url: "https://example.com/updated" }, + { index: 1, targetId: "tab-a", url: "about:blank" }, + ]; + const reorderedPages = await session.listPages(); + expect(reorderedPages[0]).toBe(secondPage); + expect(secondPage.url()).toBe("https://example.com/updated"); + + await session.selectPage(secondPage.id); + expect(runJson).toHaveBeenLastCalledWith(["tab", "switch", "tab-b"]); + await session.closePage(secondPage.id); + expect(runJson).toHaveBeenCalledWith(["tab", "close", "tab-b"]); + await expect(session.listPages()).resolves.toEqual([expect.objectContaining({ id: "tab-a" })]); + }); + + it("rejects tab results that cannot identify a page reliably", async () => { + const session = new BrowseCliSession("eval-session"); + vi.spyOn(session.runtime, "runJson").mockResolvedValue({ + tabs: [{ index: 0, url: "about:blank" }], + }); + + await expect(session.listPages()).rejects.toThrow( + "browse tab list returned no targetId for tab index 0", + ); + }); +}); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5acd0d939a..c214dd8729 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -110,11 +110,14 @@ describe("claude code tool adapter resolution", () => { expect(getBrowseCliAllowedTools()).toEqual(["Skill", "Bash"]); }); - it("exposes browse cli metadata for Braintrust rows", () => { + it("exposes workspace browse cli metadata for Braintrust rows", async () => { + const { version } = JSON.parse( + await fsp.readFile(new URL("../../../cli/package.json", import.meta.url), "utf8"), + ); expect(getBrowseCliToolMetadata()).toMatchObject({ toolCommand: "browse", - browseCliVersion: "0.9.5", - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliVersion: version, + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), }); }); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index 4a0cab5e5b..2d7f73c21c 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -259,7 +259,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), harnessConfig: { harness: "claude_code", model: "anthropic/claude-sonnet-4-20250514", @@ -317,7 +317,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), harnessConfig: { harness: "codex", model: "openai/gpt-5.4-mini", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c769a88398..4e74460909 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -596,8 +596,8 @@ importers: specifier: ^3.21.0 version: 3.28.0(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3) browse: - specifier: 0.9.5 - version: 0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1) + specifier: workspace:* + version: link:../cli dotenv: specifier: ^17.3.1 version: 17.4.2 From 629fe4b5b39bd3b100badc2b850a29dd70444821 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 17 Sep 2026 17:45:16 +0000 Subject: [PATCH 2/4] fix(evals): address Browse adapter review findings Follow the CLI-selected tab after closing, use cryptographic session names, and throw a typed error for missing target IDs. Add tab-selection regressions and format the inherited README to unblock CI. --- README.md | 45 ++++++++----------- packages/evals/browseCliPaths.ts | 6 +-- packages/evals/core/tools/browse_cli.ts | 13 ++++-- .../tests/core/browse-cli-adapter.test.ts | 35 +++++++++++++++ 4 files changed, 65 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 724e85c549..de46521a2b 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,9 @@

- ## AI that uses the browser like humans. -Sign in once, keep the session, and pull structured data out the other side. - - +Sign in once, keep the session, and pull structured data out the other side. ```typescript import { localBrowser, Stagehand } from "@browserbasehq/stagehand"; @@ -72,9 +69,7 @@ await stagehand.act("open the billing page"); const { data } = await stagehand.extract( "extract every invoice in the table", z.object({ - invoices: z.array( - z.object({ number: z.string(), amount: z.number(), paid: z.boolean() }), - ), + invoices: z.array(z.object({ number: z.string(), amount: z.number(), paid: z.boolean() })), }), ); @@ -258,7 +253,6 @@ func run(ctx context.Context) (err error) { - ## Install ```bash @@ -285,18 +279,16 @@ go get github.com/browserbase/stagehand/packages/sdk-go/v4@v4.0.0 Local runs need [Chrome](https://www.google.com/chrome/) installed. Full setup: [Quickstart](https://docs.stagehand.dev/v4/first-steps/quickstart). - ## Why Stagehand -| | | -| --- | --- | -| **Familiar APIs** | The Playwright-style methods you and your agents already know: `goto`, `click`, `locator`, `screenshot`. | -| **Token efficiency** | Hybrid accessibility-tree trimming gives agents exactly the page context they need and nothing more. | -| **Faster in production** | Stagehand runs as an extension next to the browser, cutting round-trip latency on every action. | -| **Self-healing** | `act`, `observe`, and `extract` refresh how an action happens when the site changes underneath it. | -| **Built for agents** | WebMCP, clipboard support, batch commands, deep locators for nested iframes and closed Shadow DOMs, OTel traces. | -| **Three languages** | One complete browser driver across TypeScript, Python, and Go. | - +| | | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------- | +| **Familiar APIs** | The Playwright-style methods you and your agents already know: `goto`, `click`, `locator`, `screenshot`. | +| **Token efficiency** | Hybrid accessibility-tree trimming gives agents exactly the page context they need and nothing more. | +| **Faster in production** | Stagehand runs as an extension next to the browser, cutting round-trip latency on every action. | +| **Self-healing** | `act`, `observe`, and `extract` refresh how an action happens when the site changes underneath it. | +| **Built for agents** | WebMCP, clipboard support, batch commands, deep locators for nested iframes and closed Shadow DOMs, OTel traces. | +| **Three languages** | One complete browser driver across TypeScript, Python, and Go. | ## Run it on Browserbase @@ -408,17 +400,16 @@ console.log(fetched.content); [Search](https://docs.stagehand.dev/v4/add-ons/search) · [Fetch](https://docs.stagehand.dev/v4/add-ons/fetch) - ## Docs and resources -| | | -| --- | --- | -| [Quickstart](https://docs.stagehand.dev/v4/first-steps/quickstart) | Empty directory to working automation in three steps | -| [act](https://docs.stagehand.dev/v4/basics/act) · [extract](https://docs.stagehand.dev/v4/basics/extract) · [observe](https://docs.stagehand.dev/v4/basics/observe) | The three primitives | -| [Migrate from Playwright](https://docs.stagehand.dev/v4/migrations/playwright) | Port an existing suite | -| [Integrations](https://docs.stagehand.dev/v4/integrations/overview) | CrewAI, Mastra, Deep Agents, Vercel AI SDK, Claude Code, Codex | -| [Python SDK](./packages/sdk-python/README.md) · [Go SDK](./packages/sdk-go/README.md) | Language-specific guides | -| [Ask DeepWiki](https://deepwiki.com/browserbase/stagehand) | Ask questions about this codebase | +| | | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| [Quickstart](https://docs.stagehand.dev/v4/first-steps/quickstart) | Empty directory to working automation in three steps | +| [act](https://docs.stagehand.dev/v4/basics/act) · [extract](https://docs.stagehand.dev/v4/basics/extract) · [observe](https://docs.stagehand.dev/v4/basics/observe) | The three primitives | +| [Migrate from Playwright](https://docs.stagehand.dev/v4/migrations/playwright) | Port an existing suite | +| [Integrations](https://docs.stagehand.dev/v4/integrations/overview) | CrewAI, Mastra, Deep Agents, Vercel AI SDK, Claude Code, Codex | +| [Python SDK](./packages/sdk-python/README.md) · [Go SDK](./packages/sdk-go/README.md) | Language-specific guides | +| [Ask DeepWiki](https://deepwiki.com/browserbase/stagehand) | Ask questions about this codebase | ## Join the community diff --git a/packages/evals/browseCliPaths.ts b/packages/evals/browseCliPaths.ts index 817b1bbb04..d6e4fbc561 100644 --- a/packages/evals/browseCliPaths.ts +++ b/packages/evals/browseCliPaths.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import { fileURLToPath } from "node:url"; import path from "node:path"; @@ -12,7 +13,6 @@ export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json"); export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md"); export function createBrowseCliSessionName(): string { - const timestamp = Date.now().toString(36); - const random = Math.random().toString(36).slice(2, 6); - return `eval-${process.pid}-${timestamp}-${random}`; + const random = randomBytes(8).toString("hex"); + return `eval-${process.pid}-${random}`; } diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index 38a7a9ba4b..61ff7fc43d 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -20,6 +20,7 @@ import { createBrowseCliSessionName, } from "../../browseCliPaths.js"; import { getRepoRootDir } from "../../runtimePaths.js"; +import { EvalsError } from "../../errors.js"; const execFileAsync = promisify(execFile); @@ -550,7 +551,7 @@ export class BrowseCliSession implements CoreSession { const result = await this.runtime.runJson(["tab", "list"]); const pages = (result.tabs ?? []).map((tab) => { if (!tab.targetId) { - throw new Error(`browse tab list returned no targetId for tab index ${tab.index}`); + throw new EvalsError(`browse tab list returned no targetId for tab index ${tab.index}`); } return { ...tab, targetId: tab.targetId }; }); @@ -624,10 +625,14 @@ export class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab", "close", page.targetId]); + const result = await this.runtime.runJson<{ selectedTargetId?: string }>([ + "tab", + "close", + page.targetId, + ]); this.handles.delete(pageId); - const remaining = await this.fetchPages(); - this.activePageId = remaining[0]?.targetId ?? null; + this.activePageId = result.selectedTargetId ?? null; + await this.fetchPages(); } async close(): Promise { diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts index 0ac198ae5b..beca0f31e2 100644 --- a/packages/evals/tests/core/browse-cli-adapter.test.ts +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { BrowseCliSession } from "../../core/tools/browse_cli.js"; +import { EvalsError } from "../../errors.js"; describe("workspace Browse adapter", () => { it("preserves page identity when tabs are reordered and addresses commands by target ID", async () => { @@ -12,6 +13,7 @@ describe("workspace Browse adapter", () => { if (args[0] === "tab" && args[1] === "list") return { tabs }; if (args[0] === "tab" && args[1] === "close") { tabs = tabs.filter((tab) => tab.targetId !== args[2]); + return { selectedTargetId: tabs[0]?.targetId }; } return {}; }); @@ -32,12 +34,45 @@ describe("workspace Browse adapter", () => { await expect(session.listPages()).resolves.toEqual([expect.objectContaining({ id: "tab-a" })]); }); + it.each([ + { active: "tab-b", closed: "tab-b", selected: "tab-c" }, + { active: "tab-c", closed: "tab-b", selected: "tab-c" }, + { active: "tab-c", closed: "tab-c", selected: "tab-b" }, + ])( + "tracks the CLI-selected tab $selected after closing $closed with $active active", + async ({ active, closed, selected }) => { + const session = new BrowseCliSession("eval-session"); + let tabs = ["tab-a", "tab-b", "tab-c"].map((targetId, index) => ({ + index, + targetId, + url: "about:blank", + })); + const runJson = vi.spyOn(session.runtime, "runJson").mockImplementation(async (args) => { + if (args[0] === "tab" && args[1] === "list") return { tabs }; + if (args[0] === "tab" && args[1] === "close") { + tabs = tabs.filter((tab) => tab.targetId !== args[2]); + return { selectedTargetId: selected }; + } + return {}; + }); + + await session.selectPage(active); + await session.closePage(closed); + expect((await session.activePage()).id).toBe(selected); + + // A command targeting a different surviving tab must first switch the CLI to it. + await session.selectIfNeeded("tab-a"); + expect(runJson).toHaveBeenLastCalledWith(["tab", "switch", "tab-a"]); + }, + ); + it("rejects tab results that cannot identify a page reliably", async () => { const session = new BrowseCliSession("eval-session"); vi.spyOn(session.runtime, "runJson").mockResolvedValue({ tabs: [{ index: 0, url: "about:blank" }], }); + await expect(session.listPages()).rejects.toBeInstanceOf(EvalsError); await expect(session.listPages()).rejects.toThrow( "browse tab list returned no targetId for tab index 0", ); From 65702dc1ee354cbfc5cab67c10af135e75b9b6f0 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 17 Sep 2026 17:54:28 +0000 Subject: [PATCH 3/4] test(evals): cover Browse session name safety and uniqueness --- packages/evals/tests/browseCliPaths.test.ts | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 packages/evals/tests/browseCliPaths.test.ts diff --git a/packages/evals/tests/browseCliPaths.test.ts b/packages/evals/tests/browseCliPaths.test.ts new file mode 100644 index 0000000000..1c635951e1 --- /dev/null +++ b/packages/evals/tests/browseCliPaths.test.ts @@ -0,0 +1,22 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createBrowseCliSessionName } from "../browseCliPaths.js"; + +describe("Browse CLI session names", () => { + afterEach(() => vi.restoreAllMocks()); + + it("includes the PID and stays short with socket-safe characters", () => { + const name = createBrowseCliSessionName(); + + expect(name).toMatch(new RegExp(`^eval-${process.pid}-[A-Za-z0-9_-]+$`)); + expect(Buffer.byteLength(name)).toBeLessThanOrEqual(32); + }); + + it("keeps repeated calls distinct at the same timestamp without Math.random entropy", () => { + vi.spyOn(Date, "now").mockReturnValue(1_000); + vi.spyOn(Math, "random").mockReturnValue(0.5); + + const names = Array.from({ length: 1_000 }, () => createBrowseCliSessionName()); + + expect(new Set(names).size).toBe(names.length); + }); +}); From a7711cac689d60a44472afef555f68d1ba3a82bd Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 17 Sep 2026 18:15:51 +0000 Subject: [PATCH 4/4] fix(evals): use workspace Browse to unblock release builds --- README.md | 45 ++++++++----------- packages/evals/package.json | 2 +- .../framework/claudeCodeToolAdapter.test.ts | 9 ++-- packages/evals/tests/tui/run.test.ts | 4 +- pnpm-lock.yaml | 4 +- 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 724e85c549..de46521a2b 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,9 @@

- ## AI that uses the browser like humans. -Sign in once, keep the session, and pull structured data out the other side. - - +Sign in once, keep the session, and pull structured data out the other side. ```typescript import { localBrowser, Stagehand } from "@browserbasehq/stagehand"; @@ -72,9 +69,7 @@ await stagehand.act("open the billing page"); const { data } = await stagehand.extract( "extract every invoice in the table", z.object({ - invoices: z.array( - z.object({ number: z.string(), amount: z.number(), paid: z.boolean() }), - ), + invoices: z.array(z.object({ number: z.string(), amount: z.number(), paid: z.boolean() })), }), ); @@ -258,7 +253,6 @@ func run(ctx context.Context) (err error) { - ## Install ```bash @@ -285,18 +279,16 @@ go get github.com/browserbase/stagehand/packages/sdk-go/v4@v4.0.0 Local runs need [Chrome](https://www.google.com/chrome/) installed. Full setup: [Quickstart](https://docs.stagehand.dev/v4/first-steps/quickstart). - ## Why Stagehand -| | | -| --- | --- | -| **Familiar APIs** | The Playwright-style methods you and your agents already know: `goto`, `click`, `locator`, `screenshot`. | -| **Token efficiency** | Hybrid accessibility-tree trimming gives agents exactly the page context they need and nothing more. | -| **Faster in production** | Stagehand runs as an extension next to the browser, cutting round-trip latency on every action. | -| **Self-healing** | `act`, `observe`, and `extract` refresh how an action happens when the site changes underneath it. | -| **Built for agents** | WebMCP, clipboard support, batch commands, deep locators for nested iframes and closed Shadow DOMs, OTel traces. | -| **Three languages** | One complete browser driver across TypeScript, Python, and Go. | - +| | | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------- | +| **Familiar APIs** | The Playwright-style methods you and your agents already know: `goto`, `click`, `locator`, `screenshot`. | +| **Token efficiency** | Hybrid accessibility-tree trimming gives agents exactly the page context they need and nothing more. | +| **Faster in production** | Stagehand runs as an extension next to the browser, cutting round-trip latency on every action. | +| **Self-healing** | `act`, `observe`, and `extract` refresh how an action happens when the site changes underneath it. | +| **Built for agents** | WebMCP, clipboard support, batch commands, deep locators for nested iframes and closed Shadow DOMs, OTel traces. | +| **Three languages** | One complete browser driver across TypeScript, Python, and Go. | ## Run it on Browserbase @@ -408,17 +400,16 @@ console.log(fetched.content); [Search](https://docs.stagehand.dev/v4/add-ons/search) · [Fetch](https://docs.stagehand.dev/v4/add-ons/fetch) - ## Docs and resources -| | | -| --- | --- | -| [Quickstart](https://docs.stagehand.dev/v4/first-steps/quickstart) | Empty directory to working automation in three steps | -| [act](https://docs.stagehand.dev/v4/basics/act) · [extract](https://docs.stagehand.dev/v4/basics/extract) · [observe](https://docs.stagehand.dev/v4/basics/observe) | The three primitives | -| [Migrate from Playwright](https://docs.stagehand.dev/v4/migrations/playwright) | Port an existing suite | -| [Integrations](https://docs.stagehand.dev/v4/integrations/overview) | CrewAI, Mastra, Deep Agents, Vercel AI SDK, Claude Code, Codex | -| [Python SDK](./packages/sdk-python/README.md) · [Go SDK](./packages/sdk-go/README.md) | Language-specific guides | -| [Ask DeepWiki](https://deepwiki.com/browserbase/stagehand) | Ask questions about this codebase | +| | | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | +| [Quickstart](https://docs.stagehand.dev/v4/first-steps/quickstart) | Empty directory to working automation in three steps | +| [act](https://docs.stagehand.dev/v4/basics/act) · [extract](https://docs.stagehand.dev/v4/basics/extract) · [observe](https://docs.stagehand.dev/v4/basics/observe) | The three primitives | +| [Migrate from Playwright](https://docs.stagehand.dev/v4/migrations/playwright) | Port an existing suite | +| [Integrations](https://docs.stagehand.dev/v4/integrations/overview) | CrewAI, Mastra, Deep Agents, Vercel AI SDK, Claude Code, Codex | +| [Python SDK](./packages/sdk-python/README.md) · [Go SDK](./packages/sdk-go/README.md) | Language-specific guides | +| [Ask DeepWiki](https://deepwiki.com/browserbase/stagehand) | Ask questions about this codebase | ## Join the community diff --git a/packages/evals/package.json b/packages/evals/package.json index 3b6c143ead..0e26ace598 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -40,7 +40,7 @@ "@opentelemetry/sdk-trace-node": "^2.9.0", "ai": "^5.0.133", "braintrust": "^3.21.0", - "browse": "0.9.5", + "browse": "workspace:*", "dotenv": "^17.3.1", "langsmith": "^0.5.16", "openai": "^4.104.0", diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5acd0d939a..c214dd8729 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -110,11 +110,14 @@ describe("claude code tool adapter resolution", () => { expect(getBrowseCliAllowedTools()).toEqual(["Skill", "Bash"]); }); - it("exposes browse cli metadata for Braintrust rows", () => { + it("exposes workspace browse cli metadata for Braintrust rows", async () => { + const { version } = JSON.parse( + await fsp.readFile(new URL("../../../cli/package.json", import.meta.url), "utf8"), + ); expect(getBrowseCliToolMetadata()).toMatchObject({ toolCommand: "browse", - browseCliVersion: "0.9.5", - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliVersion: version, + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), }); }); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index 4a0cab5e5b..2d7f73c21c 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -259,7 +259,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), harnessConfig: { harness: "claude_code", model: "anthropic/claude-sonnet-4-20250514", @@ -317,7 +317,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: expect.stringMatching(/packages[/\\]cli[/\\]bin[/\\]run\.js$/u), harnessConfig: { harness: "codex", model: "openai/gpt-5.4-mini", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c769a88398..4e74460909 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -596,8 +596,8 @@ importers: specifier: ^3.21.0 version: 3.28.0(@aws-sdk/credential-provider-web-identity@3.972.74)(zod@4.4.3) browse: - specifier: 0.9.5 - version: 0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1) + specifier: workspace:* + version: link:../cli dotenv: specifier: ^17.3.1 version: 17.4.2