fix(evals): use workspace Browse and repair CLI adapter

packages/evals/browseCliPaths.ts

@@ -1,3 +1,4 @@
+import { randomBytes } from "node:crypto";
 import { fileURLToPath } from "node:url";
 import path from "node:path";
 
@@ -10,3 +11,8 @@ export const BROWSE_CLI_BUILD_ARTIFACTS = [
 ];
 export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json");
 export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md");
+
+export function createBrowseCliSessionName(): string {
+  const random = randomBytes(8).toString("hex");
+  return `eval-${process.pid}-${random}`;
+}

packages/evals/core/tools/browse_cli.ts

@@ -14,8 +14,13 @@ import type {
   ToolStartInput,
   ToolStartResult,
 } from "../contracts/tool.js";
-import { BROWSE_CLI_BUILD_ARTIFACTS, BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js";
+import {
+  BROWSE_CLI_BUILD_ARTIFACTS,
+  BROWSE_CLI_ENTRYPOINT,
+  createBrowseCliSessionName,
+} from "../../browseCliPaths.js";
 import { getRepoRootDir } from "../../runtimePaths.js";
+import { EvalsError } from "../../errors.js";
 
 const execFileAsync = promisify(execFile);
 
@@ -79,11 +84,11 @@ function buildSelectorQuery(selector: string): string {
   `;
 }
 
-type BrowseCliPagesResult = {
-  pages: Array<{
+type BrowseCliTabsResult = {
+  tabs: Array<{
     index: number;
     url: string;
-    targetId: string;
+    targetId?: string;
   }>;
 };
 
@@ -93,7 +98,7 @@ class BrowseCliRuntime {
   async runJson<T>(args: string[]): Promise<T> {
     const { stdout, stderr } = await execFileAsync(
       process.execPath,
-      [resolveBrowseCliEntrypoint(), "--json", "--session", this.session, ...args],
+      [resolveBrowseCliEntrypoint(), ...args, "--session", this.session],
       {
         cwd: getRepoRootDir(),
         env: process.env,
@@ -208,7 +213,7 @@ class BrowseCliPageHandle implements CorePageHandle {
       args.push("--wait", opts.waitUntil);
     }
     if (typeof opts?.timeoutMs === "number") {
-      args.push("-t", String(opts.timeoutMs));
+      args.push("--timeout", String(opts.timeoutMs));
     }
     const result = await this.runCommandAfterSelecting<{ url: string }>(args);
     this.cachedUrl = result.url;
@@ -280,15 +285,15 @@ class BrowseCliPageHandle implements CorePageHandle {
     type?: "png" | "jpeg";
     quality?: number;
   }): Promise<Buffer> {
-    const args = ["screenshot"];
+    const args = ["screenshot", "--base64"];
     if (opts?.fullPage) {
-      args.push("-f");
+      args.push("--full-page");
     }
     if (opts?.type) {
-      args.push("-t", opts.type);
+      args.push("--type", opts.type);
     }
     if (typeof opts?.quality === "number") {
-      args.push("-q", String(opts.quality));
+      args.push("--quality", String(opts.quality));
     }
 
     const result = await this.runCommandAfterSelecting<{ base64: string }>(args);
@@ -310,9 +315,9 @@ class BrowseCliPageHandle implements CorePageHandle {
           "wait",
           "selector",
           spec.selector,
-          "-t",
+          "--timeout",
           String(spec.timeoutMs ?? 30_000),
-          "-s",
+          "--state",
           spec.state ?? "visible",
         ]);
         return;
@@ -324,7 +329,7 @@ class BrowseCliPageHandle implements CorePageHandle {
           "wait",
           "load",
           spec.state,
-          "-t",
+          "--timeout",
           String(spec.timeoutMs ?? 30_000),
         ]);
         return;
@@ -372,7 +377,7 @@ class BrowseCliPageHandle implements CorePageHandle {
       if (typeof y !== "number") {
         throw new Error("click(x, y) requires both numeric coordinates");
       }
-      await this.runCommandAfterSelecting(["click_xy", String(targetOrX), String(y)]);
+      await this.runCommandAfterSelecting(["mouse", "click", String(targetOrX), String(y)]);
       return;
     }
 
@@ -387,7 +392,7 @@ class BrowseCliPageHandle implements CorePageHandle {
         await this.runCommandAfterSelecting(["click", this.refSelector(target.value)]);
         return;
       case "coords":
-        await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]);
+        await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]);
         return;
       default:
         throw new Error(`browse_cli does not support click target kind "${target.kind}" yet`);
@@ -399,7 +404,7 @@ class BrowseCliPageHandle implements CorePageHandle {
       if (typeof y !== "number") {
         throw new Error("hover(x, y) requires both numeric coordinates");
       }
-      await this.runCommandAfterSelecting(["hover", String(targetOrX), String(y)]);
+      await this.runCommandAfterSelecting(["mouse", "hover", String(targetOrX), String(y)]);
       return;
     }
 
@@ -409,11 +414,11 @@ class BrowseCliPageHandle implements CorePageHandle {
     switch (target.kind) {
       case "selector": {
         const point = await this.resolveHoverPoint(target.value);
-        await this.runCommandAfterSelecting(["hover", String(point.x), String(point.y)]);
+        await this.runCommandAfterSelecting(["mouse", "hover", String(point.x), String(point.y)]);
         return;
       }
       case "coords":
-        await this.runCommandAfterSelecting(["hover", String(target.x), String(target.y)]);
+        await this.runCommandAfterSelecting(["mouse", "hover", String(target.x), String(target.y)]);
         return;
       default:
         throw new Error(`browse_cli does not support hover target kind "${target.kind}" yet`);
@@ -422,6 +427,7 @@ class BrowseCliPageHandle implements CorePageHandle {
 
   async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise<void> {
     await this.runCommandAfterSelecting([
+      "mouse",
       "scroll",
       String(x),
       String(y),
@@ -450,7 +456,7 @@ class BrowseCliPageHandle implements CorePageHandle {
         await this.runCommandAfterSelecting(["type", text]);
         return;
       case "selector":
-        await this.runCommandAfterSelecting(["fill", target.value, text, "--no-press-enter"]);
+        await this.runCommandAfterSelecting(["fill", target.value, text]);
         return;
       default:
         throw new Error(`browse_cli does not support type target kind "${target.kind}" yet`);
@@ -485,7 +491,7 @@ class BrowseCliPageHandle implements CorePageHandle {
         await this.runCommandAfterSelecting(["press", key]);
         return;
       case "coords":
-        await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]);
+        await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]);
         await this.runCommandAfterSelecting(["press", key]);
         return;
       default:
@@ -519,7 +525,7 @@ class BrowseCliPageHandle implements CorePageHandle {
   }
 }
 
-class BrowseCliSession implements CoreSession {
+export class BrowseCliSession implements CoreSession {
   readonly runtime: BrowseCliRuntime;
   private readonly handles = new Map<string, BrowseCliPageHandle>();
   private activePageId: string | null = null;
@@ -541,9 +547,14 @@ class BrowseCliSession implements CoreSession {
     return handle;
   }
 
-  private async fetchPages(): Promise<BrowseCliPagesResult["pages"]> {
-    const result = await this.runtime.runJson<BrowseCliPagesResult>(["pages"]);
-    const pages = result.pages ?? [];
+  private async fetchPages(): Promise<Array<{ index: number; url: string; targetId: string }>> {
+    const result = await this.runtime.runJson<BrowseCliTabsResult>(["tab", "list"]);
+    const pages = (result.tabs ?? []).map((tab) => {
+      if (!tab.targetId) {
+        throw new EvalsError(`browse tab list returned no targetId for tab index ${tab.index}`);
+      }
+      return { ...tab, targetId: tab.targetId };
+    });
 
     for (const page of pages) {
       this.wrap(page);
@@ -583,12 +594,11 @@ class BrowseCliSession implements CoreSession {
   }
 
   async newPage(url?: string): Promise<CorePageHandle> {
-    const args = ["newpage"];
+    const args = ["tab", "new"];
     if (url) {
       args.push(url);
     }
     const result = await this.runtime.runJson<{
-      created: boolean;
       url: string;
       targetId: string;
     }>(args);
@@ -604,7 +614,7 @@ class BrowseCliSession implements CoreSession {
       throw new Error(`Unknown page id "${pageId}"`);
     }
 
-    await this.runtime.runJson(["tab_switch", String(page.index)]);
+    await this.runtime.runJson(["tab", "switch", page.targetId]);
     this.activePageId = pageId;
   }
 
@@ -615,10 +625,14 @@ class BrowseCliSession implements CoreSession {
       throw new Error(`Unknown page id "${pageId}"`);
     }
 
-    await this.runtime.runJson(["tab_close", String(page.index)]);
+    const result = await this.runtime.runJson<{ selectedTargetId?: string }>([
+      "tab",
+      "close",
+      page.targetId,
+    ]);
     this.handles.delete(pageId);
-    const remaining = await this.fetchPages();
-    this.activePageId = remaining[0]?.targetId ?? null;
+    this.activePageId = result.selectedTargetId ?? null;
+    await this.fetchPages();
   }
 
   async close(): Promise<void> {
@@ -655,10 +669,6 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo
   return "launch";
 }
 
-function createSessionName(): string {
-  return `evals-browse-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
-}
-
 export class BrowseCliTool implements CoreTool {
   readonly id = "browse_cli";
   readonly surface = "cli";
@@ -687,11 +697,17 @@ export class BrowseCliTool implements CoreTool {
       );
     }
 
-    const session = new BrowseCliSession(createSessionName());
-    await session.runtime.runJson([
-      "env",
-      input.environment === "BROWSERBASE" ? "remote" : "local",
-    ]);
+    const session = new BrowseCliSession(createBrowseCliSessionName());
+    try {
+      await session.runtime.runJson([
+        "open",
+        "about:blank",
+        input.environment === "BROWSERBASE" ? "--remote" : "--local",
+      ]);
+    } catch (error) {
+      await session.close().catch(() => {});
+      throw error;
+    }
 
     return {
       session,

packages/evals/framework/claudeCodeToolAdapter.ts

@@ -11,6 +11,7 @@ import {
   BROWSE_CLI_ENTRYPOINT,
   BROWSE_CLI_PACKAGE_JSON,
   BROWSE_SKILL_SOURCE,
+  createBrowseCliSessionName,
 } from "../browseCliPaths.js";
 import {
   AGENT_RUN_TOOL_NAME,
@@ -277,7 +278,7 @@ export async function prepareBrowseCliHarnessAdapter(
     );
   }
 
-  const session = createBrowseSessionName();
+  const session = createBrowseCliSessionName();
   const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-browse-"));
   const wrapperPath = path.join(cwd, "browse");
   await installBrowseSkill(cwd);
@@ -726,10 +727,6 @@ function clip(value: string, maxLength: number): string {
   return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`;
 }
 
-function createBrowseSessionName(): string {
-  return `evals-claude-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
-}
-
 async function runBrowseCommand(
   wrapperPath: string,
   args: string[],

packages/evals/package.json

@@ -40,7 +40,7 @@
     "@opentelemetry/sdk-trace-node": "^2.9.0",
     "ai": "^5.0.133",
     "braintrust": "^3.21.0",
-    "browse": "0.9.5",
+    "browse": "workspace:*",
     "dotenv": "^17.3.1",
     "langsmith": "^0.5.16",
     "openai": "^4.104.0",

packages/evals/tests/browseCliPaths.test.ts

@@ -0,0 +1,22 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createBrowseCliSessionName } from "../browseCliPaths.js";
+
+describe("Browse CLI session names", () => {
+  afterEach(() => vi.restoreAllMocks());
+
+  it("includes the PID and stays short with socket-safe characters", () => {
+    const name = createBrowseCliSessionName();
+
+    expect(name).toMatch(new RegExp(`^eval-${process.pid}-[A-Za-z0-9_-]+$`));
+    expect(Buffer.byteLength(name)).toBeLessThanOrEqual(32);
+  });
+
+  it("keeps repeated calls distinct at the same timestamp without Math.random entropy", () => {
+    vi.spyOn(Date, "now").mockReturnValue(1_000);
+    vi.spyOn(Math, "random").mockReturnValue(0.5);
+
+    const names = Array.from({ length: 1_000 }, () => createBrowseCliSessionName());
+
+    expect(new Set(names).size).toBe(names.length);
+  });
+});