From bea2d5b65883e0c886c33f8698a36fbd578c1308 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 30 Jul 2026 14:48:58 -0700 Subject: [PATCH 1/5] refactor(evals): target the workspace V4 CLI --- packages/evals/core/tools/browse_cli.ts | 86 +++--- packages/evals/package.json | 2 +- .../tests/core/browse-cli-adapter.test.ts | 112 ++++++++ .../framework/claudeCodeToolAdapter.test.ts | 3 +- packages/evals/tests/tui/run.test.ts | 5 +- pnpm-lock.yaml | 248 ++++++------------ 6 files changed, 247 insertions(+), 209 deletions(-) create mode 100644 packages/evals/tests/core/browse-cli-adapter.test.ts diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index 4e3d7371f3..5d1744a625 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -79,21 +79,33 @@ function buildSelectorQuery(selector: string): string { `; } -type BrowseCliPagesResult = { - pages: Array<{ +type BrowseCliTabsResult = { + tabs: Array<{ index: number; url: string; - targetId: string; + targetId?: string; }>; }; +export function buildBrowseCliProcessArgs( + entrypoint: string, + session: string, + args: string[], +): string[] { + return [entrypoint, ...args, "--session", session]; +} + +export function browseCliStartupArgs(environment: ToolStartInput["environment"]): string[] { + return ["open", "about:blank", environment === "BROWSERBASE" ? "--remote" : "--local"]; +} + class BrowseCliRuntime { constructor(private readonly session: string) {} async runJson(args: string[]): Promise { const { stdout, stderr } = await execFileAsync( process.execPath, - [resolveBrowseCliEntrypoint(), "--json", "--session", this.session, ...args], + buildBrowseCliProcessArgs(resolveBrowseCliEntrypoint(), this.session, args), { cwd: getRepoRootDir(), env: process.env, @@ -162,7 +174,7 @@ class BrowseCliLocatorHandle implements CoreLocatorHandle { } } -class BrowseCliPageHandle implements CorePageHandle { +export class BrowseCliPageHandle implements CorePageHandle { constructor( private readonly session: BrowseCliSession, readonly id: string, @@ -208,7 +220,7 @@ class BrowseCliPageHandle implements CorePageHandle { args.push("--wait", opts.waitUntil); } if (typeof opts?.timeoutMs === "number") { - args.push("-t", String(opts.timeoutMs)); + args.push("--timeout", String(opts.timeoutMs)); } const result = await this.runCommandAfterSelecting<{ url: string }>(args); this.cachedUrl = result.url; @@ -280,15 +292,15 @@ class BrowseCliPageHandle implements CorePageHandle { type?: "png" | "jpeg"; quality?: number; }): Promise { - const args = ["screenshot"]; + const args = ["screenshot", "--base64"]; if (opts?.fullPage) { - args.push("-f"); + args.push("--full-page"); } if (opts?.type) { - args.push("-t", opts.type); + args.push("--type", opts.type); } if (typeof opts?.quality === "number") { - args.push("-q", String(opts.quality)); + args.push("--quality", String(opts.quality)); } const result = await this.runCommandAfterSelecting<{ base64: string }>(args); @@ -310,9 +322,9 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "selector", spec.selector, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), - "-s", + "--state", spec.state ?? "visible", ]); return; @@ -324,7 +336,7 @@ class BrowseCliPageHandle implements CorePageHandle { "wait", "load", spec.state, - "-t", + "--timeout", String(spec.timeoutMs ?? 30_000), ]); return; @@ -372,7 +384,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("click(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["click_xy", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(targetOrX), String(y)]); return; } @@ -387,7 +399,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["click", this.refSelector(target.value)]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support click target kind "${target.kind}" yet`); @@ -399,7 +411,7 @@ class BrowseCliPageHandle implements CorePageHandle { if (typeof y !== "number") { throw new Error("hover(x, y) requires both numeric coordinates"); } - await this.runCommandAfterSelecting(["hover", String(targetOrX), String(y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(targetOrX), String(y)]); return; } @@ -409,11 +421,11 @@ class BrowseCliPageHandle implements CorePageHandle { switch (target.kind) { case "selector": { const point = await this.resolveHoverPoint(target.value); - await this.runCommandAfterSelecting(["hover", String(point.x), String(point.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(point.x), String(point.y)]); return; } case "coords": - await this.runCommandAfterSelecting(["hover", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "hover", String(target.x), String(target.y)]); return; default: throw new Error(`browse_cli does not support hover target kind "${target.kind}" yet`); @@ -422,6 +434,7 @@ class BrowseCliPageHandle implements CorePageHandle { async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise { await this.runCommandAfterSelecting([ + "mouse", "scroll", String(x), String(y), @@ -450,7 +463,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["type", text]); return; case "selector": - await this.runCommandAfterSelecting(["fill", target.value, text, "--no-press-enter"]); + await this.runCommandAfterSelecting(["fill", target.value, text]); return; default: throw new Error(`browse_cli does not support type target kind "${target.kind}" yet`); @@ -485,7 +498,7 @@ class BrowseCliPageHandle implements CorePageHandle { await this.runCommandAfterSelecting(["press", key]); return; case "coords": - await this.runCommandAfterSelecting(["click_xy", String(target.x), String(target.y)]); + await this.runCommandAfterSelecting(["mouse", "click", String(target.x), String(target.y)]); await this.runCommandAfterSelecting(["press", key]); return; default: @@ -519,7 +532,7 @@ class BrowseCliPageHandle implements CorePageHandle { } } -class BrowseCliSession implements CoreSession { +export class BrowseCliSession implements CoreSession { readonly runtime: BrowseCliRuntime; private readonly handles = new Map(); private activePageId: string | null = null; @@ -541,9 +554,14 @@ class BrowseCliSession implements CoreSession { return handle; } - private async fetchPages(): Promise { - const result = await this.runtime.runJson(["pages"]); - const pages = result.pages ?? []; + private async fetchPages(): Promise> { + const result = await this.runtime.runJson(["tab", "list"]); + const pages = (result.tabs ?? []).map((tab) => { + if (!tab.targetId) { + throw new Error(`browse tab list returned no targetId for tab index ${tab.index}`); + } + return { ...tab, targetId: tab.targetId }; + }); for (const page of pages) { this.wrap(page); @@ -583,12 +601,11 @@ class BrowseCliSession implements CoreSession { } async newPage(url?: string): Promise { - const args = ["newpage"]; + const args = ["tab", "new"]; if (url) { args.push(url); } const result = await this.runtime.runJson<{ - created: boolean; url: string; targetId: string; }>(args); @@ -604,7 +621,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_switch", String(page.index)]); + await this.runtime.runJson(["tab", "switch", page.targetId]); this.activePageId = pageId; } @@ -615,7 +632,7 @@ class BrowseCliSession implements CoreSession { throw new Error(`Unknown page id "${pageId}"`); } - await this.runtime.runJson(["tab_close", String(page.index)]); + await this.runtime.runJson(["tab", "close", page.targetId]); this.handles.delete(pageId); const remaining = await this.fetchPages(); this.activePageId = remaining[0]?.targetId ?? null; @@ -655,8 +672,10 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo return "launch"; } -function createSessionName(): string { - return `evals-browse-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +export function createBrowseCliSessionName(): string { + const timestamp = Date.now().toString(36); + const random = Math.random().toString(36).slice(2, 6); + return `eval-${process.pid}-${timestamp}-${random}`; } export class BrowseCliTool implements CoreTool { @@ -687,11 +706,8 @@ export class BrowseCliTool implements CoreTool { ); } - const session = new BrowseCliSession(createSessionName()); - await session.runtime.runJson([ - "env", - input.environment === "BROWSERBASE" ? "remote" : "local", - ]); + const session = new BrowseCliSession(createBrowseCliSessionName()); + await session.runtime.runJson(browseCliStartupArgs(input.environment)); return { session, diff --git a/packages/evals/package.json b/packages/evals/package.json index f59e1dd5be..78dc75c461 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -26,7 +26,7 @@ "@browserbasehq/stagehand": "workspace:*", "@openai/codex-sdk": "0.125.0", "ai": "^5.0.133", - "browse": "0.9.5", + "browse": "workspace:*", "dotenv": "^17.3.1", "openai": "^4.104.0", "playwright": ">=1.55.1 <1.57.0", diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts new file mode 100644 index 0000000000..f509fff5d1 --- /dev/null +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + BrowseCliPageHandle, + BrowseCliSession, + browseCliStartupArgs, + buildBrowseCliProcessArgs, + createBrowseCliSessionName, +} from "../../core/tools/browse_cli.js"; + +describe("browse_cli V4 adapter", () => { + it("places session selection on each V4 command and uses mode-specific startup commands", () => { + expect(buildBrowseCliProcessArgs("/tmp/browse", "eval-session", ["tab", "list"])).toEqual([ + "/tmp/browse", + "tab", + "list", + "--session", + "eval-session", + ]); + expect(browseCliStartupArgs("LOCAL")).toEqual(["open", "about:blank", "--local"]); + expect(browseCliStartupArgs("BROWSERBASE")).toEqual(["open", "about:blank", "--remote"]); + }); + + it("keeps eval session names short enough for temporary Unix socket paths", () => { + const sessionName = createBrowseCliSessionName(); + + expect(sessionName).toMatch(/^eval-\d+-[a-z0-9]+-[a-z0-9]{4}$/u); + expect(sessionName.length).toBeLessThanOrEqual(32); + }); + + it("maps page capabilities onto the V4 command tree and flags", async () => { + const runJson = vi.fn(async (args: string[]) => { + if (args[0] === "open") return { url: args[1] }; + if (args[0] === "screenshot") { + return { base64: Buffer.from("screenshot").toString("base64") }; + } + if (args[0] === "get" && args[1] === "box") return { x: 12, y: 34 }; + return {}; + }); + const session = { + runtime: { runJson }, + selectIfNeeded: vi.fn(), + } as unknown as BrowseCliSession; + const page = new BrowseCliPageHandle(session, "page-1"); + + await page.goto("https://example.com", { + timeoutMs: 1_234, + waitUntil: "networkidle", + }); + await expect(page.screenshot({ fullPage: true, quality: 80, type: "jpeg" })).resolves.toEqual( + Buffer.from("screenshot"), + ); + await page.wait({ + kind: "selector", + selector: "#ready", + state: "attached", + timeoutMs: 2_345, + }); + await page.click(10, 20); + await page.hover({ kind: "selector", value: "#hover-target" }); + await page.scroll(30, 40, 0, 500); + await page.type({ kind: "selector", value: "#text-input" }, "hello"); + await page.press({ kind: "coords", x: 50, y: 60 }, "Enter"); + + expect(runJson.mock.calls.map(([args]) => args)).toEqual([ + ["open", "https://example.com", "--wait", "networkidle", "--timeout", "1234"], + ["screenshot", "--base64", "--full-page", "--type", "jpeg", "--quality", "80"], + ["wait", "selector", "#ready", "--timeout", "2345", "--state", "attached"], + ["mouse", "click", "10", "20"], + ["get", "box", "#hover-target"], + ["mouse", "hover", "12", "34"], + ["mouse", "scroll", "30", "40", "0", "500"], + ["fill", "#text-input", "hello"], + ["mouse", "click", "50", "60"], + ["press", "Enter"], + ]); + }); + + it("reads V4 tab-list results and addresses tabs by stable targetId", async () => { + const session = new BrowseCliSession("eval-session"); + const runJson = vi.spyOn(session.runtime, "runJson").mockImplementation(async (args) => { + if (args[0] === "tab" && args[1] === "list") { + return { + tabs: [ + { index: 0, targetId: "tab-a", url: "about:blank" }, + { index: 1, targetId: "tab-b", url: "https://example.com" }, + ], + }; + } + return {}; + }); + + await expect(session.listPages()).resolves.toEqual([ + expect.objectContaining({ id: "tab-a" }), + expect.objectContaining({ id: "tab-b" }), + ]); + await session.selectPage("tab-b"); + + expect(runJson).toHaveBeenLastCalledWith(["tab", "switch", "tab-b"]); + }); + + it("rejects V4 tab results without a stable targetId", async () => { + const session = new BrowseCliSession("eval-session"); + vi.spyOn(session.runtime, "runJson").mockResolvedValue({ + tabs: [{ index: 0, url: "about:blank" }], + }); + + await expect(session.listPages()).rejects.toThrow( + "browse tab list returned no targetId for tab index 0", + ); + }); +}); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 754292f7b7..420777baa8 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -17,6 +17,7 @@ import { resolveCodexToolSurface, } from "../../framework/codexToolAdapter.js"; import type { CdpEventMessage } from "../../core/tools/cdp_code.js"; +import { BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; describe("claude code tool adapter resolution", () => { afterEach(() => { @@ -86,7 +87,7 @@ describe("claude code tool adapter resolution", () => { expect(getBrowseCliToolMetadata()).toMatchObject({ toolCommand: "browse", browseCliVersion: "0.9.5", - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, }); }); diff --git a/packages/evals/tests/tui/run.test.ts b/packages/evals/tests/tui/run.test.ts index 2c6057ff38..5625273b31 100644 --- a/packages/evals/tests/tui/run.test.ts +++ b/packages/evals/tests/tui/run.test.ts @@ -5,6 +5,7 @@ import { deriveCategoryFilter, runCommand, } from "../../tui/commands/run.js"; +import { BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; const runEvalsMock = vi.hoisted(() => vi.fn(async () => ({ @@ -273,7 +274,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, agentMode: null, harnessConfig: { harness: "claude_code", @@ -333,7 +334,7 @@ describe("deriveCategoryFilter", () => { startupProfile: "tool_create_browserbase", toolCommand: "browse", browseCliVersion: expect.any(String), - browseCliEntrypoint: expect.stringMatching(/browse[/\\]bin[/\\]run\.js$/u), + browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, agentMode: null, harnessConfig: { harness: "codex", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aea9edde86..c85706ab23 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -459,7 +459,7 @@ importers: devDependencies: '@eslint/js': specifier: ^10.0.1 - version: 10.0.1(eslint@10.8.0(jiti@1.21.7)) + version: 10.0.1(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1)) '@types/archiver': specifier: ^6.0.3 version: 6.0.4 @@ -474,7 +474,7 @@ importers: version: 8.18.1 eslint: specifier: ^10.4.0 - version: 10.8.0(jiti@1.21.7) + version: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) eslint-plugin-security: specifier: ^4.0.0 version: 4.0.1 @@ -483,7 +483,7 @@ importers: version: 17.8.0 oclif: specifier: ^4.23.1 - version: 4.23.29(@types/node@20.19.43) + version: 4.23.29(@types/node@20.19.43)(supports-color@8.1.1) prettier: specifier: 3.5.3 version: 3.5.3 @@ -492,7 +492,7 @@ importers: version: 5.9.3 typescript-eslint: specifier: ^8.59.3 - version: 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) + version: 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3) vitest: specifier: ^4.1.9 version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.43)(vite@8.1.3(@types/node@20.19.43)(esbuild@0.28.1)(jiti@1.21.7)(tsx@4.23.1)(yaml@2.9.0)) @@ -504,7 +504,7 @@ importers: version: 3.1.1 mint: specifier: 'catalog:' - version: 4.2.742(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) + version: 4.2.742(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3) packages/evals: dependencies: @@ -527,8 +527,8 @@ importers: specifier: ^5.0.133 version: 5.0.220(zod@4.4.3) browse: - specifier: 0.9.5 - version: 0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1) + specifier: workspace:* + version: link:../cli dotenv: specifier: ^17.3.1 version: 17.4.2 @@ -559,7 +559,7 @@ importers: version: 24.13.2 braintrust: specifier: ^0.4.10 - version: 0.4.10(@aws-sdk/credential-provider-web-identity@3.972.70)(zod@4.4.3) + version: 0.4.10(@aws-sdk/credential-provider-web-identity@3.972.70)(supports-color@8.1.1)(zod@4.4.3) chalk: specifier: ^5.4.1 version: 5.6.2 @@ -1154,22 +1154,6 @@ packages: '@browserbasehq/sdk@2.16.0': resolution: {integrity: sha512-mPAuLRU9jWR7o0KJi9+gQnOBDUSIkoKbbFv4HjrA+80qWVcFacrNPlZmf4mguQnfZ0oP2t5c3ws6yuFyAX9vpA==} - '@browserbasehq/stagehand@3.6.0': - resolution: {integrity: sha512-wQ3Mv5b2xy64SntWNUbvgd7liWPbQRvGa8KEb1LorwheTkpUZFkt3BpZE8ovMY/lIleV9DfYjbvqghbeOOla4w==} - engines: {node: ^20.19.0 || >=22.12.0} - peerDependencies: - patchright-core: ^1.55.2 - playwright-core: ^1.55.1 - puppeteer-core: ^24.43.0 - zod: ^3.25.76 || ^4.2.0 - peerDependenciesMeta: - patchright-core: - optional: true - playwright-core: - optional: true - puppeteer-core: - optional: true - '@browserbasehq/stagehand@3.7.1': resolution: {integrity: sha512-vAuYSZWIhh3d76BxwppNVE3dB0ztEBLBi85G6TWulZNiebdWptNoANOMuprOB/cw5dE+80b/ZZQo4G33Pc9i6w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3836,11 +3820,6 @@ packages: peerDependencies: zod: ^3.25.34 - browse@0.9.5: - resolution: {integrity: sha512-5icxS7urivYaSaAfFxj+vSWU2zNPol8aBIU3S+3m1ervovG3qnQ0nfHEG9BhV2YXdAAACo5jT+sA0xYkugwFjA==} - engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true - buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} @@ -8644,47 +8623,6 @@ snapshots: transitivePeerDependencies: - encoding - '@browserbasehq/stagehand@3.6.0(playwright-core@1.56.1)(zod@4.4.3)': - dependencies: - '@ai-sdk/provider': 2.0.3 - '@anthropic-ai/sdk': 0.39.0 - '@browserbasehq/sdk': 2.16.0 - '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(bufferutil@4.1.0) - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - ai: 5.0.220(zod@4.4.3) - devtools-protocol: 0.0.1642743 - fetch-cookie: 3.2.0 - openai: 4.104.0(ws@8.21.0(bufferutil@4.1.0))(zod@4.4.3) - pino: 9.14.0 - pino-pretty: 13.1.3 - uuid: 11.1.1 - ws: 8.21.0(bufferutil@4.1.0) - zod: 4.4.3 - zod-to-json-schema: 3.25.2(zod@4.4.3) - optionalDependencies: - '@ai-sdk/amazon-bedrock': 3.0.111(zod@4.4.3) - '@ai-sdk/anthropic': 2.0.91(zod@4.4.3) - '@ai-sdk/azure': 2.0.120(zod@4.4.3) - '@ai-sdk/cerebras': 1.0.51(zod@4.4.3) - '@ai-sdk/deepseek': 1.0.48(zod@4.4.3) - '@ai-sdk/google': 2.0.85(zod@4.4.3) - '@ai-sdk/google-vertex': 3.0.158(zod@4.4.3) - '@ai-sdk/groq': 2.0.45(zod@4.4.3) - '@ai-sdk/mistral': 2.0.40(zod@4.4.3) - '@ai-sdk/openai': 2.0.115(zod@4.4.3) - '@ai-sdk/perplexity': 2.0.36(zod@4.4.3) - '@ai-sdk/togetherai': 1.0.49(zod@4.4.3) - '@ai-sdk/xai': 2.0.82(zod@4.4.3) - bufferutil: 4.1.0 - chrome-launcher: 1.2.1 - ollama-ai-provider-v2: 1.5.5(zod@4.4.3) - playwright-core: 1.56.1 - transitivePeerDependencies: - - '@cfworker/json-schema' - - encoding - - supports-color - - utf-8-validate - '@browserbasehq/stagehand@3.7.1(playwright-core@1.56.1)(zod@4.4.3)': dependencies: '@ai-sdk/provider': 2.0.3 @@ -9058,14 +8996,14 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true - '@eslint-community/eslint-utils@4.10.1(eslint@10.8.0(jiti@1.21.7))': + '@eslint-community/eslint-utils@4.10.1(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))': dependencies: - eslint: 10.8.0(jiti@1.21.7) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) eslint-visitor-keys: 3.4.3 '@eslint-community/regexpp@4.12.2': {} - '@eslint/config-array@0.23.5': + '@eslint/config-array@0.23.5(supports-color@8.1.1)': dependencies: '@eslint/object-schema': 3.0.5 debug: 4.4.3(supports-color@8.1.1) @@ -9081,9 +9019,9 @@ snapshots: dependencies: '@types/json-schema': 7.0.15 - '@eslint/js@10.0.1(eslint@10.8.0(jiti@1.21.7))': + '@eslint/js@10.0.1(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))': optionalDependencies: - eslint: 10.8.0(jiti@1.21.7) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) '@eslint/object-schema@3.0.5': {} @@ -9639,7 +9577,7 @@ snapshots: dependencies: jsep: 1.4.0 - '@kwsites/file-exists@1.1.1': + '@kwsites/file-exists@1.1.1(supports-color@8.1.1)': dependencies: debug: 4.4.3(supports-color@8.1.1) transitivePeerDependencies: @@ -9701,10 +9639,10 @@ snapshots: '@types/react': 19.2.17 react: 19.2.3 - '@mintlify/cli@4.0.1345(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3)': + '@mintlify/cli@4.0.1345(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@inquirer/prompts': 7.9.0(@types/node@24.13.2) - '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/link-rot': 3.0.1239(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) '@mintlify/models': 0.0.343 '@mintlify/prebuild': 1.0.1194(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) @@ -9713,7 +9651,7 @@ snapshots: adm-zip: 0.5.16 chalk: 5.2.0 color: 4.2.3 - detect-port: 1.5.1 + detect-port: 1.5.1(supports-color@8.1.1) front-matter: 4.0.2 fs-extra: 11.2.0 ink: 6.3.0(@types/react@19.2.17)(bufferutil@4.1.0)(react@19.2.3) @@ -9756,11 +9694,11 @@ snapshots: - typescript - utf-8-validate - '@mintlify/common@1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3)': + '@mintlify/common@1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@asyncapi/parser': 3.4.0 '@asyncapi/specs': 6.8.1 - '@mintlify/mdx': 3.0.4(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/mdx': 3.0.4(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/models': 0.0.343 '@mintlify/openapi-parser': 0.0.8 '@mintlify/validation': 0.1.801(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) @@ -9821,7 +9759,7 @@ snapshots: '@mintlify/link-rot@3.0.1239(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3)': dependencies: - '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/models': 0.0.343 '@mintlify/prebuild': 1.0.1194(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) '@mintlify/previewing': 4.0.1263(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) @@ -9846,11 +9784,11 @@ snapshots: - typescript - utf-8-validate - '@mintlify/mdx@3.0.4(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3)': + '@mintlify/mdx@3.0.4(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@radix-ui/react-popover': 1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3) '@shikijs/transformers': 3.23.0 - '@shikijs/twoslash': 3.23.0(typescript@5.9.3) + '@shikijs/twoslash': 3.23.0(supports-color@8.1.1)(typescript@5.9.3) arktype: 2.2.3 hast-util-to-string: 3.0.1 mdast-util-from-markdown: 2.0.3 @@ -9891,7 +9829,7 @@ snapshots: '@mintlify/prebuild@1.0.1194(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3)': dependencies: - '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/openapi-parser': 0.0.8 '@mintlify/scraping': 4.0.911(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) '@mintlify/validation': 0.1.801(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) @@ -9924,7 +9862,7 @@ snapshots: '@mintlify/previewing@4.0.1263(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3)': dependencies: - '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/prebuild': 1.0.1194(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) '@mintlify/validation': 0.1.801(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) adm-zip: 0.5.16 @@ -9963,7 +9901,7 @@ snapshots: '@mintlify/scraping@4.0.911(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3)': dependencies: - '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/common': 1.0.1046(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/openapi-parser': 0.0.8 fast-xml-parser: 5.7.3 fs-extra: 11.1.1 @@ -10000,7 +9938,7 @@ snapshots: '@mintlify/validation@0.1.801(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3)': dependencies: - '@mintlify/mdx': 3.0.4(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(typescript@5.9.3) + '@mintlify/mdx': 3.0.4(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3)(supports-color@8.1.1)(typescript@5.9.3) '@mintlify/models': 0.0.343 arktype: 2.1.27 fractional-indexing: 3.2.0 @@ -10032,8 +9970,8 @@ snapshots: cross-spawn: 7.0.6 eventsource: 3.0.7 eventsource-parser: 3.1.0 - express: 5.2.1 - express-rate-limit: 8.6.0(express@5.2.1) + express: 5.2.1(supports-color@8.1.1) + express-rate-limit: 8.6.0(express@5.2.1(supports-color@8.1.1)) hono: 4.12.32 jose: 6.2.4 json-schema-typed: 8.0.2 @@ -10101,12 +10039,12 @@ snapshots: transitivePeerDependencies: - '@types/node' - '@oclif/plugin-warn-if-update-available@3.1.70': + '@oclif/plugin-warn-if-update-available@3.1.70(supports-color@8.1.1)': dependencies: '@oclif/core': 4.13.0 ansis: 3.17.0 debug: 4.4.3(supports-color@8.1.1) - http-call: 5.3.0 + http-call: 5.3.0(supports-color@8.1.1) lodash: 4.18.1 registry-auth-token: 5.1.1 transitivePeerDependencies: @@ -10686,11 +10624,11 @@ snapshots: '@shikijs/core': 3.23.0 '@shikijs/types': 3.23.0 - '@shikijs/twoslash@3.23.0(typescript@5.9.3)': + '@shikijs/twoslash@3.23.0(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@shikijs/core': 3.23.0 '@shikijs/types': 3.23.0 - twoslash: 0.3.9(typescript@5.9.3) + twoslash: 0.3.9(supports-color@8.1.1)(typescript@5.9.3) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -11080,15 +11018,15 @@ snapshots: '@types/node': 25.9.4 optional: true - '@typescript-eslint/eslint-plugin@8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3))(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3)': + '@typescript-eslint/eslint-plugin@8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3))(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3) '@typescript-eslint/scope-manager': 8.65.0 - '@typescript-eslint/type-utils': 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/type-utils': 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3) + '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.65.0 - eslint: 10.8.0(jiti@1.21.7) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) ignore: 7.0.5 natural-compare: 1.4.0 ts-api-utils: 2.5.0(typescript@5.9.3) @@ -11096,14 +11034,14 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3)': + '@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@typescript-eslint/scope-manager': 8.65.0 '@typescript-eslint/types': 8.65.0 '@typescript-eslint/typescript-estree': 8.65.0(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.65.0 debug: 4.4.3(supports-color@8.1.1) - eslint: 10.8.0(jiti@1.21.7) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -11126,13 +11064,13 @@ snapshots: dependencies: typescript: 5.9.3 - '@typescript-eslint/type-utils@8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3)': + '@typescript-eslint/type-utils@8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3)': dependencies: '@typescript-eslint/types': 8.65.0 '@typescript-eslint/typescript-estree': 8.65.0(typescript@5.9.3) - '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(typescript@5.9.3) debug: 4.4.3(supports-color@8.1.1) - eslint: 10.8.0(jiti@1.21.7) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 transitivePeerDependencies: @@ -11155,13 +11093,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3)': + '@typescript-eslint/utils@8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(typescript@5.9.3)': dependencies: - '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@1.21.7)) + '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1)) '@typescript-eslint/scope-manager': 8.65.0 '@typescript-eslint/types': 8.65.0 '@typescript-eslint/typescript-estree': 8.65.0(typescript@5.9.3) - eslint: 10.8.0(jiti@1.21.7) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -11171,7 +11109,7 @@ snapshots: '@typescript-eslint/types': 8.65.0 eslint-visitor-keys: 5.0.1 - '@typescript/vfs@1.6.4(typescript@5.9.3)': + '@typescript/vfs@1.6.4(supports-color@8.1.1)(typescript@5.9.3)': dependencies: debug: 4.4.3(supports-color@8.1.1) typescript: 5.9.3 @@ -11579,7 +11517,7 @@ snapshots: transitivePeerDependencies: - supports-color - body-parser@2.3.0: + body-parser@2.3.0(supports-color@8.1.1): dependencies: bytes: 3.1.2 content-type: 2.0.0 @@ -11614,7 +11552,7 @@ snapshots: dependencies: fill-range: 7.1.1 - braintrust@0.4.10(@aws-sdk/credential-provider-web-identity@3.972.70)(zod@4.4.3): + braintrust@0.4.10(@aws-sdk/credential-provider-web-identity@3.972.70)(supports-color@8.1.1)(zod@4.4.3): dependencies: '@ai-sdk/provider': 1.1.3 '@next/env': 14.2.35 @@ -11632,7 +11570,7 @@ snapshots: minimatch: 9.0.9 mustache: 4.2.0 pluralize: 8.0.0 - simple-git: 3.36.0 + simple-git: 3.36.0(supports-color@8.1.1) slugify: 1.6.9 source-map: 0.7.6 uuid: 9.0.1 @@ -11642,36 +11580,6 @@ snapshots: - '@aws-sdk/credential-provider-web-identity' - supports-color - browse@0.9.5(bufferutil@4.1.0)(playwright-core@1.56.1): - dependencies: - '@browserbasehq/sdk': 2.16.0 - '@browserbasehq/stagehand': 3.6.0(playwright-core@1.56.1)(zod@4.4.3) - '@oclif/core': 4.13.0 - '@vercel/detect-agent': 1.2.3 - archiver: 7.0.1 - deepmerge: 4.3.1 - dotenv: 16.6.1 - fastest-levenshtein: 1.0.16 - http-status-codes: 2.3.0 - ignore: 7.0.5 - node-html-markdown: 1.3.0 - semver: 7.8.5 - tsx: 4.23.1 - ws: 8.21.0(bufferutil@4.1.0) - zod: 4.4.3 - transitivePeerDependencies: - - '@cfworker/json-schema' - - bare-abort-controller - - bare-buffer - - bufferutil - - encoding - - patchright-core - - playwright-core - - puppeteer-core - - react-native-b4a - - supports-color - - utf-8-validate - buffer-crc32@0.2.13: {} buffer-crc32@1.0.0: {} @@ -12100,7 +12008,7 @@ snapshots: detect-node-es@1.1.0: {} - detect-port@1.5.1: + detect-port@1.5.1(supports-color@8.1.1): dependencies: address: 1.2.2 debug: 4.4.3(supports-color@8.1.1) @@ -12440,11 +12348,11 @@ snapshots: eslint-visitor-keys@5.0.1: {} - eslint@10.8.0(jiti@1.21.7): + eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1): dependencies: - '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@1.21.7)) + '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1)) '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.23.5 + '@eslint/config-array': 0.23.5(supports-color@8.1.1) '@eslint/config-helpers': 0.7.0 '@eslint/core': 1.2.1 '@eslint/plugin-kit': 0.7.2 @@ -12557,10 +12465,10 @@ snapshots: expr-eval-fork@3.0.3: {} - express-rate-limit@8.6.0(express@5.2.1): + express-rate-limit@8.6.0(express@5.2.1(supports-color@8.1.1)): dependencies: debug: 4.4.3(supports-color@8.1.1) - express: 5.2.1 + express: 5.2.1(supports-color@8.1.1) ip-address: 10.2.0 transitivePeerDependencies: - supports-color @@ -12601,10 +12509,10 @@ snapshots: transitivePeerDependencies: - supports-color - express@5.2.1: + express@5.2.1(supports-color@8.1.1): dependencies: accepts: 2.0.0 - body-parser: 2.3.0 + body-parser: 2.3.0(supports-color@8.1.1) content-disposition: 1.1.0 content-type: 1.0.5 cookie: 0.7.2 @@ -12614,7 +12522,7 @@ snapshots: encodeurl: 2.0.0 escape-html: 1.0.3 etag: 1.8.1 - finalhandler: 2.1.1 + finalhandler: 2.1.1(supports-color@8.1.1) fresh: 2.0.0 http-errors: 2.0.1 merge-descriptors: 2.0.0 @@ -12625,8 +12533,8 @@ snapshots: proxy-addr: 2.0.7 qs: 6.15.3 range-parser: 1.2.1 - router: 2.2.0 - send: 1.2.1 + router: 2.2.0(supports-color@8.1.1) + send: 1.2.1(supports-color@8.1.1) serve-static: 2.2.1 statuses: 2.0.2 type-is: 2.1.0 @@ -12752,7 +12660,7 @@ snapshots: transitivePeerDependencies: - supports-color - finalhandler@2.1.1: + finalhandler@2.1.1(supports-color@8.1.1): dependencies: debug: 4.4.3(supports-color@8.1.1) encodeurl: 2.0.0 @@ -13284,7 +13192,7 @@ snapshots: http-cache-semantics@4.2.0: {} - http-call@5.3.0: + http-call@5.3.0(supports-color@8.1.1): dependencies: content-type: 1.0.5 debug: 4.4.3(supports-color@8.1.1) @@ -14425,9 +14333,9 @@ snapshots: dependencies: minipass: 7.1.3 - mint@4.2.742(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3): + mint@4.2.742(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3): dependencies: - '@mintlify/cli': 4.0.1345(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(typescript@5.9.3) + '@mintlify/cli': 4.0.1345(@radix-ui/react-popover@1.1.21(@types/react@19.2.17)(react-dom@18.3.1(react@19.2.3))(react@19.2.3))(@types/node@24.13.2)(@types/react@19.2.17)(bufferutil@4.1.0)(react-dom@18.3.1(react@19.2.3))(supports-color@8.1.1)(typescript@5.9.3) transitivePeerDependencies: - '@radix-ui/react-popover' - '@types/node' @@ -14594,7 +14502,7 @@ snapshots: obug@2.1.3: {} - oclif@4.23.29(@types/node@20.19.43): + oclif@4.23.29(@types/node@20.19.43)(supports-color@8.1.1): dependencies: '@aws-sdk/client-cloudfront': 3.1098.0 '@aws-sdk/client-s3': 3.1098.0 @@ -14604,7 +14512,7 @@ snapshots: '@oclif/core': 4.13.0 '@oclif/plugin-help': 6.2.55 '@oclif/plugin-not-found': 3.2.90(@types/node@20.19.43) - '@oclif/plugin-warn-if-update-available': 3.1.70 + '@oclif/plugin-warn-if-update-available': 3.1.70(supports-color@8.1.1) ansis: 3.17.0 async-retry: 1.3.3 change-case: 4.1.2 @@ -15590,7 +15498,7 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.1.5 '@rolldown/binding-win32-x64-msvc': 1.1.5 - router@2.2.0: + router@2.2.0(supports-color@8.1.1): dependencies: debug: 4.4.3(supports-color@8.1.1) depd: 2.0.0 @@ -15684,7 +15592,7 @@ snapshots: transitivePeerDependencies: - supports-color - send@1.2.1: + send@1.2.1(supports-color@8.1.1): dependencies: debug: 4.4.3(supports-color@8.1.1) encodeurl: 2.0.0 @@ -15725,7 +15633,7 @@ snapshots: encodeurl: 2.0.0 escape-html: 1.0.3 parseurl: 1.3.3 - send: 1.2.1 + send: 1.2.1(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -15879,9 +15787,9 @@ snapshots: simple-concat: 1.0.1 optional: true - simple-git@3.36.0: + simple-git@3.36.0(supports-color@8.1.1): dependencies: - '@kwsites/file-exists': 1.1.1 + '@kwsites/file-exists': 1.1.1(supports-color@8.1.1) '@kwsites/promise-deferred': 1.1.1 '@simple-git/args-pathspec': 1.0.3 '@simple-git/argv-parser': 1.1.1 @@ -16348,9 +16256,9 @@ snapshots: twoslash-protocol@0.3.9: {} - twoslash@0.3.9(typescript@5.9.3): + twoslash@0.3.9(supports-color@8.1.1)(typescript@5.9.3): dependencies: - '@typescript/vfs': 1.6.4(typescript@5.9.3) + '@typescript/vfs': 1.6.4(supports-color@8.1.1)(typescript@5.9.3) twoslash-protocol: 0.3.9 typescript: 5.9.3 transitivePeerDependencies: @@ -16414,13 +16322,13 @@ snapshots: typed-query-selector@2.12.2: {} - typescript-eslint@8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3): + typescript-eslint@8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3): dependencies: - '@typescript-eslint/eslint-plugin': 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3))(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) - '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) + '@typescript-eslint/eslint-plugin': 8.65.0(@typescript-eslint/parser@8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3))(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3) + '@typescript-eslint/parser': 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(supports-color@8.1.1)(typescript@5.9.3) '@typescript-eslint/typescript-estree': 8.65.0(typescript@5.9.3) - '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@1.21.7))(typescript@5.9.3) - eslint: 10.8.0(jiti@1.21.7) + '@typescript-eslint/utils': 8.65.0(eslint@10.8.0(jiti@1.21.7)(supports-color@8.1.1))(typescript@5.9.3) + eslint: 10.8.0(jiti@1.21.7)(supports-color@8.1.1) typescript: 5.9.3 transitivePeerDependencies: - supports-color From 58a41fc4e8d16675e35613387a54e4a6ad945f4e Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 30 Jul 2026 23:04:10 -0700 Subject: [PATCH 2/5] fix(evals): keep prompted CLI sessions socket-safe --- packages/evals/browseCliPaths.ts | 6 ++++++ packages/evals/core/tools/browse_cli.ts | 12 +++++------- packages/evals/framework/claudeCodeToolAdapter.ts | 7 ++----- packages/evals/tests/core/browse-cli-adapter.test.ts | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/packages/evals/browseCliPaths.ts b/packages/evals/browseCliPaths.ts index fe2ca89e3b..817b1bbb04 100644 --- a/packages/evals/browseCliPaths.ts +++ b/packages/evals/browseCliPaths.ts @@ -10,3 +10,9 @@ export const BROWSE_CLI_BUILD_ARTIFACTS = [ ]; export const BROWSE_CLI_PACKAGE_JSON = path.join(browseCliRoot, "package.json"); export const BROWSE_SKILL_SOURCE = path.join(browseCliRoot, "skills", "browse", "SKILL.md"); + +export function createBrowseCliSessionName(): string { + const timestamp = Date.now().toString(36); + const random = Math.random().toString(36).slice(2, 6); + return `eval-${process.pid}-${timestamp}-${random}`; +} diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index 5d1744a625..b4ade1e325 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -14,7 +14,11 @@ import type { ToolStartInput, ToolStartResult, } from "../contracts/tool.js"; -import { BROWSE_CLI_BUILD_ARTIFACTS, BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; +import { + BROWSE_CLI_BUILD_ARTIFACTS, + BROWSE_CLI_ENTRYPOINT, + createBrowseCliSessionName, +} from "../../browseCliPaths.js"; import { getRepoRootDir } from "../../runtimePaths.js"; const execFileAsync = promisify(execFile); @@ -672,12 +676,6 @@ function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMo return "launch"; } -export function createBrowseCliSessionName(): string { - const timestamp = Date.now().toString(36); - const random = Math.random().toString(36).slice(2, 6); - return `eval-${process.pid}-${timestamp}-${random}`; -} - export class BrowseCliTool implements CoreTool { readonly id = "browse_cli"; readonly surface = "cli"; diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index 886ab7ec6d..9ba8029057 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -12,6 +12,7 @@ import { BROWSE_CLI_ENTRYPOINT, BROWSE_CLI_PACKAGE_JSON, BROWSE_SKILL_SOURCE, + createBrowseCliSessionName, } from "../browseCliPaths.js"; import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js"; import { prepareCoreBrowserTarget } from "../core/targets/index.js"; @@ -296,7 +297,7 @@ export async function prepareBrowseCliHarnessAdapter( ); } - const session = createBrowseSessionName(); + const session = createBrowseCliSessionName(); const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-browse-")); const wrapperPath = path.join(cwd, "browse"); await installBrowseSkill(cwd); @@ -1145,10 +1146,6 @@ function isPromiseLike(value: unknown): value is PromiseLike & { ); } -function createBrowseSessionName(): string { - return `evals-claude-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; -} - async function runBrowseCommand( wrapperPath: string, args: string[], diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts index f509fff5d1..894a5a568f 100644 --- a/packages/evals/tests/core/browse-cli-adapter.test.ts +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -5,8 +5,8 @@ import { BrowseCliSession, browseCliStartupArgs, buildBrowseCliProcessArgs, - createBrowseCliSessionName, } from "../../core/tools/browse_cli.js"; +import { createBrowseCliSessionName } from "../../browseCliPaths.js"; describe("browse_cli V4 adapter", () => { it("places session selection on each V4 command and uses mode-specific startup commands", () => { From ece0d04b9794c5e21499518e9df6e39580655aa1 Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Thu, 6 Aug 2026 01:28:13 -0700 Subject: [PATCH 3/5] fix(evals): harden V4 CLI harness execution --- packages/evals/README.md | 1 + packages/evals/core/tools/browse_cli.ts | 17 +- packages/evals/framework/benchHarness.ts | 31 ++-- .../evals/framework/claudeCodeToolAdapter.ts | 150 ++++++++++++------ packages/evals/tests/cli.test.ts | 6 + .../tests/core/browse-cli-adapter.test.ts | 4 + .../framework/claudeCodeToolAdapter.test.ts | 45 ++++++ packages/evals/tests/tui/parse.test.ts | 5 + packages/evals/tui/commands/help.ts | 1 + packages/evals/tui/commands/parse.ts | 6 +- 10 files changed, 196 insertions(+), 70 deletions(-) diff --git a/packages/evals/README.md b/packages/evals/README.md index a102d2a783..9dee4e1fd8 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -70,6 +70,7 @@ Use `Esc` to abort an in-flight run without exiting the REPL. | `-c, --concurrency ` | Max parallel sessions | | `-m, --model ` / `-p, --provider ` | Override the model/provider matrix | | `--api` | Run via the Stagehand API instead of the SDK | +| `--verbose` | Stream detailed logs while the eval runs | | `--harness ` | Which agent harness drives the bench task | | `--agent-mode ` / `--agent-modes ` | Stagehand agent mode (or matrix) | | `-l, --limit ` / `-s, --sample ` / `-f, --filter key=value` | Suite shaping for benchmark targets | diff --git a/packages/evals/core/tools/browse_cli.ts b/packages/evals/core/tools/browse_cli.ts index b4ade1e325..83460c7419 100644 --- a/packages/evals/core/tools/browse_cli.ts +++ b/packages/evals/core/tools/browse_cli.ts @@ -106,7 +106,7 @@ export function browseCliStartupArgs(environment: ToolStartInput["environment"]) class BrowseCliRuntime { constructor(private readonly session: string) {} - async runJson(args: string[]): Promise { + async runJson(args: string[], timeoutMs?: number): Promise { const { stdout, stderr } = await execFileAsync( process.execPath, buildBrowseCliProcessArgs(resolveBrowseCliEntrypoint(), this.session, args), @@ -114,6 +114,7 @@ class BrowseCliRuntime { cwd: getRepoRootDir(), env: process.env, maxBuffer: 10 * 1024 * 1024, + ...(timeoutMs === undefined ? {} : { timeout: timeoutMs }), }, ); @@ -645,12 +646,7 @@ export class BrowseCliSession implements CoreSession { async close(): Promise { if (this.closed) return; this.closed = true; - - try { - await this.runtime.runJson(["stop", "--force"]); - } catch { - // best-effort only - } + await this.runtime.runJson(["stop", "--force"], 5_000); } async getArtifacts(): Promise { @@ -705,7 +701,12 @@ export class BrowseCliTool implements CoreTool { } const session = new BrowseCliSession(createBrowseCliSessionName()); - await session.runtime.runJson(browseCliStartupArgs(input.environment)); + try { + await session.runtime.runJson(browseCliStartupArgs(input.environment)); + } catch (error) { + await session.close().catch(() => {}); + throw error; + } return { session, diff --git a/packages/evals/framework/benchHarness.ts b/packages/evals/framework/benchHarness.ts index dac02b31b6..7085434e37 100644 --- a/packages/evals/framework/benchHarness.ts +++ b/packages/evals/framework/benchHarness.ts @@ -27,6 +27,7 @@ import { prepareCodexToolAdapter, type PreparedCodexToolAdapter } from "./codexT import { buildExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; import type { DiscoveredTask, TaskResult } from "./types.js"; import type { BenchMatrixRow, BenchTaskKind, Harness } from "./benchTypes.js"; +import { onceAsync, registerActiveRunCleanup } from "./activeRunCleanup.js"; type Page = ReturnType[number]; @@ -281,6 +282,11 @@ export const claudeCodeHarness: BenchHarness = { // the adapter and the carrier. const carrierV3 = buildVerifierCarrierV3(logger); let toolAdapter: PreparedClaudeCodeToolAdapter | undefined; + const cleanup = onceAsync(async () => { + await toolAdapter?.cleanup(); + await carrierV3.close().catch(() => {}); + }); + const unregisterCleanup = registerActiveRunCleanup(cleanup); try { toolAdapter = await prepareClaudeCodeToolAdapter({ toolSurface: row.config.toolSurface, @@ -302,11 +308,11 @@ export const claudeCodeHarness: BenchHarness = { }, }); } finally { - await toolAdapter?.cleanup(); - // Deregister the never-init()-ed carrier (instance registry, event - // store, logger binding) so long matrix runs don't accumulate one - // V3 object graph per task. - await carrierV3.close().catch(() => {}); + try { + await cleanup(); + } finally { + unregisterCleanup(); + } } }, async start(): Promise { @@ -330,6 +336,11 @@ export const codexHarness: BenchHarness = { // the adapter and the carrier. const carrierV3 = buildVerifierCarrierV3(logger); let toolAdapter: PreparedCodexToolAdapter | undefined; + const cleanup = onceAsync(async () => { + await toolAdapter?.cleanup(); + await carrierV3.close().catch(() => {}); + }); + const unregisterCleanup = registerActiveRunCleanup(cleanup); try { toolAdapter = await prepareCodexToolAdapter({ toolSurface: row.config.toolSurface, @@ -351,11 +362,11 @@ export const codexHarness: BenchHarness = { }, }); } finally { - await toolAdapter?.cleanup(); - // Deregister the never-init()-ed carrier (instance registry, event - // store, logger binding) so long matrix runs don't accumulate one - // V3 object graph per task. - await carrierV3.close().catch(() => {}); + try { + await cleanup(); + } finally { + unregisterCleanup(); + } } }, async start(): Promise { diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index 9ba8029057..8d8f307676 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -299,55 +299,85 @@ export async function prepareBrowseCliHarnessAdapter( const session = createBrowseCliSessionName(); const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-browse-")); - const wrapperPath = path.join(cwd, "browse"); - await installBrowseSkill(cwd); - input.logger.log({ - category: input.logCategory, - message: `Installed browse skill at ${path.join(cwd, ".claude", "skills", "browse", "SKILL.md")}`, - level: 1, - }); - const env = { - ...process.env, - BROWSE_SESSION: session, - PATH: `${cwd}${path.delimiter}${process.env.PATH ?? ""}`, - } as Record; + try { + const wrapperPath = path.join(cwd, "browse"); + await installBrowseSkill(cwd); + input.logger.log({ + category: input.logCategory, + message: `Installed browse skill at ${path.join(cwd, ".claude", "skills", "browse", "SKILL.md")}`, + level: 1, + }); + const env = { + ...process.env, + BROWSE_SESSION: session, + PATH: `${cwd}${path.delimiter}${process.env.PATH ?? ""}`, + } as Record; + + const modeFlag = input.environment === "BROWSERBASE" ? "--remote" : "--local"; + await fsp.writeFile( + wrapperPath, + buildBrowseCliWrapperScript({ + entrypoint: BROWSE_CLI_ENTRYPOINT, + modeFlag, + nodePath: process.execPath, + session, + }), + { mode: 0o755 }, + ); - const modeFlag = input.environment === "BROWSERBASE" ? "--remote" : "--local"; - await fsp.writeFile( - wrapperPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - // The mode flag (--local/--remote) selects the environment when the daemon - // is first started and must be explicit so a set BROWSERBASE_API_KEY does - // not silently auto-select remote. It is only accepted by the driver - // commands, so skip it for the few subcommands that reject it (stop, - // status). The session name is safe on every command. - "cmd=${1:-}", - "mode=()", - 'if [[ "$cmd" != "stop" && "$cmd" != "status" ]]; then', - ` mode=(${JSON.stringify(modeFlag)})`, - "fi", - `exec ${JSON.stringify(process.execPath)} ${JSON.stringify(BROWSE_CLI_ENTRYPOINT)} "$@" "\${mode[@]+\${mode[@]}}" --session ${JSON.stringify(session)}`, - "", - ].join("\n"), - { mode: 0o755 }, - ); + return { + toolSurface: "browse_cli", + startupProfile: input.startupProfile, + cwd, + env, + promptInstructions: buildBrowseCliPromptInstructions(input.plan), + metadata: getBrowseCliToolMetadata(), + cleanup: async () => { + try { + await runBrowseCommand(wrapperPath, ["stop", "--force"], input.logger, env, cwd, 5_000); + } catch (error) { + input.logger.warn({ + category: input.logCategory, + message: `browse_cli cleanup failed: ${error instanceof Error ? error.message : String(error)}`, + level: 0, + }); + } finally { + await fsp.rm(cwd, { recursive: true, force: true }); + } + }, + }; + } catch (error) { + await fsp.rm(cwd, { recursive: true, force: true }); + throw error; + } +} - return { - toolSurface: "browse_cli", - startupProfile: input.startupProfile, - cwd, - env, - promptInstructions: buildBrowseCliPromptInstructions(input.plan), - metadata: getBrowseCliToolMetadata(), - cleanup: async () => { - await runBrowseCommand(wrapperPath, ["stop", "--force"], input.logger, env, cwd).catch( - (): undefined => undefined, - ); - await fsp.rm(cwd, { recursive: true, force: true }); - }, - }; +export function buildBrowseCliWrapperScript({ + entrypoint, + modeFlag, + nodePath, + session, +}: { + entrypoint: string; + modeFlag: "--local" | "--remote"; + nodePath: string; + session: string; +}): string { + return [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "cmd=${1:-}", + 'case "$cmd" in', + " back|click|cursor|doctor|eval|fill|forward|get|highlight|is|mouse|network|open|press|reload|screenshot|select|snapshot|status|stop|tab|type|upload|viewport|wait) ;;", + ' *) echo "The browse eval harness only supports browser driver commands (received: ${cmd:-})." >&2; exit 64 ;;', + "esac", + "mode=()", + 'if [[ "$cmd" != "stop" && "$cmd" != "status" ]]; then', + ` mode=(${JSON.stringify(modeFlag)})`, + "fi", + `exec ${JSON.stringify(nodePath)} ${JSON.stringify(entrypoint)} "$@" "\${mode[@]+\${mode[@]}}" --session ${JSON.stringify(session)}`, + "", + ].join("\n"); } async function preparePlaywrightCodeAdapter( @@ -1152,6 +1182,7 @@ async function runBrowseCommand( logger: EvalLogger, env: Record, cwd: string, + timeoutMs: number, ): Promise { const { spawn } = await import("node:child_process"); await new Promise((resolve, reject) => { @@ -1161,6 +1192,19 @@ async function runBrowseCommand( stdio: ["ignore", "pipe", "pipe"], }); let stderr = ""; + let timedOut = false; + let settled = false; + const finish = (error?: Error): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + if (error) reject(error); + else resolve(); + }; + const timeout = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk: string) => { logger.log({ category: "browse_cli", message: chunk, level: 1 }); @@ -1170,13 +1214,19 @@ async function runBrowseCommand( stderr += chunk; logger.log({ category: "browse_cli", message: chunk, level: 1 }); }); - child.on("error", reject); + child.on("error", (error) => finish(error)); child.on("close", (code) => { + if (timedOut) { + finish( + new EvalsError(`browse_cli command timed out after ${timeoutMs}ms (${args.join(" ")})`), + ); + return; + } if (code === 0) { - resolve(); + finish(); return; } - reject(new EvalsError(`browse_cli command failed (${args.join(" ")}): ${stderr.trim()}`)); + finish(new EvalsError(`browse_cli command failed (${args.join(" ")}): ${stderr.trim()}`)); }); }); } diff --git a/packages/evals/tests/cli.test.ts b/packages/evals/tests/cli.test.ts index 27cbf0f2ab..be4381a73a 100644 --- a/packages/evals/tests/cli.test.ts +++ b/packages/evals/tests/cli.test.ts @@ -249,6 +249,12 @@ describe("CLI entrypoint", () => { expect(() => JSON.parse(stdout)).toThrow(); }); + it("accepts --verbose for a one-shot run", async () => { + const { stdout, code } = await runCli(["run", "act", "--dry-run", "--verbose"]); + expect(code).toBe(0); + expect(JSON.parse(stdout).runOptions.verbose).toBe(true); + }); + it("rejects --preview combined with --dry-run", async () => { const { stdout, stderr, code } = await runCli(["run", "act", "--dry-run", "--preview"]); expect(code).toBe(1); diff --git a/packages/evals/tests/core/browse-cli-adapter.test.ts b/packages/evals/tests/core/browse-cli-adapter.test.ts index 894a5a568f..f23b2f958c 100644 --- a/packages/evals/tests/core/browse-cli-adapter.test.ts +++ b/packages/evals/tests/core/browse-cli-adapter.test.ts @@ -7,6 +7,7 @@ import { buildBrowseCliProcessArgs, } from "../../core/tools/browse_cli.js"; import { createBrowseCliSessionName } from "../../browseCliPaths.js"; +import { getSocketPath } from "../../../cli/src/lib/driver/daemon/paths.js"; describe("browse_cli V4 adapter", () => { it("places session selection on each V4 command and uses mode-specific startup commands", () => { @@ -26,6 +27,9 @@ describe("browse_cli V4 adapter", () => { expect(sessionName).toMatch(/^eval-\d+-[a-z0-9]+-[a-z0-9]{4}$/u); expect(sessionName.length).toBeLessThanOrEqual(32); + if (process.platform !== "win32") { + expect(Buffer.byteLength(getSocketPath(sessionName))).toBeLessThanOrEqual(103); + } }); it("maps page capabilities onto the V4 command tree and flags", async () => { diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 420777baa8..1f9291dbe3 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -1,8 +1,11 @@ +import { execFile } from "node:child_process"; import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { + buildBrowseCliWrapperScript, getBrowseCliAllowedTools, getBrowseCliToolMetadata, insertAfterFrontmatter, @@ -19,6 +22,8 @@ import { import type { CdpEventMessage } from "../../core/tools/cdp_code.js"; import { BROWSE_CLI_ENTRYPOINT } from "../../browseCliPaths.js"; +const execFileAsync = promisify(execFile); + describe("claude code tool adapter resolution", () => { afterEach(() => { delete process.env.EVAL_CLAUDE_CODE_ALLOW_UNSANDBOXED_LOCAL; @@ -214,6 +219,46 @@ describe("claude code tool adapter resolution", () => { } }); + it("routes only driver commands through the pinned browse wrapper", async () => { + const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-wrapper-test-")); + const entrypoint = path.join(cwd, "capture.mjs"); + const wrapper = path.join(cwd, "browse"); + try { + await fsp.writeFile( + entrypoint, + "process.stdout.write(JSON.stringify(process.argv.slice(2)));\n", + ); + await fsp.writeFile( + wrapper, + buildBrowseCliWrapperScript({ + entrypoint, + modeFlag: "--local", + nodePath: process.execPath, + session: "eval-session", + }), + { mode: 0o755 }, + ); + + const open = await execFileAsync(wrapper, ["open", "about:blank"]); + expect(JSON.parse(open.stdout)).toEqual([ + "open", + "about:blank", + "--local", + "--session", + "eval-session", + ]); + + const stop = await execFileAsync(wrapper, ["stop", "--force"]); + expect(JSON.parse(stop.stdout)).toEqual(["stop", "--force", "--session", "eval-session"]); + + await expect(execFileAsync(wrapper, ["cloud", "projects", "list"])).rejects.toMatchObject({ + code: 64, + }); + } finally { + await fsp.rm(cwd, { recursive: true, force: true }); + } + }); + it("keeps abandoned CDP event waits from becoming unhandled rejections", async () => { const listeners = new Set<(event: CdpEventMessage) => void>(); const connection = { diff --git a/packages/evals/tests/tui/parse.test.ts b/packages/evals/tests/tui/parse.test.ts index 85cf38942f..6f9a3434ea 100644 --- a/packages/evals/tests/tui/parse.test.ts +++ b/packages/evals/tests/tui/parse.test.ts @@ -17,6 +17,11 @@ describe("resolveRunOptions", () => { expect(resolved.verbose).toBe(true); }); + it("lets --verbose override the config default", () => { + const resolved = resolveRunOptions(parseRunArgs(["act", "--verbose"]), { verbose: false }, {}); + expect(resolved.verbose).toBe(true); + }); + it("defaults to the stagehand bench harness", () => { const resolved = resolveRunOptions({}, {}, {}); expect(resolved.harness).toBe("stagehand"); diff --git a/packages/evals/tui/commands/help.ts b/packages/evals/tui/commands/help.ts index 316721089e..dcaea8823e 100644 --- a/packages/evals/tui/commands/help.ts +++ b/packages/evals/tui/commands/help.ts @@ -67,6 +67,7 @@ export function printRunHelp(): void { `Provider: ${gray("openai | anthropic | google | ...")}`, ), row(cyan("--api"), "Use Stagehand API mode"), + row(cyan("--verbose"), "Stream detailed logs instead of the animated progress view"), "", ` ${bold("Core options:")}`, "", diff --git a/packages/evals/tui/commands/parse.ts b/packages/evals/tui/commands/parse.ts index 4e047d43f2..7450c27875 100644 --- a/packages/evals/tui/commands/parse.ts +++ b/packages/evals/tui/commands/parse.ts @@ -38,6 +38,7 @@ export interface RunFlags { filter?: Array<[string, string]>; dryRun?: boolean; preview?: boolean; + verbose?: boolean; /** * Rubric success mode for the verifier — outcome | process | both. * outcome (default): binary EvaluationResult.outcomeSuccess. @@ -104,7 +105,7 @@ const SUPPORTED_BENCHMARKS = new Set([ const LEGACY_ONLY_BENCHMARKS = new Set(["gaia", "osworld"]); -const BOOLEAN_FLAGS = new Set(["api", "dry-run", "preview", "legacy"]); +const BOOLEAN_FLAGS = new Set(["api", "dry-run", "preview", "verbose", "legacy"]); const VALUE_FLAGS = new Set([ "trials", "concurrency", @@ -219,6 +220,7 @@ export function parseRunArgs(tokens: string[]): RunFlags { if (name === "api") flags.api = true; else if (name === "dry-run") flags.dryRun = true; else if (name === "preview") flags.preview = true; + else if (name === "verbose") flags.verbose = true; else if (name === "legacy") flags.legacy = true; i++; continue; @@ -461,7 +463,7 @@ export function resolveRunOptions( envOverrides, dryRun: flags.dryRun ?? false, preview: flags.preview ?? false, - verbose: defaults.verbose ?? false, + verbose: flags.verbose ?? defaults.verbose ?? false, }; } From e5daba1f9e7d4e4f73cc0ec1a06223e7b5228afd Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Sun, 9 Aug 2026 21:49:50 +0000 Subject: [PATCH 4/5] test(evals): expect published browse version --- packages/evals/tests/framework/claudeCodeToolAdapter.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 5f8213648a..6a5ba6a3c6 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -130,7 +130,7 @@ describe("claude code tool adapter resolution", () => { it("exposes browse cli metadata for Braintrust rows", () => { expect(getBrowseCliToolMetadata()).toMatchObject({ toolCommand: "browse", - browseCliVersion: "0.9.5", + browseCliVersion: "0.9.6", browseCliEntrypoint: BROWSE_CLI_ENTRYPOINT, }); }); From 013cb4d6d95a830256073adc32c45bb8264f62fd Mon Sep 17 00:00:00 2001 From: Shrey Pandya Date: Sun, 9 Aug 2026 21:53:19 +0000 Subject: [PATCH 5/5] fix(evals): drop stale merge helper --- packages/evals/framework/claudeCodeToolAdapter.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index 1d9a3537a8..fd28c51cb8 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -789,19 +789,6 @@ function clip(value: string, maxLength: number): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; } -function isPromiseLike(value: unknown): value is PromiseLike & { - catch: (handler: (error: unknown) => void) => unknown; -} { - return ( - value !== null && - typeof value === "object" && - "then" in value && - typeof (value as { then?: unknown }).then === "function" && - "catch" in value && - typeof (value as { catch?: unknown }).catch === "function" - ); -} - async function runBrowseCommand( wrapperPath: string, args: string[],