diff --git a/CHANGELOG.md b/CHANGELOG.md index 123213f78..d8f765470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +- Let remote AG-UI Bots use the browser in chat: `computer_*` calls a caller offers now pass through for the caller to execute instead of being refused as "unavailable for this headless turn" ([#804](https://github.com/CopilotKit/OpenBot/issues/804)). **Before upgrading.** Five things change for an existing deployment: - A hosted computer's browser is now headed by default. It uses a little more memory than headless did; set `COMPUTER_BROWSER_MODE=headless` (Helm: `computers.browserMode: headless`) to keep the old diff --git a/toolkit/src/headless-tools.ts b/toolkit/src/headless-tools.ts index 58958a2f9..cedfae246 100644 --- a/toolkit/src/headless-tools.ts +++ b/toolkit/src/headless-tools.ts @@ -73,6 +73,15 @@ export class HeadlessToolsMiddleware extends Middleware { const tools = new Map( this.tools.map((tool) => [tool.definition.name, tool]), ); + /* + * A computer_* call is claimed only when this turn cannot reach a browser for it. When + * the caller offered the tool itself (a chat with a browser attached), the call passes + * through unanswered so the caller executes it; the governed refusal below is reserved + * for genuinely browserless turns, where the call would otherwise die silently. + */ + const callerTools = new Set( + (originalInput.tools ?? []).map((tool) => tool.name), + ); const executed = new Set(); let subscription: Subscription | undefined; let started = false; @@ -161,7 +170,8 @@ export class HeadlessToolsMiddleware extends Middleware { const owned = pending.filter( (call) => tools.has(call.function.name) || - call.function.name.startsWith("computer_"), + (call.function.name.startsWith("computer_") && + !callerTools.has(call.function.name)), ); if (owned.length === 0) { subscriber.next(terminal); diff --git a/toolkit/tests/headless-tools.test.ts b/toolkit/tests/headless-tools.test.ts new file mode 100644 index 000000000..5c7e2f70d --- /dev/null +++ b/toolkit/tests/headless-tools.test.ts @@ -0,0 +1,128 @@ +import { expect, test } from "bun:test"; +import { + type BaseEvent, + EventType, + HttpAgent, + type RunAgentInput, +} from "@ag-ui/client"; +import { HeadlessToolsMiddleware } from "../src/headless-tools"; + +function remoteTurn(events: (input: RunAgentInput, index: number) => BaseEvent[]) { + const requests: RunAgentInput[] = []; + const agent = new HttpAgent({ + agentId: "bot", + url: "https://agent.example/ag-ui", + fetch: async (_url, init) => { + const request = JSON.parse(String(init?.body)) as RunAgentInput; + requests.push(request); + return new Response( + events(request, requests.length - 1) + .map((event) => `data: ${JSON.stringify(event)}\n\n`) + .join(""), + { headers: { "content-type": "text/event-stream" } }, + ); + }, + }); + return { requests, agent }; +} + +function lifecycle(input: RunAgentInput, body: BaseEvent[]): BaseEvent[] { + return [ + { type: EventType.RUN_STARTED, threadId: input.threadId, runId: input.runId }, + ...body, + { type: EventType.RUN_FINISHED, threadId: input.threadId, runId: input.runId }, + ]; +} + +function toolCall(name: string, id: string): BaseEvent[] { + return [ + { + type: EventType.TOOL_CALL_START, + toolCallId: id, + toolCallName: name, + parentMessageId: `message-${id}`, + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: id, delta: "{}" }, + { type: EventType.TOOL_CALL_END, toolCallId: id }, + ]; +} + +function reply(text: string): BaseEvent[] { + return [ + { type: EventType.TEXT_MESSAGE_START, messageId: "reply", role: "assistant" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "reply", delta: text }, + { type: EventType.TEXT_MESSAGE_END, messageId: "reply" }, + ]; +} + +function input(tools: RunAgentInput["tools"]): RunAgentInput { + return { + threadId: "thread", + runId: "run", + messages: [], + state: {}, + tools, + context: [], + forwardedProps: {}, + } as RunAgentInput; +} + +async function collect(middleware: HeadlessToolsMiddleware, runInput: RunAgentInput, agent: HttpAgent) { + const events: BaseEvent[] = []; + await new Promise((resolve, reject) => + middleware.run(runInput, agent).subscribe({ + next: (event) => events.push(event), + error: reject, + complete: resolve, + }), + ); + return events; +} + +test("a computer tool the caller offers passes through for the caller to execute", async () => { + const remote = remoteTurn((runInput, _index) => + lifecycle(runInput, [ + ...toolCall("computer_navigate", "nav-call"), + ...reply("Opening the page."), + ]), + ); + const middleware = new HeadlessToolsMiddleware([]); + const events = await collect( + middleware, + input([ + { + name: "computer_navigate", + description: "Open a page in the person's browser", + parameters: { type: "object", properties: {} }, + }, + ]), + remote.agent, + ); + + expect(events.map((event) => event.type)).toContain(EventType.RUN_FINISHED); + expect(events.map((event) => event.type)).not.toContain( + EventType.TOOL_CALL_RESULT, + ); + expect(remote.requests).toHaveLength(1); +}); + +test("a computer tool the caller did not offer still gets the governed refusal", async () => { + const remote = remoteTurn((runInput, index) => + lifecycle( + runInput, + index === 0 + ? toolCall("computer_steal", "steal-call") + : reply("That tool is unavailable."), + ), + ); + const middleware = new HeadlessToolsMiddleware([]); + await collect(middleware, input([]), remote.agent); + + expect(remote.requests).toHaveLength(2); + expect(remote.requests[1]?.messages).toContainEqual( + expect.objectContaining({ + role: "tool", + error: expect.stringContaining("unavailable"), + }), + ); +});