diff --git a/tests/integration/harness.ts b/tests/integration/harness.ts index 30c1ad42b..4312900ee 100644 --- a/tests/integration/harness.ts +++ b/tests/integration/harness.ts @@ -27,11 +27,15 @@ import type { ContextTransform, ContextStore, InferenceSource, + ToolDefinition, } from "@intx/types/runtime"; import { type } from "arktype"; import { createAgentWithLiveToolDispatch } from "../../src/agent/live-tool-dispatch.js"; -import { createChatDirector } from "../../src/agent/director.js"; +import { + createChatDirector, + type ChatDirector, +} from "../../src/agent/director.js"; import { OPERATOR_ORIGINATED_FLAG } from "../../src/agent/message-provenance.js"; import { readSourceCredentialMaterial, @@ -82,6 +86,7 @@ export interface IntegrationSession { workdir: string; agent: Agent; toolset: Awaited>; + updateToolDefinitions: (definitions: ToolDefinition[]) => void; } export interface OpenIntegrationSessionOpts { @@ -114,6 +119,9 @@ export async function openIntegrationSession( const storageHolder: { current: ContextStore | undefined } = { current: undefined, }; + const directorHolder: { current: ChatDirector | undefined } = { + current: undefined, + }; const toolset = await createAgentToolset({ cwd, @@ -140,6 +148,7 @@ export async function openIntegrationSession( }, ); d.setClearDenials(() => opts.permissionGate.clearDenials()); + directorHolder.current = d; return d; }, }); @@ -262,7 +271,22 @@ export async function openIntegrationSession( ? innerAgent : createPrimaryDeliveryAdmission(innerAgent, primaryArchive); - return { harness, cwd, workdir, agent, toolset, storage: storageForAgent }; + const updateToolDefinitions = (definitions: ToolDefinition[]): void => { + const director = directorHolder.current; + if (director === undefined) + throw new Error("chat director is not available"); + director.updateToolDefinitions(definitions); + }; + + return { + harness, + cwd, + workdir, + agent, + toolset, + storage: storageForAgent, + updateToolDefinitions, + }; } export async function closeIntegrationSession( diff --git a/tests/integration/mcp-late-dispatch.test.ts b/tests/integration/mcp-late-dispatch.test.ts index 88a3e3096..f991e15bb 100644 --- a/tests/integration/mcp-late-dispatch.test.ts +++ b/tests/integration/mcp-late-dispatch.test.ts @@ -3,6 +3,8 @@ import { createAgent } from "@intx/agent"; import type { ReactorEmittedEvent } from "@intx/inference"; import { createAgentWithLiveToolDispatch } from "../../src/agent/live-tool-dispatch.js"; +import type { MCPClient } from "../../src/mcp/client.js"; +import { mcpClientTools } from "../../src/mcp/plugin.js"; import { createPermissionGate } from "../../src/permission/gate.js"; import { closeIntegrationSession, @@ -11,6 +13,20 @@ import { } from "./harness.js"; const LATE_MCP = "mcp__linear__list_issues"; +const LATE_MCP_SCHEMA = { + type: "object" as const, + properties: { + limit: { type: "integer" }, + team: { type: "string" }, + }, +}; + +interface AnthropicRequestBody { + tools?: { + name: string; + input_schema: Record; + }[]; +} function permissionGate() { return createPermissionGate({ @@ -21,20 +37,27 @@ function permissionGate() { }); } -function lateMcpTool() { - return { - kind: "string" as const, - definition: { - name: LATE_MCP, - description: "list issues", - inputSchema: { - type: "object" as const, - properties: {}, - required: [] as string[], +function lateMcpTools( + onCall?: (toolName: string, args: Record) => void, +) { + const client: MCPClient = { + serverName: "linear", + tools: [ + { + name: "list_issues", + description: "list issues", + inputSchema: LATE_MCP_SCHEMA, }, + ], + async call(toolName, args) { + onCall?.(toolName, args); + return "ISSUE-1"; + }, + async close() { + return undefined; }, - handler: async () => "ISSUE-1", }; + return mcpClientTools(client); } function toolDoneContents(events: ReactorEmittedEvent[]): string[] { @@ -62,18 +85,23 @@ describe("integration — late MCP dispatch", () => { }); try { - session.toolset.dynamicRunner.addTools([lateMcpTool()]); + session.toolset.dynamicRunner.addTools(lateMcpTools()); session.harness.scenario.replyOnce("anthropic", { toolCalls: [{ name: LATE_MCP, args: {} }], }); session.harness.scenario.replyOnce("anthropic", { text: "listed" }); const { events } = await runUntilDone(session, "list linear issues"); - expect( - toolDoneContents(events).some((content) => - content.includes(`unknown tool: ${LATE_MCP}`), - ), - ).toBe(true); + const loud = events.find( + ( + event, + ): event is Extract => + event.type === "tool.done" && + typeof event.data.result.content === "string" && + event.data.result.content.includes(`unknown tool: ${LATE_MCP}`), + ); + expect(loud).toBeDefined(); + expect(loud?.data.result.isError).toBe(true); } finally { await closeIntegrationSession(session); } @@ -89,7 +117,7 @@ describe("integration — late MCP dispatch", () => { }); try { - session.toolset.dynamicRunner.addTools([lateMcpTool()]); + session.toolset.dynamicRunner.addTools(lateMcpTools()); session.harness.scenario.replyOnce("anthropic", { toolCalls: [{ name: LATE_MCP, args: {} }], }); @@ -106,4 +134,308 @@ describe("integration — late MCP dispatch", () => { } }, ); + + test.serial( + "unknown MCP tool dispatch fails loudly instead of altering results", + async () => { + const session = await openIntegrationSession({ + permissionGate: permissionGate(), + createAgentFn: createAgentWithLiveToolDispatch, + }); + + try { + session.toolset.dynamicRunner.addTools(lateMcpTools()); + const missing = "mcp__linear__no_such_tool"; + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [{ name: missing, args: {} }], + }); + session.harness.scenario.replyOnce("anthropic", { text: "refused" }); + + const { events } = await runUntilDone(session, "delete everything"); + const loud = events.find( + ( + event, + ): event is Extract => + event.type === "tool.done" && + typeof event.data.result.content === "string" && + event.data.result.content.includes(`unknown tool: ${missing}`), + ); + expect(loud).toBeDefined(); + expect(loud?.data.result.isError).toBe(true); + } finally { + await closeIntegrationSession(session); + } + }, + ); + + test.serial( + "out-of-scope MCP arguments fail loudly with an actionable error", + async () => { + const session = await openIntegrationSession({ + permissionGate: permissionGate(), + createAgentFn: createAgentWithLiveToolDispatch, + }); + + try { + const client: MCPClient = { + serverName: "linear", + tools: [ + { + name: "list_issues", + description: "list issues", + inputSchema: LATE_MCP_SCHEMA, + }, + ], + async call(toolName, args) { + if (toolName === "list_issues" && args.team === "rogue") { + throw new Error( + 'scope denied: team "rogue" is not in scope for this connection', + ); + } + return "ISSUE-1"; + }, + async close() { + return undefined; + }, + }; + session.toolset.dynamicRunner.addTools(mcpClientTools(client)); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [{ name: LATE_MCP, args: { limit: 1, team: "rogue" } }], + }); + session.harness.scenario.replyOnce("anthropic", { text: "refused" }); + + const { events } = await runUntilDone( + session, + "list rogue team issues", + ); + const loud = events.find( + ( + event, + ): event is Extract => + event.type === "tool.done" && + typeof event.data.result.content === "string" && + event.data.result.content.includes("scope denied"), + ); + expect(loud).toBeDefined(); + expect(loud?.data.result.isError).toBe(true); + expect(loud?.data.result.content).toContain("not in scope"); + } finally { + await closeIntegrationSession(session); + } + }, + ); + + test.serial( + "tool_search promotion preserves optional MCP arguments", + async () => { + const session = await openIntegrationSession({ + permissionGate: permissionGate(), + createAgentFn: createAgentWithLiveToolDispatch, + }); + let receivedArgs: Record | undefined; + + try { + const tools = lateMcpTools((toolName, args) => { + if (toolName === "list_issues") { + receivedArgs = args; + } + }); + expect(tools).toHaveLength(1); + expect(tools[0]?.kind).toBe("full"); + const expectedSchema = structuredClone(LATE_MCP_SCHEMA); + session.toolset.dynamicRunner.addTools(tools); + session.toolset.setToolPromoter(() => { + session.updateToolDefinitions( + session.toolset.dynamicRunner.currentDefinitions(), + ); + }); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [ + { name: "tool_search", args: { query: "linear list issues" } }, + ], + }); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [{ name: LATE_MCP, args: { limit: 1, team: "eng" } }], + }); + session.harness.scenario.replyOnce("anthropic", { text: "listed" }); + + const { events } = await runUntilDone(session, "list one linear issue"); + const bodies = await Promise.all( + session.harness.scenario + .matchedRequests() + .map( + async (request) => + JSON.parse( + await (request.clone() as unknown as Request).text(), + ) as AnthropicRequestBody, + ), + ); + const publishedTool = bodies + .flatMap((body) => body.tools ?? []) + .find((tool) => tool.name === LATE_MCP); + + expect(publishedTool?.input_schema.properties).toEqual( + expectedSchema.properties, + ); + expect( + Object.hasOwn(publishedTool?.input_schema ?? {}, "required"), + ).toBe(false); + expect(receivedArgs).toEqual({ limit: 1, team: "eng" }); + expect(toolDoneContents(events)).toContain("ISSUE-1"); + } finally { + await closeIntegrationSession(session); + } + }, + ); + + test.serial( + "tool_search promotion does not inject omitted optional MCP arguments", + async () => { + const session = await openIntegrationSession({ + permissionGate: permissionGate(), + createAgentFn: createAgentWithLiveToolDispatch, + }); + let receivedArgs: Record | undefined; + + try { + const tools = lateMcpTools((toolName, args) => { + if (toolName === "list_issues") { + receivedArgs = args; + } + }); + expect(tools).toHaveLength(1); + expect(tools[0]?.kind).toBe("full"); + const expectedSchema = structuredClone(LATE_MCP_SCHEMA); + session.toolset.dynamicRunner.addTools(tools); + session.toolset.setToolPromoter(() => { + session.updateToolDefinitions( + session.toolset.dynamicRunner.currentDefinitions(), + ); + }); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [ + { name: "tool_search", args: { query: "linear list issues" } }, + ], + }); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [{ name: LATE_MCP, args: { limit: 1 } }], + }); + session.harness.scenario.replyOnce("anthropic", { text: "listed" }); + + const { events } = await runUntilDone(session, "list one linear issue"); + const bodies = await Promise.all( + session.harness.scenario + .matchedRequests() + .map( + async (request) => + JSON.parse( + await (request.clone() as unknown as Request).text(), + ) as AnthropicRequestBody, + ), + ); + const publishedTool = bodies + .flatMap((body) => body.tools ?? []) + .find((tool) => tool.name === LATE_MCP); + + expect(publishedTool?.input_schema.properties).toEqual( + expectedSchema.properties, + ); + expect( + Object.hasOwn(publishedTool?.input_schema ?? {}, "required"), + ).toBe(false); + expect(receivedArgs).toEqual({ limit: 1 }); + expect(toolDoneContents(events)).toContain("ISSUE-1"); + } finally { + await closeIntegrationSession(session); + } + }, + ); + + test.serial( + "tool_search promotion preserves required and optional MCP arguments", + async () => { + const session = await openIntegrationSession({ + permissionGate: permissionGate(), + createAgentFn: createAgentWithLiveToolDispatch, + }); + let receivedArgs: Record | undefined; + const schema = { + type: "object" as const, + properties: { + limit: { type: "integer" }, + team: { type: "string" }, + customView: { type: "string" }, + }, + required: ["limit"], + }; + + try { + const client: MCPClient = { + serverName: "linear", + tools: [ + { + name: "list_issues", + description: "list issues", + inputSchema: schema, + }, + ], + async call(toolName, args) { + if (toolName === "list_issues") { + receivedArgs = args; + } + return "ISSUE-1"; + }, + async close() { + return undefined; + }, + }; + const expectedSchema = structuredClone(schema); + session.toolset.dynamicRunner.addTools(mcpClientTools(client)); + session.toolset.setToolPromoter(() => { + session.updateToolDefinitions( + session.toolset.dynamicRunner.currentDefinitions(), + ); + }); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [ + { name: "tool_search", args: { query: "linear list issues" } }, + ], + }); + session.harness.scenario.replyOnce("anthropic", { + toolCalls: [ + { + name: LATE_MCP, + args: { limit: 1, team: "eng", customView: "mine" }, + }, + ], + }); + session.harness.scenario.replyOnce("anthropic", { text: "listed" }); + + const { events } = await runUntilDone(session, "list one linear issue"); + const bodies = await Promise.all( + session.harness.scenario + .matchedRequests() + .map( + async (request) => + JSON.parse( + await (request.clone() as unknown as Request).text(), + ) as AnthropicRequestBody, + ), + ); + const publishedTool = bodies + .flatMap((body) => body.tools ?? []) + .find((tool) => tool.name === LATE_MCP); + + expect(publishedTool?.input_schema).toEqual(expectedSchema); + expect(receivedArgs).toEqual({ + limit: 1, + team: "eng", + customView: "mine", + }); + expect(toolDoneContents(events)).toContain("ISSUE-1"); + } finally { + await closeIntegrationSession(session); + } + }, + ); });