diff --git a/.changeset/bright-calendars-search.md b/.changeset/bright-calendars-search.md new file mode 100644 index 0000000000..d3e06e78c4 --- /dev/null +++ b/.changeset/bright-calendars-search.md @@ -0,0 +1,5 @@ +--- +"@executor-js/execution": minor +--- + +Return available integrations from `tools.search`, including integrations that do not have a connection yet. Search results now use `kind: "tool" | "integration"` so callers can distinguish callable paths from integration discovery metadata. diff --git a/packages/core/execution/src/skills.ts b/packages/core/execution/src/skills.ts index deeb31cb3b..45ee3ce4ec 100644 --- a/packages/core/execution/src/skills.ts +++ b/packages/core/execution/src/skills.ts @@ -33,7 +33,7 @@ const EXECUTE_SKILL_BODY = [ "## Workflow", "", '1. `const { items: matches } = await tools.search({ query: "", limit: 12 });`', - '2. `const path = matches[0]?.path; if (!path) return "No matching tools found.";`', + '2. Inspect `matches[0].kind`: `"tool"` results have a callable `path`; `"integration"` results identify an available integration that may need a connection.', "3. `const details = await tools.describe.tool({ path });`", "4. Use `details.inputTypeScript` / `details.outputTypeScript` and `details.typeScriptDefinitions` for compact shapes.", "5. Use `tools.executor.coreTools.connections.list({})` when you need live saved-connection inventory.", @@ -41,7 +41,7 @@ const EXECUTE_SKILL_BODY = [ "", "## Rules", "", - "- `tools.search()` returns paginated, ranked matches: `{ items, total, hasMore, nextOffset }`. Best-first. Use short intent phrases like `github issues`, `repo details`, or `create calendar event`.", + '- `tools.search()` returns connected tools (`kind: "tool"`) and available integrations (`kind: "integration"`), including integrations with no connection. Use short intent phrases like `github issues`, `repo details`, or `create calendar event`.', '- When you already know the namespace, narrow with `tools.search({ namespace: "github", query: "issues" })`.', "- `tools.executor.coreTools.connections.list({})` returns saved connections with `{ address, integration, owner, name, ... }`. The `address` field includes the leading `tools.` root.", "- Tool calls return a value union: `{ ok: true, data }` for success or `{ ok: false, error: { code, message, status?, details?, retryable? } }` for expected tool/domain failures. Branch on `result.ok`.", @@ -54,7 +54,7 @@ const EXECUTE_SKILL_BODY = [ "- `return` is only for ordinary structured data. Returning a `ToolFile`, a `ToolResult`, an MCP content block, or a bare base64 string does not emit content to the MCP client.", "- Some providers, including Gmail, return attachment bytes as a `ToolFile` with no public URL to hand off — the bytes themselves are the payload, so `emit(result.data)` to display it, or pass its base64 `data` as another tool's `bodyBase64` to forward it.", "- If `tools.search()` returns `hasMore: true` and you didn't find what you need, fetch the next page: `tools.search({ query, offset: nextOffset, limit })`.", - "- Always use the full address when calling tools: `tools....(args)`. The `path` returned by `tools.search()` / `tools.describe.tool()` is already the exact path under `tools` — call `tools[path]` rather than guessing segments.", + '- Always use the full address when calling tools: `tools....(args)`. A `kind: "tool"` search result contains the exact callable `path`; a `kind: "integration"` result is discovery metadata and has no callable path.', "- The `tools` object is a lazy proxy — enumerating it (`Object.keys(tools)`, spread, `for...in`) throws. Use `tools.search()` or `tools.executor.coreTools.connections.list({})` instead.", '- Pass an object to system tools, e.g. `tools.search({ query: "..." })`, `tools.executor.coreTools.connections.list({})`, and `tools.describe.tool({ path })`.', '- `tools.describe.tool()` returns compact TypeScript shapes. Use `inputTypeScript`, `outputTypeScript`, and `typeScriptDefinitions`. If the path doesn\'t resolve, the result carries `error: { code: "tool_not_found", suggestions }` — use a suggestion instead of retrying the same path.', diff --git a/packages/core/execution/src/tool-invoker.test.ts b/packages/core/execution/src/tool-invoker.test.ts index 747bd23105..a97ad5219a 100644 --- a/packages/core/execution/src/tool-invoker.test.ts +++ b/packages/core/execution/src/tool-invoker.test.ts @@ -36,6 +36,8 @@ import { makeExecutorToolInvoker, searchTools, type ToolDiscoveryProvider, + type ToolDiscoveryResult, + type ToolDiscoveryToolResult, } from "./tool-invoker"; // --------------------------------------------------------------------------- @@ -51,6 +53,9 @@ import { const codeExecutor = makeQuickJsExecutor(); +const toolMatches = (items: readonly ToolDiscoveryResult[]): readonly ToolDiscoveryToolResult[] => + items.filter((item) => item.kind === "tool"); + // Standard-schema validators — used by `invokeTool` to validate args and emit // the `Missing key` issues that surface as `invalid_tool_arguments`. type Validator = ReturnType; @@ -477,22 +482,41 @@ describe("tool discovery", () => { const executor = yield* makeSearchExecutor(); const githubMatches = yield* searchTools(executor, "github issues", 5); - expect(githubMatches.items.map((match) => match.path)).toEqual([ + expect(toolMatches(githubMatches.items).map((match) => match.path)).toEqual([ "github.org.main.listRepositoryIssues", ]); expect(githubMatches.items[0]?.score ?? 0).toBeGreaterThan(0); + expect(githubMatches.items.some((match) => match.kind === "integration")).toBe(false); expect(githubMatches.hasMore).toBe(false); expect(githubMatches.nextOffset).toBeNull(); const repoMatches = yield* searchTools(executor, "repo details", 5); - expect(repoMatches.items[0]?.path).toBe("github.org.main.getRepositoryDetails"); + expect(toolMatches(repoMatches.items)[0]?.path).toBe("github.org.main.getRepositoryDetails"); const crmMatches = yield* searchTools(executor, "crm create contact", 5); - expect(crmMatches.items[0]?.path).toBe("crm.org.main.createContact"); + expect(toolMatches(crmMatches.items)[0]?.path).toBe("crm.org.main.createContact"); expect(crmMatches.items[0]?.score ?? 0).toBeGreaterThan(crmMatches.items[1]?.score ?? 0); }), ); + it.effect("returns integrations that do not have a connection", () => + Effect.gen(function* () { + const executor = yield* makeExecutorWith([githubPlugin] as const); + yield* executor["github-test"]!.seed(); + + const matches = yield* searchTools(executor, "github", 5); + expect(matches.items).toContainEqual( + expect.objectContaining({ + kind: "integration", + id: "github", + integration: "github", + toolCount: 0, + }), + ); + expect(matches.items.some((item) => item.kind === "tool")).toBe(false); + }), + ); + it.effect("returns no matches for empty queries instead of listing arbitrary tools", () => Effect.gen(function* () { const executor = yield* makeSearchExecutor(); @@ -514,8 +538,10 @@ describe("tool discovery", () => { const enumerated = yield* searchTools(executor, "", 100, { namespace: "github" }); expect(enumerated.items.length).toBeGreaterThan(0); expect(enumerated.total).toBe(enumerated.items.length); - expect(enumerated.items.map((item) => item.path)).toEqual( - [...enumerated.items.map((item) => item.path)].sort((a, b) => a.localeCompare(b)), + expect(toolMatches(enumerated.items).map((item) => item.path)).toEqual( + [...toolMatches(enumerated.items).map((item) => item.path)].sort((a, b) => + a.localeCompare(b), + ), ); expect(enumerated.items.every((item) => item.integration === "github")).toBe(true); expect(enumerated.items.every((item) => item.score === 0)).toBe(true); @@ -649,14 +675,16 @@ describe("tool discovery", () => { const githubOnly = yield* searchTools(executor, "list", 5, { namespace: "github", }); - expect(githubOnly.items.map((match) => match.path)).toEqual([ + expect(toolMatches(githubOnly.items).map((match) => match.path)).toEqual([ "github.org.main.listRepositoryIssues", ]); const crmOnly = yield* searchTools(executor, "list", 5, { namespace: "crm", }); - expect(crmOnly.items.map((match) => match.path)).toEqual(["crm.org.main.listContacts"]); + expect(toolMatches(crmOnly.items).map((match) => match.path)).toEqual([ + "crm.org.main.listContacts", + ]); const sandboxResult = yield* createExecutionEngine({ executor, codeExecutor }).execute( 'return await tools.search({ namespace: "crm", query: "create contact", limit: 5 });', @@ -690,6 +718,7 @@ describe("tool discovery", () => { return { items: [ { + kind: "tool", path: "custom.org.main.searchResult", name: "searchResult", description: "Provided by the host", @@ -725,6 +754,7 @@ describe("tool discovery", () => { expect(result.result).toEqual({ items: [ { + kind: "tool", path: "custom.org.main.searchResult", name: "searchResult", description: "Provided by the host", diff --git a/packages/core/execution/src/tool-invoker.ts b/packages/core/execution/src/tool-invoker.ts index 2df47644ed..d564be9096 100644 --- a/packages/core/execution/src/tool-invoker.ts +++ b/packages/core/execution/src/tool-invoker.ts @@ -106,7 +106,7 @@ const BUILTIN_TOOL_DESCRIPTIONS: ReadonlyMap = new Map< "{ items: ToolDiscoveryResult[]; total: number; hasMore: boolean; nextOffset: number | null; }", typeScriptDefinitions: { ToolDiscoveryResult: - "{ path: string; name: string; description?: string; integration: string; score: number; }", + '{ kind: "tool"; path: string; name: string; description?: string; integration: string; score: number; } | { kind: "integration"; id: string; name: string; description?: string; integration: string; integrationKind: string; toolCount: number; score: number; }', }, }, ], @@ -394,7 +394,8 @@ const isElicitationDeclinedError = ( "action" in value && (value.action === "cancel" || value.action === "decline"); -export type ToolDiscoveryResult = { +export type ToolDiscoveryToolResult = { + readonly kind: "tool"; readonly path: string; readonly name: string; readonly description?: string; @@ -402,6 +403,19 @@ export type ToolDiscoveryResult = { readonly score: number; }; +export type IntegrationDiscoveryResult = { + readonly kind: "integration"; + readonly id: string; + readonly name: string; + readonly description?: string; + readonly integration: string; + readonly integrationKind: string; + readonly toolCount: number; + readonly score: number; +}; + +export type ToolDiscoveryResult = ToolDiscoveryToolResult | IntegrationDiscoveryResult; + export type ExecutorIntegrationListItem = { readonly id: string; readonly name: string; @@ -585,7 +599,7 @@ const matchesNamespace = (tool: SearchableTool, namespace?: string): boolean => return isPrefixMatch(integrationTokens) || isPrefixMatch(pathTokens); }; -const scoreToolMatch = (tool: SearchableTool, query: string): ToolDiscoveryResult | null => { +const scoreToolMatch = (tool: SearchableTool, query: string): ToolDiscoveryToolResult | null => { const normalizedQuery = normalizeSearchText(query); const queryTokens = tokenizeSearchText(query); @@ -646,6 +660,7 @@ const scoreToolMatch = (tool: SearchableTool, query: string): ToolDiscoveryResul } return { + kind: "tool", path: tool.path, name: tool.name, description: tool.description, @@ -654,6 +669,34 @@ const scoreToolMatch = (tool: SearchableTool, query: string): ToolDiscoveryResul }; }; +const scoreIntegrationMatch = ( + integration: Integration, + toolCount: number, + query: string, +): IntegrationDiscoveryResult | null => { + const id = String(integration.slug); + const match = scoreToolMatch( + { + path: id, + integration: id, + name: integration.name, + description: integration.description, + }, + query, + ); + if (match === null) return null; + return { + kind: "integration", + id, + name: integration.name, + description: integration.description, + integration: id, + integrationKind: integration.kind, + toolCount, + score: match.score, + }; +}; + /** What `tools.search()` calls inside the sandbox. */ export const searchTools = Effect.fn("executor.tools.search")(function* ( executor: Executor, @@ -685,16 +728,32 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( } satisfies PagedResult; } - const all = yield* executor.tools.list({ includeAnnotations: false }).pipe( - Effect.mapError( - (cause) => - new ExecutionToolError({ - message: "Failed to list tools for search", - cause, - }), + const [all, integrations] = yield* Effect.all([ + executor.tools.list({ includeAnnotations: false }).pipe( + Effect.mapError( + (cause) => + new ExecutionToolError({ + message: "Failed to list tools for search", + cause, + }), + ), ), - ); + executor.integrations.list().pipe( + Effect.mapError( + (cause) => + new ExecutionToolError({ + message: "Failed to list integrations for search", + cause, + }), + ), + ), + ]); const searchable = all.map(toSearchableTool); + const toolCountByIntegration = new Map(); + for (const tool of all) { + const key = String(tool.integration); + toolCountByIntegration.set(key, (toolCountByIntegration.get(key) ?? 0) + 1); + } // An empty query WITH a namespace is enumeration, not search: there is no // ranking signal, so the namespace's whole catalog comes back sorted by @@ -704,11 +763,12 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( // google_gmail and google_sheets), which would silently break the census // guarantee: `total` here must reconcile against // `executor.integrations.list`'s per-integration toolCount. - const ranked: readonly ToolDiscoveryResult[] = emptyQuery + const rankedTools: readonly ToolDiscoveryToolResult[] = emptyQuery ? searchable .filter((tool) => tool.integration === options?.namespace?.trim()) .sort((left, right) => left.path.localeCompare(right.path)) .map((tool) => ({ + kind: "tool" as const, path: tool.path, name: tool.name, integration: tool.integration, @@ -721,10 +781,42 @@ export const searchTools = Effect.fn("executor.tools.search")(function* ( .filter(Predicate.isNotNull) .sort((left, right) => right.score - left.score || left.path.localeCompare(right.path)); + const rankedIntegrations = emptyQuery + ? [] + : integrations + .filter((integration) => (toolCountByIntegration.get(String(integration.slug)) ?? 0) === 0) + .filter((integration) => + matchesNamespace( + { + path: String(integration.slug), + integration: String(integration.slug), + name: integration.name, + description: integration.description, + }, + options?.namespace, + ), + ) + .map((integration) => + scoreIntegrationMatch( + integration, + toolCountByIntegration.get(String(integration.slug)) ?? 0, + query, + ), + ) + .filter(Predicate.isNotNull); + + const ranked: readonly ToolDiscoveryResult[] = [...rankedTools, ...rankedIntegrations].sort( + (left, right) => + right.score - left.score || + (left.kind === "tool" ? left.path : left.id).localeCompare( + right.kind === "tool" ? right.path : right.id, + ), + ); + const page = paginate(ranked, offset, limit); yield* Effect.annotateCurrentSpan({ - "executor.search.candidate_count": all.length, + "executor.search.candidate_count": all.length + integrations.length, "executor.search.match_count": ranked.length, "executor.search.result_count": page.items.length, "executor.search.has_more": page.hasMore, @@ -850,7 +942,7 @@ export const describeTool = Effect.fn("executor.tools.describe")(function* ( scoped.items.length > 0 ? scoped.items : (yield* searchTools(executor, leaf, TOOL_DESCRIBE_SUGGESTION_LIMIT)).items; - const suggestions = matches.map((item) => item.path); + const suggestions = matches.flatMap((item) => (item.kind === "tool" ? [item.path] : [])); const notFound: DescribedTool = { path, name: path,