diff --git a/apps/e2e/demo-e2e-direct/e2e/tool-output-schema-modes.e2e.spec.ts b/apps/e2e/demo-e2e-direct/e2e/tool-output-schema-modes.e2e.spec.ts index 1e69744ac..def70bf9e 100644 --- a/apps/e2e/demo-e2e-direct/e2e/tool-output-schema-modes.e2e.spec.ts +++ b/apps/e2e/demo-e2e-direct/e2e/tool-output-schema-modes.e2e.spec.ts @@ -13,8 +13,24 @@ * - 'none': expose it nowhere. * * Driven through the real `tools/list` flow via `server.listTools()`. + * + * `output.undeclaredKeys` decides what `tools/call` does with keys an answer carries that its + * `outputSchema` does not declare: 'strip', 'warn' (strip and log once), 'keep' or 'fail'. + * A call that renders the tool's UI sends declared keys only, whatever the policy. */ -import { App, FrontMcpInstance, Tool, ToolContext, z, type DirectMcpServer } from '@frontmcp/sdk'; +import { + App, + connect, + FrontMcpInstance, + LogLevel, + LogTransport, + LogTransportInterface, + Tool, + ToolContext, + z, + type DirectMcpServer, + type LogRecord, +} from '@frontmcp/sdk'; // A structured (top-level object) output so the schema is eligible for both // outputSchema advertisement and the description rendering. @@ -204,3 +220,147 @@ describe('tools/list output.schemaMode (E2E)', () => { }); }); }); + +const UNDECLARED_VALUE = 'QCC-4410'; +const quoteAnswer = { price: 10, quoteCostCenter: UNDECLARED_VALUE } as { price: number }; +const quoteOutput = { price: z.number() }; + +@Tool({ name: 'quote_default', inputSchema: {}, outputSchema: quoteOutput }) +class DefaultQuoteTool extends ToolContext { + async execute() { + return quoteAnswer; + } +} + +@Tool({ name: 'quote_warned', inputSchema: {}, outputSchema: quoteOutput, output: { undeclaredKeys: 'warn' } }) +class WarnedQuoteTool extends ToolContext { + async execute() { + return quoteAnswer; + } +} + +@Tool({ name: 'quote_kept', inputSchema: {}, outputSchema: quoteOutput, output: { undeclaredKeys: 'keep' } }) +class KeptQuoteTool extends ToolContext { + async execute() { + return quoteAnswer; + } +} + +@Tool({ name: 'quote_strict', inputSchema: {}, outputSchema: quoteOutput, output: { undeclaredKeys: 'fail' } }) +class StrictQuoteTool extends ToolContext { + async execute() { + return quoteAnswer; + } +} + +@Tool({ + name: 'quote_view', + inputSchema: {}, + outputSchema: quoteOutput, + output: { undeclaredKeys: 'keep' }, + ui: { template: (ctx) => `
${ctx.helpers.escapeHtml(JSON.stringify(ctx.output))}
` }, +}) +class QuoteViewTool extends ToolContext { + async execute() { + return quoteAnswer; + } +} + +@App({ + name: 'UndeclaredKeys', + description: 'Tools exercising output.undeclaredKeys', + tools: [DefaultQuoteTool, WarnedQuoteTool, KeptQuoteTool, StrictQuoteTool, QuoteViewTool], +}) +class UndeclaredKeysApp {} + +const policyWarnings: string[] = []; + +@LogTransport({ name: 'UndeclaredKeysE2eCapture', description: 'Keeps warnings for assertions' }) +class PolicyWarningCapture extends LogTransportInterface { + log(record: LogRecord): void { + if (record.level === LogLevel.Warn) policyWarnings.push(record.message); + } +} + +const undeclaredKeysServerConfig = { + info: { name: 'undeclared-keys-e2e', version: '0.0.0' }, + apps: [UndeclaredKeysApp], + auth: { mode: 'public' as const }, + // A client named quote-viewer is treated as Claude, which renders the UI without structuredContent. + transport: { platformDetection: { mappings: [{ pattern: 'quote-viewer', platform: 'claude' as const }] } }, + logging: { level: LogLevel.Warn, enableConsole: false, transports: [PolicyWarningCapture] }, +}; + +describe('tools/call output.undeclaredKeys (E2E)', () => { + let server: DirectMcpServer; + + beforeAll(async () => { + server = await FrontMcpInstance.createDirect(undeclaredKeysServerConfig); + }); + + afterAll(async () => { + await server.dispose(); + }); + + it("strips undeclared keys silently by default outside development ('strip')", async () => { + const result = await server.callTool('quote_default', {}); + + expect(result.structuredContent).toEqual({ price: 10 }); + expect(JSON.stringify(result)).not.toContain(UNDECLARED_VALUE); + expect(policyWarnings.filter((line) => line.includes('quote_default'))).toEqual([]); + }); + + it("'warn' strips undeclared keys and warns once for the tool and key set", async () => { + const first = await server.callTool('quote_warned', {}); + await server.callTool('quote_warned', {}); + + expect(first.structuredContent).toEqual({ price: 10 }); + const warnings = policyWarnings.filter((line) => line.includes('quote_warned')); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain('quoteCostCenter'); + }); + + it("'keep' passes undeclared keys through, and an MCP client accepts them against the listed schema", async () => { + const client = await connect(undeclaredKeysServerConfig, { + clientInfo: { name: 'policy-probe', version: '1.0.0' }, + }); + try { + await client.listTools(); + const result = (await client.callTool('quote_kept', {})) as { + structuredContent?: unknown; + content?: Array<{ text?: string }>; + }; + + expect(result.structuredContent).toEqual({ price: 10, quoteCostCenter: UNDECLARED_VALUE }); + expect(result.content?.[0]?.text).toContain(UNDECLARED_VALUE); + } finally { + await client.close(); + } + }); + + it("'fail' refuses the answer with INVALID_OUTPUT", async () => { + await expect(server.callTool('quote_strict', {})).rejects.toMatchObject({ code: 'INVALID_OUTPUT' }); + }); + + it('a call that renders the UI sends declared keys only, even under keep', async () => { + const result = await server.callTool('quote_view', {}); + + expect(result._meta?.['ui/html']).toBeDefined(); + expect(result.structuredContent).toEqual({ price: 10 }); + expect(JSON.stringify(result)).not.toContain(UNDECLARED_VALUE); + }); + + it('a UI render for a client that takes no structuredContent sends declared keys only, even under keep', async () => { + const client = await connect(undeclaredKeysServerConfig, { + clientInfo: { name: 'quote-viewer', version: '1.0.0' }, + }); + try { + const result = await client.callTool('quote_view', {}); + + expect(JSON.stringify(result)).toContain('ui/html'); + expect(JSON.stringify(result)).not.toContain(UNDECLARED_VALUE); + } finally { + await client.close(); + } + }); +}); diff --git a/apps/e2e/demo-e2e-direct/e2e/tools-list-output-schema.e2e.spec.ts b/apps/e2e/demo-e2e-direct/e2e/tools-list-output-schema.e2e.spec.ts index 9f5b095d0..804cde3c5 100644 --- a/apps/e2e/demo-e2e-direct/e2e/tools-list-output-schema.e2e.spec.ts +++ b/apps/e2e/demo-e2e-direct/e2e/tools-list-output-schema.e2e.spec.ts @@ -9,8 +9,9 @@ * Verifies the SDK now converts structured object outputs (symmetric with * inputSchema), while primitive outputs correctly remain unadvertised (they flow * through `content`, and MCP requires `outputSchema` to be a top-level object). + * A union whose members are all objects is listed too, typed `object` at the top. */ -import { App, FrontMcpInstance, Tool, ToolContext, z, type DirectMcpServer } from '@frontmcp/sdk'; +import { App, connect, FrontMcpInstance, Tool, ToolContext, z, type DirectMcpServer } from '@frontmcp/sdk'; const weatherOutput = { city: z.string(), tempF: z.number() }; @@ -50,24 +51,83 @@ class CountPrimitiveTool extends ToolContext { } } +const documentHit = z.object({ kind: z.literal('document'), title: z.string() }); +const personHit = z.object({ kind: z.literal('person'), name: z.string() }); + +@Tool({ + name: 'search_union', + description: 'Tool with a z.union() of objects outputSchema', + inputSchema: { query: z.string() }, + outputSchema: z.union([documentHit, personHit]), +}) +class SearchUnionTool extends ToolContext { + async execute(input: { query: string }) { + return { kind: 'document' as const, title: input.query }; + } +} + +@Tool({ + name: 'resolve_discriminated', + description: 'Tool with a z.discriminatedUnion() outputSchema', + inputSchema: { handle: z.string() }, + outputSchema: z.discriminatedUnion('kind', [documentHit, personHit]), +}) +class ResolveDiscriminatedTool extends ToolContext { + async execute(input: { handle: string }) { + return { kind: 'person' as const, name: input.handle }; + } +} + +// Schemas registered with an id serialize to `$ref`s into `$defs` +const registeredHit = z.union([ + documentHit.meta({ id: 'ToolsListE2eDocumentHit' }), + personHit.meta({ id: 'ToolsListE2ePersonHit' }), +]); + +@Tool({ + name: 'lookup_registered', + description: 'Tool with a z.union() of objects registered with an id', + inputSchema: { name: z.string() }, + outputSchema: registeredHit, +}) +class LookupRegisteredTool extends ToolContext { + async execute(input: { name: string }) { + return { kind: 'person' as const, name: input.name }; + } +} + @App({ name: 'OutputSchema', description: 'Tools exercising outputSchema advertisement', - tools: [WeatherStructTool, ZodObjectOutTool, CountPrimitiveTool], + tools: [ + WeatherStructTool, + ZodObjectOutTool, + CountPrimitiveTool, + SearchUnionTool, + ResolveDiscriminatedTool, + LookupRegisteredTool, + ], }) class OutputSchemaApp {} -type JsonSchemaLike = { type?: string; properties?: Record }; +const outputSchemaServerConfig = { + info: { name: 'output-schema-e2e', version: '0.0.0' }, + apps: [OutputSchemaApp], + auth: { mode: 'public' as const }, +}; + +type JsonSchemaLike = { + type?: string; + properties?: Record; + anyOf?: JsonSchemaLike[]; + oneOf?: JsonSchemaLike[]; +}; describe('tools/list outputSchema advertisement (E2E)', () => { let server: DirectMcpServer; beforeAll(async () => { - server = await FrontMcpInstance.createDirect({ - info: { name: 'output-schema-e2e', version: '0.0.0' }, - apps: [OutputSchemaApp], - auth: { mode: 'public' as const }, - }); + server = await FrontMcpInstance.createDirect(outputSchemaServerConfig); }); afterAll(async () => { @@ -98,4 +158,73 @@ describe('tools/list outputSchema advertisement (E2E)', () => { expect(tool).toBeDefined(); expect(tool?.outputSchema).toBeUndefined(); }); + + it('advertises a z.union() of objects with a top-level object type and anyOf', async () => { + const { tools } = await server.listTools(); + const out = tools.find((t) => t.name === 'search_union')?.outputSchema as JsonSchemaLike | undefined; + + expect(out?.type).toBe('object'); + expect(out?.anyOf?.map((member) => member.type)).toEqual(['object', 'object']); + expect(out?.anyOf?.[0]?.properties?.['title']).toBeDefined(); + }); + + it('advertises a z.discriminatedUnion() with a top-level object type and oneOf', async () => { + const { tools } = await server.listTools(); + const out = tools.find((t) => t.name === 'resolve_discriminated')?.outputSchema as JsonSchemaLike | undefined; + + expect(out?.type).toBe('object'); + expect(out?.oneOf?.map((member) => member.type)).toEqual(['object', 'object']); + }); + + it('answers a union tool with structuredContent an MCP client validates against the listed schema', async () => { + const client = await connect(outputSchemaServerConfig, { clientInfo: { name: 'schema-probe', version: '1.0.0' } }); + try { + await client.listTools(); + const result = (await client.callTool('resolve_discriminated', { handle: 'Ada' })) as { + structuredContent?: unknown; + }; + + expect(result.structuredContent).toEqual({ kind: 'person', name: 'Ada' }); + } finally { + await client.close(); + } + }); + + it('advertises a union of objects registered with an id, and an MCP client validates its answer', async () => { + const { tools } = await server.listTools(); + const out = tools.find((t) => t.name === 'lookup_registered')?.outputSchema as + | (JsonSchemaLike & { $defs?: Record }) + | undefined; + expect(out?.type).toBe('object'); + expect(Object.keys(out?.$defs ?? {})).toEqual(['ToolsListE2eDocumentHit', 'ToolsListE2ePersonHit']); + + const client = await connect(outputSchemaServerConfig, { clientInfo: { name: 'schema-probe', version: '1.0.0' } }); + try { + await client.listTools(); + const result = (await client.callTool('lookup_registered', { name: 'Ada' })) as { structuredContent?: unknown }; + + expect(result.structuredContent).toEqual({ kind: 'person', name: 'Ada' }); + } finally { + await client.close(); + } + }); + + it('lists a union registered with an id that an MCP client compiles when elicitation wraps it', async () => { + const client = await connect( + { ...outputSchemaServerConfig, elicitation: { enabled: true } }, + { clientInfo: { name: 'schema-probe', version: '1.0.0' } }, + ); + try { + const tools = (await client.listTools()) as Array<{ name: string; outputSchema?: unknown }>; + expect(tools.find((t) => t.name === 'lookup_registered')?.outputSchema).toMatchObject({ + type: 'object', + $defs: { ToolsListE2eDocumentHit: { type: 'object' } }, + }); + + const result = (await client.callTool('lookup_registered', { name: 'Ada' })) as { structuredContent?: unknown }; + expect(result.structuredContent).toEqual({ kind: 'person', name: 'Ada' }); + } finally { + await client.close(); + } + }); }); diff --git a/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts b/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts index e5bbbe45e..ecdf0a0f3 100644 --- a/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts +++ b/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts @@ -30,6 +30,7 @@ const fixtureFiles = [ 'hooks-must-compile.ts', 'host-must-compile.ts', 'skills-must-compile.ts', + 'output-schemas-must-compile.ts', ].map((fileName) => path.join(fixtureDir, fileName)); function compileFixture(): { diagnostics: FixtureDiagnostic[]; sourceFiles: string[] } { @@ -102,6 +103,10 @@ describe('@frontmcp/sdk declarations in a strict consumer project', () => { expect(describeEach(inFixtureFile('skills-must-compile.ts'))).toEqual([]); }); + it('types a z.union() or z.discriminatedUnion() outputSchema with no cast, and output.undeclaredKeys', () => { + expect(describeEach(inFixtureFile('output-schemas-must-compile.ts'))).toEqual([]); + }); + it('ships declaration files that type-check with skipLibCheck disabled', () => { const sdkDeclarationDiagnostics = diagnostics.filter((diagnostic) => diagnostic.fileName.startsWith(sdkDeclarationsDir + path.sep), diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/output-schemas-must-compile.ts b/apps/e2e/demo-e2e-dts-consumer/fixture/output-schemas-must-compile.ts new file mode 100644 index 000000000..3d30388f5 --- /dev/null +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/output-schemas-must-compile.ts @@ -0,0 +1,38 @@ +import 'reflect-metadata'; + +import { App, FrontMcp, Tool, ToolContext, z } from '@frontmcp/sdk'; + +const searchHit = z.union([ + z.object({ kind: z.literal('document'), title: z.string() }), + z.object({ kind: z.literal('person'), name: z.string() }), +]); + +@Tool({ name: 'search', inputSchema: { query: z.string() }, outputSchema: searchHit }) +export class SearchTool extends ToolContext { + async execute(input: { query: string }) { + return { kind: 'document' as const, title: input.query }; + } +} + +const principal = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('user'), id: z.string() }), + z.object({ kind: z.literal('group'), members: z.array(z.string()) }), +]); + +@Tool({ + name: 'resolve_principal', + inputSchema: {}, + outputSchema: principal, + output: { undeclaredKeys: 'fail' }, +}) +export class ResolvePrincipalTool extends ToolContext { + async execute() { + return { kind: 'group' as const, members: ['u_1'] }; + } +} + +@App({ name: 'Directory', tools: [SearchTool, ResolvePrincipalTool], output: { undeclaredKeys: 'warn' } }) +export class DirectoryApp {} + +@FrontMcp({ info: { name: 'directory', version: '1.0.0' }, apps: [DirectoryApp], output: { undeclaredKeys: 'keep' } }) +export class DirectoryServer {} diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/tool-must-fail.ts b/apps/e2e/demo-e2e-dts-consumer/fixture/tool-must-fail.ts index f847e524d..d4d3dc25c 100644 --- a/apps/e2e/demo-e2e-dts-consumer/fixture/tool-must-fail.ts +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/tool-must-fail.ts @@ -1,6 +1,6 @@ import 'reflect-metadata'; -import { Agent, Tool, ToolContext, z } from '@frontmcp/sdk'; +import { Tool, ToolContext, z, type Agent } from '@frontmcp/sdk'; type IsAny = 0 extends 1 & T ? true : false; @@ -33,3 +33,11 @@ export class Total extends ToolContext { return { total: '12.50 EUR' }; } } + +// @ts-expect-error output.undeclaredKeys accepts 'strip' | 'warn' | 'keep' | 'fail' only +@Tool({ name: 'drop_extra', inputSchema: {}, output: { undeclaredKeys: 'drop' } }) +export class DropExtra extends ToolContext { + async execute() { + return 1; + } +} diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json b/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json index 282aa3e60..023df07ba 100644 --- a/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json @@ -15,6 +15,7 @@ "prompts-must-compile.ts", "hooks-must-compile.ts", "host-must-compile.ts", - "skills-must-compile.ts" + "skills-must-compile.ts", + "output-schemas-must-compile.ts" ] } diff --git a/libs/lazy-zod/README.md b/libs/lazy-zod/README.md index c1cc9cecd..ed8fc1f33 100644 --- a/libs/lazy-zod/README.md +++ b/libs/lazy-zod/README.md @@ -46,6 +46,7 @@ A stand-in behaves like the schema it stands for: it parses the same, `instanceo | `lazyZ(build)` | A schema that calls `build()` the first time it's used, and only then. | | `isLazy(schema)` | `true` for a stand-in, from `z` or from `lazyZ()`. | | `forceMaterialize(schema)` | The real Zod schema, with every stand-in inside it built. | +| `hasTransform(schema)` | `true` when the schema runs a transform anywhere in its tree (`.transform()`, `z.preprocess()`, `.pipe()`, a codec), so what it parses to can differ in shape from its input. | | `toJSONSchema(schema)` | Zod's JSON Schema converter, run on `forceMaterialize(schema)`. Use it instead of Zod's own `z.toJSONSchema()`, which throws `TypeError: Cannot set properties of undefined (setting 'ref')` when an object or union in the schema is `.optional()` or has a `.default()`. | | `ZodObject`, `ZodError`, … | Zod's classes and types (`ZodTypeAny`, `infer`, `input`, `output`), so your code needs no import from `zod`. | diff --git a/libs/lazy-zod/src/__tests__/has-transform.spec.ts b/libs/lazy-zod/src/__tests__/has-transform.spec.ts new file mode 100644 index 000000000..244f08d05 --- /dev/null +++ b/libs/lazy-zod/src/__tests__/has-transform.spec.ts @@ -0,0 +1,43 @@ +import { z as realZ } from 'zod'; + +import { z } from '../lazy-z'; +import { hasTransform } from '../utils'; + +describe('hasTransform', () => { + it.each([ + ['a .transform()', z.object({ a: z.string() }).transform((value) => ({ renamed: value.a }))], + ['a z.preprocess()', z.preprocess((value) => value, z.object({ a: z.string() }))], + ['a .pipe()', z.string().pipe(z.coerce.number())], + ['a codec', z.codec(z.string(), z.number(), { decode: Number, encode: String })], + ['a transform on an object field', z.object({ at: z.string().transform((value) => new Date(value)) })], + ['a transform in an array element', z.array(z.object({ a: z.string() }).transform((value) => value.a))], + ['a transform in a union option', z.union([z.object({ a: z.string() }), z.string().transform(Number)])], + ['a transform under optional and nullable', z.object({ a: z.string().transform(Number).optional().nullable() })], + ['a transform in a record value', z.record(z.string(), z.string().transform(Number))], + ['a transform behind z.lazy()', z.object({ next: z.lazy(() => z.string().transform(Number)) })], + ['a real zod transform', realZ.object({ a: realZ.string() }).transform((value) => value.a)], + ])('is true for %s', (_label, schema) => { + expect(hasTransform(schema)).toBe(true); + }); + + it.each([ + ['an object', z.object({ a: z.string(), b: z.array(z.number()) })], + [ + 'a discriminated union', + z.discriminatedUnion('k', [z.object({ k: z.literal('a') }), z.object({ k: z.literal('b') })]), + ], + ['refinements and defaults', z.object({ a: z.string().refine(Boolean).default('x') })], + ['an intersection and a tuple', z.intersection(z.object({ a: z.string() }), z.tuple([z.number()]))], + ['a real zod object', realZ.object({ a: realZ.string() })], + ['something that is not a schema', { a: 'b' }], + ])('is false for %s', (_label, schema) => { + expect(hasTransform(schema)).toBe(false); + }); + + it('terminates on a recursive schema', () => { + type Node = { children: Node[] }; + const node: z.ZodType = z.object({ children: z.array(z.lazy(() => node)) }); + + expect(hasTransform(node)).toBe(false); + }); +}); diff --git a/libs/lazy-zod/src/index.ts b/libs/lazy-zod/src/index.ts index 3c56ae162..a306e63af 100644 --- a/libs/lazy-zod/src/index.ts +++ b/libs/lazy-zod/src/index.ts @@ -21,7 +21,7 @@ export { z } from './lazy-z'; export { z as default } from './lazy-z'; export { eagerZ } from './eager-z'; export { lazyZ, LazyZodSchema, LAZY_BRAND, LAZY_TARGET, type InferLazy } from './lazy-schema'; -export { isLazy, forceMaterialize } from './utils'; +export { isLazy, forceMaterialize, hasTransform } from './utils'; // JSON-Schema conversion. Kept here so consumer code never reaches past the // `@frontmcp/lazy-zod` boundary into `zod/v4` directly — one place to swap the diff --git a/libs/lazy-zod/src/utils.ts b/libs/lazy-zod/src/utils.ts index 269f1e8de..f14d3d091 100644 --- a/libs/lazy-zod/src/utils.ts +++ b/libs/lazy-zod/src/utils.ts @@ -11,6 +11,32 @@ export function isLazy(value: unknown): boolean { return (value as Record)[LAZY_BRAND] === true; } +/** + * True when a schema runs a transform anywhere in its tree: `.transform()`, `z.preprocess()`, + * `.pipe()` and codecs all parse through a pipe, so what the schema returns can differ in shape + * from what it was given. + */ +export function hasTransform(schema: unknown): boolean { + return containsPipe(schema, new WeakSet()); +} + +function containsPipe(value: unknown, seen: WeakSet): boolean { + const def = (value as { _zod?: { def?: Record } } | null | undefined)?._zod?.def; + if (!def || seen.has(def)) return false; + seen.add(def); + if (def['type'] === 'pipe') return true; + if (def['type'] === 'lazy') return containsPipe((def['getter'] as () => unknown)(), seen); + return Object.values(def).some((child) => nestedSchemas(child).some((nested) => containsPipe(nested, seen))); +} + +/** The schemas a definition field can hold: one schema, a list (options, items) or a shape. */ +function nestedSchemas(field: unknown): unknown[] { + if (Array.isArray(field)) return field; + const isShape = field !== null && typeof field === 'object' && (field as { _zod?: unknown })._zod === undefined; + if (isShape) return Object.values(field); + return [field]; +} + /** * Materialize a lazy schema and return the underlying real zod schema, * recursively materializing any lazy wrappers found in the schema tree diff --git a/libs/sdk/src/common/decorators/tool.decorator.ts b/libs/sdk/src/common/decorators/tool.decorator.ts index f73b939da..e4101e6d2 100644 --- a/libs/sdk/src/common/decorators/tool.decorator.ts +++ b/libs/sdk/src/common/decorators/tool.decorator.ts @@ -200,8 +200,8 @@ type __StructuredOutputType = | z.ZodRawShape | z.ZodObject | z.ZodArray - | z.ZodUnion<[z.ZodObject, ...z.ZodObject[]]> - | z.ZodDiscriminatedUnion<[z.ZodObject, ...z.ZodObject[]]>; + | z.ZodUnion, ...z.ZodObject[]]> + | z.ZodDiscriminatedUnion, ...z.ZodObject[]]>; type __ToolSingleOutputType = | __PrimitiveOutputType diff --git a/libs/sdk/src/common/entries/tool.entry.ts b/libs/sdk/src/common/entries/tool.entry.ts index 9ea5d97bc..c9e2e99fc 100644 --- a/libs/sdk/src/common/entries/tool.entry.ts +++ b/libs/sdk/src/common/entries/tool.entry.ts @@ -15,7 +15,7 @@ import type ProviderRegistry from '../../provider/provider.registry'; import { type ToolInputOf, type ToolOutputOf } from '../decorators'; import { type ToolContext } from '../interfaces'; import { type ProviderRegistryInterface } from '../interfaces/internal'; -import { type ToolInputType, type ToolMetadata, type ToolOutputType } from '../metadata'; +import { type ToolInputType, type ToolMetadata, type ToolOutputType, type UndeclaredKeysPolicy } from '../metadata'; import { type ToolRecord } from '../records'; import { BaseEntry, type EntryOwnerRef } from './base.entry'; @@ -42,6 +42,51 @@ export type ToolCallExtra = RequestHandlerExtra & { export type ParsedToolResult = CallToolResult; export type SafeTransformResult = { success: true; data: T } | { success: false; error: Error }; +/** How a tool's answer is parsed against its `outputSchema`. */ +export interface OutputParseOptions { + /** What happens to keys the answer carries that the schema does not declare. @default 'strip' */ + undeclaredKeys?: UndeclaredKeysPolicy; + /** Called with the undeclared key paths an answer carried, under the `'warn'` policy. */ + onUndeclaredKeys?: (keyPaths: string[]) => void; +} + +type OutputJsonSchemaResolution = { schema: Record | null; unlistedReason?: string }; + +/** Zod output types that are primitive outputs: they flow through `content` and are never listed. */ +const PRIMITIVE_ZOD_OUTPUT_TYPES = new Set(['string', 'number', 'boolean', 'bigint', 'date']); + +/** How zod refers to a schema registered with an id, or to a reused one: `#/$defs/`. */ +const DEFINITION_REF_PREFIX = '#/$defs/'; + +/** + * The JSON Schema MCP accepts as a tool's `outputSchema`: an object schema as is, or a union + * (`anyOf` / `oneOf`) or `$ref` that resolves to object schemas only, typed `object` at the top level. + */ +function asObjectOutputSchema(json: Record): Record | null { + if (json['type'] === 'object') return json; + return isObjectSchema(json, json['$defs'], new Set()) ? { type: 'object', ...json } : null; +} + +/** True for an object schema, a `#/$defs/` reference to one, or a union of those. */ +function isObjectSchema(schema: unknown, definitions: unknown, followedRefs: ReadonlySet): boolean { + if (typeof schema !== 'object' || schema === null) return false; + const node = schema as Record; + if (node['type'] === 'object') return true; + const ref = node['$ref']; + if (typeof ref === 'string') { + // Each `#/$defs/` reference is followed once: a recursive union can loop a validator forever + if (!ref.startsWith(DEFINITION_REF_PREFIX) || followedRefs.has(ref)) return false; + const definition = (definitions as Record | undefined)?.[ref.slice(DEFINITION_REF_PREFIX.length)]; + return isObjectSchema(definition, definitions, new Set([...followedRefs, ref])); + } + const members = node['anyOf'] ?? node['oneOf']; + return ( + Array.isArray(members) && + members.length > 0 && + members.every((member) => isObjectSchema(member, definitions, followedRefs)) + ); +} + export abstract class ToolEntry< InSchema extends ToolInputType = ToolInputType, OutSchema extends ToolOutputType = ToolOutputType, @@ -145,38 +190,47 @@ export abstract class ToolEntry< return null; } - /** Cached output JSON Schema result (undefined = not yet computed, null = none) */ - private _cachedOutputJsonSchema?: Record | null; + /** The output JSON Schema and, when a declared schema is left out of tools/list, why (undefined = not yet computed) */ + private outputJsonSchemaResolution?: OutputJsonSchemaResolution; /** * Get the tool's output schema as JSON Schema (cached after first call). * * Prefers an explicit `rawOutputSchema` (the JSON Schema set by the OpenAPI adapter / - * remote tools); otherwise converts a Zod-shape or `z.object()` `outputSchema` to JSON - * Schema — symmetric with {@link getInputJsonSchema}. Returns `null` for output forms - * that have no object-typed schema to advertise: primitive / media string literals - * (`'string'`, `'image'`, …), multi-content arrays (`['string', 'image']`), and any - * schema that does not serialize to a top-level `type: 'object'` (e.g. a union) — those - * flow through `content`, not `structuredContent`, per the MCP spec. + * remote tools); otherwise converts a Zod-shape or Zod `outputSchema` to JSON Schema — + * symmetric with {@link getInputJsonSchema}. MCP requires a top-level `type: 'object'`, so + * an object schema is returned as is and a union whose members are all object schemas is + * returned typed `object` (`{ type: 'object', anyOf | oneOf }`). Returns `null` for + * primitive / media outputs and multi-content arrays, which flow through `content`, and for + * any other schema (see {@link getUnlistedOutputSchemaReason}). * * @returns JSON Schema object, or null if there is no advertisable output schema. */ getOutputJsonSchema(): Record | null { - if (this._cachedOutputJsonSchema !== undefined) return this._cachedOutputJsonSchema; + return this.resolveOutputJsonSchema().schema; + } - const result = this.computeOutputJsonSchema(); - this._cachedOutputJsonSchema = result; - return result; + /** + * Why the declared `outputSchema` is left out of tools/list, or undefined when it is listed + * or is a primitive / media / multi-content output, which is never listed by design. + */ + getUnlistedOutputSchemaReason(): string | undefined { + return this.resolveOutputJsonSchema().unlistedReason; + } + + private resolveOutputJsonSchema(): OutputJsonSchemaResolution { + this.outputJsonSchemaResolution ??= this.computeOutputJsonSchema(); + return this.outputJsonSchemaResolution; } - private computeOutputJsonSchema(): Record | null { + private computeOutputJsonSchema(): OutputJsonSchemaResolution { // Prefer rawOutputSchema when it is already a JSON Schema object (OpenAPI / remote passthrough). if ( typeof this.rawOutputSchema === 'object' && this.rawOutputSchema !== null && !Array.isArray(this.rawOutputSchema) ) { - return this.rawOutputSchema as Record; + return { schema: this.rawOutputSchema as Record }; } const schema = this.outputSchema as unknown; @@ -184,36 +238,37 @@ export abstract class ToolEntry< // Primitive / media literals ('string' | 'number' | 'image' | …) and multi-content // arrays (['string', 'image']) have no object outputSchema — they flow via `content`. if (!schema || typeof schema === 'string' || Array.isArray(schema)) { - return null; + return { schema: null }; } - try { - // Duck-type a Zod schema (z.object(), z.discriminatedUnion(), …) vs a raw shape - // ({ field: z.string() }); identity-based instanceof is avoided so a duplicated zod - // module copy cannot break detection. - const isZodSchema = - typeof (schema as { safeParse?: unknown }).safeParse === 'function' && - typeof (schema as { parse?: unknown }).parse === 'function'; - - if (!isZodSchema && Object.keys(schema as Record).length === 0) { - // Empty raw shape — nothing meaningful to advertise. - return null; - } + // Duck-type a Zod schema (z.object(), z.discriminatedUnion(), …) vs a raw shape + // ({ field: z.string() }); identity-based instanceof is avoided so a duplicated zod + // module copy cannot break detection. + const isZodSchema = + typeof (schema as { safeParse?: unknown }).safeParse === 'function' && + typeof (schema as { parse?: unknown }).parse === 'function'; + + if (!isZodSchema && Object.keys(schema as Record).length === 0) { + // Empty raw shape — nothing meaningful to advertise. + return { schema: null }; + } + + if (isZodSchema && PRIMITIVE_ZOD_OUTPUT_TYPES.has((schema as { type?: unknown }).type)) { + return { schema: null }; + } + try { // `schema` is a Zod type (z.object()/discriminatedUnion/…) or a raw shape at // runtime, but `unknown` to TS. Narrow to the Zod-object type both z.object() and // toJSONSchema() accept (the raw-shape cast mirrors the inputSchema path above). const zodSchema = isZodSchema ? (schema as ReturnType) : z.object(schema as ToolInputType); - const json = toJSONSchema(zodSchema) as Record; - - // MCP requires outputSchema to be a top-level object schema; skip anything else - // (e.g. a discriminated union serializes to `anyOf`, not `type: 'object'`). - return json && json['type'] === 'object' ? json : null; + const objectSchema = asObjectOutputSchema(toJSONSchema(zodSchema) as Record); + return objectSchema + ? { schema: objectSchema } + : { schema: null, unlistedReason: 'it is not an object schema or a union of object schemas' }; } catch (error) { - if (isDebug() || isDevelopment()) { - console.warn('[ToolEntry] Failed to convert output Zod schema to JSON Schema:', error); - } - return null; + const message = error instanceof Error ? error.message : String(error); + return { schema: null, unlistedReason: `it cannot be converted to JSON Schema (${message})` }; } } @@ -231,10 +286,13 @@ export abstract class ToolEntry< * Convert the raw tool return value (Out) into an MCP CallToolResult-shaped object. * Concrete logic is implemented in ToolInstance. */ - abstract parseOutput(result: Out | Partial | any): ParsedToolResult; + abstract parseOutput(result: Out | Partial | any, options?: OutputParseOptions): ParsedToolResult; /** * Convert the raw tool return value (Out) into an MCP CallToolResult-shaped object. * Concrete logic is implemented in ToolInstance. */ - abstract safeParseOutput(raw: Out | Partial | any): SafeTransformResult; + abstract safeParseOutput( + raw: Out | Partial | any, + options?: OutputParseOptions, + ): SafeTransformResult; } diff --git a/libs/sdk/src/common/metadata/__tests__/output-policy.spec.ts b/libs/sdk/src/common/metadata/__tests__/output-policy.spec.ts index cee5be394..9c83ff674 100644 --- a/libs/sdk/src/common/metadata/__tests__/output-policy.spec.ts +++ b/libs/sdk/src/common/metadata/__tests__/output-policy.spec.ts @@ -196,6 +196,88 @@ describe('output-policy', () => { const out = formatOutputSchemaForDescription({ type: 'array', items: { type: 'number' } }, 'summary'); expect(out).toBe('\n\n**Returns:** `number[]`'); }); + + it('renders each member of an object union', () => { + const schema = { + type: 'object', + anyOf: [ + { type: 'object', properties: { id: { type: 'string' } }, required: ['id'] }, + { type: 'object', properties: { error: { type: 'string' } } }, + ], + }; + + const out = formatOutputSchemaForDescription(schema, 'summary'); + + expect(out).toBe( + '\n\n**Returns** one of:\n- Option 1:\n - `id`: string (required)\n- Option 2:\n - `error`: string (optional)', + ); + }); + + it('follows $ref members of a oneOf union into the root $defs', () => { + const schema = { + type: 'object', + oneOf: [{ $ref: '#/$defs/Invoice' }, { $ref: '#/$defs/Refund' }], + $defs: { + Invoice: { type: 'object', properties: { total: { type: 'number' } }, required: ['total'] }, + Refund: { type: 'object', properties: { reason: { type: 'string' } } }, + }, + }; + + const out = formatOutputSchemaForDescription(schema, 'summary'); + + expect(out).toContain('- Option 1:\n - `total`: number (required)'); + expect(out).toContain('- Option 2:\n - `reason`: string (optional)'); + }); + + it('follows a root $ref into the root $defs', () => { + const schema = { + $ref: '#/$defs/Invoice', + $defs: { Invoice: { type: 'object', properties: { total: { type: 'number' } }, required: ['total'] } }, + }; + + expect(formatOutputSchemaForDescription(schema, 'summary')).toBe( + '\n\n**Returns:**\n- `total`: number (required)', + ); + }); + + it('renders the shared properties and each union member when the root has both', () => { + const schema = { + type: 'object', + properties: { kind: { type: 'string' } }, + required: ['kind'], + oneOf: [ + { type: 'object', properties: { id: { type: 'string' } } }, + { type: 'object', properties: { error: { type: 'string' } } }, + ], + }; + + expect(formatOutputSchemaForDescription(schema, 'summary')).toBe( + '\n\n**Returns:**\n- `kind`: string (required)\n\nand one of:\n' + + '- Option 1:\n - `id`: string (optional)\n- Option 2:\n - `error`: string (optional)', + ); + }); + + it('leaves the summary out when the root has properties and a union member cannot be summarized', () => { + const schema = { + type: 'object', + properties: { kind: { type: 'string' } }, + anyOf: [{ type: 'object', properties: { id: { type: 'string' } } }, { type: 'string' }], + }; + + expect(formatOutputSchemaForDescription(schema, 'summary')).toBe(''); + }); + + it('leaves the summary out when a union member cannot be summarized', () => { + const schema = { + type: 'object', + anyOf: [ + { type: 'object', properties: { id: { type: 'string' } } }, + { $ref: 'https://example.com/remote.json' }, + ], + }; + + expect(formatOutputSchemaForDescription(schema, 'summary')).toBe(''); + }); }); describe('formatOutputSchemaForDescription — jsonSchema', () => { diff --git a/libs/sdk/src/common/metadata/front-mcp.metadata.ts b/libs/sdk/src/common/metadata/front-mcp.metadata.ts index fc053c161..623278b36 100644 --- a/libs/sdk/src/common/metadata/front-mcp.metadata.ts +++ b/libs/sdk/src/common/metadata/front-mcp.metadata.ts @@ -323,6 +323,9 @@ export interface FrontMcpBaseMetadata { * `InvalidOutputError`, surfacing silent `JSON.stringify` corruption. * - `schemaMode`: how `outputSchema` is exposed in `tools/list` — `'definition'` * (default) / `'description'` / `'both'` / `'none'`. + * - `undeclaredKeys`: what `tools/call` does with keys an answer carries that its + * `outputSchema` does not declare — `'strip'` / `'warn'` / `'keep'` / `'fail'`. Default + * `'warn'` when `NODE_ENV` is `'development'`, otherwise `'strip'`. * * @default { allowNonFinite: false, schemaMode: 'definition' } */ diff --git a/libs/sdk/src/common/metadata/output-policy.ts b/libs/sdk/src/common/metadata/output-policy.ts index 18d14614b..7c50b0947 100644 --- a/libs/sdk/src/common/metadata/output-policy.ts +++ b/libs/sdk/src/common/metadata/output-policy.ts @@ -1,5 +1,6 @@ // file: libs/sdk/src/common/metadata/output-policy.ts import { z } from '@frontmcp/lazy-zod'; +import { isDevelopment } from '@frontmcp/utils'; /** * How a tool's output schema is exposed to clients in `tools/list`. @@ -28,6 +29,22 @@ export type OutputSchemaMode = (typeof OUTPUT_SCHEMA_MODES)[number]; export const SCHEMA_DESCRIPTION_FORMATS = ['summary', 'jsonSchema'] as const; export type SchemaDescriptionFormat = (typeof SCHEMA_DESCRIPTION_FORMATS)[number]; +/** + * What `tools/call` does with keys an answer carries that its `outputSchema` does not declare. + * + * - `'strip'`: drop them. + * - `'warn'`: drop them and log a warning naming each key once per tool (up to 100 keys per tool). + * - `'keep'`: send them in `structuredContent` and the text content; `tools/list` then lists the + * schema's objects open to undeclared keys. + * - `'fail'`: refuse the answer with `InvalidOutputError` naming the keys. + * + * A call that renders the tool's UI sends declared keys only, whatever the policy, and so does an + * `outputSchema` with a transform (`.transform()`, `z.preprocess()`, `.pipe()`), whose output cannot + * be compared with the answer key by key. + */ +export const UNDECLARED_KEYS_POLICIES = ['strip', 'warn', 'keep', 'fail'] as const; +export type UndeclaredKeysPolicy = (typeof UNDECLARED_KEYS_POLICIES)[number]; + /** Default exposure mode when unset at every level. */ export const DEFAULT_OUTPUT_SCHEMA_MODE: OutputSchemaMode = 'definition'; /** Default description rendering when unset at every level. */ @@ -60,6 +77,14 @@ export interface OutputPolicy { * @default 'summary' */ schemaDescriptionFormat?: SchemaDescriptionFormat; + + /** + * What `tools/call` does with keys an answer carries that its `outputSchema` does not + * declare. See {@link UndeclaredKeysPolicy}. + * + * @default 'warn' when NODE_ENV is 'development', otherwise 'strip' + */ + undeclaredKeys?: UndeclaredKeysPolicy; } /** @@ -70,6 +95,7 @@ export const outputPolicySchema = z.object({ allowNonFinite: z.boolean().optional(), schemaMode: z.enum(OUTPUT_SCHEMA_MODES).optional(), schemaDescriptionFormat: z.enum(SCHEMA_DESCRIPTION_FORMATS).optional(), + undeclaredKeys: z.enum(UNDECLARED_KEYS_POLICIES).optional(), }); /** Resolve the effective output-schema mode: Tool > App > server > default. */ @@ -95,6 +121,26 @@ export function resolveSchemaDescriptionFormat( ); } +/** Resolve the undeclared-keys policy: Tool > App > server > 'warn' in development, 'strip' otherwise. */ +export function resolveUndeclaredKeysPolicy( + tool?: OutputPolicy, + app?: OutputPolicy, + server?: OutputPolicy, +): UndeclaredKeysPolicy { + return tool?.undeclaredKeys ?? app?.undeclaredKeys ?? server?.undeclaredKeys ?? (isDevelopment() ? 'warn' : 'strip'); +} + +/** A copy of a JSON Schema whose objects accept undeclared keys: every `additionalProperties: false` is left out. */ +export function withUndeclaredKeysAllowed(schema: unknown): unknown { + if (Array.isArray(schema)) return schema.map(withUndeclaredKeysAllowed); + if (typeof schema !== 'object' || schema === null) return schema; + return Object.fromEntries( + Object.entries(schema) + .filter(([keyword, value]) => !(keyword === 'additionalProperties' && value === false)) + .map(([keyword, value]) => [keyword, withUndeclaredKeysAllowed(value)]), + ); +} + /** Render a JSON Schema's type as a short string (e.g. `string`, `number[]`, `a | b`). */ function schemaTypeOf(schema: Record): string { const type = schema['type']; @@ -120,15 +166,45 @@ export function formatOutputSchemaForDescription( return `\n\n**Output schema:**\n\`\`\`json\n${JSON.stringify(schema, null, 2)}\n\`\`\``; } - const properties = schema['properties'] as Record> | undefined; - if (!properties || Object.keys(properties).length === 0) { - return `\n\n**Returns:** \`${schemaTypeOf(schema)}\``; + const target = resolveLocalRef(schema, schema) ?? schema; + const lines = propertyLinesOf(target); + const members = target['anyOf'] ?? target['oneOf']; + if (!Array.isArray(members)) { + return lines ? `\n\n**Returns:**\n${lines.join('\n')}` : `\n\n**Returns:** \`${schemaTypeOf(target)}\``; } + const memberLines = members.map((member: Record) => { + const resolvedMember = resolveLocalRef(member, schema); + return resolvedMember ? propertyLinesOf(resolvedMember) : undefined; + }); + if (memberLines.some((memberLine) => memberLine === undefined)) return ''; + + const options = memberLines.map( + (memberLine, index) => `- Option ${index + 1}:\n${(memberLine ?? []).map((line) => ` ${line}`).join('\n')}`, + ); + const heading = lines ? `**Returns:**\n${lines.join('\n')}\n\nand one of:` : '**Returns** one of:'; + return `\n\n${heading}\n${options.join('\n')}`; +} + +function propertyLinesOf(schema: Record): string[] | undefined { + const properties = schema['properties'] as Record> | undefined; + if (!properties || Object.keys(properties).length === 0) return undefined; + const required = new Set(Array.isArray(schema['required']) ? (schema['required'] as string[]) : []); - const lines = Object.entries(properties).map(([key, prop]) => { + return Object.entries(properties).map(([key, prop]) => { const desc = typeof prop['description'] === 'string' ? ` — ${prop['description']}` : ''; return `- \`${key}\`: ${schemaTypeOf(prop)} (${required.has(key) ? 'required' : 'optional'})${desc}`; }); - return `\n\n**Returns:**\n${lines.join('\n')}`; +} + +function resolveLocalRef( + schema: Record, + root: Record, +): Record | undefined { + const ref = schema['$ref']; + if (typeof ref !== 'string') return schema; + const definitionsPrefix = '#/$defs/'; + if (!ref.startsWith(definitionsPrefix)) return undefined; + const definitions = root['$defs'] as Record> | undefined; + return definitions?.[ref.slice(definitionsPrefix.length)]; } diff --git a/libs/sdk/src/common/metadata/tool.metadata.ts b/libs/sdk/src/common/metadata/tool.metadata.ts index 0f6707b57..457689d78 100644 --- a/libs/sdk/src/common/metadata/tool.metadata.ts +++ b/libs/sdk/src/common/metadata/tool.metadata.ts @@ -236,8 +236,8 @@ type StructuredOutputType = | z.ZodRawShape | z.ZodObject | z.ZodArray - | z.ZodUnion<[z.ZodObject, ...z.ZodObject[]]> - | z.ZodDiscriminatedUnion<[z.ZodObject, ...z.ZodObject[]]>; + | z.ZodUnion, ...z.ZodObject[]]> + | z.ZodDiscriminatedUnion, ...z.ZodObject[]]>; export type ToolSingleOutputType = | PrimitiveOutputType @@ -504,9 +504,10 @@ export interface ToolMetadata { }); }); + describe('with original schema whose references point into its $defs', () => { + const registeredUnion: Record = { + type: 'object', + anyOf: [{ $ref: '#/$defs/Document' }, { $ref: '#/$defs/Person' }], + $defs: { + Document: { type: 'object', properties: { title: { type: 'string' } } }, + Person: { type: 'object', properties: { name: { type: 'string' } } }, + }, + }; + + it('keeps $defs at the root, where the references resolve from', () => { + const extended = extendOutputSchemaForElicitation(registeredUnion); + + expect(extended['$defs']).toEqual(registeredUnion['$defs']); + expect(extended['oneOf']).toEqual([ + { type: 'object', anyOf: registeredUnion['anyOf'] }, + ELICITATION_FALLBACK_JSON_SCHEMA, + ]); + }); + }); + describe('MCP SDK ToolSchema compatibility', () => { function buildToolDescriptor(outputSchema: Record) { return { diff --git a/libs/sdk/src/elicitation/helpers/extend-output-schema.ts b/libs/sdk/src/elicitation/helpers/extend-output-schema.ts index bf8b9900e..a5438fd80 100644 --- a/libs/sdk/src/elicitation/helpers/extend-output-schema.ts +++ b/libs/sdk/src/elicitation/helpers/extend-output-schema.ts @@ -70,8 +70,11 @@ export function extendOutputSchemaForElicitation( // Wrap in oneOf: either original output OR elicitation fallback // type: 'object' is required by the MCP spec's ToolSchema for outputSchema + // $defs stays at the root, where the original's `#/$defs/...` references resolve from + const { $defs: definitions, ...wrappedSchema } = originalSchema; return { type: 'object', - oneOf: [originalSchema, ELICITATION_FALLBACK_JSON_SCHEMA], + oneOf: [wrappedSchema, ELICITATION_FALLBACK_JSON_SCHEMA], + ...(definitions !== undefined && { $defs: definitions }), }; } diff --git a/libs/sdk/src/scope/scope.instance.ts b/libs/sdk/src/scope/scope.instance.ts index e333e035d..e9e8533ff 100644 --- a/libs/sdk/src/scope/scope.instance.ts +++ b/libs/sdk/src/scope/scope.instance.ts @@ -71,6 +71,7 @@ import { pluginsRegisterDynamicSkills, registerSkillCapabilities } from '../skil import SkillRegistry from '../skill/skill.registry'; import type { InProcessTaskRunner, TaskRegistry, TaskStore } from '../task'; import { resolvesToVercelKvTaskBackend } from '../task/store/task-backend.utils'; +import { describeUnlistedOutputSchemas } from '../tool/tool-output-policy'; import { ToolInstance } from '../tool/tool.instance'; import ToolRegistry from '../tool/tool.registry'; import { normalizeTool } from '../tool/tool.utils'; @@ -692,6 +693,9 @@ export class Scope extends ScopeEntry { // Fail-fast: entries with metadata only a plugin enforces (`approval`, `featureFlag`, ...) that no such plugin covers this.validateEnforcedMetadata(); + // One warning per tool whose declared outputSchema tools/list has to leave out + for (const message of describeUnlistedOutputSchemas(this)) this.logger.warn(message); + await this.initGuardForDeclaredLimits(); // ═══ BATCH 3: Cross-registry finalization (sequential) ═══ diff --git a/libs/sdk/src/tool/__tests__/tool-instance.output-schema.spec.ts b/libs/sdk/src/tool/__tests__/tool-instance.output-schema.spec.ts new file mode 100644 index 000000000..80ba84283 --- /dev/null +++ b/libs/sdk/src/tool/__tests__/tool-instance.output-schema.spec.ts @@ -0,0 +1,296 @@ +import { z } from '@frontmcp/lazy-zod'; + +import { createMockOwner, createMockProviderRegistry } from '../../__test-utils__/mocks'; +import { ToolContext, ToolKind, type ToolMetadata, type ToolRecord } from '../../common'; +import { InvalidOutputError } from '../../errors'; +import { ToolInstance } from '../tool.instance'; + +class AnswerTool extends ToolContext { + async execute() { + return {}; + } +} + +function toolWithOutput(outputSchema: unknown): { tool: ToolInstance } { + const record = { + kind: ToolKind.CLASS_TOKEN, + provide: AnswerTool, + metadata: { name: 'answer', inputSchema: {}, outputSchema } as unknown as ToolMetadata, + } as unknown as ToolRecord; + return { tool: new ToolInstance(record, createMockProviderRegistry(), createMockOwner()) }; +} + +const documentHit = z.object({ kind: z.literal('document'), title: z.string() }); +const personHit = z.object({ kind: z.literal('person'), name: z.string() }); + +describe('ToolInstance output JSON Schema', () => { + it('lists a union of object schemas as an object schema with anyOf', () => { + const { tool } = toolWithOutput(z.union([documentHit, personHit])); + + expect(tool.getOutputJsonSchema()).toMatchObject({ + type: 'object', + anyOf: [ + { type: 'object', required: ['kind', 'title'] }, + { type: 'object', required: ['kind', 'name'] }, + ], + }); + expect(tool.getUnlistedOutputSchemaReason()).toBeUndefined(); + }); + + it('lists a discriminated union as an object schema with oneOf', () => { + const { tool } = toolWithOutput(z.discriminatedUnion('kind', [documentHit, personHit])); + + expect(tool.getOutputJsonSchema()).toMatchObject({ + type: 'object', + oneOf: [{ type: 'object' }, { type: 'object' }], + }); + }); + + it('lists a union whose members are unions of object schemas', () => { + const nested = z.union([z.union([documentHit, personHit]), z.object({ kind: z.literal('team') })]); + + expect(toolWithOutput(nested).tool.getOutputJsonSchema()).toMatchObject({ type: 'object' }); + }); + + it('lists a union whose members are object schemas registered with an id', () => { + const registeredUnion = z.union([ + documentHit.meta({ id: 'ToolInstanceSpecDocument' }), + personHit.meta({ id: 'ToolInstanceSpecPerson' }), + ]); + const { tool } = toolWithOutput(registeredUnion); + + expect(tool.getOutputJsonSchema()).toMatchObject({ + type: 'object', + anyOf: [{ $ref: '#/$defs/ToolInstanceSpecDocument' }, { $ref: '#/$defs/ToolInstanceSpecPerson' }], + $defs: { ToolInstanceSpecDocument: { type: 'object' }, ToolInstanceSpecPerson: { type: 'object' } }, + }); + expect(tool.getUnlistedOutputSchemaReason()).toBeUndefined(); + }); + + it('lists an object schema registered with an id', () => { + const { tool } = toolWithOutput(documentHit.meta({ id: 'ToolInstanceSpecHit' })); + + expect(tool.getOutputJsonSchema()).toMatchObject({ type: 'object', $ref: '#/$defs/ToolInstanceSpecHit' }); + }); + + it('leaves out a union of a registered object schema and a non-object member', () => { + const { tool } = toolWithOutput(z.union([documentHit.meta({ id: 'ToolInstanceSpecMixed' }), z.string()])); + + expect(tool.getOutputJsonSchema()).toBeNull(); + expect(tool.getUnlistedOutputSchemaReason()).toContain('not an object schema or a union of object schemas'); + }); + + it('leaves out a recursive union instead of following its references forever', () => { + const recursiveUnion: z.ZodType = z.union([z.object({ leaf: z.string() }), z.lazy(() => recursiveUnion)]); + const outerUnion = z.union([z.object({ root: z.string() }), recursiveUnion]); + + expect(toolWithOutput(recursiveUnion).tool.getOutputJsonSchema()).toBeNull(); + expect(toolWithOutput(outerUnion).tool.getOutputJsonSchema()).toBeNull(); + expect(toolWithOutput(outerUnion).tool.getUnlistedOutputSchemaReason()).toContain('not an object schema'); + }); + + it.each([ + ['a union with a non-object member', z.union([documentHit, z.string()])], + ['a nullable object', documentHit.nullable()], + ['an array of objects', z.array(documentHit)], + ])('leaves out %s and says why', (_label, outputSchema) => { + const { tool } = toolWithOutput(outputSchema); + + expect(tool.getOutputJsonSchema()).toBeNull(); + expect(tool.getUnlistedOutputSchemaReason()).toBe('it is not an object schema or a union of object schemas'); + }); + + it('says why when the output schema cannot be converted to JSON Schema', () => { + const { tool } = toolWithOutput(z.object({ at: z.date() })); + + expect(tool.getOutputJsonSchema()).toBeNull(); + expect(tool.getUnlistedOutputSchemaReason()).toContain('Date cannot be represented in JSON Schema'); + }); + + it.each([ + ['a primitive literal', 'number'], + ['a Zod primitive', z.string()], + ['a Zod date', z.date()], + ['a media literal', 'image'], + ['a multi-content array', ['string', 'image']], + ['an empty raw shape', {}], + ['no output schema', undefined], + ])('lists nothing for %s, by design and with no reason to report', (_label, outputSchema) => { + const { tool } = toolWithOutput(outputSchema); + + expect(tool.getOutputJsonSchema()).toBeNull(); + expect(tool.getUnlistedOutputSchemaReason()).toBeUndefined(); + }); +}); + +describe('ToolInstance.parseOutput and undeclared keys', () => { + const invoiceShape = { id: z.string(), lines: z.array(z.object({ sku: z.string() })) }; + const answer = { + id: 'INV-1', + costCenter: 'CC-7', + lines: [ + { sku: 'A', margin: 0.4 }, + { sku: 'B', margin: 0.1 }, + ], + }; + + it("strips undeclared keys by default and under 'strip'", () => { + const { tool } = toolWithOutput(invoiceShape); + const declared = { id: 'INV-1', lines: [{ sku: 'A' }, { sku: 'B' }] }; + + expect(tool.parseOutput(answer).structuredContent).toEqual(declared); + expect(tool.parseOutput(answer, { undeclaredKeys: 'strip' }).structuredContent).toEqual(declared); + }); + + it("strips under 'warn' and reports each undeclared key path once", () => { + const { tool } = toolWithOutput(invoiceShape); + const onUndeclaredKeys = jest.fn(); + + const result = tool.parseOutput(answer, { undeclaredKeys: 'warn', onUndeclaredKeys }); + + expect(result.structuredContent).toEqual({ id: 'INV-1', lines: [{ sku: 'A' }, { sku: 'B' }] }); + expect(onUndeclaredKeys).toHaveBeenCalledWith(['costCenter', 'lines[].margin']); + }); + + it("does not report under 'warn' when the answer has no undeclared key", () => { + const { tool } = toolWithOutput(invoiceShape); + const onUndeclaredKeys = jest.fn(); + + tool.parseOutput({ id: 'INV-2', lines: [] }, { undeclaredKeys: 'warn', onUndeclaredKeys }); + + expect(onUndeclaredKeys).not.toHaveBeenCalled(); + }); + + it("passes undeclared keys through to structuredContent and the text under 'keep'", () => { + const { tool } = toolWithOutput(invoiceShape); + + const result = tool.parseOutput(answer, { undeclaredKeys: 'keep' }); + + expect(result.structuredContent).toEqual(answer); + expect(JSON.parse((result.content[0] as { text: string }).text)).toEqual(answer); + }); + + it("refuses the answer under 'fail' with InvalidOutputError naming the keys", () => { + const { tool } = toolWithOutput(invoiceShape); + + const parsed = tool.safeParseOutput(answer, { undeclaredKeys: 'fail' }); + + expect(parsed.success).toBe(false); + const error = (parsed as { error: Error }).error; + expect(error).toBeInstanceOf(InvalidOutputError); + expect(error.message).toContain('costCenter, lines[].margin'); + }); + + it("accepts an answer with declared keys only under 'fail'", () => { + const { tool } = toolWithOutput(invoiceShape); + + expect(tool.safeParseOutput({ id: 'INV-2', lines: [{ sku: 'A' }] }, { undeclaredKeys: 'fail' })).toMatchObject({ + success: true, + }); + }); + + it('applies the policy to a union output schema', () => { + const { tool } = toolWithOutput(z.union([documentHit, personHit])); + const hit = { kind: 'person', name: 'Ada', email: 'ada@example.com' }; + + expect(tool.parseOutput(hit).structuredContent).toEqual({ kind: 'person', name: 'Ada' }); + expect(tool.parseOutput(hit, { undeclaredKeys: 'keep' }).structuredContent).toEqual(hit); + }); + + it('keeps an undeclared key next to a non-finite number left for the finalize stage', () => { + const { tool } = toolWithOutput({ ratio: z.number() }); + + expect(tool.parseOutput({ ratio: Infinity, note: 'n/a' }, { undeclaredKeys: 'keep' }).structuredContent).toEqual({ + ratio: Infinity, + note: 'n/a', + }); + }); + + describe('with an object schema that sets its own unknown-key handling, which wins over the policy', () => { + const policies = ['strip', 'warn', 'keep', 'fail'] as const; + + it.each(policies)("lets z.strictObject() refuse an undeclared key under '%s'", (undeclaredKeys) => { + const { tool } = toolWithOutput(z.strictObject({ id: z.string() })); + + const parsed = tool.safeParseOutput({ id: '1', extra: true }, { undeclaredKeys }); + + expect(parsed.success).toBe(false); + expect((parsed as { error: Error }).error).toBeInstanceOf(InvalidOutputError); + }); + + it.each(policies)("lets z.looseObject() keep an undeclared key under '%s'", (undeclaredKeys) => { + const { tool } = toolWithOutput(z.looseObject({ id: z.string() })); + + expect(tool.parseOutput({ id: '1', extra: true }, { undeclaredKeys }).structuredContent).toEqual({ + id: '1', + extra: true, + }); + }); + }); + + it("refuses under 'keep' an undeclared key whose object the schema coerced to a primitive", () => { + const { tool } = toolWithOutput({ payload: z.coerce.string() }); + + const parsed = tool.safeParseOutput({ payload: { secret: 'x' } }, { undeclaredKeys: 'keep' }); + + expect(parsed.success).toBe(false); + const error = (parsed as { error: Error }).error; + expect(error).toBeInstanceOf(InvalidOutputError); + expect(error.message).toContain('payload.secret'); + }); + + describe('with a cycle under a z.any() field, which the parse keeps by reference', () => { + function cyclicAnswer() { + const meta: Record = { name: 'node' }; + meta['self'] = meta; + return { id: '1', note: 'n/a', meta }; + } + + it("reports the undeclared key under 'warn' without following the cycle", () => { + const { tool } = toolWithOutput({ id: z.string(), meta: z.any() }); + const onUndeclaredKeys = jest.fn(); + + tool.parseOutput(cyclicAnswer(), { undeclaredKeys: 'warn', onUndeclaredKeys }); + + expect(onUndeclaredKeys).toHaveBeenCalledWith(['note']); + }); + + it("keeps the undeclared key under 'keep' without following the cycle", () => { + const { tool } = toolWithOutput({ id: z.string(), meta: z.any() }); + + expect(tool.parseOutput(cyclicAnswer(), { undeclaredKeys: 'keep' }).structuredContent).toEqual({ + id: '1', + note: 'n/a', + meta: { name: 'node' }, + }); + }); + }); + + describe('with a transform, whose output cannot be compared with the answer key by key', () => { + const renamingSchema = z.object({ a: z.string() }).transform((value) => ({ renamed: value.a })); + const droppingSchema = z + .object({ id: z.string(), internal: z.string() }) + .transform(({ internal: _internal, ...rest }) => rest); + + it.each(['warn', 'keep', 'fail'] as const)( + "returns the transformed output and strips an undeclared key under '%s'", + (undeclaredKeys) => { + const { tool } = toolWithOutput(renamingSchema); + const onUndeclaredKeys = jest.fn(); + + const result = tool.parseOutput({ a: 'x', extra: 1 }, { undeclaredKeys, onUndeclaredKeys }); + + expect(result.structuredContent).toEqual({ renamed: 'x' }); + expect(onUndeclaredKeys).not.toHaveBeenCalled(); + }, + ); + + it("never puts back a key the transform dropped under 'keep'", () => { + const { tool } = toolWithOutput(droppingSchema); + + const result = tool.parseOutput({ id: '1', internal: 'secret' }, { undeclaredKeys: 'keep' }); + + expect(result.structuredContent).toEqual({ id: '1' }); + }); + }); +}); diff --git a/libs/sdk/src/tool/flows/__tests__/call-tool.output-schema.spec.ts b/libs/sdk/src/tool/flows/__tests__/call-tool.output-schema.spec.ts index 979d805b4..3017b480d 100644 --- a/libs/sdk/src/tool/flows/__tests__/call-tool.output-schema.spec.ts +++ b/libs/sdk/src/tool/flows/__tests__/call-tool.output-schema.spec.ts @@ -1,13 +1,26 @@ import 'reflect-metadata'; import { z } from '@frontmcp/lazy-zod'; +import { MCP_20260728_META } from '@frontmcp/protocol'; import { createTestFetchServer, rpc20260728, type TestFetchServer, } from '../../../__test-utils__/helpers/mcp-20260728.helpers'; -import { App, FlowHooksOf, Plugin, Tool, ToolContext, type FlowCtxOf } from '../../../common'; +import { + App, + FlowHooksOf, + LogLevel, + LogTransport, + LogTransportInterface, + Plugin, + Tool, + ToolContext, + type FlowCtxOf, + type LogRecord, + type TemplateContext, +} from '../../../common'; import { InvalidOutputError } from '../../../errors'; const ToolHook = FlowHooksOf('tools:call-tool'); @@ -138,3 +151,227 @@ describe('tools/call output schema enforcement', () => { }); }); }); + +const quoteWarnings: string[] = []; + +@LogTransport({ name: 'UndeclaredKeysCapture', description: 'Keeps warnings for assertions' }) +class WarningCapture extends LogTransportInterface { + log(record: LogRecord): void { + if (record.level === LogLevel.Warn) quoteWarnings.push(record.message); + } +} + +const quoteAnswer = { price: 10, quoteCostCenter: 'QCC-4410' }; + +@Tool({ name: 'get_quote', inputSchema: {}, outputSchema: { price: z.number() } }) +class GetQuoteTool extends ToolContext { + async execute() { + return quoteAnswer as { price: number }; + } +} + +@Tool({ + name: 'get_strict_quote', + inputSchema: {}, + outputSchema: { price: z.number() }, + output: { undeclaredKeys: 'fail' }, +}) +class GetStrictQuoteTool extends ToolContext { + async execute() { + return quoteAnswer as { price: number }; + } +} + +@Tool({ + name: 'show_quote', + inputSchema: {}, + outputSchema: { price: z.number() }, + ui: { template: (ctx: TemplateContext) => JSON.stringify(ctx.output) }, +}) +class ShowQuoteTool extends ToolContext { + async execute() { + return quoteAnswer as { price: number }; + } +} + +@Tool({ name: 'get_partner_quote', inputSchema: {}, outputSchema: { price: z.number() } }) +class GetPartnerQuoteTool extends ToolContext { + async execute() { + return quoteAnswer as { price: number }; + } +} + +@Tool({ + name: 'get_tagged_quote', + inputSchema: { tags: z.array(z.string()) }, + outputSchema: { price: z.number() }, + output: { undeclaredKeys: 'warn' }, +}) +class GetTaggedQuoteTool extends ToolContext { + async execute(input: { tags: string[] }) { + return { price: 10, ...Object.fromEntries(input.tags.map((tag) => [tag, true])) } as { price: number }; + } +} + +@App({ id: 'quotes', name: 'Quotes', tools: [GetQuoteTool, GetStrictQuoteTool, ShowQuoteTool, GetTaggedQuoteTool] }) +class QuotesApp {} + +@Tool({ name: 'get_plugin_quote', inputSchema: {}, outputSchema: { price: z.number() } }) +class GetPluginQuoteTool extends ToolContext { + async execute() { + return quoteAnswer as { price: number }; + } +} + +@Plugin({ name: 'partner-pricing', tools: [GetPluginQuoteTool] }) +class PartnerPricingPlugin {} + +@App({ + id: 'partners', + name: 'Partners', + tools: [GetPartnerQuoteTool], + plugins: [PartnerPricingPlugin], + output: { undeclaredKeys: 'keep' }, +}) +class PartnersApp {} + +describe('tools/call undeclared output keys', () => { + const originalNodeEnv = process.env['NODE_ENV']; + + afterEach(() => { + process.env['NODE_ENV'] = originalNodeEnv; + quoteWarnings.length = 0; + }); + + function startServer(output?: { undeclaredKeys?: 'strip' | 'keep' | 'warn' | 'fail' }) { + return createTestFetchServer({ + info: { name: 'undeclared-keys', version: '1.0.0' }, + apps: [QuotesApp, PartnersApp], + logging: { level: LogLevel.Warn, enableConsole: false, transports: [WarningCapture] }, + ...(output && { output }), + }); + } + + async function callQuote(target: TestFetchServer, name: string, clientInfo?: { name: string; version: string }) { + const { message } = await rpc20260728( + target.handler, + 'tools/call', + { name, arguments: {} }, + clientInfo && { meta: { [MCP_20260728_META.clientInfo]: clientInfo } }, + ); + const result = message.result ?? {}; + const resultMeta = result['_meta'] as Record | undefined; + return { + isError: result['isError'] === true, + code: resultMeta?.['code'], + structuredContent: result['structuredContent'], + wire: JSON.stringify(result), + }; + } + + it('strips undeclared keys without a warning by default outside development', async () => { + process.env['NODE_ENV'] = 'production'; + const server = await startServer(); + + expect(await callQuote(server, 'get_quote')).toMatchObject({ isError: false, structuredContent: { price: 10 } }); + expect(quoteWarnings.filter((line) => line.includes('quoteCostCenter'))).toEqual([]); + }); + + it('strips undeclared keys and warns once per tool and key set by default in development', async () => { + process.env['NODE_ENV'] = 'development'; + const server = await startServer(); + + const first = await callQuote(server, 'get_quote'); + await callQuote(server, 'get_quote'); + + expect(first).toMatchObject({ isError: false, structuredContent: { price: 10 } }); + expect(first.wire).not.toContain('QCC-4410'); + const warnings = quoteWarnings.filter((line) => line.includes('quoteCostCenter')); + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain('get_quote'); + }); + + async function callTaggedQuote(target: TestFetchServer, tags: string[]): Promise { + await rpc20260728(target.handler, 'tools/call', { name: 'get_tagged_quote', arguments: { tags } }); + } + + function taggedQuoteWarnings(): string[] { + return quoteWarnings.filter((line) => line.includes('get_tagged_quote')); + } + + it("names under 'warn' only the undeclared keys not reported before", async () => { + const server = await startServer(); + + await callTaggedQuote(server, ['rush']); + await callTaggedQuote(server, ['rush', 'gift']); + await callTaggedQuote(server, ['gift', 'rush']); + + expect(taggedQuoteWarnings()).toHaveLength(2); + expect(taggedQuoteWarnings()[0]).toContain('stripped: rush.'); + expect(taggedQuoteWarnings()[1]).toContain('stripped: gift.'); + }); + + it("stops reporting under 'warn' once a tool has named 100 undeclared keys", async () => { + const server = await startServer(); + const manyTags = Array.from({ length: 120 }, (_unused, index) => `tag${String(index).padStart(3, '0')}`); + + await callTaggedQuote(server, manyTags); + await callTaggedQuote(server, ['late-tag']); + + expect(taggedQuoteWarnings()).toHaveLength(1); + expect(taggedQuoteWarnings()[0]).toContain('tag099.'); + expect(taggedQuoteWarnings()[0]).not.toContain('tag100'); + }); + + it("passes undeclared keys through under a server-level 'keep'", async () => { + const server = await startServer({ undeclaredKeys: 'keep' }); + + const result = await callQuote(server, 'get_quote'); + + expect(result).toMatchObject({ isError: false, structuredContent: quoteAnswer }); + }); + + it("lets a tool's 'fail' override the server and refuse the answer with INVALID_OUTPUT", async () => { + const server = await startServer({ undeclaredKeys: 'keep' }); + + const result = await callQuote(server, 'get_strict_quote'); + + expect(result).toMatchObject({ isError: true, code: 'INVALID_OUTPUT' }); + expect(result.wire).not.toContain('QCC-4410'); + }); + + it("lets an app's policy override the server", async () => { + const server = await startServer({ undeclaredKeys: 'fail' }); + + expect(await callQuote(server, 'get_partner_quote')).toMatchObject({ structuredContent: quoteAnswer }); + expect(await callQuote(server, 'get_quote')).toMatchObject({ isError: true, code: 'INVALID_OUTPUT' }); + }); + + it("applies an app's policy to a tool a plugin of that app contributes", async () => { + const server = await startServer({ undeclaredKeys: 'fail' }); + + expect(await callQuote(server, 'get_plugin_quote')).toMatchObject({ + isError: false, + structuredContent: quoteAnswer, + }); + }); + + it.each([ + ['a client that takes structuredContent', undefined], + ['a client that takes no structuredContent', { name: 'claude-desktop', version: '1.0.0' }], + ])("renders a tool's UI from declared keys only under 'keep', for %s", async (_label, clientInfo) => { + const server = await startServer({ undeclaredKeys: 'keep' }); + + const result = await callQuote(server, 'show_quote', clientInfo); + + expect(result.isError).toBe(false); + expect(result.wire).toContain('ui/html'); + expect(result.wire).not.toContain('QCC-4410'); + }); + + it('refuses an unknown undeclaredKeys policy at boot', async () => { + await expect(startServer({ undeclaredKeys: 'drop' } as unknown as { undeclaredKeys: 'strip' })).rejects.toThrow( + /undeclaredKeys/, + ); + }); +}); diff --git a/libs/sdk/src/tool/flows/__tests__/tools-list.output-schema.spec.ts b/libs/sdk/src/tool/flows/__tests__/tools-list.output-schema.spec.ts new file mode 100644 index 000000000..b8d0e2b49 --- /dev/null +++ b/libs/sdk/src/tool/flows/__tests__/tools-list.output-schema.spec.ts @@ -0,0 +1,191 @@ +import 'reflect-metadata'; + +import { z } from '@frontmcp/lazy-zod'; + +import { createTestFetchServer, rpc20260728 } from '../../../__test-utils__/helpers/mcp-20260728.helpers'; +import { + App, + LogLevel, + LogTransport, + LogTransportInterface, + Plugin, + Tool, + ToolContext, + type LogRecord, + type OutputPolicy, +} from '../../../common'; + +const documentHit = z.object({ kind: z.literal('document'), title: z.string() }); +const personHit = z.object({ kind: z.literal('person'), name: z.string() }); + +@Tool({ name: 'search', inputSchema: {}, outputSchema: z.union([documentHit, personHit]) }) +class SearchTool extends ToolContext { + async execute() { + return { kind: 'document' as const, title: 'Roadmap' }; + } +} + +@Tool({ name: 'resolve', inputSchema: {}, outputSchema: z.discriminatedUnion('kind', [documentHit, personHit]) }) +class ResolveTool extends ToolContext { + async execute() { + return { kind: 'person' as const, name: 'Ada' }; + } +} + +@Tool({ + name: 'invoice', + inputSchema: {}, + outputSchema: { id: z.string(), lines: z.array(z.object({ sku: z.string() })) }, +}) +class InvoiceTool extends ToolContext { + async execute() { + return { id: 'INV-1', lines: [] }; + } +} + +@App({ id: 'directory', name: 'Directory', tools: [SearchTool, ResolveTool, InvoiceTool] }) +class DirectoryApp {} + +@Tool({ + name: 'archive_entry', + inputSchema: {}, + outputSchema: { id: z.string(), tags: z.array(z.object({ label: z.string() })) }, +}) +class ArchiveEntryTool extends ToolContext { + async execute() { + return { id: 'A-1', tags: [] }; + } +} + +@Plugin({ name: 'archive-index', tools: [ArchiveEntryTool] }) +class ArchiveIndexPlugin {} + +@App({ id: 'archive', name: 'Archive', plugins: [ArchiveIndexPlugin], output: { undeclaredKeys: 'keep' } }) +class ArchiveApp {} + +async function listOutputSchemas(output?: OutputPolicy): Promise | undefined>> { + const server = await createTestFetchServer({ + info: { name: 'tools-list-output-schema', version: '1.0.0' }, + apps: [DirectoryApp, ArchiveApp], + ...(output && { output }), + }); + const { message } = await rpc20260728(server.handler, 'tools/list'); + const tools = (message.result?.['tools'] as Array<{ name: string; outputSchema?: Record }>) ?? []; + return new Map(tools.map((tool) => [tool.name, tool.outputSchema])); +} + +describe('tools/list output schema', () => { + it('lists a union of object schemas with a top-level object type', async () => { + const schemas = await listOutputSchemas(); + + expect(schemas.get('search')).toMatchObject({ + type: 'object', + anyOf: [{ properties: { title: { type: 'string' } } }, { properties: { name: { type: 'string' } } }], + }); + }); + + it('lists a discriminated union with a top-level object type', async () => { + const schemas = await listOutputSchemas(); + + expect(schemas.get('resolve')).toMatchObject({ type: 'object', oneOf: [{ type: 'object' }, { type: 'object' }] }); + }); + + it('lists object schemas closed to undeclared keys by default', async () => { + const schemas = await listOutputSchemas(); + + expect(JSON.stringify(schemas.get('invoice'))).toContain('"additionalProperties":false'); + }); + + it("lists object schemas open to undeclared keys under 'keep', at every level", async () => { + const schemas = await listOutputSchemas({ undeclaredKeys: 'keep' }); + + expect(schemas.get('invoice')).toMatchObject({ type: 'object', properties: { id: { type: 'string' } } }); + expect(JSON.stringify([...schemas.values()])).not.toContain('"additionalProperties":false'); + }); + + it("lists a plugin's tool open under its app's 'keep', and other apps' tools closed", async () => { + const schemas = await listOutputSchemas(); + + expect(schemas.get('archive_entry')).toMatchObject({ type: 'object', properties: { id: { type: 'string' } } }); + expect(JSON.stringify(schemas.get('archive_entry'))).not.toContain('"additionalProperties":false'); + expect(JSON.stringify(schemas.get('invoice'))).toContain('"additionalProperties":false'); + }); +}); + +const startupWarnings: string[] = []; + +@LogTransport({ name: 'UnlistedOutputSchemaCapture', description: 'Keeps warnings for assertions' }) +class StartupWarningCapture extends LogTransportInterface { + log(record: LogRecord): void { + if (record.level === LogLevel.Warn) startupWarnings.push(record.message); + } +} + +@Tool({ name: 'list_hits', inputSchema: {}, outputSchema: z.array(documentHit) }) +class ListHitsTool extends ToolContext { + async execute() { + return []; + } +} + +@Tool({ + name: 'list_hits_quietly', + inputSchema: {}, + outputSchema: z.array(documentHit), + output: { schemaMode: 'none' }, +}) +class ListHitsQuietlyTool extends ToolContext { + async execute() { + return []; + } +} + +@Tool({ name: 'list_archived_hits', inputSchema: {}, outputSchema: z.array(documentHit) }) +class ListArchivedHitsTool extends ToolContext { + async execute() { + return []; + } +} + +@Plugin({ name: 'archived-hits', tools: [ListArchivedHitsTool] }) +class ArchivedHitsPlugin {} + +@App({ id: 'hits', name: 'Hits', tools: [ListHitsTool, ListHitsQuietlyTool, SearchTool] }) +class HitsApp {} + +@App({ id: 'quiet-hits', name: 'QuietHits', plugins: [ArchivedHitsPlugin], output: { schemaMode: 'none' } }) +class QuietHitsApp {} + +describe('startup warning for an outputSchema tools/list leaves out', () => { + afterEach(() => { + startupWarnings.length = 0; + }); + + function unlistedOutputSchemaWarnings(): string[] { + return startupWarnings.filter((line) => line.includes('leaves out its outputSchema')); + } + + async function startServer(output?: OutputPolicy) { + await createTestFetchServer({ + info: { name: 'unlisted-output-schema', version: '1.0.0' }, + apps: [HitsApp, QuietHitsApp], + logging: { level: LogLevel.Warn, enableConsole: false, transports: [StartupWarningCapture] }, + ...(output && { output }), + }); + } + + it("names the tool, once, and stays quiet for a tool or an app whose schemaMode is 'none'", async () => { + await startServer(); + + expect(unlistedOutputSchemaWarnings()).toEqual([ + 'Tool "list_hits": tools/list leaves out its outputSchema because it is not an object schema or a union of ' + + 'object schemas. Declare a z.object() or a union of z.object() schemas to list it.', + ]); + }); + + it("stays quiet for every tool when the server's schemaMode is 'none'", async () => { + await startServer({ schemaMode: 'none' }); + + expect(unlistedOutputSchemaWarnings()).toEqual([]); + }); +}); diff --git a/libs/sdk/src/tool/flows/call-tool.flow.ts b/libs/sdk/src/tool/flows/call-tool.flow.ts index 2b12cecd6..b72a12e41 100644 --- a/libs/sdk/src/tool/flows/call-tool.flow.ts +++ b/libs/sdk/src/tool/flows/call-tool.flow.ts @@ -26,6 +26,7 @@ import { FlowHooksOf, GLOBAL_RATE_LIMIT_CHECKED, isOrchestratedMode, + resolveUndeclaredKeysPolicy, ToolContext, ToolEntry, type FlowPlan, @@ -71,6 +72,7 @@ import { generateTaskId } from '../../task/helpers/task-id'; import { TaskNotifier } from '../../task/helpers/task-notifier'; import { TASK_DEFAULTS, toWireShape, type TaskRecord } from '../../task/task.types'; import { hookOwnerIdOf, isServerEntryOwner } from '../../utils/lineage.utils'; +import { toolOutputPolicies } from '../tool-output-policy'; import { toolServingMode } from '../ui/serving-mode-default'; import { hasUIConfig, WIDGET_CALL_META_KEY } from '../ui/ui-shared'; import { evaluateToolCredentialGate } from './tool-credentials.gate'; @@ -173,6 +175,8 @@ const stateSchema = z.object({ .optional(), }); +type CallToolEntry = z.infer['tool']; + const plan = { pre: [ 'parseInput', @@ -222,6 +226,12 @@ const { Stage } = FlowHooksOf<'tools:call-tool'>(name); /** Tools `resolveHookOwnerId` found, reused by the same run's `findTool`. */ const resolvedTools = new ResolvedEntries(); +/** The undeclared key paths already reported per tool, so the 'warn' policy names each one once. */ +const reportedUndeclaredKeyPaths = new WeakMap>(); + +/** Answers keyed by ids yield new key paths forever, so a tool stops reporting after this many. */ +const MAX_REPORTED_UNDECLARED_KEY_PATHS = 100; + @Flow({ name, plan, @@ -1531,7 +1541,12 @@ export default class CallToolFlow extends FlowBase { } // Parse and construct the MCP-compliant output using safeParseOutput - const parseResult = tool.safeParseOutput(rawOutput); + const undeclaredKeys = resolveUndeclaredKeysPolicy(...toolOutputPolicies(tool, this.scope)); + const parseResult = tool.safeParseOutput(rawOutput, { + // A call that renders the tool's UI sends declared keys only, whatever the policy + undeclaredKeys: uiResult && undeclaredKeys === 'keep' ? 'strip' : undeclaredKeys, + onUndeclaredKeys: (keyPaths) => this.warnUndeclaredKeysOnce(tool, keyPaths), + }); if (!parseResult.success) { this.logger.error('finalize: output validation failed', { @@ -1620,6 +1635,19 @@ export default class CallToolFlow extends FlowBase { this.respond(result); this.logger.verbose('finalize:done'); } + + private warnUndeclaredKeysOnce(tool: CallToolEntry, keyPaths: string[]): void { + const reportedKeyPaths = reportedUndeclaredKeyPaths.get(tool) ?? new Set(); + reportedUndeclaredKeyPaths.set(tool, reportedKeyPaths); + const roomLeft = MAX_REPORTED_UNDECLARED_KEY_PATHS - reportedKeyPaths.size; + const newKeyPaths = keyPaths.filter((keyPath) => !reportedKeyPaths.has(keyPath)).slice(0, roomLeft); + if (newKeyPaths.length === 0) return; + newKeyPaths.forEach((keyPath) => reportedKeyPaths.add(keyPath)); + this.logger.warn( + `Tool "${tool.metadata.name}" answered with keys its outputSchema does not declare, so they were stripped: ` + + `${newKeyPaths.join(', ')}. Declare them, or set output.undeclaredKeys to 'strip', 'keep' or 'fail'.`, + ); + } } /** diff --git a/libs/sdk/src/tool/flows/tools-list.flow.ts b/libs/sdk/src/tool/flows/tools-list.flow.ts index 3e3bc5e4d..1e4e2b87e 100644 --- a/libs/sdk/src/tool/flows/tools-list.flow.ts +++ b/libs/sdk/src/tool/flows/tools-list.flow.ts @@ -7,10 +7,11 @@ import { formatOutputSchemaForDescription, resolveOutputSchemaMode, resolveSchemaDescriptionFormat, + resolveUndeclaredKeysPolicy, ToolEntry, + withUndeclaredKeysAllowed, type FlowPlan, type FlowRunOptions, - type OutputPolicy, } from '../../common'; import 'reflect-metadata'; @@ -27,6 +28,7 @@ import { DEFAULT_TOOL_PAGINATION, type ToolPaginationOptions } from '../../commo import { InternalMcpError, InvalidInputError, InvalidMethodError } from '../../errors'; import { type Scope } from '../../scope/scope.instance'; import { isSkillsOnlyRequest } from '../../skill/skill-mode.utils'; +import { toolOutputPolicies } from '../tool-output-policy'; import { buildStaticWidgetUri, hasUIConfig } from '../ui/ui-shared'; const inputSchema = z.object({ @@ -475,12 +477,6 @@ export default class ToolsListFlow extends FlowBase { // All platforms now use ui/* keys per the MCP Apps specification - // Resolve the output-schema exposure policy once (per-tool: Tool > App > server). - const serverOutput = (this.scope.metadata as { output?: OutputPolicy }).output; - const appOutputById = new Map( - this.scope.apps.getApps().map((app) => [app.id, (app.metadata as { output?: OutputPolicy }).output]), - ); - const uiRenderer = resolved.some(({ tool }) => hasUIConfig(tool.metadata)) ? await this.loadToolUIRenderer() : undefined; @@ -511,21 +507,24 @@ export default class ToolsListFlow extends FlowBase { // Expose the output schema per the resolved mode (Tool > App > server; default // 'definition'). 'definition'/'both' advertise it as item.outputSchema; // 'description'/'both' fold a readable rendering into the description; 'none' skips it. - const toolOutput = (tool.metadata as { output?: OutputPolicy }).output; - const appOutput = appOutputById.get(tool.owner.id); - const schemaMode = resolveOutputSchemaMode(toolOutput, appOutput, serverOutput); + const outputPolicies = toolOutputPolicies(tool, this.scope); + const schemaMode = resolveOutputSchemaMode(...outputPolicies); if (schemaMode === 'definition' || schemaMode === 'both') { // getRawOutputSchema() returns the explicit rawOutputSchema (OpenAPI / remote - // passthrough) or a JSON Schema derived from the declared Zod-shape / z.object() - // outputSchema (elicitation-extended when elicitation is enabled). + // passthrough) or a JSON Schema derived from the declared Zod-shape, z.object() or + // union-of-objects outputSchema (elicitation-extended when elicitation is enabled). const rawOutput = tool.getRawOutputSchema(); if (rawOutput) { // MCP spec requires outputSchema to have type: 'object' at the top level. // Strip non-compliant schemas to prevent a single tool from breaking the entire tools/list response. const schema = rawOutput as Record; if (schema['type'] === 'object') { - item.outputSchema = schema as typeof item.outputSchema; + // An answer that keeps undeclared keys must still validate against the listed schema + const keepsUndeclaredKeys = resolveUndeclaredKeysPolicy(...outputPolicies) === 'keep'; + item.outputSchema = ( + keepsUndeclaredKeys ? withUndeclaredKeysAllowed(schema) : schema + ) as typeof item.outputSchema; } else { this.logger.warn( `parseTools: tool "${finalName}" has outputSchema without type: 'object' — stripping to maintain MCP compliance`, @@ -538,7 +537,7 @@ export default class ToolsListFlow extends FlowBase { // Use the clean (non-elicitation-extended) schema for the human-readable description. const outputJsonSchema = tool.getOutputJsonSchema(); if (outputJsonSchema) { - const format = resolveSchemaDescriptionFormat(toolOutput, appOutput, serverOutput); + const format = resolveSchemaDescriptionFormat(...outputPolicies); item.description = (item.description ?? '') + formatOutputSchemaForDescription(outputJsonSchema, format); } } diff --git a/libs/sdk/src/tool/tool-output-policy.ts b/libs/sdk/src/tool/tool-output-policy.ts new file mode 100644 index 000000000..1265d2dd7 --- /dev/null +++ b/libs/sdk/src/tool/tool-output-policy.ts @@ -0,0 +1,57 @@ +// file: libs/sdk/src/tool/tool-output-policy.ts + +import { + resolveOutputSchemaMode, + type EntryLineage, + type OutputPolicy, + type ScopeEntry, + type ToolEntry, +} from '../common'; +import { appOwnerIdOf } from '../utils/lineage.utils'; + +/** A tool's `output` policies, most specific first, for the `resolve*` cascades in `output-policy.ts`. */ +export type ToolOutputPolicies = [tool?: OutputPolicy, app?: OutputPolicy, server?: OutputPolicy]; + +/** The parts of a tool the output-policy lookup reads. */ +type OutputPolicyTool = Pick; + +/** What the output-policy lookup reads from a scope: its metadata, its apps and its tools' lineage. */ +export interface OutputPolicyScope { + readonly metadata: { readonly output?: OutputPolicy }; + readonly apps: Pick; + readonly tools: { lineageOf(entry: Tool): EntryLineage | undefined }; +} + +/** + * The `output` policies that apply to `tool`: its own, its app's and the server's. The app is the + * one the tool is in, so a tool a plugin or an adapter of that app contributes follows it too. + * `tools/list` and `tools/call` both read it, so a tool is listed and answered under one policy. + */ +export function toolOutputPolicies( + tool: Tool, + scope: OutputPolicyScope, +): ToolOutputPolicies { + const appId = appOwnerIdOf(scope.tools.lineageOf(tool) ?? [], tool.owner); + const app = appId === undefined ? undefined : scope.apps.getApps().find((candidate) => candidate.id === appId); + // Remote apps have no `output` option + const appOutput = (app?.metadata as { output?: OutputPolicy } | undefined)?.output; + return [tool.metadata.output, appOutput, scope.metadata.output]; +} + +/** + * One startup warning for each listed tool whose declared `outputSchema` tools/list has to leave + * out, naming the tool and the reason. A tool whose `output.schemaMode` resolves to `'none'` asked + * for no schema, so it stays quiet. + */ +export function describeUnlistedOutputSchemas( + scope: OutputPolicyScope & { readonly tools: { getTools(): ToolEntry[] } }, +): string[] { + return scope.tools.getTools().flatMap((tool) => { + const reason = tool.getUnlistedOutputSchemaReason(); + if (!reason || resolveOutputSchemaMode(...toolOutputPolicies(tool, scope)) === 'none') return []; + return [ + `Tool "${tool.metadata.name}": tools/list leaves out its outputSchema because ${reason}. ` + + 'Declare a z.object() or a union of z.object() schemas to list it.', + ]; + }); +} diff --git a/libs/sdk/src/tool/tool.instance.ts b/libs/sdk/src/tool/tool.instance.ts index a1284d92c..84bb750f1 100644 --- a/libs/sdk/src/tool/tool.instance.ts +++ b/libs/sdk/src/tool/tool.instance.ts @@ -8,6 +8,7 @@ import { ToolEntry, ToolKind, type EntryOwnerRef, + type OutputParseOptions, type ParsedToolResult, type SafeTransformResult, type ScopeEntry, @@ -221,16 +222,19 @@ export class ToolInstance< * - image/audio/resource/resource_link → passed through as-is. * - JSON / structured → JSON.stringify for text, and full sanitized JSON in structuredContent. */ - override parseOutput(raw: Out | Partial | any): ParsedToolResult { + override parseOutput(raw: Out | Partial | any, options?: OutputParseOptions): ParsedToolResult { const descriptor = this.outputSchema as any; - return buildParsedToolResult(descriptor, raw); + return buildParsedToolResult(descriptor, raw, options); } - override safeParseOutput(raw: Out | Partial | any): SafeTransformResult { + override safeParseOutput( + raw: Out | Partial | any, + options?: OutputParseOptions, + ): SafeTransformResult { const descriptor = this.outputSchema as any; try { - return { success: true, data: buildParsedToolResult(descriptor, raw) }; + return { success: true, data: buildParsedToolResult(descriptor, raw, options) }; } catch (error: any) { return { success: false, error }; } diff --git a/libs/sdk/src/tool/tool.utils.ts b/libs/sdk/src/tool/tool.utils.ts index 3cdcb1e5a..8ee8364ea 100644 --- a/libs/sdk/src/tool/tool.utils.ts +++ b/libs/sdk/src/tool/tool.utils.ts @@ -1,6 +1,6 @@ // file: libs/sdk/src/tool/tool.utils.ts import { depsOfClass, depsOfFunc, getMetadata, isClass, type Token, type Type } from '@frontmcp/di'; -import { z, ZodBigInt, ZodBoolean, ZodDate, ZodNumber, ZodString } from '@frontmcp/lazy-zod'; +import { hasTransform, z, ZodBigInt, ZodBoolean, ZodDate, ZodNumber, ZodString } from '@frontmcp/lazy-zod'; import { type AudioContent, type ContentBlock, @@ -9,12 +9,14 @@ import { type ResourceLink, type TextContent, } from '@frontmcp/protocol'; +import { findDroppedKeys } from '@frontmcp/utils'; import { extendedToolMetadata, FrontMcpToolTokens, ToolKind, type AgentToolDefinition, + type OutputParseOptions, type ParsedToolResult, type ToolContext, type ToolEntry, @@ -143,7 +145,11 @@ function isCallToolResult(value: unknown): value is ParsedToolResult { return true; } -export function buildParsedToolResult(descriptor: any, raw: unknown): ParsedToolResult { +export function buildParsedToolResult( + descriptor: any, + raw: unknown, + options: OutputParseOptions = {}, +): ParsedToolResult { // If raw is already a valid CallToolResult (e.g., from remote tools), return it as-is // This prevents double-wrapping of MCP-formatted responses if (isCallToolResult(raw)) { @@ -169,7 +175,7 @@ export function buildParsedToolResult(descriptor: any, raw: unknown): ParsedTool // Parse all items first descriptor.forEach((singleDescriptor: any, idx: number) => { const value = values[idx]; - const { blocks, parsedValue, isPrimitive } = parseSingleValue(singleDescriptor, value); + const { blocks, parsedValue, isPrimitive } = parseSingleValue(singleDescriptor, value, options); parsedItems.push({ blocks, parsedValue, isPrimitive }); content.push(...blocks); }); @@ -206,7 +212,7 @@ export function buildParsedToolResult(descriptor: any, raw: unknown): ParsedTool }); } } else { - const { blocks, parsedValue, isPrimitive } = parseSingleValue(descriptor, raw); + const { blocks, parsedValue, isPrimitive } = parseSingleValue(descriptor, raw, options); content.push(...blocks); // Single item: wrap primitives in {content: value}, use objects directly @@ -244,6 +250,7 @@ export function buildParsedToolResult(descriptor: any, raw: unknown): ParsedTool function parseSingleValue( descriptor: any, value: unknown, + options: OutputParseOptions, ): { blocks: ContentBlock[]; parsedValue?: any; isPrimitive: boolean } { // Literal primitives + special content types if (typeof descriptor === 'string') { @@ -373,9 +380,9 @@ function parseSingleValue( let parsedValue: any; if (descriptor instanceof z.ZodType) { - parsedValue = toStructuredContent(parseAgainstOutputSchema(descriptor, value)); + parsedValue = toStructuredContent(parseAgainstOutputSchema(descriptor, value, options)); } else if (isZodRawShape(descriptor)) { - parsedValue = toStructuredContent(parseAgainstOutputSchema(z.object(descriptor), value)); + parsedValue = toStructuredContent(parseAgainstOutputSchema(z.object(descriptor), value, options)); } else { parsedValue = toStructuredContent(value); } @@ -388,9 +395,57 @@ function parseSingleValue( } /** - * The value an output schema accepts, or an InvalidOutputError naming the first mismatch. + * The value an output schema accepts, with undeclared keys handled per `options.undeclaredKeys`, + * or an InvalidOutputError naming the first mismatch, the undeclared keys `'fail'` refuses, or those `'keep'` cannot + * put back because the schema replaced the object holding them (a coercion to a primitive). + * A schema with a transform is always stripped: its output cannot be compared with the answer key by key. */ -function parseAgainstOutputSchema(schema: z.ZodType, value: unknown): unknown { +function parseAgainstOutputSchema(schema: z.ZodType, value: unknown, options: OutputParseOptions = {}): unknown { + const declared = parseDeclaredOutput(schema, value); + const policy = options.undeclaredKeys ?? 'strip'; + if (policy === 'strip' || hasTransform(schema)) return declared; + const undeclaredKeys = findDroppedKeys(value, declared); + if (undeclaredKeys.length === 0) return declared; + switch (policy) { + case 'keep': { + const kept = withUndeclaredKeys(value, declared); + const unkeptKeys = findDroppedKeys(value, kept); + if (unkeptKeys.length > 0) { + throw new InvalidOutputError({ + reason: 'undeclared keys the outputSchema cannot keep', + value: unkeptKeys.join(', '), + }); + } + return kept; + } + case 'fail': + throw new InvalidOutputError({ reason: 'undeclared keys', value: undeclaredKeys.join(', ') }); + case 'warn': + options.onUndeclaredKeys?.(undeclaredKeys); + return declared; + } +} + +/** `declared` with the keys of `original` its schema dropped put back, at every level. */ +function withUndeclaredKeys(original: unknown, declared: unknown): unknown { + if (original === declared) return declared; + if (Array.isArray(original) && Array.isArray(declared)) { + return declared.map((item, index) => withUndeclaredKeys(original[index], item)); + } + if (!isRecord(original) || !isRecord(declared)) return declared; + const declaredEntries = Object.entries(declared).map(([key, item]) => [key, withUndeclaredKeys(original[key], item)]); + const undeclaredEntries = Object.entries(original).filter( + ([key, item]) => item !== undefined && !Object.prototype.hasOwnProperty.call(declared, key), + ); + return Object.fromEntries([...declaredEntries, ...undeclaredEntries]); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** The value an output schema accepts, undeclared keys stripped, or an InvalidOutputError naming the first mismatch. */ +function parseDeclaredOutput(schema: z.ZodType, value: unknown): unknown { const parseResult = schema.safeParse(value); if (parseResult.success) { return parseResult.data; diff --git a/libs/skills/catalog/create-tool/examples/04-tool-with-zod-schema-output.md b/libs/skills/catalog/create-tool/examples/04-tool-with-zod-schema-output.md index 97f4ae36f..200caa240 100644 --- a/libs/skills/catalog/create-tool/examples/04-tool-with-zod-schema-output.md +++ b/libs/skills/catalog/create-tool/examples/04-tool-with-zod-schema-output.md @@ -14,7 +14,7 @@ features: Tool returning a discriminated union via a full `z.discriminatedUnion(...)` outputSchema — for outputs that branch on a kind field. -When the output has more than one shape (e.g. `user` vs `group`), use a full Zod schema instead of a raw shape. `z.discriminatedUnion` is the cleanest pattern. +When the output has more than one shape (e.g. `user` vs `group`), use a full Zod schema instead of a raw shape. `z.discriminatedUnion` is the cleanest pattern. A union whose members are all objects is advertised in `tools/list` as `{ type: 'object', oneOf: [...] }` (`anyOf` for a plain `z.union`), and needs no cast in `@Tool({ outputSchema })`. ## Code @@ -92,6 +92,8 @@ export class ResolvePrincipalTool extends ToolContext { | Use raw shape | Use full Zod schema | | ---------------------------------------- | -------------------------------------------------------- | | Single object with a fixed set of fields | Union of multiple shapes (`z.discriminatedUnion`) | -| All fields known statically | Arrays of complex objects (`z.array(z.object(...))`) | +| All fields known statically | Arrays of complex objects (`z.array(z.object(...))`)¹ | | No transforms / refinements needed | Need `z.transform(...)`, `z.refine(...)`, `z.brand(...)` | | Default and recommended | When the raw shape can't express the contract | + +¹ A top-level array is not an object, so `tools/list` leaves it out (with a startup warning naming the tool). Wrap it, e.g. `z.object({ items: z.array(...) })`, to advertise it. diff --git a/libs/skills/catalog/create-tool/references/decorator-options.md b/libs/skills/catalog/create-tool/references/decorator-options.md index 01a97a949..6960e4248 100644 --- a/libs/skills/catalog/create-tool/references/decorator-options.md +++ b/libs/skills/catalog/create-tool/references/decorator-options.md @@ -7,23 +7,23 @@ description: Every field on @Tool({...}) — what it does, default, when to set Full surface of the `@Tool` decorator. Mandatory fields are bolded. -| Field | Type | Default | When to set | -| ----------------- | ------------------------------------------------------------------------------ | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`name`** | `string` (snake_case) | — | Always. MCP protocol convention. `get_weather`, not `getWeather`. | -| `title` | `string` | — | When the tool needs a display name in `tools/list`. Without it, `annotations.title` is the only title. | -| `description` | `string` | — | Almost always. Shows in `tools/list` and helps clients choose the right tool. | -| **`inputSchema`** | Zod raw shape | — | Always. `{ field: z.string() }` — never wrapped in `z.object`. See [`input-schema.md`](./input-schema.md). | -| `outputSchema` | Zod raw shape / Zod schema / primitive literal / media literal / array | — | **Always** — prevents data leaks and enables CodeCall chaining. See [`output-schema.md`](./output-schema.md). | -| `output` | `{ allowNonFinite?, schemaMode?, schemaDescriptionFormat? }` | `definition` | Control how `outputSchema` is exposed in `tools/list` (`'definition'` \| `'description'` \| `'both'` \| `'none'`) and `allowNonFinite`. Cascades Tool > App > server. See [`output-schema.md`](./output-schema.md#exposure-mode). | -| `annotations` | `{ title?, readOnlyHint?, destructiveHint?, idempotentHint?, openWorldHint? }` | — | When the tool has notable behavioral semantics clients should be hinted about. See [`annotations.md`](./annotations.md). | -| `rateLimit` | `{ maxRequests, windowMs }` | — | Expensive / external-dependent / abuse-prone tools. See [`throttling.md`](./throttling.md). | -| `concurrency` | `{ maxConcurrent }` | — | Tools that hold a scarce resource (DB connection, GPU). See [`throttling.md`](./throttling.md). | -| `timeout` | `{ executeMs }` | — | Any tool that could legitimately hang. See [`throttling.md`](./throttling.md). | -| `authProviders` | `string[]` \| `Array<{ name, scopes?, required?, alias? }>` | — | Tool requires user OAuth credentials. See [`auth-providers.md`](./auth-providers.md). | -| `availableWhen` | `{ os?, runtime?, deployment?, provider?, target?, surface?, env? }` | — | Hard registry-level constraint — tool is filtered out of `tools/list` and execution when context doesn't match. See [`availability.md`](./availability.md). | -| `examples` | `Array<{ description, input, output? }>` | — | Discovery / docs surfacing. The client may show these to the user. | -| `ui` | `ToolUIConfig` | — | Tool result should render as a widget in the host UI. See [`ui-widgets.md`](./ui-widgets.md). | -| `visibility` | `'public' \| 'hidden' \| 'internal'` | `'public'` | Control discoverability/reachability. `'public'`: listed in `tools/list` + callable via `tools/call`. `'hidden'`: not listed, still callable by name. `'internal'`: not listed AND not externally callable — only via `this.callTool(...)` from inside the SDK. | +| Field | Type | Default | When to set | +| ----------------- | ------------------------------------------------------------------------------ | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`name`** | `string` (snake_case) | — | Always. MCP protocol convention. `get_weather`, not `getWeather`. | +| `title` | `string` | — | When the tool needs a display name in `tools/list`. Without it, `annotations.title` is the only title. | +| `description` | `string` | — | Almost always. Shows in `tools/list` and helps clients choose the right tool. | +| **`inputSchema`** | Zod raw shape | — | Always. `{ field: z.string() }` — never wrapped in `z.object`. See [`input-schema.md`](./input-schema.md). | +| `outputSchema` | Zod raw shape / Zod schema / primitive literal / media literal / array | — | **Always** — prevents data leaks and enables CodeCall chaining. See [`output-schema.md`](./output-schema.md). | +| `output` | `{ allowNonFinite?, schemaMode?, schemaDescriptionFormat?, undeclaredKeys? }` | `definition` | Control how `outputSchema` is exposed in `tools/list` (`'definition'` \| `'description'` \| `'both'` \| `'none'`), `allowNonFinite`, and what happens to undeclared answer keys (`undeclaredKeys`: `'strip'` \| `'warn'` \| `'keep'` \| `'fail'`; `'warn'` in development, else `'strip'`). Cascades Tool > App > server. See [`output-schema.md`](./output-schema.md#exposure-mode). | +| `annotations` | `{ title?, readOnlyHint?, destructiveHint?, idempotentHint?, openWorldHint? }` | — | When the tool has notable behavioral semantics clients should be hinted about. See [`annotations.md`](./annotations.md). | +| `rateLimit` | `{ maxRequests, windowMs }` | — | Expensive / external-dependent / abuse-prone tools. See [`throttling.md`](./throttling.md). | +| `concurrency` | `{ maxConcurrent }` | — | Tools that hold a scarce resource (DB connection, GPU). See [`throttling.md`](./throttling.md). | +| `timeout` | `{ executeMs }` | — | Any tool that could legitimately hang. See [`throttling.md`](./throttling.md). | +| `authProviders` | `string[]` \| `Array<{ name, scopes?, required?, alias? }>` | — | Tool requires user OAuth credentials. See [`auth-providers.md`](./auth-providers.md). | +| `availableWhen` | `{ os?, runtime?, deployment?, provider?, target?, surface?, env? }` | — | Hard registry-level constraint — tool is filtered out of `tools/list` and execution when context doesn't match. See [`availability.md`](./availability.md). | +| `examples` | `Array<{ description, input, output? }>` | — | Discovery / docs surfacing. The client may show these to the user. | +| `ui` | `ToolUIConfig` | — | Tool result should render as a widget in the host UI. See [`ui-widgets.md`](./ui-widgets.md). | +| `visibility` | `'public' \| 'hidden' \| 'internal'` | `'public'` | Control discoverability/reachability. `'public'`: listed in `tools/list` + callable via `tools/call`. `'hidden'`: not listed, still callable by name. `'internal'`: not listed AND not externally callable — only via `this.callTool(...)` from inside the SDK. | > `hideFromDiscovery: boolean` is a **deprecated** alias for `visibility`: when `visibility` is unset, `hideFromDiscovery: true` is treated as `visibility: 'hidden'`. Prefer `visibility` in new code. diff --git a/libs/skills/catalog/create-tool/references/output-schema.md b/libs/skills/catalog/create-tool/references/output-schema.md index 79d09aa72..039a27ca7 100644 --- a/libs/skills/catalog/create-tool/references/output-schema.md +++ b/libs/skills/catalog/create-tool/references/output-schema.md @@ -5,7 +5,7 @@ description: Define the tool's output contract — Zod shape, primitives, media, # `outputSchema` reference -`outputSchema` is **always required** ([rule](../rules/always-define-output-schema.md)). It declares what `execute()` returns and gives the framework permission to strip any fields you didn't declare, which guards against accidental PII / token / debug-trace leaks in the result. The exception is a `_meta` object on the returned value: it is copied into the response's `_meta` without validation, so never put secrets there. A result that does not match fails the call with `_meta.code: 'INVALID_OUTPUT'`, and nothing from the rejected result is sent. A number field holding `Infinity`, `-Infinity`, or `NaN` fails the same way, unless the server sets `output: { allowNonFinite: true }`. JSON cannot encode those values, so with that setting the client receives `null` in their place. +`outputSchema` is **always required** ([rule](../rules/always-define-output-schema.md)). It declares what `execute()` returns, and the framework strips any fields you didn't declare, which guards against accidental PII / token / debug-trace leaks in the result. In development (`NODE_ENV=development`) it also warns once per tool about the keys it stripped; see [Undeclared keys](#undeclared-keys) to change that. The exception is a `_meta` object on the returned value: it is copied into the response's `_meta` without validation, so never put secrets there. A result that does not match fails the call with `_meta.code: 'INVALID_OUTPUT'`, and nothing from the rejected result is sent. A number field holding `Infinity`, `-Infinity`, or `NaN` fails the same way, unless the server sets `output: { allowNonFinite: true }`. JSON cannot encode those values, so with that setting the client receives `null` in their place. ## Supported shapes @@ -44,7 +44,7 @@ class GetWeatherTool extends ToolContext { ## Zod schema (full Zod) -When the output is a union, discriminated union, or array of objects: +Use a full Zod schema when the output is a union, discriminated union, or array of objects. `z.union([...])` and `z.discriminatedUnion(...)` type-check as `outputSchema` with no cast: ```typescript const outputSchema = z.discriminatedUnion('kind', [ @@ -129,10 +129,11 @@ class AnalyzeImageTool extends ToolContext { Structured object outputs are converted to JSON Schema and advertised on the tool's `outputSchema` in `tools/list` — symmetric with how `inputSchema` is advertised: -- **Advertised:** a Zod raw shape (`{ field: z.string() }`) or a top-level `z.object({...})` — these serialize to a `type: 'object'` JSON Schema. -- **Not advertised:** primitives (`'string'` / `'number'` / `'boolean'` / `'date'`), media (`'image'` / `'audio'` / `'resource'` / `'resource_link'`), multi-content arrays (`['string', 'image']`), and unions (`z.discriminatedUnion(...)`). These flow through `content` rather than `structuredContent`, and the MCP spec requires `outputSchema` to be a top-level `type: 'object'` — so there is nothing object-shaped to advertise. Runtime output validation still applies to every form. +- **Advertised:** a Zod raw shape (`{ field: z.string() }`) or a top-level `z.object({...})` — these serialize to a `type: 'object'` JSON Schema. A `z.union([...])` or `z.discriminatedUnion(...)` whose members are all objects is advertised too, as `{ type: 'object', anyOf: [...] }` (or `oneOf` for a discriminated union), since MCP requires a top-level `type: 'object'`. A member registered with an id (`.meta({ id })`) serializes to a `$ref` into `$defs` and counts as an object too. +- **Not advertised, by design:** primitives (`'string'` / `'number'` / `'boolean'` / `'date'`, `z.string()`, …), media (`'image'` / `'audio'` / `'resource'` / `'resource_link'`), and multi-content arrays (`['string', 'image']`). These flow through `content` rather than `structuredContent`. +- **Not advertised, with a startup warning:** any other Zod schema — an array (`z.array(...)`), a nullable object, a union with a non-object member, or a schema that has no JSON Schema form (a `z.date()` field). The server logs `Tool "": tools/list leaves out its outputSchema because …` once at startup, naming the tool, unless the tool's `schemaMode` resolves to `'none'`. Wrap the value in a `z.object({...})` to advertise it. -If a client reports _"Output schema recommended,"_ declare a structured object `outputSchema` (a raw shape or `z.object`) — that's the form that gets advertised. +Runtime output validation still applies to every form. If a client reports _"Output schema recommended,"_ declare a structured object `outputSchema` (a raw shape, a `z.object`, or a union of `z.object`s) — that's the form that gets advertised. ### Exposure mode @@ -142,6 +143,7 @@ A cascading `output` policy controls _how_ the (object-shaped) schema reaches cl output?: { schemaMode?: 'definition' | 'description' | 'both' | 'none'; // default 'definition' schemaDescriptionFormat?: 'summary' | 'jsonSchema'; // default 'summary' + undeclaredKeys?: 'strip' | 'warn' | 'keep' | 'fail'; // see Undeclared keys }; ``` @@ -154,7 +156,7 @@ output?: { `schemaDescriptionFormat` controls the rendering for `'description'` / `'both'`: `'summary'` (default) is a compact property list; `'jsonSchema'` is a fenced JSON Schema block. -The effective value resolves with a **Tool > App > server > default** cascade — set a server-wide default on `@FrontMcp` and override per tool. This mirrors the OpenAPI adapter's `outputSchema.mode`. +The effective value resolves with a **Tool > App > server > default** cascade — set a server-wide default on `@FrontMcp` and override per tool. A tool that a plugin or an adapter of an app contributes follows that app's `output`. This mirrors the OpenAPI adapter's `outputSchema.mode`. ```typescript // This tool folds its schema into the description instead of advertising `outputSchema` @@ -166,9 +168,43 @@ class GetStatusTool extends ToolContext { The default (`'definition'`) preserves the current behavior described above — omit `output` entirely and nothing changes. +## Undeclared keys + +`output.undeclaredKeys` decides what `tools/call` does with keys an answer carries that `outputSchema` does not declare, at any depth (`costCenter`, `lines[].margin`). It cascades **Tool > App > server > default**, like `schemaMode`. + +| `undeclaredKeys` | Effect | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `'strip'` | Drop them. **Default** outside development. | +| `'warn'` | Drop them and log a warning naming each key once per tool (up to 100 keys per tool). **Default** when `NODE_ENV=development`. | +| `'keep'` | Send them in `structuredContent` and the text content. `tools/list` lists the schema's objects without `additionalProperties: false`, so clients accept them. A key it cannot put back, because the schema turned the object holding it into a primitive (`z.coerce.string()`), refuses the answer as `'fail'` does. | +| `'fail'` | Refuse the answer with `_meta.code: 'INVALID_OUTPUT'`; the error message in the server log names the keys. Nothing from the answer is sent. | + +```typescript +// Fail loudly when an upstream API starts returning fields the tool never declared +@Tool({ name: 'get_invoice', inputSchema, outputSchema, output: { undeclaredKeys: 'fail' } }) +class GetInvoiceTool extends ToolContext { + /* … */ +} +``` + +A call that renders the tool's UI (`@Tool({ ui })`) is parsed with `'strip'`, whatever the policy: the widget, the text content and `structuredContent` are built from that parsed output. An `outputSchema` with a transform anywhere (`.transform()`, `z.preprocess()`, `.pipe()`) is always parsed with `'strip'` too, and the answer is what the schema returns: its output cannot be compared with the answer key by key, so `'warn'`, `'keep'` and `'fail'` do not apply to it. A JSON Schema output (the OpenAPI adapter, a remote app) is not parsed, so its answer is sent as the upstream returned it, whatever the policy. An object schema that sets its own unknown-key handling also wins over the policy: `z.strictObject()` (`.strict()`) refuses an answer with undeclared keys (`INVALID_OUTPUT`), even under `'keep'`, and `z.looseObject()` (`.passthrough()`) keeps them, even under `'strip'` or `'fail'` and on the UI path. + +A `z.union([...])` keeps the first member that matches, so a key that only a later member declares counts as undeclared: `'strip'` drops it, `'warn'` drops and logs it, `'fail'` refuses the answer, and `'keep'` sends it. Put the widest member first, or use `z.discriminatedUnion(...)`, which picks the member by its discriminator. + +To check in a test that an answer has no undeclared keys, use the `@frontmcp/testing` matcher: + +```typescript +import { expect } from '@frontmcp/testing'; + +expect(await tool.execute({ id: 'INV-1' })).toPassOutputSchemaWhole(outputSchema); +// fails with: … parsing drops keys it does not declare: costCenter, lines[].margin +``` + +The matcher fails for a schema with a transform, for the same reason; assert on the parsed answer there. + ## Why this matters -- **Data leak prevention** — without `outputSchema`, accidentally returning `{ result, internalApiKey }` leaks the key. With it, only `result` flows. +- **Data leak prevention** — without `outputSchema`, accidentally returning `{ result, internalApiKey }` leaks the key. With it, only `result` flows (unless `output.undeclaredKeys` is `'keep'`). - **CodeCall compatibility** — the CodeCall plugin uses `outputSchema` to chain tool calls in its VM. Tools without it degrade chain-ability. - **Compile-time type safety** — `ToolContext` infers `execute()`'s return type from `outputSchema` (when `ToolOutputOf<>` is used). The compiler catches divergence at build time. - **Self-documenting** — a structured object `outputSchema` is converted to JSON Schema and exposed in `tools/list` (see [Advertisement in `tools/list`](#advertisement-in-toolslist)); AI clients pick tools partly based on what they return. diff --git a/libs/skills/catalog/create-tool/rules/always-define-output-schema.md b/libs/skills/catalog/create-tool/rules/always-define-output-schema.md index f5c858375..f58329170 100644 --- a/libs/skills/catalog/create-tool/rules/always-define-output-schema.md +++ b/libs/skills/catalog/create-tool/rules/always-define-output-schema.md @@ -52,10 +52,10 @@ class GetWeatherTool extends ToolContext { ## Why -1. **Output validation prevents data leaks.** Without `outputSchema`, every field your code accidentally includes (or that an upstream API drops in unsolicited — auth tokens, internal IDs, debug traces, PII) reaches the MCP client. With it, only declared fields pass through; everything else is stripped. +1. **Output validation prevents data leaks.** Without `outputSchema`, every field your code accidentally includes (or that an upstream API drops in unsolicited — auth tokens, internal IDs, debug traces, PII) reaches the MCP client. With it, only declared fields pass through; everything else is stripped (and, in development, named in a warning). `output: { undeclaredKeys: 'fail' }` refuses such an answer instead. 2. **CodeCall plugin compatibility.** The CodeCall plugin uses `outputSchema` to understand what a tool returns, enabling correct VM-based orchestration and pass-by-reference. Tools without `outputSchema` degrade CodeCall's ability to chain calls. 3. **Type safety on `execute()`'s return type.** With `outputSchema` declared, `ToolContext` infers the expected return type from it. The compiler tells you when your return value diverges from the declared shape. -4. **Self-documenting tools.** A structured object `outputSchema` (a Zod raw shape or `z.object`) is converted to JSON Schema and advertised in `tools/list`, so AI clients can choose the right tool by what it returns. Primitive / media / multi-content-array / union outputs flow through `content` instead and aren't advertised as `outputSchema` — that's expected (the MCP `outputSchema` must be a top-level object). +4. **Self-documenting tools.** A structured object `outputSchema` (a Zod raw shape, a `z.object`, or a union of `z.object`s) is converted to JSON Schema and advertised in `tools/list`, so AI clients can choose the right tool by what it returns. Primitive / media / multi-content-array outputs flow through `content` instead and aren't advertised as `outputSchema` — that's expected (the MCP `outputSchema` must be a top-level object). Any other schema (e.g. `z.array(...)`) is left out with a startup warning naming the tool. ## How to apply diff --git a/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md b/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md index 57d97ad1b..a09efd56b 100644 --- a/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md +++ b/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md @@ -45,7 +45,7 @@ These checks apply to ALL deployment targets. Run them first, then proceed to yo ### Input Validation - [ ] All tool inputs use Zod schemas (never trust raw input) -- [ ] All tool outputs use `outputSchema` to prevent data leaks +- [ ] All tool outputs use `outputSchema` to prevent data leaks; `output.undeclaredKeys` is not `'keep'` for a tool whose answers can carry secrets (the default strips undeclared keys) - [ ] Path parameters and query params are validated - [ ] File paths are sanitized to prevent directory traversal - [ ] SQL queries use parameterized statements (never string interpolation) diff --git a/libs/skills/catalog/frontmcp-testing/references/setup-testing.md b/libs/skills/catalog/frontmcp-testing/references/setup-testing.md index b562a67af..806426572 100644 --- a/libs/skills/catalog/frontmcp-testing/references/setup-testing.md +++ b/libs/skills/catalog/frontmcp-testing/references/setup-testing.md @@ -454,15 +454,16 @@ A nested `test.describe` inherits an outer skip. The `(name, fn)` form still ski import { expect } from '@frontmcp/testing'; ``` -| Matcher | Asserts | -| ------------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `toContainTool(name)` | Tools list includes a tool with the given name | -| `toContainResource(uri)` | Resources list includes a resource with the given URI | -| `toContainPrompt(name)` | Prompts list includes a prompt with the given name | -| `toBeSuccessful()` | Tool call result is not an error | -| `toBeError(code?)` | Result is an error; a string code matches `_meta.code` (`'INVALID_INPUT'`), a number matches a JSON-RPC error code | -| `toHaveTextContent(text)` | Result contains text content matching the string | -| `toHaveMimeType(mime)` | Resource content has the expected MIME type | +| Matcher | Asserts | +| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `toContainTool(name)` | Tools list includes a tool with the given name | +| `toContainResource(uri)` | Resources list includes a resource with the given URI | +| `toContainPrompt(name)` | Prompts list includes a prompt with the given name | +| `toBeSuccessful()` | Tool call result is not an error | +| `toBeError(code?)` | Result is an error; a string code matches `_meta.code` (`'INVALID_INPUT'`), a number matches a JSON-RPC error code | +| `toHaveTextContent(text)` | Result contains text content matching the string | +| `toHaveMimeType(mime)` | Resource content has the expected MIME type | +| `toPassOutputSchemaWhole(outputSchema)` | A tool's answer matches its `outputSchema` (Zod schema or raw shape) and has no undeclared key the server would strip; the failure names the keys | ## Running Tests with Nx diff --git a/libs/testing/README.md b/libs/testing/README.md index 36ccdc16e..9daf048e5 100644 --- a/libs/testing/README.md +++ b/libs/testing/README.md @@ -50,7 +50,7 @@ npx frontmcp test ## Matchers -**Tools**: `toContainTool`, `toBeSuccessful`, `toBeError`, `toHaveTextContent`, `toHaveImageContent`, `toHaveResourceContent` +**Tools**: `toContainTool`, `toBeSuccessful`, `toBeError`, `toHaveTextContent`, `toHaveImageContent`, `toHaveResourceContent`, `toPassOutputSchemaWhole` **Resources**: `toContainResource`, `toContainResourceTemplate`, `toHaveMimeType` diff --git a/libs/testing/src/matchers/__tests__/mcp-matchers.output-schema.spec.ts b/libs/testing/src/matchers/__tests__/mcp-matchers.output-schema.spec.ts new file mode 100644 index 000000000..66f232e0f --- /dev/null +++ b/libs/testing/src/matchers/__tests__/mcp-matchers.output-schema.spec.ts @@ -0,0 +1,61 @@ +import { z } from '@frontmcp/lazy-zod'; + +import { mcpMatchers } from '../mcp-matchers'; + +import type {} from '../matcher-types'; + +expect.extend(mcpMatchers); + +const invoiceShape = { id: z.string(), lines: z.array(z.object({ sku: z.string() })) }; + +describe('toPassOutputSchemaWhole', () => { + it('passes an answer that holds declared keys only, for a raw shape', () => { + expect({ id: 'INV-1', lines: [{ sku: 'A' }] }).toPassOutputSchemaWhole(invoiceShape); + }); + + it('passes an answer that holds declared keys only, for a Zod schema', () => { + const hit = z.union([z.object({ title: z.string() }), z.object({ name: z.string() })]); + + expect({ name: 'Ada' }).toPassOutputSchemaWhole(hit); + }); + + it('fails an answer whose undeclared keys the schema would drop, naming them', () => { + const answer = { id: 'INV-1', costCenter: 'CC-7', lines: [{ sku: 'A', margin: 0.4 }] }; + + expect(() => expect(answer).toPassOutputSchemaWhole(invoiceShape)).toThrow( + 'drops keys it does not declare: costCenter, lines[].margin', + ); + }); + + it('fails an answer that does not match the schema, naming the first mismatch', () => { + expect(() => expect({ id: 7, lines: [] }).toPassOutputSchemaWhole(invoiceShape)).toThrow( + /does not match its output schema at id/, + ); + }); + + it('supports .not for an answer with undeclared keys', () => { + expect({ id: 'INV-1', lines: [], debug: true }).not.toPassOutputSchemaWhole(invoiceShape); + }); + + it('fails .not for an answer that passes whole', () => { + expect(() => expect({ id: 'INV-1', lines: [] }).not.toPassOutputSchemaWhole(invoiceShape)).toThrow( + 'Expected the answer not to pass its output schema whole', + ); + }); + + it('fails an answer whose keys parsing loses by coercing their object to a primitive', () => { + const coercing = z.object({ payload: z.coerce.string() }); + + expect(() => expect({ payload: { secret: 'x' } }).toPassOutputSchemaWhole(coercing)).toThrow( + 'drops keys it does not declare: payload.secret', + ); + }); + + it('fails for a schema with a transform, whose output cannot be compared key by key', () => { + const renaming = z.object({ a: z.string() }).transform((value) => ({ renamed: value.a })); + + expect(() => expect({ a: 'x' }).toPassOutputSchemaWhole(renaming)).toThrow( + /cannot compare keys through a transform/, + ); + }); +}); diff --git a/libs/testing/src/matchers/matcher-types.ts b/libs/testing/src/matchers/matcher-types.ts index 60c4c8123..35d63be14 100644 --- a/libs/testing/src/matchers/matcher-types.ts +++ b/libs/testing/src/matchers/matcher-types.ts @@ -6,13 +6,12 @@ * Import this file or ensure it's included in your tsconfig to get type checking. */ -// Note: These imports are used for documentation/JSDoc purposes in the interface comments -// The actual runtime types are in mcp-matchers.ts +import type { ZodRawShape, ZodType } from '@frontmcp/lazy-zod'; /** - * Custom MCP matchers for Jest + * Custom MCP matchers for Jest: tool, resource, prompt and protocol results */ -export interface McpMatchers { +export interface McpResultMatchers { // ═══════════════════════════════════════════════════════════════════ // TOOL MATCHERS // ═══════════════════════════════════════════════════════════════════ @@ -89,6 +88,21 @@ export interface McpMatchers { */ toHaveResourceContent(): R; + /** + * Check that a tool's answer passes its output schema whole: it matches the schema, and parsing + * drops no key the schema does not declare. The failure message names the undeclared keys, which + * the server would strip, warn about or refuse, per its `output.undeclaredKeys` policy. A schema with + * a transform (`.transform()`, `z.preprocess()`, `.pipe()`) cannot be compared key by key and fails. + * @param outputSchema - The tool's `outputSchema`: a Zod schema or a raw shape + * + * @example + * ```typescript + * const answer = await tool.execute({ id: 'INV-1' }); + * expect(answer).toPassOutputSchemaWhole(invoiceOutputSchema); + * ``` + */ + toPassOutputSchemaWhole(outputSchema: ZodType | ZodRawShape): R; + // ═══════════════════════════════════════════════════════════════════ // RESOURCE MATCHERS // ═══════════════════════════════════════════════════════════════════ @@ -205,7 +219,12 @@ export interface McpMatchers { * ``` */ toHaveErrorCode(code: number): R; +} +/** + * Custom MCP matchers for Jest: rendered tool UI + */ +export interface McpUiMatchers { // ═══════════════════════════════════════════════════════════════════ // UI MATCHERS // ═══════════════════════════════════════════════════════════════════ @@ -301,25 +320,30 @@ export interface McpMatchers { toHaveProperHtmlStructure(): R; } +/** + * Custom MCP matchers for Jest + */ +export interface McpMatchers extends McpResultMatchers, McpUiMatchers {} + // ═══════════════════════════════════════════════════════════════════ // JEST TYPE AUGMENTATION // ═══════════════════════════════════════════════════════════════════ -/* eslint-disable @typescript-eslint/no-namespace, @typescript-eslint/no-empty-object-type */ +/* eslint-disable @typescript-eslint/no-namespace */ declare global { namespace jest { // Extend expect matchers - interface Matchers extends McpMatchers {} + interface Matchers extends McpResultMatchers, McpUiMatchers {} // Extend asymmetric matchers (for expect.toContainTool etc.) - interface Expect extends McpMatchers {} + interface Expect extends McpResultMatchers, McpUiMatchers {} // Extend inverse matchers (for expect.not.toContainTool etc.) - interface InverseAsymmetricMatchers extends McpMatchers {} + interface InverseAsymmetricMatchers extends McpResultMatchers, McpUiMatchers {} } } -/* eslint-enable @typescript-eslint/no-namespace, @typescript-eslint/no-empty-object-type */ +/* eslint-enable @typescript-eslint/no-namespace */ // This export is needed to make this a module export {}; diff --git a/libs/testing/src/matchers/mcp-matchers.ts b/libs/testing/src/matchers/mcp-matchers.ts index 2ec923c92..56f60d3b2 100644 --- a/libs/testing/src/matchers/mcp-matchers.ts +++ b/libs/testing/src/matchers/mcp-matchers.ts @@ -18,8 +18,12 @@ */ import type { MatcherFunction } from 'expect'; -import type { Tool, Resource, ResourceTemplate, Prompt } from '@frontmcp/protocol'; -import type { ToolResultWrapper, ResourceContentWrapper, PromptResultWrapper } from '../client/mcp-test-client.types'; + +import { hasTransform, z, type ZodRawShape, type ZodType } from '@frontmcp/lazy-zod'; +import type { Prompt, Resource, ResourceTemplate, Tool } from '@frontmcp/protocol'; +import { findDroppedKeys } from '@frontmcp/utils'; + +import type { PromptResultWrapper, ResourceContentWrapper, ToolResultWrapper } from '../client/mcp-test-client.types'; import { uiMatchers } from '../ui/ui-matchers'; // ═══════════════════════════════════════════════════════════════════ @@ -195,6 +199,46 @@ const toHaveResourceContent: MatcherFunction<[]> = function (received) { }; }; +/** + * Check that a tool's answer passes its output schema whole: it matches, and parsing drops no + * key the schema does not declare (which the server would strip, warn about or refuse) + */ +const toPassOutputSchemaWhole: MatcherFunction<[outputSchema: ZodType | ZodRawShape]> = function ( + received, + outputSchema, +) { + const schema = outputSchema instanceof z.ZodType ? outputSchema : z.object(outputSchema); + if (hasTransform(schema)) { + return { + pass: false, + message: () => + 'toPassOutputSchemaWhole cannot compare keys through a transform (.transform(), z.preprocess(), .pipe()): ' + + 'assert on the parsed answer instead', + }; + } + const parsed = schema.safeParse(received); + if (!parsed.success) { + const [firstIssue] = parsed.error.issues; + const location = firstIssue?.path.length ? ` at ${firstIssue.path.join('.')}` : ''; + return { + pass: false, + message: () => + `Expected the answer to pass its output schema whole, but it does not match its output schema${location}: ${firstIssue?.message}`, + }; + } + + const droppedKeys = findDroppedKeys(received, parsed.data); + const pass = droppedKeys.length === 0; + + return { + pass, + message: () => + pass + ? 'Expected the answer not to pass its output schema whole' + : `Expected the answer to pass its output schema whole, but parsing drops keys it does not declare: ${droppedKeys.join(', ')}`, + }; +}; + // ═══════════════════════════════════════════════════════════════════ // RESOURCE MATCHERS // ═══════════════════════════════════════════════════════════════════ @@ -457,6 +501,7 @@ export const mcpMatchers = { toHaveTextContent, toHaveImageContent, toHaveResourceContent, + toPassOutputSchemaWhole, // Resource matchers toContainResource, diff --git a/libs/utils/src/content/content.dropped-keys.spec.ts b/libs/utils/src/content/content.dropped-keys.spec.ts new file mode 100644 index 000000000..528d8e5d6 --- /dev/null +++ b/libs/utils/src/content/content.dropped-keys.spec.ts @@ -0,0 +1,59 @@ +import { findDroppedKeys } from './content'; + +describe('findDroppedKeys', () => { + it('names the keys the parsed value lacks, nested ones by their dotted path', () => { + const original = { id: 'INV-1', costCenter: 'CC-7', customer: { name: 'Ada', taxId: 'T-1' } }; + const parsed = { id: 'INV-1', customer: { name: 'Ada' } }; + + expect(findDroppedKeys(original, parsed)).toEqual(['costCenter', 'customer.taxId']); + }); + + it('names a key dropped from array items once, with an [] segment', () => { + const original = { lines: [{ sku: 'A', margin: 0.4 }, { sku: 'B', margin: 0.1 }, { sku: 'C' }] }; + const parsed = { lines: [{ sku: 'A' }, { sku: 'B' }, { sku: 'C' }] }; + + expect(findDroppedKeys(original, parsed)).toEqual(['lines[].margin']); + }); + + it('walks a top-level array', () => { + expect(findDroppedKeys([{ id: 1, debug: true }], [{ id: 1 }])).toEqual(['[].debug']); + }); + + it('returns the paths sorted', () => { + expect(findDroppedKeys({ zeta: 1, alpha: 2, kept: 3 }, { kept: 3 })).toEqual(['alpha', 'zeta']); + }); + + it('ignores keys holding undefined, which never reach JSON', () => { + expect(findDroppedKeys({ id: 1, note: undefined }, { id: 1 })).toEqual([]); + }); + + it('finds nothing when the parsed value keeps every key', () => { + const value = { id: 1, tags: ['a'], owner: { name: 'Ada' } }; + + expect(findDroppedKeys(value, structuredClone(value))).toEqual([]); + }); + + it('does not count inherited properties of the parsed value as kept', () => { + expect(findDroppedKeys({ id: 1, toString: 'shadow' }, { id: 1 })).toEqual(['toString']); + }); + + it('counts every key of an object that parsing replaced with a non-object as dropped', () => { + expect(findDroppedKeys({ payload: { secret: 'x', nested: { pin: 1 } } }, { payload: '[object Object]' })).toEqual([ + 'payload.nested', + 'payload.secret', + ]); + expect(findDroppedKeys([{ id: 1 }], { 0: { id: 1 } })).toEqual(['[].id']); + }); + + it('stops at a value the parsed answer holds by reference, so a cycle under it is not followed', () => { + const node: Record = { name: 'node' }; + node['self'] = node; + + expect(findDroppedKeys({ id: 1, extra: true, meta: node }, { id: 1, meta: node })).toEqual(['extra']); + }); + + it('finds nothing to drop in a primitive answer', () => { + expect(findDroppedKeys('text', { value: 'text' })).toEqual([]); + expect(findDroppedKeys(['a', 'b'], 'a,b')).toEqual([]); + }); +}); diff --git a/libs/utils/src/content/content.ts b/libs/utils/src/content/content.ts index 9791ccce6..3e90fd70c 100644 --- a/libs/utils/src/content/content.ts +++ b/libs/utils/src/content/content.ts @@ -139,6 +139,40 @@ export function findNonFiniteNumber(value: unknown): { path: string; value: numb return walk(value, ''); } +/** + * The key paths `original` holds that `parsed` lacks: what parsing `original` against a schema + * dropped. Nested keys are dotted (`customer.taxId`), array items share an `[]` segment + * (`lines[].margin`), keys holding `undefined` are skipped, and the paths come back sorted and + * unique. The walk follows `parsed`, so it stops wherever the two values differ in shape, and + * wherever `parsed` holds the very same object, which also keeps it out of cycles. + */ +export function findDroppedKeys(original: unknown, parsed: unknown): string[] { + const droppedPaths = new Set(); + + function walk(source: unknown, target: unknown, path: string): void { + if (source === target) return; + if (Array.isArray(source)) { + source.forEach((item, index) => walk(item, Array.isArray(target) ? target[index] : undefined, `${path}[]`)); + return; + } + if (!isPlainRecord(source)) return; + const targetRecord = isPlainRecord(target) ? target : {}; + for (const [key, value] of Object.entries(source)) { + if (value === undefined) continue; + const keyPath = path ? `${path}.${key}` : key; + if (Object.prototype.hasOwnProperty.call(targetRecord, key)) walk(value, targetRecord[key], keyPath); + else droppedPaths.add(keyPath); + } + } + + walk(original, parsed, ''); + return [...droppedPaths].sort(); +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + /** * Common MIME types by file extension. */ diff --git a/libs/utils/src/content/index.ts b/libs/utils/src/content/index.ts index 909f0a16c..6c7659621 100644 --- a/libs/utils/src/content/index.ts +++ b/libs/utils/src/content/index.ts @@ -1 +1 @@ -export { sanitizeToJson, inferMimeType, findNonFiniteNumber } from './content'; +export { sanitizeToJson, inferMimeType, findNonFiniteNumber, findDroppedKeys } from './content'; diff --git a/libs/utils/src/index.ts b/libs/utils/src/index.ts index ab1255ff4..bf100c814 100644 --- a/libs/utils/src/index.ts +++ b/libs/utils/src/index.ts @@ -38,7 +38,7 @@ export { } from './path'; // Content utilities -export { sanitizeToJson, inferMimeType, findNonFiniteNumber } from './content'; +export { sanitizeToJson, inferMimeType, findNonFiniteNumber, findDroppedKeys } from './content'; // HTTP utilities export { validateBaseUrl, buildSetCookie, parseCookies, getCookie, isLocalhost, isSecureRequest } from './http';