diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index fa2880f9c364..ebc084ad78ce 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -1,9 +1,17 @@ import { expect, it } from "@effect/vitest"; import { NodeHttpServer } from "@effect/platform-node"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import { EnvironmentId, PreviewTabId, ProviderInstanceId, ThreadId } from "@t3tools/contracts"; +import { + EnvironmentId, + PreviewTabId, + ProviderInstanceId, + ThreadId, + type PreviewAutomationElement, + type PreviewAutomationSnapshot, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; import { McpProtocol, McpSchema, McpServer } from "effect/unstable/ai"; import { HttpBody, HttpClient, HttpRouter, HttpServerResponse } from "effect/unstable/http"; @@ -97,6 +105,194 @@ it.effect("returns bounded structural preview snapshot failures", () => ).pipe(Effect.provide(TestLayer)), ); +const snapshotFixture = ( + overrides: Partial = {}, +): PreviewAutomationSnapshot => ({ + url: "http://example.test/", + title: "Example", + loading: false, + visibleText: "Example", + interactiveElements: [], + accessibilityTree: {}, + consoleEntries: [], + networkEntries: [], + actionTimeline: [], + screenshot: { + mimeType: "image/png", + data: Buffer.from("png").toString("base64"), + width: 10, + height: 5, + }, + ...overrides, +}); +const element = (name: string): PreviewAutomationElement => ({ + tag: "button", + role: null, + name, + selector: "button", + x: 0, + y: 0, + width: 1, + height: 1, +}); +const serializedByteLength = (value: unknown) => Buffer.byteLength(JSON.stringify(value), "utf8"); +const parseJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); + +it("passes snapshot metadata through untouched while it fits", () => { + const snapshot = snapshotFixture({ accessibilityTree: { nodes: [{ role: "main" }] } }); + const { metadata, serialized } = McpHttpServer.boundPreviewSnapshotMetadata(snapshot); + + expect(metadata).not.toHaveProperty("truncation"); + expect(metadata).toMatchObject({ + accessibilityTree: { nodes: [{ role: "main" }] }, + screenshot: { mimeType: "image/png", width: 10, height: 5 }, + }); + expect(metadata.screenshot).not.toHaveProperty("data"); + expect(serialized).toBe(JSON.stringify(metadata)); +}); + +it("drops the accessibility tree first and keeps the rest of an oversized snapshot", () => { + const snapshot = snapshotFixture({ + visibleText: "visible ".repeat(2_000), + interactiveElements: [element("Save")], + consoleEntries: [{ level: "log", text: "hello", timestamp: "2026-09-04T00:00:00Z" }], + accessibilityTree: { nodes: [{ name: "accessible ".repeat(50_000) }] }, + }); + const { metadata, serialized } = McpHttpServer.boundPreviewSnapshotMetadata(snapshot); + + expect(Buffer.byteLength(serialized, "utf8")).toBeLessThanOrEqual( + McpHttpServer.PREVIEW_SNAPSHOT_METADATA_MAX_BYTES, + ); + expect(metadata.accessibilityTree).toBeNull(); + expect(metadata.truncation).toEqual({ + originalBytes: serializedByteLength({ + ...snapshot, + screenshot: { mimeType: "image/png", width: 10, height: 5 }, + }), + omitted: ["accessibilityTree"], + trimmed: [], + }); + expect(metadata.visibleText).toBe(snapshot.visibleText); + expect(metadata.interactiveElements).toEqual(snapshot.interactiveElements); + expect(metadata.consoleEntries).toEqual(snapshot.consoleEntries); + expect(parseJson(serialized)).toEqual(metadata); +}); + +it("cuts the visible text short before giving up the interactive elements", () => { + // Multi-byte text: the byte budget, not the character count, is what has to fit. + const snapshot = snapshotFixture({ + visibleText: "görünür metin 🙂 ".repeat(20_000), + interactiveElements: [element("Save"), element("Cancel")], + accessibilityTree: { nodes: [{ name: "accessible ".repeat(50_000) }] }, + }); + const { metadata, serialized } = McpHttpServer.boundPreviewSnapshotMetadata(snapshot); + + expect(Buffer.byteLength(serialized, "utf8")).toBeLessThanOrEqual( + McpHttpServer.PREVIEW_SNAPSHOT_METADATA_MAX_BYTES, + ); + expect(metadata.truncation).toMatchObject({ + omitted: ["accessibilityTree"], + trimmed: ["visibleText"], + }); + expect(metadata.visibleText.length).toBeGreaterThan(0); + expect(metadata.visibleText.isWellFormed()).toBe(true); + expect(snapshot.visibleText.startsWith(metadata.visibleText)).toBe(true); + expect(metadata.interactiveElements).toEqual(snapshot.interactiveElements); +}); + +it("caps a runaway URL before touching page content", () => { + const snapshot = snapshotFixture({ + url: `data:text/html,${"x".repeat(200_000)}`, + visibleText: "visible ".repeat(100), + interactiveElements: [element("Save")], + accessibilityTree: { nodes: [{ role: "main" }] }, + }); + const { metadata, serialized } = McpHttpServer.boundPreviewSnapshotMetadata(snapshot); + + expect(Buffer.byteLength(serialized, "utf8")).toBeLessThanOrEqual( + McpHttpServer.PREVIEW_SNAPSHOT_METADATA_MAX_BYTES, + ); + expect(metadata.url).toBe(snapshot.url.slice(0, 2_048)); + expect(metadata.truncation?.trimmed).toEqual(["url"]); + expect(metadata.truncation?.omitted).toEqual([]); + expect(metadata.visibleText).toBe(snapshot.visibleText); + expect(metadata.interactiveElements).toEqual(snapshot.interactiveElements); + expect(metadata.accessibilityTree).toEqual(snapshot.accessibilityTree); +}); + +it("gives up the interactive elements last when they alone exceed the limit", () => { + const snapshot = snapshotFixture({ + visibleText: "visible ".repeat(2_000), + interactiveElements: [element("x".repeat(200_000))], + }); + const { metadata, serialized } = McpHttpServer.boundPreviewSnapshotMetadata(snapshot); + + expect(Buffer.byteLength(serialized, "utf8")).toBeLessThanOrEqual( + McpHttpServer.PREVIEW_SNAPSHOT_METADATA_MAX_BYTES, + ); + expect(metadata.interactiveElements).toEqual([]); + // Empty logs are not reported as omitted: only what the agent actually lost is listed. + expect(metadata.truncation?.omitted).toEqual(["accessibilityTree", "interactiveElements"]); + expect(metadata.truncation?.trimmed).toEqual(["visibleText"]); + expect(metadata.visibleText).toBe(""); +}); + +it.effect("returns bounded preview snapshot metadata with the screenshot and a note", () => + Effect.scoped( + Effect.gen(function* () { + const server = yield* McpServer.McpServer; + const broker = yield* PreviewAutomationBroker.PreviewAutomationBroker; + const events = yield* broker.connect({ + clientId: "mcp-oversized-snapshot-client", + environmentId, + }); + yield* Stream.runForEach(events, (event) => + event.type === "connected" + ? Effect.void + : broker.respond({ + clientId: "mcp-oversized-snapshot-client", + connectionId: event.connectionId, + requestId: event.request.requestId, + ok: true, + result: snapshotFixture({ + url: "http://example.test/large", + title: "Large page", + interactiveElements: [element("Save")], + accessibilityTree: { nodes: [{ name: "accessible ".repeat(50_000) }] }, + }), + }), + ).pipe(Effect.forkScoped); + yield* Effect.yieldNow; + + const snapshot = yield* server + .callTool({ name: "preview_snapshot", arguments: {} }) + .pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, invocation), + Effect.provideService(McpSchema.McpServerClient, client), + ); + + expect(snapshot.isError).toBe(false); + const texts = snapshot.content.flatMap((content) => + content.type === "text" ? [content.text] : [], + ); + expect(texts).toHaveLength(2); + expect(Buffer.byteLength(texts[0]!, "utf8")).toBeLessThanOrEqual( + McpHttpServer.PREVIEW_SNAPSHOT_METADATA_MAX_BYTES, + ); + expect(parseJson(texts[0]!)).toEqual(snapshot.structuredContent); + expect(texts[1]).toContain("Omitted: accessibilityTree."); + expect(snapshot.structuredContent).toMatchObject({ + title: "Large page", + accessibilityTree: null, + interactiveElements: [element("Save")], + truncation: { omitted: ["accessibilityTree"], trimmed: [] }, + screenshot: { mimeType: "image/png", width: 10, height: 5 }, + }); + expect(snapshot.content.some((content) => content.type === "image")).toBe(true); + }), + ).pipe(Effect.provide(TestLayer)), +); + it.effect("terminates HTTP MCP sessions with DELETE", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index 44ca928e63bb..9205c62c6da0 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -8,6 +8,7 @@ import * as Stream from "effect/Stream"; import type * as Types from "effect/Types"; import { McpProtocol, McpSchema, McpServer, Tool } from "effect/unstable/ai"; import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; +import type { PreviewAutomationSnapshot } from "@t3tools/contracts"; import packageJson from "../../package.json" with { type: "json" }; import * as McpInvocationContext from "./McpInvocationContext.ts"; @@ -23,6 +24,138 @@ import { PreviewStandardToolkit, } from "./toolkits/preview/tools.ts"; +/** + * Ceiling for the JSON metadata a `preview_snapshot` result hands the agent. + * Chrome's full accessibility tree has no natural bound and alone ran to + * hundreds of kilobytes on content-heavy pages, enough to crowd out the rest + * of a thread's context, so it is the first thing to go. The ceiling sits + * above what the producer's own caps (20k characters of visible text, 200 + * interactive elements) add up to on an ordinary page, so a snapshot without + * the tree is normally sent whole. The screenshot travels beside the metadata + * as an image and is never reduced. + */ +export const PREVIEW_SNAPSHOT_METADATA_MAX_BYTES = 100_000; +/** Longest title or URL kept once a snapshot has to be reduced at all. */ +const PREVIEW_SNAPSHOT_IDENTIFIER_MAX_CHARS = 2_048; + +type PreviewSnapshotMetadata = Omit & { + readonly screenshot: Omit; +}; + +type OmittableField = + | "accessibilityTree" + | "networkEntries" + | "consoleEntries" + | "actionTimeline" + | "interactiveElements"; + +/** Cuts a string to at most `length` UTF-16 units without splitting a surrogate pair. */ +const cutText = (text: string, length: number): string => { + const cut = text.slice(0, Math.max(0, length)); + const last = cut.charCodeAt(cut.length - 1); + return last >= 0xd800 && last <= 0xdbff ? cut.slice(0, -1) : cut; +}; + +const serializedBytes = (value: unknown) => { + const serialized = JSON.stringify(value); + return { serialized, bytes: Buffer.byteLength(serialized, "utf8") }; +}; + +/** + * Reduces snapshot metadata until it fits the limit, least useful field + * first, and records what went. The order is what an agent can best do + * without: the accessibility tree (redundant with the elements and the + * screenshot), then the diagnostics logs, then the visible text is cut short, + * and only then the interactive elements, which carry the locators the other + * tools need. Every step keeps the result inside the tool's output schema. + */ +export function boundPreviewSnapshotMetadata(snapshot: PreviewAutomationSnapshot): { + readonly metadata: PreviewSnapshotMetadata; + readonly serialized: string; +} { + const { + screenshot: { data: _data, ...screenshot }, + ...page + } = snapshot; + let metadata: PreviewSnapshotMetadata = { ...page, screenshot }; + let { serialized, bytes } = serializedBytes(metadata); + if (bytes <= PREVIEW_SNAPSHOT_METADATA_MAX_BYTES) { + return { metadata, serialized }; + } + + const originalBytes = bytes; + const omitted: string[] = []; + const trimmed: string[] = []; + const measure = (): number => { + metadata = { ...metadata, truncation: { originalBytes, omitted, trimmed } }; + const measured = serializedBytes(metadata); + serialized = measured.serialized; + return measured.bytes; + }; + const omit = (field: OmittableField): number => { + metadata = { ...metadata, [field]: field === "accessibilityTree" ? null : [] }; + omitted.push(field); + return measure(); + }; + + // The title and URL are identifiers, not content. Past this length (a data: + // URL, a runaway title) they carry nothing the agent needs, and either could + // be the whole overrun by itself, so they are capped before any content goes. + for (const field of ["title", "url"] as const) { + if (metadata[field].length > PREVIEW_SNAPSHOT_IDENTIFIER_MAX_CHARS) { + metadata = { + ...metadata, + [field]: cutText(metadata[field], PREVIEW_SNAPSHOT_IDENTIFIER_MAX_CHARS), + }; + trimmed.push(field); + bytes = measure(); + } + } + if (bytes <= PREVIEW_SNAPSHOT_METADATA_MAX_BYTES) { + return { metadata, serialized }; + } + + if (metadata.accessibilityTree !== null && metadata.accessibilityTree !== undefined) { + bytes = omit("accessibilityTree"); + } + for (const field of ["networkEntries", "consoleEntries", "actionTimeline"] as const) { + if (bytes <= PREVIEW_SNAPSHOT_METADATA_MAX_BYTES) break; + if (metadata[field].length > 0) bytes = omit(field); + } + if (bytes > PREVIEW_SNAPSHOT_METADATA_MAX_BYTES && metadata.visibleText.length > 0) { + trimmed.push("visibleText"); + while (bytes > PREVIEW_SNAPSHOT_METADATA_MAX_BYTES && metadata.visibleText.length > 0) { + // Each pass keeps at most the share of the text the budget allows, so a + // few passes converge even when escaping and multi-byte characters make + // the serialized form larger than the character count. + const share = Math.min(0.9, PREVIEW_SNAPSHOT_METADATA_MAX_BYTES / bytes); + metadata = { + ...metadata, + visibleText: cutText(metadata.visibleText, Math.floor(metadata.visibleText.length * share)), + }; + bytes = measure(); + } + } + if (bytes > PREVIEW_SNAPSHOT_METADATA_MAX_BYTES && metadata.interactiveElements.length > 0) { + omit("interactiveElements"); + } + return { metadata, serialized }; +} + +const formatKilobytes = (bytes: number) => `${Math.round(bytes / 1000)} KB`; + +/** The plain-language half of a reduced result, beside the JSON the agent parses. */ +const truncationNote = (metadata: PreviewSnapshotMetadata): string | undefined => { + const truncation = metadata.truncation; + if (truncation === undefined) return undefined; + return [ + `Snapshot metadata was ${formatKilobytes(truncation.originalBytes)}, above the ${formatKilobytes(PREVIEW_SNAPSHOT_METADATA_MAX_BYTES)} limit sent to agents.`, + ...(truncation.omitted.length > 0 ? [`Omitted: ${truncation.omitted.join(", ")}.`] : []), + ...(truncation.trimmed.length > 0 ? [`Cut short: ${truncation.trimmed.join(", ")}.`] : []), + "Use preview_evaluate for targeted reads of what was left out.", + ].join(" "); +}; + const unauthorized = HttpServerResponse.jsonUnsafe( { error: "invalid_mcp_credential", @@ -163,34 +296,20 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot Effect.matchCauseEffect({ onFailure: previewSnapshotFailure, onSuccess: ({ encodedResult }) => { - const snapshot = encodedResult as { - readonly screenshot: { - readonly mimeType: "image/png"; - readonly data: string; - readonly width: number; - readonly height: number; - }; - readonly [key: string]: unknown; - }; - const { screenshot, ...page } = snapshot; - const metadata = { - ...page, - screenshot: { - mimeType: screenshot.mimeType, - width: screenshot.width, - height: screenshot.height, - }, - }; + const snapshot = encodedResult as PreviewAutomationSnapshot; + const { metadata, serialized } = boundPreviewSnapshotMetadata(snapshot); + const note = truncationNote(metadata); return Effect.succeed( new McpSchema.CallToolResult({ isError: false, structuredContent: metadata, content: [ - { type: "text", text: JSON.stringify(metadata) }, + { type: "text", text: serialized }, + ...(note === undefined ? [] : [{ type: "text" as const, text: note }]), { type: "image", - data: new Uint8Array(Buffer.from(screenshot.data, "base64")), - mimeType: screenshot.mimeType, + data: new Uint8Array(Buffer.from(snapshot.screenshot.data, "base64")), + mimeType: snapshot.screenshot.mimeType, }, ], }), diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 33528d8bb38c..af19742f2dae 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -111,7 +111,7 @@ export const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.", + "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. On very large pages the metadata is reduced to fit a size limit, least useful fields first, and `truncation` lists what was omitted or trimmed.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationSnapshot, failure: PreviewAutomationError, diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index e33615fa4c05..a21151280fa3 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -543,6 +543,16 @@ export const PreviewAutomationSnapshot = Schema.Struct({ width: Schema.Int, height: Schema.Int, }), + /** Present only when the metadata handed to an agent had to be reduced to + fit its size limit: which fields were replaced with null or an empty + list, which were cut short, and how large the whole was before. */ + truncation: Schema.optional( + Schema.Struct({ + originalBytes: Schema.Int, + omitted: Schema.Array(Schema.String), + trimmed: Schema.Array(Schema.String), + }), + ), }); export type PreviewAutomationSnapshot = typeof PreviewAutomationSnapshot.Type;