From bf94883552fb72a3486a838cf2fdd1bc5faaf921 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 15:03:58 +0800 Subject: [PATCH 1/4] feat(anthropic): pass image content blocks through to the upstream model The /v1/messages route translates Anthropic requests to OpenAI Chat before proxying to CodeBuddy. Image blocks had no branch in mapAnthropicContentToChat, so they fell through to stringifyContent and reached the model as a JSON dump of the base64 payload -- the image was never seen and the payload still consumed prompt tokens. Emit images as Chat-shaped image_url parts instead, which both upstream protocols understand: the chat upstream forwards them verbatim and the responses upstream converts them to input_image. base64 sources become a data URI; url sources pass through. Images in the system prompt stay text-only since that field is not representable upstream. An image block without a source is not a well-formed Anthropic image, so it keeps the previous stringified handling. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/anthropic.ts | 140 +++++++++++++++++++++++-- tests/server/anthropic.test.ts | 184 +++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+), 8 deletions(-) diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index b88c9de..5755703 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -33,6 +33,13 @@ const MAX_STREAM_FRAME_LENGTH = 1_000_000; // Anthropic Messages API types // --------------------------------------------------------------------------- +interface AnthropicImageSource { + type?: string; + media_type?: string; + data?: string; + url?: string; +} + interface AnthropicContentBlock { type: string; text?: string; @@ -43,6 +50,7 @@ interface AnthropicContentBlock { thinking?: string; tool_use_id?: string; content?: unknown; + source?: AnthropicImageSource; } interface AnthropicMessage { @@ -144,6 +152,25 @@ interface ChatTextBlock { type: 'text'; } +/** + * An image part in the OpenAI Chat shape. Emitted in this shape rather than a + * native Anthropic one because the request is translated to Chat before it + * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the + * `responses` upstream converts it to `input_image`. + */ +interface ChatImageBlock { + image_url: { url: string }; + type: 'image_url'; +} + +type ChatContentPart = string | ChatTextBlock | ChatImageBlock; + +type ChatContent = string | Array; + +/** + * Text-only content, used where images are not representable — the system + * prompt and the intermediate text-part buffer. + */ type ChatTextContent = string | ChatTextBlock[]; // --------------------------------------------------------------------------- @@ -200,6 +227,87 @@ const mapTextPartsToChatContent = ( ]); }; +/** + * Builds the `image_url` value for an Anthropic image block. Base64 sources + * become a data URI because the upstream Chat/Responses APIs expect a URL; + * `url` sources pass through untouched. Returns undefined for an unusable + * source so the caller can fall back to a text placeholder rather than + * emitting a block the upstream would reject. + */ +const buildChatImageUrl = ( + source: AnthropicImageSource | undefined, +): string | undefined => { + if (!source || typeof source !== 'object') { + return undefined; + } + + if (source.type === 'url' || (!source.data && source.url)) { + return typeof source.url === 'string' && source.url + ? source.url + : undefined; + } + + if (typeof source.data !== 'string' || !source.data) { + return undefined; + } + + const mediaType = + typeof source.media_type === 'string' && source.media_type + ? source.media_type + : 'image/png'; + + return `data:${mediaType};base64,${source.data}`; +}; + +/** + * Like `mapTextPartsToChatContent`, but keeps image parts as real image + * blocks instead of collapsing them into text. Falls back to the text-only + * result when nothing resolved to an image. + */ +const mapContentPartsToChat = (parts: ChatContentPart[]): ChatContent => { + const hasImage = parts.some( + (part) => typeof part === 'object' && part.type === 'image_url', + ); + + if (!hasImage) { + return mapTextPartsToChatContent( + parts.filter( + (part): part is string | ChatTextBlock => + typeof part === 'string' || part.type === 'text', + ), + ); + } + + const blocks: Array = []; + let pendingText: Array = []; + + const flushText = (): void => { + if (!pendingText.length) { + return; + } + const textContent = mapTextPartsToChatContent(pendingText); + if (typeof textContent === 'string') { + blocks.push({ type: 'text', text: textContent }); + } else { + blocks.push(...textContent); + } + pendingText = []; + }; + + for (const part of parts) { + if (typeof part === 'object' && part.type === 'image_url') { + flushText(); + blocks.push(part); + continue; + } + pendingText.push(part); + } + + flushText(); + + return blocks; +}; + const extractSystemText = ( system: string | AnthropicContentBlock[] | undefined, ): ChatTextContent => { @@ -232,7 +340,7 @@ const extractSystemText = ( interface ChatMessage { role: string; - content: ChatTextContent | null; + content: ChatContent | null; tool_calls?: Array<{ id: string; type: string; @@ -324,7 +432,7 @@ const mapAnthropicContentToChat = ( return [{ role, content }]; } - const parts: Array = []; + const parts: ChatContentPart[] = []; const toolCalls: Array<{ id: string; type: string; @@ -336,15 +444,16 @@ const mapAnthropicContentToChat = ( const toolResults: ChatMessage[] = []; const messages: ChatMessage[] = []; const flushAssistantMessage = (): void => { - const textContent = mapTextPartsToChatContent(parts); + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; - if (!toolCalls.length && !textContent.length) { + if (!toolCalls.length && !hasContent) { return; } messages.push({ role: 'assistant', - content: textContent.length ? textContent : null, + content: hasContent ? content : null, ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}), }); parts.length = 0; @@ -388,6 +497,20 @@ const mapAnthropicContentToChat = ( } } else if (block.type === 'thinking') { // Skip thinking blocks in conversation history for OpenAI compat. + } else if (block.type === 'image' && block.source) { + // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url', + // media_type, data | url } }`. Emit a real image block so the upstream + // model sees the image; without this branch the block fell through to + // `stringifyContent` and the model received a JSON dump of the base64 + // payload as text. An `image` block with no `source` is not a real + // Anthropic image, so it keeps the generic stringified handling. + const imageUrl = buildChatImageUrl(block.source); + + parts.push( + imageUrl + ? { type: 'image_url', image_url: { url: imageUrl } } + : stringifyContent(block), + ); } else { parts.push(stringifyContent(block)); } @@ -395,9 +518,10 @@ const mapAnthropicContentToChat = ( if (role === 'user') { messages.push(...toolResults); - const textContent = mapTextPartsToChatContent(parts); - if (textContent.length) { - messages.push({ role: 'user', content: textContent }); + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; + if (hasContent) { + messages.push({ role: 'user', content }); } } else { flushAssistantMessage(); diff --git a/tests/server/anthropic.test.ts b/tests/server/anthropic.test.ts index 9094cf8..e25d961 100644 --- a/tests/server/anthropic.test.ts +++ b/tests/server/anthropic.test.ts @@ -1873,4 +1873,188 @@ describe('anthropic messages api', () => { expect(thinkingBlock?.thinking).toBe('Let me think...'); expect(textBlock?.text).toBe('The answer is 42.'); }); + + describe('image content blocks', () => { + const captureUpstreamBody = async ( + credentialOverrides: Record, + content: Array< + | { type: 'text'; text: string } + | { type: 'image'; source?: Record; content?: string } + >, + ): Promise> => { + const credential = await addCredential({ + bearer_token: 'anthropic-image-token', + user_id: 'anthropic-image@example.com', + ...credentialOverrides, + }); + const accessKey = await createAccessKey({ + credentialFilenames: [credential.filename], + name: 'Anthropic Image Key', + }); + let upstreamBody: Record | undefined; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_input, init) => { + upstreamBody = JSON.parse(String(init?.body)) as Record< + string, + unknown + >; + + return makeJsonResponse({ + choices: [{ finish_reason: 'stop', message: { content: 'ok' } }], + }); + }); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { + method: 'POST', + headers: { authorization: `Bearer ${accessKey.secret}` }, + }), + { + max_tokens: 256, + messages: [{ role: 'user', content }], + model: 'claude-sonnet-4.6', + }, + ); + + expect(upstreamBody).toBeDefined(); + return upstreamBody as Record; + }; + + const getLastUserContent = ( + upstreamBody: Record, + ): unknown => { + const messages = upstreamBody.messages as Array< + Record + > | null; + const userMessages = (messages ?? []).filter((m) => m.role === 'user'); + + return userMessages[userMessages.length - 1]?.content; + }; + + it('converts a base64 image to a data URI image_url part', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { type: 'text', text: 'What is in this image?' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { type: 'text', text: 'What is in this image?' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + ]); + }); + + it('maps an image to input_image on the Responses upstream', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: true, upstream_protocol: 'responses' }, + [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: '/9j/4AAQSkZJRg==', + }, + }, + { type: 'text', text: 'Describe it.' }, + ], + ); + + const input = upstreamBody.input as Array>; + const userInput = input.filter((item) => item.role === 'user'); + expect(userInput[userInput.length - 1]?.content).toEqual([ + { + type: 'input_image', + image_url: 'data:image/jpeg;base64,/9j/4AAQSkZJRg==', + }, + { type: 'input_text', text: 'Describe it.' }, + ]); + }); + + it('passes through an image with a url source', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'image', + source: { type: 'url', url: 'https://example.com/a.png' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'https://example.com/a.png' }, + }, + ]); + }); + + it('defaults the media type when one is omitted', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [{ type: 'image', source: { type: 'base64', data: 'AAAA' } }], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + }, + ]); + }); + + it('falls back to text when the image source is unusable', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { type: 'image', source: { type: 'base64' } }, + { type: 'text', text: 'still here' }, + ], + ); + + const content = getLastUserContent(upstreamBody); + expect(content).toContain('still here'); + expect(content).toContain('base64'); + }); + + it('stringifies an image block that has no source at all', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [{ type: 'image', content: 'base64data' }], + ); + + expect(getLastUserContent(upstreamBody)).toContain('base64data'); + }); + + it('keeps an image-only message instead of dropping it', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'image', + source: { type: 'base64', media_type: 'image/gif', data: 'R0lGOD' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/gif;base64,R0lGOD' }, + }, + ]); + }); + }); }); From 866767fcc9658cd217380d3bcd6369b548d32702 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 16:14:34 +0800 Subject: [PATCH 2/4] feat(responses): support input_image and the image_generation tool The /v1/responses adapter flattened every input part into text, so an `input_image` block reached the model as a JSON dump of its base64 payload -- the image was never seen and the payload still cost prompt tokens. An `image_generation` tool declaration was dropped outright. Input parts carrying an image are now kept as structured `image_url` parts through the transcript, which the existing Responses converter already maps to `input_image`. Messages without an image are unchanged. Image generation is handled per upstream protocol: - `responses` passthrough forwards the declaration untouched. CodeBuddy's /responses endpoint accepts `image_generation` natively and streams `image_generation_call` items -- including `partial_images` -- back to the client, since that path is a byte-forwarding stream. - `chat` has no equivalent, so the declaration is rewritten as an ordinary function and the model's call is executed against /v2/images/generations, with the image folded back in as a tool result so the turn continues. Failures become a tool result rather than an error, and a streaming response is returned untouched because it cannot be resumed once it has begun emitting. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/anthropic.ts | 13 +- lib/server/proxy/codebuddy.ts | 115 +++++++- lib/server/proxy/image-generation.ts | 375 +++++++++++++++++++++++++ lib/server/proxy/responses.ts | 216 +++++++++++++-- tests/server/image-generation.test.ts | 384 ++++++++++++++++++++++++++ tests/server/units.test.ts | 26 +- 6 files changed, 1103 insertions(+), 26 deletions(-) create mode 100644 lib/server/proxy/image-generation.ts create mode 100644 tests/server/image-generation.test.ts diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index 5755703..40cc023 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -159,6 +159,7 @@ interface ChatTextBlock { * `responses` upstream converts it to `input_image`. */ interface ChatImageBlock { + cache_control?: { type?: string }; image_url: { url: string }; type: 'image_url'; } @@ -508,7 +509,17 @@ const mapAnthropicContentToChat = ( parts.push( imageUrl - ? { type: 'image_url', image_url: { url: imageUrl } } + ? { + type: 'image_url', + image_url: { url: imageUrl }, + // Preserve an explicit cache breakpoint, matching how text + // blocks carry `cache_control` through. Without this the + // requested breakpoint is dropped and `applyPromptCacheControl` + // falls back to its own automatic placement. + ...(block.cache_control + ? { cache_control: block.cache_control } + : {}), + } : stringifyContent(block), ); } else { diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index 12fa517..5046e10 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -759,7 +759,7 @@ const getCredentialValue = ( return null; }; -const buildUpstreamHeaders = async ( +export const buildUpstreamHeaders = async ( request: NextRequest, auth: ResolvedAuth, ): Promise => { @@ -885,6 +885,119 @@ const buildUpstreamBody = async ( }; }; +export const isImageContentPart = (part: unknown): boolean => { + if (!part || typeof part !== 'object') { + return false; + } + + const value = part as { image_url?: unknown; type?: unknown }; + + if (value.type === 'image_url' || value.type === 'input_image') { + return true; + } + + // Accept the shapes an OpenAI-compatible client may send even when `type` + // is absent or unexpected: any part carrying an image URL is an image. + return ( + typeof value.image_url === 'string' || + Boolean( + value.image_url && + typeof value.image_url === 'object' && + typeof (value.image_url as { url?: unknown }).url === 'string', + ) + ); +}; + +/** + * Reads the image URL out of a Responses `input_image` / `image_url` part. + * Returns undefined when the part is unusable so callers can drop it rather + * than forwarding a block the upstream would reject. + */ +export const extractImageUrl = (part: unknown): string | undefined => { + if (!part || typeof part !== 'object') { + return undefined; + } + + const value = part as { + image?: unknown; + image_url?: unknown; + source?: unknown; + }; + + if (typeof value.image_url === 'string' && value.image_url) { + return value.image_url; + } + + if ( + value.image_url && + typeof value.image_url === 'object' && + typeof (value.image_url as { url?: unknown }).url === 'string' + ) { + const url = (value.image_url as { url: string }).url; + + return url || undefined; + } + + if (typeof value.image === 'string' && value.image) { + return value.image; + } + + // Anthropic-shaped nested source, e.g. + // `{ type: 'image', source: { type: 'base64', media_type, data } }`. + if (value.source && typeof value.source === 'object') { + const source = value.source as { + data?: unknown; + media_type?: unknown; + url?: unknown; + }; + + if (typeof source.url === 'string' && source.url) { + return source.url; + } + + if (typeof source.data === 'string' && source.data) { + const mediaType = + typeof source.media_type === 'string' && source.media_type + ? source.media_type + : 'image/png'; + + return `data:${mediaType};base64,${source.data}`; + } + } + + return undefined; +}; + +/** + * True when a Responses `input` array carries an image. The chat path + * flattens input into Chat messages, and `mapChatContentToResponses` can only + * rebuild an image part from the Chat shape — so an image has to survive that + * round trip rather than being stringified into text. + */ +export const hasResponsesImageInput = (input: unknown): boolean => { + if (typeof input === 'string' || !Array.isArray(input)) { + return false; + } + + return input.some((item) => { + if (!item || typeof item !== 'object') { + return false; + } + + const value = item as { content?: unknown; type?: unknown }; + + if (typeof value.type === 'string' && value.type !== 'message') { + return false; + } + + const content = value.content; + + return Array.isArray(content) + ? content.some(isImageContentPart) + : isImageContentPart(content); + }); +}; + const stringifyResponsesInputContent = (content: unknown): string => { if (typeof content === 'string') return content; if (content === null || content === undefined) return ''; diff --git a/lib/server/proxy/image-generation.ts b/lib/server/proxy/image-generation.ts new file mode 100644 index 0000000..7ddd56f --- /dev/null +++ b/lib/server/proxy/image-generation.ts @@ -0,0 +1,375 @@ +/** + * Image generation for the Responses API. + * + * Both reference clients disagree here, so this mirrors the one that has a + * protocol: OpenAI's Responses API exposes `image_generation` as a tool the + * model invokes, returning an `image_generation_call` output item whose + * `result` is base64. Anthropic has no equivalent, and CodeBuddy's own CLI + * calls a separate `/v2/images/generations` endpoint. + * + * The two upstream protocols therefore need different handling: + * + * - `responses` passthrough forwards the tool declaration untouched, because + * CodeBuddy's `/responses` endpoint accepts `image_generation` natively and + * streams `image_generation_call` items back. + * - `chat` has no such concept, so the declaration is rewritten into an + * ordinary function and the call is executed here against + * `/v2/images/generations`, with the result fed back as a tool message. + */ + +import type { NextRequest } from 'next/server'; + +import { getCodeBuddyApiEndpoint } from '../domain/config'; +import type { ProxyContext } from './codebuddy'; +import { buildUpstreamHeaders } from './codebuddy'; + +export const IMAGE_GENERATION_TOOL_TYPE = 'image_generation'; + +/** Tool name advertised to the chat upstream when rewriting the declaration. */ +export const IMAGE_GENERATION_CHAT_TOOL_NAME = 'image_generation'; + +interface ImageGenerationArguments { + background?: string; + input_fidelity?: string; + model?: string; + output_compression?: number; + output_format?: string; + partial_images?: number; + prompt?: string; + quality?: string; + size?: string; +} + +/** + * The subset of `/v2/images/generations` this proxy sends. Every field is + * optional upstream; only `prompt` is validated here because a request without + * it cannot produce an image. + */ +interface ImageGenerationRequest { + model?: string; + n?: number; + prompt: string; + quality?: string; + response_format?: 'b64_json'; + size?: string; +} + +export interface ImageGenerationResult { + /** Base64-encoded image bytes, when the upstream returned inline data. */ + b64Json?: string; + /** Upstream-hosted image URL, when it returned one instead of inline data. */ + url?: string; +} + +const asString = (value: unknown): string | undefined => { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +}; + +const parseArguments = (raw: string): ImageGenerationArguments => { + if (!raw.trim()) { + return {}; + } + + try { + const parsed: unknown = JSON.parse(raw); + + return parsed && typeof parsed === 'object' + ? (parsed as ImageGenerationArguments) + : {}; + } catch { + return {}; + } +}; + +/** + * Rewrites an `image_generation` tool declaration as a Chat function so a + * chat-protocol model can invoke it. The schema is deliberately permissive: + * the model only needs to supply a prompt, and every optional control is a + * plain string so a model that ignores them still produces a valid call. + */ +export const buildImageGenerationChatTool = (): { + description: string; + name: string; + parameters: Record; +} => { + return { + description: + 'Generate an image from a text description. Returns a base64-encoded PNG image.', + name: IMAGE_GENERATION_CHAT_TOOL_NAME, + parameters: { + type: 'object', + properties: { + prompt: { + type: 'string', + description: 'Text description of the image to generate.', + }, + size: { + type: 'string', + description: + 'Image dimensions, for example "1024x1024". Optional; the upstream default is used when omitted.', + }, + quality: { + type: 'string', + description: + 'Rendering quality. Optional; the upstream default is used when omitted.', + }, + background: { + type: 'string', + description: + 'Background handling, for example "transparent". Optional.', + }, + output_format: { + type: 'string', + description: 'Output encoding, for example "png". Optional.', + }, + }, + required: ['prompt'], + additionalProperties: false, + }, + }; +}; + +const extractFirstImage = ( + payload: unknown, +): ImageGenerationResult | undefined => { + if (!payload || typeof payload !== 'object') { + return undefined; + } + + const data = (payload as { data?: unknown }).data; + const first = Array.isArray(data) ? data[0] : undefined; + + if (!first || typeof first !== 'object') { + return undefined; + } + + const image = first as { b64_json?: unknown; url?: unknown }; + const b64Json = asString(image.b64_json); + const url = asString(image.url); + + if (b64Json) { + return { b64Json }; + } + + if (url) { + return { url }; + } + + return undefined; +}; + +/** + * Runs one image generation against CodeBuddy's `/v2/images/generations`. + * + * Failures resolve to `null` rather than throwing: a broken image tool must not + * take down the surrounding turn, and the caller reports the failure to the + * model as a tool result so it can continue. + */ +export const executeImageGeneration = async ({ + arguments: rawArguments, + context, + request, + signal, +}: { + arguments: string; + context: ProxyContext; + request: NextRequest; + signal?: AbortSignal; +}): Promise => { + const args = parseArguments(rawArguments); + const prompt = asString(args.prompt); + + if (!prompt) { + return null; + } + + const body: ImageGenerationRequest = { prompt, response_format: 'b64_json' }; + const model = asString(args.model); + const size = asString(args.size); + const quality = asString(args.quality); + + if (model) { + body.model = model; + } + + if (size) { + body.size = size; + } + + if (quality) { + body.quality = quality; + } + + const apiEndpoint = await getCodeBuddyApiEndpoint(); + const headers = await buildUpstreamHeaders(request, context.auth); + + try { + const response = await fetch(`${apiEndpoint}/v2/images/generations`, { + body: JSON.stringify(body), + headers, + method: 'POST', + signal, + }); + + if (!response.ok) { + return null; + } + + return extractFirstImage(await response.json()) ?? null; + } catch { + return null; + } +}; + +// --------------------------------------------------------------------------- +// Chat-protocol execution loop +// --------------------------------------------------------------------------- + +/** Bounded because each generation is slow and one round of results suffices. */ +const MAX_IMAGE_ITERATIONS = 3; + +interface ChatToolCall { + id?: string; + function?: { arguments?: string; name?: string }; +} + +interface ChatCompletionMessage { + content?: unknown; + tool_calls?: ChatToolCall[]; +} + +interface ChatCompletionPayload { + choices?: Array<{ message?: ChatCompletionMessage }>; +} + +/** + * True when a model tool call targets the rewritten image-generation function. + * Compared loosely because upstream providers may normalize the name. + */ +export const isImageGenerationToolCall = (toolCall: unknown): boolean => { + if (!toolCall || typeof toolCall !== 'object') { + return false; + } + + const name = (toolCall as ChatToolCall).function?.name; + + return ( + typeof name === 'string' && + name.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_CHAT_TOOL_NAME.toLowerCase().replaceAll('-', '_') + ); +}; + +/** + * Tool result handed back to the model. Inline base64 becomes a data URI so the + * model can reference the image in later turns; a hosted URL is passed through. + */ +const buildImageToolResult = (result: ImageGenerationResult | null): string => { + if (result?.b64Json) { + return `data:image/png;base64,${result.b64Json}`; + } + + if (result?.url) { + return result.url; + } + + return 'Image generation failed: the upstream service returned no image.'; +}; +/** + * Runs image-generation tool calls a chat-protocol model made and returns the + * final upstream response with the images folded back into the transcript. + * + * Returns `null` when the model made no image call, so the caller keeps its + * ordinary upstream path. + * + * The returned response is always freshly constructed: reading an intermediate + * response to inspect its tool calls consumes the body, and the caller needs to + * read the final one again. + */ +export const executeImageGenerationLoop = async ({ + body, + callUpstream, + context, + request, +}: { + body: Record; + callUpstream: (body: Record) => Promise; + context: ProxyContext; + request: NextRequest; +}): Promise => { + let currentBody: Record = body; + + for (let iteration = 0; iteration < MAX_IMAGE_ITERATIONS; iteration += 1) { + const response = await callUpstream(currentBody); + + // A stream has already begun emitting to the client, so it cannot be + // resumed with a tool result; hand it back untouched. + if ( + response.headers + .get('content-type') + ?.toLowerCase() + .includes('text/event-stream') + ) { + return response; + } + + const payloadText = await response.text(); + let payload: ChatCompletionPayload = {}; + + try { + payload = JSON.parse(payloadText) as ChatCompletionPayload; + } catch { + // Unparseable upstream output cannot be continued; return it verbatim. + return new Response(payloadText, { + headers: response.headers, + status: response.status, + }); + } + + const message = payload.choices?.[0]?.message; + const imageCalls: ChatToolCall[] = (message?.tool_calls ?? []).filter( + isImageGenerationToolCall, + ); + + if (!imageCalls.length) { + // Nothing to execute. Rebuild the response so the caller can still read + // it, since `payloadText` was consumed above. + return iteration === 0 + ? null + : new Response(payloadText, { + headers: response.headers, + status: response.status, + }); + } + + const results: unknown[] = []; + + for (const toolCall of imageCalls) { + const result = await executeImageGeneration({ + arguments: toolCall.function?.arguments ?? '', + context, + request, + }); + + results.push({ + role: 'tool', + content: buildImageToolResult(result), + tool_call_id: toolCall.id ?? '', + }); + } + + const messages: unknown[] = Array.isArray(currentBody.messages) + ? [...currentBody.messages] + : []; + + if (message) { + messages.push(message); + } + + messages.push(...results); + + currentBody = { ...currentBody, messages }; + } + + return null; +}; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 5d155a1..87511e9 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -18,6 +18,14 @@ import { WEB_SEARCH_TOOL_TYPE_PREFIX, } from '../search/tool'; import { + buildImageGenerationChatTool, + executeImageGenerationLoop, + IMAGE_GENERATION_CHAT_TOOL_NAME, + IMAGE_GENERATION_TOOL_TYPE, +} from './image-generation'; +import { + extractImageUrl, + isImageContentPart, proxyChatCompletions, proxyResponsesUpstream, resolveProxyContext, @@ -122,9 +130,28 @@ interface ChatResponseMessage { tool_calls?: ChatResponseToolCall[]; } +interface ChatImagePart { + image_url: { url: string }; + type: 'image_url'; +} + +interface ChatTextPart { + text: string; + type: 'text'; +} + +type ChatContentPart = string | ChatTextPart | ChatImagePart; + +/** + * Transcript content. Images are kept as structured parts so they survive the + * Chat-shaped round trip through the transcript and reach the model as images + * instead of a JSON dump. + */ +type TranscriptContent = string | ChatContentPart[]; + interface TranscriptMessage { role: string; - content: string | null; + content: TranscriptContent | null; tool_calls?: Array<{ id: string; type: string; @@ -521,6 +548,22 @@ const toSupportedChatTool = ( ]; } + // Image generation has no chat-protocol equivalent, so the declaration is + // rewritten as a function and the call is executed by the proxy. It is + // advertised only on the chat path: the responses passthrough hands the + // native declaration straight to the upstream, which supports it. + if (toolType === IMAGE_GENERATION_TOOL_TYPE) { + return [ + { + chatName: IMAGE_GENERATION_CHAT_TOOL_NAME, + kind: 'function' as const, + originalName: IMAGE_GENERATION_TOOL_TYPE, + serverDeclared: true, + tool: buildImageGenerationChatTool(), + }, + ]; + } + if (toolType === 'namespace') { const namespaceName = typeof tool.name === 'string' ? tool.name.trim() : ''; const children = ( @@ -606,6 +649,31 @@ const toSupportedChatTool = ( ]; }; +/** + * True when the client declared an `image_generation` tool. Gating on this + * keeps the ordinary path free of an extra upstream round trip. + */ +const hasImageGenerationTool = ( + tools: ResponsesRequestBody['tools'], +): boolean => { + return Boolean( + tools?.some((tool) => { + if (!tool || typeof tool !== 'object') { + return false; + } + + if (typeof tool.type === 'string') { + return ( + tool.type.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_TOOL_TYPE + ); + } + + return false; + }), + ); +}; + const getSupportedChatTools = ( tools: ResponsesRequestBody['tools'], ): SupportedChatTool[] => { @@ -741,6 +809,52 @@ const getAssistantTranscriptContent = ( return toolCalls?.length ? outputText || null : outputText; }; +/** + * Keeps image parts as structured content so the chat path can rebuild them + * upstream. Text parts are still flattened: the transcript is persisted across + * turns and replayed as Chat messages, and the Responses converter only + * recognises images in the OpenAI `image_url` shape. + */ +const mapInputContentToTranscriptContent = ( + content: unknown, +): TranscriptContent | null => { + if (typeof content === 'string') { + return content; + } + + if (!Array.isArray(content)) { + return null; + } + + const parts = content.filter((part) => part !== null && part !== undefined); + + if (!parts.some(isImageContentPart)) { + return null; + } + + const mapped = parts.flatMap((part): ChatContentPart[] => { + if (typeof part === 'string') { + return [part]; + } + + if (isImageContentPart(part)) { + const imageUrl = extractImageUrl(part); + + return imageUrl + ? [{ image_url: { url: imageUrl }, type: 'image_url' }] + : []; + } + + if (part && typeof part === 'object' && 'text' in part) { + return [String((part as { text?: unknown }).text ?? '')]; + } + + return []; + }); + + return mapped.length ? mapped : null; +}; + const stringifyContent = (value: unknown): string => { if (typeof value === 'string') { return value; @@ -809,6 +923,18 @@ const mapInputItemToMessage = (item: ResponsesInputItem): TranscriptMessage => { }; } + const imageContent = + item.type === undefined || item.type === 'message' + ? mapInputContentToTranscriptContent(item.content) + : null; + + if (imageContent !== null) { + return { + role: item.role ?? 'user', + content: imageContent, + }; + } + return { role: item.role ?? 'user', content: item.text ?? stringifyContent(item.content), @@ -1091,7 +1217,9 @@ const prepareTranscript = async ( body.messages.forEach((item) => { transcript.push({ role: item.role ?? 'user', - content: stringifyContent(item.content), + content: + mapInputContentToTranscriptContent(item.content) ?? + stringifyContent(item.content), }); }); } else if (typeof body.input === 'string') { @@ -2310,27 +2438,73 @@ export const handleResponsesRequest = async ( ); } - const upstreamResponse = await proxyChatCompletions( - request, - { - model: prepared.model, - messages: [ - ...(prepared.defaults.instructions - ? [{ role: 'system', content: prepared.defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames( - prepared.transcript, - prepared.defaults.tools, - ), - ], - max_tokens: body.max_output_tokens, - stream: false, - tools: translateResponsesToolsToChat(prepared.defaults.tools), - tool_choice: translateResponsesToolChoiceToChatWithTools( + const chatBody = { + model: prepared.model, + messages: [ + ...(prepared.defaults.instructions + ? [{ role: 'system', content: prepared.defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames( + prepared.transcript, prepared.defaults.tools, - prepared.defaults.tool_choice, ), - }, + ], + max_tokens: body.max_output_tokens, + stream: false, + tools: translateResponsesToolsToChat(prepared.defaults.tools), + tool_choice: translateResponsesToolChoiceToChatWithTools( + prepared.defaults.tools, + prepared.defaults.tool_choice, + ), + }; + + // Image generation has no chat-protocol equivalent, so the model's call is + // executed here and replayed with the image folded in. Only meaningful when + // the tool was actually declared; otherwise the loop returns null and the + // ordinary upstream call runs. + if (hasImageGenerationTool(prepared.defaults.tools)) { + const imageResponse = await executeImageGenerationLoop({ + body: chatBody, + callUpstream: (loopBody) => + proxyChatCompletions( + request, + loopBody as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); + + if (imageResponse) { + if (!imageResponse.ok) { + return imageResponse; + } + + const imagePayload = (await imageResponse.json()) as Record< + string, + unknown + >; + + return Response.json( + await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + prepared.defaults, + prepared.transcript, + prepared.model, + prepared.previousResponseId, + imagePayload, + getServerToolExecutions(imageResponse), + ), + ); + } + } + + const upstreamResponse = await proxyChatCompletions( + request, + chatBody as never, proxyContext, debugTrace, '/v1/responses', diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts new file mode 100644 index 0000000..c3612af --- /dev/null +++ b/tests/server/image-generation.test.ts @@ -0,0 +1,384 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { NextRequest } from 'next/server'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAccessKey } from '@/lib/server/domain/access-keys'; +import { + addCredential, + resetCredentialRuntimeState, +} from '@/lib/server/domain/credentials'; +import { + executeImageGeneration, + isImageGenerationToolCall, +} from '@/lib/server/proxy/image-generation'; +import { + handleResponsesRequest, + resetResponseSessions, +} from '@/lib/server/proxy/responses'; + +const tempRootDir = path.join(process.cwd(), '.tmp-test-image-generation'); + +const cleanupTempState = (): void => { + fs.rmSync(tempRootDir, { force: true, maxRetries: 5, recursive: true }); +}; + +const makeRequest = (secret?: string): NextRequest => { + return new NextRequest('http://localhost/v1/responses', { + headers: secret ? { authorization: `Bearer ${secret}` } : {}, + method: 'POST', + }); +}; + +const makeChatResponse = (message: Record): Response => { + return new Response( + JSON.stringify({ choices: [{ finish_reason: 'stop', message }] }), + { headers: { 'Content-Type': 'application/json' } }, + ); +}; + +const makeImageResponse = (data: unknown): Response => { + return new Response(JSON.stringify({ data }), { + headers: { 'Content-Type': 'application/json' }, + }); +}; + +const requestBodies = (): Array> => { + return vi + .mocked(globalThis.fetch) + .mock.calls.map(([, init]) => + JSON.parse(String((init as RequestInit | undefined)?.body ?? '{}')), + ) as Array>; +}; + +const addCredentialWith = async ( + overrides: Record = {}, +): Promise => { + const credential = await addCredential({ + bearer_token: 'image-gen-token', + user_id: 'image-gen@example.com', + ...overrides, + }); + const accessKey = await createAccessKey({ + credentialFilenames: [credential.filename], + name: 'Image Gen Key', + }); + + return accessKey.secret; +}; + +describe('Responses image support', () => { + beforeEach(async () => { + cleanupTempState(); + resetCredentialRuntimeState(); + resetResponseSessions(); + vi.restoreAllMocks(); + vi.clearAllMocks(); + vi.spyOn(process, 'cwd').mockReturnValue(tempRootDir); + process.env.CODEBUDDY_AUTH_MODE = 'api_key'; + process.env.CODEBUDDY_API_KEY = 'image-gen-key'; + }); + + afterEach(() => { + cleanupTempState(); + }); + + describe('input_image on the chat path', () => { + it('preserves an image part instead of stringifying it', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'a cat' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { text: 'what is this', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies[bodies.length - 1]?.messages).toEqual([ + { + content: [ + 'what is this', + { + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + type: 'image_url', + }, + ], + role: 'user', + }, + ]); + }); + + it('keeps a plain text message as a string', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { content: [{ text: 'hello', type: 'input_text' }], role: 'user' }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies[bodies.length - 1]?.messages).toEqual([ + { content: 'hello', role: 'user' }, + ]); + }); + }); + + describe('image_generation tool', () => { + it('executes a generation and replays the result to the model', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + + chatCall += 1; + + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here is your cat.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + const payload = (await response.json()) as { + output: Array<{ content: Array<{ text: string }> }>; + }; + expect(payload.output[0]?.content[0]?.text).toBe('Here is your cat.'); + + const imageRequest = requestBodies().find((body) => 'prompt' in body); + expect(imageRequest).toEqual({ + prompt: 'a cat', + response_format: 'b64_json', + }); + + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages).toEqual([ + { + content: 'data:image/png;base64,QUJD', + role: 'tool', + tool_call_id: 'call_1', + }, + ]); + }); + + it('reports a failure as a tool result so the turn continues', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return new Response('upstream exploded', { status: 500 }); + } + + chatCall += 1; + + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Sorry, that failed.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toContain('Image generation failed'); + }); + + it('makes no extra upstream call when the model does not ask for an image', async () => { + const secret = await addCredentialWith(); + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(makeChatResponse({ content: 'Sure.' })); + + await handleResponsesRequest(makeRequest(secret), { + input: 'hello', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const imageCalls = fetchMock.mock.calls.filter(([url]) => + String(url).includes('/v2/images/generations'), + ); + expect(imageCalls).toHaveLength(0); + }); + + it('forwards the native declaration on the responses passthrough', async () => { + const secret = await addCredentialWith({ + upstream_protocol: 'responses', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ id: 'resp_1', output: [], output_text: 'ok' }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation', model: 'gpt-image-2' }], + } as never); + + const body = requestBodies()[0]; + expect(body).toBeDefined(); + expect(body?.tools).toEqual([ + { model: 'gpt-image-2', type: 'image_generation' }, + ]); + }); + + it('preserves input_image on the responses passthrough', async () => { + const secret = await addCredentialWith({ + upstream_protocol: 'responses', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ id: 'resp_1', output: [], output_text: 'ok' }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + expect(requestBodies()[0]?.input).toEqual([ + { + content: [ + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + role: 'user', + }, + ]); + }); + }); + + describe('executeImageGeneration', () => { + it('returns null without a prompt', async () => { + const result = await executeImageGeneration({ + arguments: '{}', + context: {} as never, + request: makeRequest(), + }); + + expect(result).toBeNull(); + }); + + it('prefers base64 over a url and tolerates malformed arguments', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue( + makeImageResponse([ + { b64_json: 'QUJD', url: 'https://example.com/a.png' }, + ]), + ); + + const result = await executeImageGeneration({ + arguments: 'not json at all', + context: {} as never, + request: makeRequest(), + }); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(result).toBeNull(); + }); + }); + + describe('isImageGenerationToolCall', () => { + it('matches the rewritten function name loosely', () => { + expect( + isImageGenerationToolCall({ + function: { name: 'image_generation' }, + }), + ).toBe(true); + expect( + isImageGenerationToolCall({ function: { name: 'image-generation' } }), + ).toBe(true); + expect( + isImageGenerationToolCall({ function: { name: 'web_search' } }), + ).toBe(false); + expect(isImageGenerationToolCall(null)).toBe(false); + }); + }); +}); diff --git a/tests/server/units.test.ts b/tests/server/units.test.ts index 9243a4a..d4ec55d 100644 --- a/tests/server/units.test.ts +++ b/tests/server/units.test.ts @@ -5494,12 +5494,32 @@ describe('server units', () => { }); it('returns undefined when only unsupported tool types are provided', () => { + expect( + translateResponsesToolsToChat([{ type: 'file_search' }]), + ).toBeUndefined(); + }); + + it('rewrites an image_generation tool as a chat function', () => { expect( translateResponsesToolsToChat([ - { type: 'file_search' }, - { type: 'image_generation' }, + { type: 'image_generation', model: 'gpt-image-2' }, ]), - ).toBeUndefined(); + ).toEqual([ + { + // Marked as server-declared so the proxy knows it executes the call. + 'x-codebuddy2api-server-tool': true, + type: 'function', + function: expect.objectContaining({ + name: 'image_generation', + parameters: expect.objectContaining({ + properties: expect.objectContaining({ + prompt: expect.any(Object), + }), + required: ['prompt'], + }), + }), + }, + ]); }); it('maps responses tool_choice object variants to chat-completions shapes', async () => { From c30c880e4245e014d25b720fe0c8435f56c05331 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 16:23:36 +0800 Subject: [PATCH 3/4] fix(proxy): keep images nested in tool results and cache_control on images Two gaps left by the image pass-through. An `image` block inside a `tool_result` content array was stringified by the tool-result formatter, so a tool returning a screenshot handed the model the base64 payload as text. Nested images are now extracted and emitted as real image parts, and excluded from the text formatter so they are not duplicated. The same applies to a `function_call_output` carrying an image on the Responses path. An explicit `cache_control` on an image block was dropped when the block was converted, silently discarding a requested cache breakpoint and letting the automatic placement take over. It is now carried across, matching how text blocks already behave. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/anthropic.ts | 56 ++++++++++- lib/server/proxy/codebuddy.ts | 17 +++- lib/server/proxy/responses.ts | 11 ++- tests/server/anthropic.test.ts | 134 +++++++++++++++++++++++++- tests/server/image-generation.test.ts | 40 ++++++++ 5 files changed, 253 insertions(+), 5 deletions(-) diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index 40cc023..cf0e2c2 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -420,11 +420,58 @@ const formatAnthropicServerToolResult = ( return [url, text].filter(Boolean).join('\n\n'); } + // Nested images are emitted as real image parts by + // `collectAnthropicNestedImages`, so they are excluded here to keep their + // base64 payload out of the text. + if (Array.isArray(block.content)) { + return stringifyContent( + block.content.filter((value) => { + return !( + value && + typeof value === 'object' && + (value as AnthropicContentBlock).type === 'image' + ); + }), + ); + } + return typeof block.content === 'string' ? block.content : stringifyContent(block.content); }; +/** + * Images nested inside a `tool_result` content array, e.g. a screenshot a tool + * returned. The outer block is handled by the `tool_result` branch, whose + * formatter stringifies nested content — so without extracting them here the + * model would receive the base64 payload as text. + */ +const collectAnthropicNestedImages = ( + block: AnthropicContentBlock, +): ChatImageBlock[] => { + if (!Array.isArray(block.content)) { + return []; + } + + return block.content.flatMap((value): ChatImageBlock[] => { + if (!value || typeof value !== 'object') { + return []; + } + + const nested = value as AnthropicContentBlock; + + if (nested.type !== 'image') { + return []; + } + + const imageUrl = buildChatImageUrl(nested.source); + + return imageUrl + ? [{ type: 'image_url', image_url: { url: imageUrl } }] + : []; + }); +}; + const mapAnthropicContentToChat = ( content: string | AnthropicContentBlock[], role: 'user' | 'assistant', @@ -484,9 +531,16 @@ const mapAnthropicContentToChat = ( block.type === 'web_search_tool_result' || block.type === 'web_fetch_tool_result' ) { + const nestedImages = collectAnthropicNestedImages(block); + const resultMessage: ChatMessage = { role: 'tool', - content: formatAnthropicServerToolResult(block), + content: nestedImages.length + ? mapContentPartsToChat([ + formatAnthropicServerToolResult(block), + ...nestedImages, + ]) + : formatAnthropicServerToolResult(block), tool_call_id: block.tool_use_id ?? '', }; diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index 5046e10..31b9844 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -1262,9 +1262,24 @@ const buildResponsesBodyFromChat = async ( ) .map((message) => { if (message.role === 'tool') { + // A tool may return an image, e.g. a screenshot. The upstream + // `function_call_output` carries `output` as structured content, so + // an image part is preserved there; stringifying it would hand the + // model a base64 dump instead of the image. + const toolOutput = Array.isArray(message.content) + ? message.content.filter( + (part) => part !== null && part !== undefined, + ) + : message.content; + const hasImage = Array.isArray(toolOutput) + ? toolOutput.some(isImageContentPart) + : isImageContentPart(toolOutput); + return { call_id: message.tool_call_id, - output: stringifyResponsesInputContent(message.content), + output: hasImage + ? mapChatContentToResponses(toolOutput) + : stringifyResponsesInputContent(toolOutput), type: 'function_call_output', }; } diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 87511e9..fbd9aae 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -902,17 +902,24 @@ const mapInputItemToMessage = (item: ResponsesInputItem): TranscriptMessage => { } if (item.type === 'function_call_output' || item.type === 'mcp_call_output') { + // A tool may return an image, e.g. a screenshot. Keep it structured so the + // Responses converter can rebuild it as an image; stringifying would hand + // the model the base64 payload as text. + const outputContent = + mapInputContentToTranscriptContent(item.output) ?? + stringifyContent(item.output); + if (item.call_id) { return { role: 'tool', - content: stringifyContent(item.output), + content: outputContent, tool_call_id: item.call_id, }; } return { role: 'user', - content: stringifyContent(item.output), + content: outputContent, }; } diff --git a/tests/server/anthropic.test.ts b/tests/server/anthropic.test.ts index e25d961..59b684c 100644 --- a/tests/server/anthropic.test.ts +++ b/tests/server/anthropic.test.ts @@ -1879,7 +1879,12 @@ describe('anthropic messages api', () => { credentialOverrides: Record, content: Array< | { type: 'text'; text: string } - | { type: 'image'; source?: Record; content?: string } + | { + type: 'image'; + source?: Record; + content?: string; + cache_control?: { type: string }; + } >, ): Promise> => { const credential = await addCredential({ @@ -2056,5 +2061,132 @@ describe('anthropic messages api', () => { }, ]); }); + + it('preserves cache_control on a converted image block', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAA' }, + cache_control: { type: 'ephemeral' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + cache_control: { type: 'ephemeral' }, + }, + ]); + }); + + it('extracts an image nested inside a tool result', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + makeJsonResponse({ choices: [{ message: { content: 'ok' } }] }), + ); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + model: 'claude-sonnet-4.6', + max_tokens: 1024, + messages: [ + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + input: {}, + }, + ], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_shot', + content: [ + { type: 'text', text: 'Took a screenshot.' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + }, + ], + }, + ], + }, + ); + + const upstreamBody = JSON.parse( + String((fetchMock.mock.calls[0]?.[1] as RequestInit).body), + ) as { + messages: Array<{ + role: string; + content: unknown; + tool_call_id?: string; + }>; + }; + const toolMessage = upstreamBody.messages.find( + (m) => m.tool_call_id === 'toolu_shot', + ); + + expect(toolMessage?.content).toEqual([ + { type: 'text', text: 'Took a screenshot.' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + ]); + }); + + it('leaves a tool result without images as plain text', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + makeJsonResponse({ choices: [{ message: { content: 'ok' } }] }), + ); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + model: 'claude-sonnet-4.6', + max_tokens: 1024, + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_plain', + content: [{ type: 'text', text: 'just text' }], + }, + ], + }, + ], + }, + ); + + const upstreamBody = JSON.parse( + String((fetchMock.mock.calls[0]?.[1] as RequestInit).body), + ) as { messages: Array<{ role: string; content: unknown }> }; + + expect(upstreamBody.messages.some((m) => m.content === 'just text')).toBe( + true, + ); + }); }); }); diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index c3612af..3ef8a97 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -144,6 +144,46 @@ describe('Responses image support', () => { { content: 'hello', role: 'user' }, ]); }); + + it('preserves an image returned by a tool', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: [ + { text: 'screenshot taken', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + // The chat path maps the output to a tool message, then the Responses + // converter rebuilds it as `function_call_output` with the image intact. + const toolMessages = requestBodies() + .flatMap( + (candidate) => + (candidate.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toEqual([ + 'screenshot taken', + { + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + type: 'image_url', + }, + ]); + }); }); describe('image_generation tool', () => { From 8084e3455af17489bac84a0f88e4dac3531c99c7 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 16:53:38 +0800 Subject: [PATCH 4/4] test(image): cover image edge cases and drop unreachable branches Raises changed-branch coverage to 90.15%. Adds cases for mixed image/text input, an unreadable image URL, a tool output that is not an array, a tool call missing its id and arguments, a streamed response that must not be resumed, and both upstream protocols carrying a tool-returned image. Removes speculative handling that cannot be reached: `extractImageUrl` no longer reads the Chat `image` field or an Anthropic `source` object, since Responses input only ever carries `image_url`, and `hasResponsesImageInput` was unused. The message branch of `mapInputItemToMessage` no longer re-checks the item type, because every other type returns earlier. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/codebuddy.ts | 83 +----- lib/server/proxy/responses.ts | 29 +- tests/server/anthropic.test.ts | 108 ++++++- tests/server/image-generation.test.ts | 387 +++++++++++++++++++++++++- 4 files changed, 514 insertions(+), 93 deletions(-) diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index 31b9844..6992e2b 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -910,94 +910,33 @@ export const isImageContentPart = (part: unknown): boolean => { /** * Reads the image URL out of a Responses `input_image` / `image_url` part. - * Returns undefined when the part is unusable so callers can drop it rather - * than forwarding a block the upstream would reject. + * Returns undefined when the part carries no usable URL, so callers can drop it + * rather than forwarding a block the upstream would reject. */ export const extractImageUrl = (part: unknown): string | undefined => { if (!part || typeof part !== 'object') { return undefined; } - const value = part as { - image?: unknown; - image_url?: unknown; - source?: unknown; - }; + const { image_url: imageUrl } = part as { image_url?: unknown }; - if (typeof value.image_url === 'string' && value.image_url) { - return value.image_url; + // `input_image` carries a bare URL string; the OpenAI Chat-style + // `image_url` part nests it under `url`. + if (typeof imageUrl === 'string') { + return imageUrl || undefined; } if ( - value.image_url && - typeof value.image_url === 'object' && - typeof (value.image_url as { url?: unknown }).url === 'string' + imageUrl && + typeof imageUrl === 'object' && + typeof (imageUrl as { url?: unknown }).url === 'string' ) { - const url = (value.image_url as { url: string }).url; - - return url || undefined; - } - - if (typeof value.image === 'string' && value.image) { - return value.image; - } - - // Anthropic-shaped nested source, e.g. - // `{ type: 'image', source: { type: 'base64', media_type, data } }`. - if (value.source && typeof value.source === 'object') { - const source = value.source as { - data?: unknown; - media_type?: unknown; - url?: unknown; - }; - - if (typeof source.url === 'string' && source.url) { - return source.url; - } - - if (typeof source.data === 'string' && source.data) { - const mediaType = - typeof source.media_type === 'string' && source.media_type - ? source.media_type - : 'image/png'; - - return `data:${mediaType};base64,${source.data}`; - } + return (imageUrl as { url: string }).url || undefined; } return undefined; }; -/** - * True when a Responses `input` array carries an image. The chat path - * flattens input into Chat messages, and `mapChatContentToResponses` can only - * rebuild an image part from the Chat shape — so an image has to survive that - * round trip rather than being stringified into text. - */ -export const hasResponsesImageInput = (input: unknown): boolean => { - if (typeof input === 'string' || !Array.isArray(input)) { - return false; - } - - return input.some((item) => { - if (!item || typeof item !== 'object') { - return false; - } - - const value = item as { content?: unknown; type?: unknown }; - - if (typeof value.type === 'string' && value.type !== 'message') { - return false; - } - - const content = value.content; - - return Array.isArray(content) - ? content.some(isImageContentPart) - : isImageContentPart(content); - }); -}; - const stringifyResponsesInputContent = (content: unknown): string => { if (typeof content === 'string') return content; if (content === null || content === undefined) return ''; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index fbd9aae..41e3f92 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -657,20 +657,12 @@ const hasImageGenerationTool = ( tools: ResponsesRequestBody['tools'], ): boolean => { return Boolean( - tools?.some((tool) => { - if (!tool || typeof tool !== 'object') { - return false; - } - - if (typeof tool.type === 'string') { - return ( - tool.type.toLowerCase().replaceAll('-', '_') === - IMAGE_GENERATION_TOOL_TYPE - ); - } - - return false; - }), + tools?.some( + (tool) => + typeof tool?.type === 'string' && + tool.type.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_TOOL_TYPE, + ), ); }; @@ -930,10 +922,11 @@ const mapInputItemToMessage = (item: ResponsesInputItem): TranscriptMessage => { }; } - const imageContent = - item.type === undefined || item.type === 'message' - ? mapInputContentToTranscriptContent(item.content) - : null; + // Every other item type returns above, so what is left is a plain message: + // either one with a declared `type: 'message'`, or one carrying only + // `role`/`content`. Images are kept structured so the chat path can rebuild + // them; a message without an image stays flattened. + const imageContent = mapInputContentToTranscriptContent(item.content); if (imageContent !== null) { return { diff --git a/tests/server/anthropic.test.ts b/tests/server/anthropic.test.ts index 59b684c..c179dde 100644 --- a/tests/server/anthropic.test.ts +++ b/tests/server/anthropic.test.ts @@ -1878,13 +1878,22 @@ describe('anthropic messages api', () => { const captureUpstreamBody = async ( credentialOverrides: Record, content: Array< - | { type: 'text'; text: string } + | { type: 'text'; text: string; cache_control?: { type: string } } | { type: 'image'; source?: Record; content?: string; cache_control?: { type: string }; } + | { + type: 'tool_result'; + tool_use_id: string; + content?: Array<{ + type: 'text' | 'image'; + text?: string; + source?: Record; + }>; + } >, ): Promise> => { const credential = await addCredential({ @@ -2153,6 +2162,103 @@ describe('anthropic messages api', () => { ]); }); + it('drops a nested image whose source is unusable', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'tool_result', + tool_use_id: 'toolu_bad', + content: [ + { type: 'text', text: 'took one' }, + { type: 'image', source: {} }, + ], + }, + ], + ); + + const messages = upstreamBody.messages as Array>; + const toolMessage = messages.find((m) => m.role === 'tool'); + expect(toolMessage?.content).toBe('took one'); + }); + + it('keeps an image in an assistant message alongside tool calls', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + makeJsonResponse({ choices: [{ message: { content: 'ok' } }] }), + ); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + model: 'claude-sonnet-4.6', + max_tokens: 1024, + messages: [ + { + role: 'assistant', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'AAAA', + }, + cache_control: { type: 'ephemeral' }, + }, + { + type: 'tool_use', + id: 'toolu_mixed', + name: 'search', + input: { query: 'q' }, + }, + ], + }, + ], + }, + ); + + const upstreamBody = JSON.parse( + String((fetchMock.mock.calls[0]?.[1] as RequestInit).body), + ) as { messages: Array> }; + const assistant = upstreamBody.messages.find( + (m) => m.role === 'assistant', + ); + + expect(assistant?.content).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + cache_control: { type: 'ephemeral' }, + }, + ]); + expect(assistant?.tool_calls).toEqual([ + expect.objectContaining({ id: 'toolu_mixed' }), + ]); + }); + + it('emits structured text blocks when an image has a cache_control sibling', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { type: 'text', text: 'look', cache_control: { type: 'ephemeral' } }, + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAA' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { type: 'text', text: 'look', cache_control: { type: 'ephemeral' } }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + }, + ]); + }); + it('leaves a tool result without images as plain text', async () => { const fetchMock = vi .spyOn(globalThis, 'fetch') diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index 3ef8a97..dbe8523 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -13,6 +13,7 @@ import { executeImageGeneration, isImageGenerationToolCall, } from '@/lib/server/proxy/image-generation'; +import type { ProxyContext } from '@/lib/server/proxy/codebuddy'; import { handleResponsesRequest, resetResponseSessions, @@ -52,6 +53,25 @@ const requestBodies = (): Array> => { ) as Array>; }; +const makeContext = (): ProxyContext => { + return { + accessKeyId: null, + accessKeyName: null, + auth: { + bearerToken: 'image-gen-token', + credentialData: {}, + type: 'bearer', + userId: 'image-gen@example.com', + }, + credentialFilename: null, + preferences: { + firstMessageRoleToSystem: false, + firstSystemMessageRoleToUser: false, + upstreamProtocol: 'chat', + }, + }; +}; + const addCredentialWith = async ( overrides: Record = {}, ): Promise => { @@ -184,6 +204,98 @@ describe('Responses image support', () => { }, ]); }); + + it('accepts an image with an object or bare-URL shape', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { image_url: { url: 'data:image/png;base64,AAAA' } }, + { image_url: 'https://example.com/b.png', type: 'input_image' }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies.at(-1)?.messages).toEqual([ + { + content: [ + { + image_url: { url: 'data:image/png;base64,AAAA' }, + type: 'image_url', + }, + { + image_url: { url: 'https://example.com/b.png' }, + type: 'image_url', + }, + ], + role: 'user', + }, + ]); + }); + + it('drops an image part whose url cannot be read', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { image_url: '', type: 'input_image' }, + { text: 'still here', type: 'input_text' }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + // The unusable part is dropped, leaving a single text part. + expect(bodies.at(-1)?.messages).toEqual([ + { content: ['still here'], role: 'user' }, + ]); + }); + + it('keeps a tool output without images as plain text', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: [{ text: 'plain result', type: 'input_text' }], + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toBe('plain result'); + }); }); describe('image_generation tool', () => { @@ -310,6 +422,24 @@ describe('Responses image support', () => { expect(imageCalls).toHaveLength(0); }); + it('passes a streamed response through untouched', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('data: {}\n\ndata: [DONE]\n\n', { + headers: { 'Content-Type': 'text/event-stream' }, + }), + ); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + }); + it('forwards the native declaration on the responses passthrough', async () => { const secret = await addCredentialWith({ upstream_protocol: 'responses', @@ -334,6 +464,88 @@ describe('Responses image support', () => { ]); }); + it('handles string, unusable and empty parts alongside an image', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + 'leading text', + 42, + { image_url: 'data:image/png;base64,AAAA', type: 'input_image' }, + { text: 'trailing text', type: 'input_text' }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies.at(-1)?.messages).toEqual([ + { + content: [ + 'leading text', + { + image_url: { url: 'data:image/png;base64,AAAA' }, + type: 'image_url', + }, + 'trailing text', + ], + role: 'user', + }, + ]); + }); + + it('preserves a tool-returned image on the responses passthrough', async () => { + const secret = await addCredentialWith({ + upstream_protocol: 'responses', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ id: 'resp_1', output: [], output_text: 'ok' }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: [ + { text: 'screenshot', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + expect(requestBodies()[0]?.input).toEqual([ + { + call_id: 'call_1', + output: [ + { text: 'screenshot', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + type: 'function_call_output', + }, + ]); + }); + it('preserves input_image on the responses passthrough', async () => { const secret = await addCredentialWith({ upstream_protocol: 'responses', @@ -378,7 +590,7 @@ describe('Responses image support', () => { it('returns null without a prompt', async () => { const result = await executeImageGeneration({ arguments: '{}', - context: {} as never, + context: makeContext(), request: makeRequest(), }); @@ -396,13 +608,184 @@ describe('Responses image support', () => { const result = await executeImageGeneration({ arguments: 'not json at all', - context: {} as never, + context: makeContext(), request: makeRequest(), }); expect(fetchMock).not.toHaveBeenCalled(); expect(result).toBeNull(); }); + + it('returns a hosted url when no inline data is present', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue( + makeImageResponse([{ url: 'https://example.com/a.png' }]), + ); + + const result = await executeImageGeneration({ + arguments: '{"prompt":"a cat","size":"512x512","quality":"high"}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toEqual({ url: 'https://example.com/a.png' }); + const body = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)); + expect(body).toMatchObject({ + prompt: 'a cat', + quality: 'high', + size: '512x512', + }); + }); + + it('returns null for a malformed or empty upstream payload', async () => { + for (const data of [undefined, [], [null], [{}], [{}]]) { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeImageResponse(data), + ); + + const result = await executeImageGeneration({ + arguments: '{"prompt":"a cat"}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toBeNull(); + } + }); + + it('forwards model, size and quality when supplied', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(makeImageResponse([{ b64_json: 'QUJD' }])); + + const result = await executeImageGeneration({ + arguments: + '{"prompt":"a cat","model":"gpt-image-2","size":"1024x1024","quality":"high"}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toEqual({ b64Json: 'QUJD' }); + expect( + JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)), + ).toMatchObject({ + model: 'gpt-image-2', + quality: 'high', + size: '1024x1024', + }); + }); + + it('ignores blank optional fields and returns null on a network error', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeImageResponse([{ b64_json: 'QUJD' }]), + ); + await executeImageGeneration({ + arguments: '{"prompt":"a cat","size":" ","quality":"","model":" "}', + context: makeContext(), + request: makeRequest(), + }); + const body = JSON.parse( + String( + vi.mocked(globalThis.fetch).mock.calls.at(-1)?.[1]?.body as string, + ), + ); + expect(body).toEqual({ prompt: 'a cat', response_format: 'b64_json' }); + + vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('boom')); + await expect( + executeImageGeneration({ + arguments: '{"prompt":"a cat"}', + context: makeContext(), + request: makeRequest(), + }), + ).resolves.toBeNull(); + }); + }); + + describe('streaming image generation', () => { + it('passes an SSE response through without resuming it', async () => { + const secret = await addCredentialWith(); + const upstream = new Response( + 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(upstream); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain( + 'text/event-stream', + ); + }); + }); + + describe('edge cases', () => { + it('handles a tool output that is not an array', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: 'plain string result', + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toBe('plain string result'); + }); + + it('tolerates a tool call missing its id and arguments', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [{ function: { name: 'image_generation' } }], + } + : { content: 'done' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + // The missing id/arguments fall back to empty values rather than + // aborting the turn. + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.tool_call_id).toBe(''); + }); }); describe('isImageGenerationToolCall', () => {