diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index b88c9de..cf0e2c2 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -33,6 +33,13 @@ const MAX_STREAM_FRAME_LENGTH = 1_000_000; // Anthropic Messages API types // --------------------------------------------------------------------------- +interface AnthropicImageSource { + type?: string; + media_type?: string; + data?: string; + url?: string; +} + interface AnthropicContentBlock { type: string; text?: string; @@ -43,6 +50,7 @@ interface AnthropicContentBlock { thinking?: string; tool_use_id?: string; content?: unknown; + source?: AnthropicImageSource; } interface AnthropicMessage { @@ -144,6 +152,26 @@ interface ChatTextBlock { type: 'text'; } +/** + * An image part in the OpenAI Chat shape. Emitted in this shape rather than a + * native Anthropic one because the request is translated to Chat before it + * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the + * `responses` upstream converts it to `input_image`. + */ +interface ChatImageBlock { + cache_control?: { type?: string }; + image_url: { url: string }; + type: 'image_url'; +} + +type ChatContentPart = string | ChatTextBlock | ChatImageBlock; + +type ChatContent = string | Array; + +/** + * Text-only content, used where images are not representable — the system + * prompt and the intermediate text-part buffer. + */ type ChatTextContent = string | ChatTextBlock[]; // --------------------------------------------------------------------------- @@ -200,6 +228,87 @@ const mapTextPartsToChatContent = ( ]); }; +/** + * Builds the `image_url` value for an Anthropic image block. Base64 sources + * become a data URI because the upstream Chat/Responses APIs expect a URL; + * `url` sources pass through untouched. Returns undefined for an unusable + * source so the caller can fall back to a text placeholder rather than + * emitting a block the upstream would reject. + */ +const buildChatImageUrl = ( + source: AnthropicImageSource | undefined, +): string | undefined => { + if (!source || typeof source !== 'object') { + return undefined; + } + + if (source.type === 'url' || (!source.data && source.url)) { + return typeof source.url === 'string' && source.url + ? source.url + : undefined; + } + + if (typeof source.data !== 'string' || !source.data) { + return undefined; + } + + const mediaType = + typeof source.media_type === 'string' && source.media_type + ? source.media_type + : 'image/png'; + + return `data:${mediaType};base64,${source.data}`; +}; + +/** + * Like `mapTextPartsToChatContent`, but keeps image parts as real image + * blocks instead of collapsing them into text. Falls back to the text-only + * result when nothing resolved to an image. + */ +const mapContentPartsToChat = (parts: ChatContentPart[]): ChatContent => { + const hasImage = parts.some( + (part) => typeof part === 'object' && part.type === 'image_url', + ); + + if (!hasImage) { + return mapTextPartsToChatContent( + parts.filter( + (part): part is string | ChatTextBlock => + typeof part === 'string' || part.type === 'text', + ), + ); + } + + const blocks: Array = []; + let pendingText: Array = []; + + const flushText = (): void => { + if (!pendingText.length) { + return; + } + const textContent = mapTextPartsToChatContent(pendingText); + if (typeof textContent === 'string') { + blocks.push({ type: 'text', text: textContent }); + } else { + blocks.push(...textContent); + } + pendingText = []; + }; + + for (const part of parts) { + if (typeof part === 'object' && part.type === 'image_url') { + flushText(); + blocks.push(part); + continue; + } + pendingText.push(part); + } + + flushText(); + + return blocks; +}; + const extractSystemText = ( system: string | AnthropicContentBlock[] | undefined, ): ChatTextContent => { @@ -232,7 +341,7 @@ const extractSystemText = ( interface ChatMessage { role: string; - content: ChatTextContent | null; + content: ChatContent | null; tool_calls?: Array<{ id: string; type: string; @@ -311,11 +420,58 @@ const formatAnthropicServerToolResult = ( return [url, text].filter(Boolean).join('\n\n'); } + // Nested images are emitted as real image parts by + // `collectAnthropicNestedImages`, so they are excluded here to keep their + // base64 payload out of the text. + if (Array.isArray(block.content)) { + return stringifyContent( + block.content.filter((value) => { + return !( + value && + typeof value === 'object' && + (value as AnthropicContentBlock).type === 'image' + ); + }), + ); + } + return typeof block.content === 'string' ? block.content : stringifyContent(block.content); }; +/** + * Images nested inside a `tool_result` content array, e.g. a screenshot a tool + * returned. The outer block is handled by the `tool_result` branch, whose + * formatter stringifies nested content — so without extracting them here the + * model would receive the base64 payload as text. + */ +const collectAnthropicNestedImages = ( + block: AnthropicContentBlock, +): ChatImageBlock[] => { + if (!Array.isArray(block.content)) { + return []; + } + + return block.content.flatMap((value): ChatImageBlock[] => { + if (!value || typeof value !== 'object') { + return []; + } + + const nested = value as AnthropicContentBlock; + + if (nested.type !== 'image') { + return []; + } + + const imageUrl = buildChatImageUrl(nested.source); + + return imageUrl + ? [{ type: 'image_url', image_url: { url: imageUrl } }] + : []; + }); +}; + const mapAnthropicContentToChat = ( content: string | AnthropicContentBlock[], role: 'user' | 'assistant', @@ -324,7 +480,7 @@ const mapAnthropicContentToChat = ( return [{ role, content }]; } - const parts: Array = []; + const parts: ChatContentPart[] = []; const toolCalls: Array<{ id: string; type: string; @@ -336,15 +492,16 @@ const mapAnthropicContentToChat = ( const toolResults: ChatMessage[] = []; const messages: ChatMessage[] = []; const flushAssistantMessage = (): void => { - const textContent = mapTextPartsToChatContent(parts); + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; - if (!toolCalls.length && !textContent.length) { + if (!toolCalls.length && !hasContent) { return; } messages.push({ role: 'assistant', - content: textContent.length ? textContent : null, + content: hasContent ? content : null, ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}), }); parts.length = 0; @@ -374,9 +531,16 @@ const mapAnthropicContentToChat = ( block.type === 'web_search_tool_result' || block.type === 'web_fetch_tool_result' ) { + const nestedImages = collectAnthropicNestedImages(block); + const resultMessage: ChatMessage = { role: 'tool', - content: formatAnthropicServerToolResult(block), + content: nestedImages.length + ? mapContentPartsToChat([ + formatAnthropicServerToolResult(block), + ...nestedImages, + ]) + : formatAnthropicServerToolResult(block), tool_call_id: block.tool_use_id ?? '', }; @@ -388,6 +552,30 @@ const mapAnthropicContentToChat = ( } } else if (block.type === 'thinking') { // Skip thinking blocks in conversation history for OpenAI compat. + } else if (block.type === 'image' && block.source) { + // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url', + // media_type, data | url } }`. Emit a real image block so the upstream + // model sees the image; without this branch the block fell through to + // `stringifyContent` and the model received a JSON dump of the base64 + // payload as text. An `image` block with no `source` is not a real + // Anthropic image, so it keeps the generic stringified handling. + const imageUrl = buildChatImageUrl(block.source); + + parts.push( + imageUrl + ? { + type: 'image_url', + image_url: { url: imageUrl }, + // Preserve an explicit cache breakpoint, matching how text + // blocks carry `cache_control` through. Without this the + // requested breakpoint is dropped and `applyPromptCacheControl` + // falls back to its own automatic placement. + ...(block.cache_control + ? { cache_control: block.cache_control } + : {}), + } + : stringifyContent(block), + ); } else { parts.push(stringifyContent(block)); } @@ -395,9 +583,10 @@ const mapAnthropicContentToChat = ( if (role === 'user') { messages.push(...toolResults); - const textContent = mapTextPartsToChatContent(parts); - if (textContent.length) { - messages.push({ role: 'user', content: textContent }); + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; + if (hasContent) { + messages.push({ role: 'user', content }); } } else { flushAssistantMessage(); diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index 12fa517..6992e2b 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -759,7 +759,7 @@ const getCredentialValue = ( return null; }; -const buildUpstreamHeaders = async ( +export const buildUpstreamHeaders = async ( request: NextRequest, auth: ResolvedAuth, ): Promise => { @@ -885,6 +885,58 @@ const buildUpstreamBody = async ( }; }; +export const isImageContentPart = (part: unknown): boolean => { + if (!part || typeof part !== 'object') { + return false; + } + + const value = part as { image_url?: unknown; type?: unknown }; + + if (value.type === 'image_url' || value.type === 'input_image') { + return true; + } + + // Accept the shapes an OpenAI-compatible client may send even when `type` + // is absent or unexpected: any part carrying an image URL is an image. + return ( + typeof value.image_url === 'string' || + Boolean( + value.image_url && + typeof value.image_url === 'object' && + typeof (value.image_url as { url?: unknown }).url === 'string', + ) + ); +}; + +/** + * Reads the image URL out of a Responses `input_image` / `image_url` part. + * Returns undefined when the part carries no usable URL, so callers can drop it + * rather than forwarding a block the upstream would reject. + */ +export const extractImageUrl = (part: unknown): string | undefined => { + if (!part || typeof part !== 'object') { + return undefined; + } + + const { image_url: imageUrl } = part as { image_url?: unknown }; + + // `input_image` carries a bare URL string; the OpenAI Chat-style + // `image_url` part nests it under `url`. + if (typeof imageUrl === 'string') { + return imageUrl || undefined; + } + + if ( + imageUrl && + typeof imageUrl === 'object' && + typeof (imageUrl as { url?: unknown }).url === 'string' + ) { + return (imageUrl as { url: string }).url || undefined; + } + + return undefined; +}; + const stringifyResponsesInputContent = (content: unknown): string => { if (typeof content === 'string') return content; if (content === null || content === undefined) return ''; @@ -1149,9 +1201,24 @@ const buildResponsesBodyFromChat = async ( ) .map((message) => { if (message.role === 'tool') { + // A tool may return an image, e.g. a screenshot. The upstream + // `function_call_output` carries `output` as structured content, so + // an image part is preserved there; stringifying it would hand the + // model a base64 dump instead of the image. + const toolOutput = Array.isArray(message.content) + ? message.content.filter( + (part) => part !== null && part !== undefined, + ) + : message.content; + const hasImage = Array.isArray(toolOutput) + ? toolOutput.some(isImageContentPart) + : isImageContentPart(toolOutput); + return { call_id: message.tool_call_id, - output: stringifyResponsesInputContent(message.content), + output: hasImage + ? mapChatContentToResponses(toolOutput) + : stringifyResponsesInputContent(toolOutput), type: 'function_call_output', }; } diff --git a/lib/server/proxy/image-generation.ts b/lib/server/proxy/image-generation.ts new file mode 100644 index 0000000..7ddd56f --- /dev/null +++ b/lib/server/proxy/image-generation.ts @@ -0,0 +1,375 @@ +/** + * Image generation for the Responses API. + * + * Both reference clients disagree here, so this mirrors the one that has a + * protocol: OpenAI's Responses API exposes `image_generation` as a tool the + * model invokes, returning an `image_generation_call` output item whose + * `result` is base64. Anthropic has no equivalent, and CodeBuddy's own CLI + * calls a separate `/v2/images/generations` endpoint. + * + * The two upstream protocols therefore need different handling: + * + * - `responses` passthrough forwards the tool declaration untouched, because + * CodeBuddy's `/responses` endpoint accepts `image_generation` natively and + * streams `image_generation_call` items back. + * - `chat` has no such concept, so the declaration is rewritten into an + * ordinary function and the call is executed here against + * `/v2/images/generations`, with the result fed back as a tool message. + */ + +import type { NextRequest } from 'next/server'; + +import { getCodeBuddyApiEndpoint } from '../domain/config'; +import type { ProxyContext } from './codebuddy'; +import { buildUpstreamHeaders } from './codebuddy'; + +export const IMAGE_GENERATION_TOOL_TYPE = 'image_generation'; + +/** Tool name advertised to the chat upstream when rewriting the declaration. */ +export const IMAGE_GENERATION_CHAT_TOOL_NAME = 'image_generation'; + +interface ImageGenerationArguments { + background?: string; + input_fidelity?: string; + model?: string; + output_compression?: number; + output_format?: string; + partial_images?: number; + prompt?: string; + quality?: string; + size?: string; +} + +/** + * The subset of `/v2/images/generations` this proxy sends. Every field is + * optional upstream; only `prompt` is validated here because a request without + * it cannot produce an image. + */ +interface ImageGenerationRequest { + model?: string; + n?: number; + prompt: string; + quality?: string; + response_format?: 'b64_json'; + size?: string; +} + +export interface ImageGenerationResult { + /** Base64-encoded image bytes, when the upstream returned inline data. */ + b64Json?: string; + /** Upstream-hosted image URL, when it returned one instead of inline data. */ + url?: string; +} + +const asString = (value: unknown): string | undefined => { + return typeof value === 'string' && value.trim() ? value.trim() : undefined; +}; + +const parseArguments = (raw: string): ImageGenerationArguments => { + if (!raw.trim()) { + return {}; + } + + try { + const parsed: unknown = JSON.parse(raw); + + return parsed && typeof parsed === 'object' + ? (parsed as ImageGenerationArguments) + : {}; + } catch { + return {}; + } +}; + +/** + * Rewrites an `image_generation` tool declaration as a Chat function so a + * chat-protocol model can invoke it. The schema is deliberately permissive: + * the model only needs to supply a prompt, and every optional control is a + * plain string so a model that ignores them still produces a valid call. + */ +export const buildImageGenerationChatTool = (): { + description: string; + name: string; + parameters: Record; +} => { + return { + description: + 'Generate an image from a text description. Returns a base64-encoded PNG image.', + name: IMAGE_GENERATION_CHAT_TOOL_NAME, + parameters: { + type: 'object', + properties: { + prompt: { + type: 'string', + description: 'Text description of the image to generate.', + }, + size: { + type: 'string', + description: + 'Image dimensions, for example "1024x1024". Optional; the upstream default is used when omitted.', + }, + quality: { + type: 'string', + description: + 'Rendering quality. Optional; the upstream default is used when omitted.', + }, + background: { + type: 'string', + description: + 'Background handling, for example "transparent". Optional.', + }, + output_format: { + type: 'string', + description: 'Output encoding, for example "png". Optional.', + }, + }, + required: ['prompt'], + additionalProperties: false, + }, + }; +}; + +const extractFirstImage = ( + payload: unknown, +): ImageGenerationResult | undefined => { + if (!payload || typeof payload !== 'object') { + return undefined; + } + + const data = (payload as { data?: unknown }).data; + const first = Array.isArray(data) ? data[0] : undefined; + + if (!first || typeof first !== 'object') { + return undefined; + } + + const image = first as { b64_json?: unknown; url?: unknown }; + const b64Json = asString(image.b64_json); + const url = asString(image.url); + + if (b64Json) { + return { b64Json }; + } + + if (url) { + return { url }; + } + + return undefined; +}; + +/** + * Runs one image generation against CodeBuddy's `/v2/images/generations`. + * + * Failures resolve to `null` rather than throwing: a broken image tool must not + * take down the surrounding turn, and the caller reports the failure to the + * model as a tool result so it can continue. + */ +export const executeImageGeneration = async ({ + arguments: rawArguments, + context, + request, + signal, +}: { + arguments: string; + context: ProxyContext; + request: NextRequest; + signal?: AbortSignal; +}): Promise => { + const args = parseArguments(rawArguments); + const prompt = asString(args.prompt); + + if (!prompt) { + return null; + } + + const body: ImageGenerationRequest = { prompt, response_format: 'b64_json' }; + const model = asString(args.model); + const size = asString(args.size); + const quality = asString(args.quality); + + if (model) { + body.model = model; + } + + if (size) { + body.size = size; + } + + if (quality) { + body.quality = quality; + } + + const apiEndpoint = await getCodeBuddyApiEndpoint(); + const headers = await buildUpstreamHeaders(request, context.auth); + + try { + const response = await fetch(`${apiEndpoint}/v2/images/generations`, { + body: JSON.stringify(body), + headers, + method: 'POST', + signal, + }); + + if (!response.ok) { + return null; + } + + return extractFirstImage(await response.json()) ?? null; + } catch { + return null; + } +}; + +// --------------------------------------------------------------------------- +// Chat-protocol execution loop +// --------------------------------------------------------------------------- + +/** Bounded because each generation is slow and one round of results suffices. */ +const MAX_IMAGE_ITERATIONS = 3; + +interface ChatToolCall { + id?: string; + function?: { arguments?: string; name?: string }; +} + +interface ChatCompletionMessage { + content?: unknown; + tool_calls?: ChatToolCall[]; +} + +interface ChatCompletionPayload { + choices?: Array<{ message?: ChatCompletionMessage }>; +} + +/** + * True when a model tool call targets the rewritten image-generation function. + * Compared loosely because upstream providers may normalize the name. + */ +export const isImageGenerationToolCall = (toolCall: unknown): boolean => { + if (!toolCall || typeof toolCall !== 'object') { + return false; + } + + const name = (toolCall as ChatToolCall).function?.name; + + return ( + typeof name === 'string' && + name.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_CHAT_TOOL_NAME.toLowerCase().replaceAll('-', '_') + ); +}; + +/** + * Tool result handed back to the model. Inline base64 becomes a data URI so the + * model can reference the image in later turns; a hosted URL is passed through. + */ +const buildImageToolResult = (result: ImageGenerationResult | null): string => { + if (result?.b64Json) { + return `data:image/png;base64,${result.b64Json}`; + } + + if (result?.url) { + return result.url; + } + + return 'Image generation failed: the upstream service returned no image.'; +}; +/** + * Runs image-generation tool calls a chat-protocol model made and returns the + * final upstream response with the images folded back into the transcript. + * + * Returns `null` when the model made no image call, so the caller keeps its + * ordinary upstream path. + * + * The returned response is always freshly constructed: reading an intermediate + * response to inspect its tool calls consumes the body, and the caller needs to + * read the final one again. + */ +export const executeImageGenerationLoop = async ({ + body, + callUpstream, + context, + request, +}: { + body: Record; + callUpstream: (body: Record) => Promise; + context: ProxyContext; + request: NextRequest; +}): Promise => { + let currentBody: Record = body; + + for (let iteration = 0; iteration < MAX_IMAGE_ITERATIONS; iteration += 1) { + const response = await callUpstream(currentBody); + + // A stream has already begun emitting to the client, so it cannot be + // resumed with a tool result; hand it back untouched. + if ( + response.headers + .get('content-type') + ?.toLowerCase() + .includes('text/event-stream') + ) { + return response; + } + + const payloadText = await response.text(); + let payload: ChatCompletionPayload = {}; + + try { + payload = JSON.parse(payloadText) as ChatCompletionPayload; + } catch { + // Unparseable upstream output cannot be continued; return it verbatim. + return new Response(payloadText, { + headers: response.headers, + status: response.status, + }); + } + + const message = payload.choices?.[0]?.message; + const imageCalls: ChatToolCall[] = (message?.tool_calls ?? []).filter( + isImageGenerationToolCall, + ); + + if (!imageCalls.length) { + // Nothing to execute. Rebuild the response so the caller can still read + // it, since `payloadText` was consumed above. + return iteration === 0 + ? null + : new Response(payloadText, { + headers: response.headers, + status: response.status, + }); + } + + const results: unknown[] = []; + + for (const toolCall of imageCalls) { + const result = await executeImageGeneration({ + arguments: toolCall.function?.arguments ?? '', + context, + request, + }); + + results.push({ + role: 'tool', + content: buildImageToolResult(result), + tool_call_id: toolCall.id ?? '', + }); + } + + const messages: unknown[] = Array.isArray(currentBody.messages) + ? [...currentBody.messages] + : []; + + if (message) { + messages.push(message); + } + + messages.push(...results); + + currentBody = { ...currentBody, messages }; + } + + return null; +}; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 5d155a1..41e3f92 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -18,6 +18,14 @@ import { WEB_SEARCH_TOOL_TYPE_PREFIX, } from '../search/tool'; import { + buildImageGenerationChatTool, + executeImageGenerationLoop, + IMAGE_GENERATION_CHAT_TOOL_NAME, + IMAGE_GENERATION_TOOL_TYPE, +} from './image-generation'; +import { + extractImageUrl, + isImageContentPart, proxyChatCompletions, proxyResponsesUpstream, resolveProxyContext, @@ -122,9 +130,28 @@ interface ChatResponseMessage { tool_calls?: ChatResponseToolCall[]; } +interface ChatImagePart { + image_url: { url: string }; + type: 'image_url'; +} + +interface ChatTextPart { + text: string; + type: 'text'; +} + +type ChatContentPart = string | ChatTextPart | ChatImagePart; + +/** + * Transcript content. Images are kept as structured parts so they survive the + * Chat-shaped round trip through the transcript and reach the model as images + * instead of a JSON dump. + */ +type TranscriptContent = string | ChatContentPart[]; + interface TranscriptMessage { role: string; - content: string | null; + content: TranscriptContent | null; tool_calls?: Array<{ id: string; type: string; @@ -521,6 +548,22 @@ const toSupportedChatTool = ( ]; } + // Image generation has no chat-protocol equivalent, so the declaration is + // rewritten as a function and the call is executed by the proxy. It is + // advertised only on the chat path: the responses passthrough hands the + // native declaration straight to the upstream, which supports it. + if (toolType === IMAGE_GENERATION_TOOL_TYPE) { + return [ + { + chatName: IMAGE_GENERATION_CHAT_TOOL_NAME, + kind: 'function' as const, + originalName: IMAGE_GENERATION_TOOL_TYPE, + serverDeclared: true, + tool: buildImageGenerationChatTool(), + }, + ]; + } + if (toolType === 'namespace') { const namespaceName = typeof tool.name === 'string' ? tool.name.trim() : ''; const children = ( @@ -606,6 +649,23 @@ const toSupportedChatTool = ( ]; }; +/** + * True when the client declared an `image_generation` tool. Gating on this + * keeps the ordinary path free of an extra upstream round trip. + */ +const hasImageGenerationTool = ( + tools: ResponsesRequestBody['tools'], +): boolean => { + return Boolean( + tools?.some( + (tool) => + typeof tool?.type === 'string' && + tool.type.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_TOOL_TYPE, + ), + ); +}; + const getSupportedChatTools = ( tools: ResponsesRequestBody['tools'], ): SupportedChatTool[] => { @@ -741,6 +801,52 @@ const getAssistantTranscriptContent = ( return toolCalls?.length ? outputText || null : outputText; }; +/** + * Keeps image parts as structured content so the chat path can rebuild them + * upstream. Text parts are still flattened: the transcript is persisted across + * turns and replayed as Chat messages, and the Responses converter only + * recognises images in the OpenAI `image_url` shape. + */ +const mapInputContentToTranscriptContent = ( + content: unknown, +): TranscriptContent | null => { + if (typeof content === 'string') { + return content; + } + + if (!Array.isArray(content)) { + return null; + } + + const parts = content.filter((part) => part !== null && part !== undefined); + + if (!parts.some(isImageContentPart)) { + return null; + } + + const mapped = parts.flatMap((part): ChatContentPart[] => { + if (typeof part === 'string') { + return [part]; + } + + if (isImageContentPart(part)) { + const imageUrl = extractImageUrl(part); + + return imageUrl + ? [{ image_url: { url: imageUrl }, type: 'image_url' }] + : []; + } + + if (part && typeof part === 'object' && 'text' in part) { + return [String((part as { text?: unknown }).text ?? '')]; + } + + return []; + }); + + return mapped.length ? mapped : null; +}; + const stringifyContent = (value: unknown): string => { if (typeof value === 'string') { return value; @@ -788,17 +894,24 @@ const mapInputItemToMessage = (item: ResponsesInputItem): TranscriptMessage => { } if (item.type === 'function_call_output' || item.type === 'mcp_call_output') { + // A tool may return an image, e.g. a screenshot. Keep it structured so the + // Responses converter can rebuild it as an image; stringifying would hand + // the model the base64 payload as text. + const outputContent = + mapInputContentToTranscriptContent(item.output) ?? + stringifyContent(item.output); + if (item.call_id) { return { role: 'tool', - content: stringifyContent(item.output), + content: outputContent, tool_call_id: item.call_id, }; } return { role: 'user', - content: stringifyContent(item.output), + content: outputContent, }; } @@ -809,6 +922,19 @@ const mapInputItemToMessage = (item: ResponsesInputItem): TranscriptMessage => { }; } + // Every other item type returns above, so what is left is a plain message: + // either one with a declared `type: 'message'`, or one carrying only + // `role`/`content`. Images are kept structured so the chat path can rebuild + // them; a message without an image stays flattened. + const imageContent = mapInputContentToTranscriptContent(item.content); + + if (imageContent !== null) { + return { + role: item.role ?? 'user', + content: imageContent, + }; + } + return { role: item.role ?? 'user', content: item.text ?? stringifyContent(item.content), @@ -1091,7 +1217,9 @@ const prepareTranscript = async ( body.messages.forEach((item) => { transcript.push({ role: item.role ?? 'user', - content: stringifyContent(item.content), + content: + mapInputContentToTranscriptContent(item.content) ?? + stringifyContent(item.content), }); }); } else if (typeof body.input === 'string') { @@ -2310,27 +2438,73 @@ export const handleResponsesRequest = async ( ); } - const upstreamResponse = await proxyChatCompletions( - request, - { - model: prepared.model, - messages: [ - ...(prepared.defaults.instructions - ? [{ role: 'system', content: prepared.defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames( - prepared.transcript, - prepared.defaults.tools, - ), - ], - max_tokens: body.max_output_tokens, - stream: false, - tools: translateResponsesToolsToChat(prepared.defaults.tools), - tool_choice: translateResponsesToolChoiceToChatWithTools( + const chatBody = { + model: prepared.model, + messages: [ + ...(prepared.defaults.instructions + ? [{ role: 'system', content: prepared.defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames( + prepared.transcript, prepared.defaults.tools, - prepared.defaults.tool_choice, ), - }, + ], + max_tokens: body.max_output_tokens, + stream: false, + tools: translateResponsesToolsToChat(prepared.defaults.tools), + tool_choice: translateResponsesToolChoiceToChatWithTools( + prepared.defaults.tools, + prepared.defaults.tool_choice, + ), + }; + + // Image generation has no chat-protocol equivalent, so the model's call is + // executed here and replayed with the image folded in. Only meaningful when + // the tool was actually declared; otherwise the loop returns null and the + // ordinary upstream call runs. + if (hasImageGenerationTool(prepared.defaults.tools)) { + const imageResponse = await executeImageGenerationLoop({ + body: chatBody, + callUpstream: (loopBody) => + proxyChatCompletions( + request, + loopBody as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); + + if (imageResponse) { + if (!imageResponse.ok) { + return imageResponse; + } + + const imagePayload = (await imageResponse.json()) as Record< + string, + unknown + >; + + return Response.json( + await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + prepared.defaults, + prepared.transcript, + prepared.model, + prepared.previousResponseId, + imagePayload, + getServerToolExecutions(imageResponse), + ), + ); + } + } + + const upstreamResponse = await proxyChatCompletions( + request, + chatBody as never, proxyContext, debugTrace, '/v1/responses', diff --git a/tests/server/anthropic.test.ts b/tests/server/anthropic.test.ts index 9094cf8..c179dde 100644 --- a/tests/server/anthropic.test.ts +++ b/tests/server/anthropic.test.ts @@ -1873,4 +1873,426 @@ describe('anthropic messages api', () => { expect(thinkingBlock?.thinking).toBe('Let me think...'); expect(textBlock?.text).toBe('The answer is 42.'); }); + + describe('image content blocks', () => { + const captureUpstreamBody = async ( + credentialOverrides: Record, + content: Array< + | { type: 'text'; text: string; cache_control?: { type: string } } + | { + type: 'image'; + source?: Record; + content?: string; + cache_control?: { type: string }; + } + | { + type: 'tool_result'; + tool_use_id: string; + content?: Array<{ + type: 'text' | 'image'; + text?: string; + source?: Record; + }>; + } + >, + ): Promise> => { + const credential = await addCredential({ + bearer_token: 'anthropic-image-token', + user_id: 'anthropic-image@example.com', + ...credentialOverrides, + }); + const accessKey = await createAccessKey({ + credentialFilenames: [credential.filename], + name: 'Anthropic Image Key', + }); + let upstreamBody: Record | undefined; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_input, init) => { + upstreamBody = JSON.parse(String(init?.body)) as Record< + string, + unknown + >; + + return makeJsonResponse({ + choices: [{ finish_reason: 'stop', message: { content: 'ok' } }], + }); + }); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { + method: 'POST', + headers: { authorization: `Bearer ${accessKey.secret}` }, + }), + { + max_tokens: 256, + messages: [{ role: 'user', content }], + model: 'claude-sonnet-4.6', + }, + ); + + expect(upstreamBody).toBeDefined(); + return upstreamBody as Record; + }; + + const getLastUserContent = ( + upstreamBody: Record, + ): unknown => { + const messages = upstreamBody.messages as Array< + Record + > | null; + const userMessages = (messages ?? []).filter((m) => m.role === 'user'); + + return userMessages[userMessages.length - 1]?.content; + }; + + it('converts a base64 image to a data URI image_url part', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { type: 'text', text: 'What is in this image?' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { type: 'text', text: 'What is in this image?' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + ]); + }); + + it('maps an image to input_image on the Responses upstream', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: true, upstream_protocol: 'responses' }, + [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: '/9j/4AAQSkZJRg==', + }, + }, + { type: 'text', text: 'Describe it.' }, + ], + ); + + const input = upstreamBody.input as Array>; + const userInput = input.filter((item) => item.role === 'user'); + expect(userInput[userInput.length - 1]?.content).toEqual([ + { + type: 'input_image', + image_url: 'data:image/jpeg;base64,/9j/4AAQSkZJRg==', + }, + { type: 'input_text', text: 'Describe it.' }, + ]); + }); + + it('passes through an image with a url source', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'image', + source: { type: 'url', url: 'https://example.com/a.png' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'https://example.com/a.png' }, + }, + ]); + }); + + it('defaults the media type when one is omitted', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [{ type: 'image', source: { type: 'base64', data: 'AAAA' } }], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + }, + ]); + }); + + it('falls back to text when the image source is unusable', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { type: 'image', source: { type: 'base64' } }, + { type: 'text', text: 'still here' }, + ], + ); + + const content = getLastUserContent(upstreamBody); + expect(content).toContain('still here'); + expect(content).toContain('base64'); + }); + + it('stringifies an image block that has no source at all', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [{ type: 'image', content: 'base64data' }], + ); + + expect(getLastUserContent(upstreamBody)).toContain('base64data'); + }); + + it('keeps an image-only message instead of dropping it', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'image', + source: { type: 'base64', media_type: 'image/gif', data: 'R0lGOD' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/gif;base64,R0lGOD' }, + }, + ]); + }); + + it('preserves cache_control on a converted image block', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAA' }, + cache_control: { type: 'ephemeral' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + cache_control: { type: 'ephemeral' }, + }, + ]); + }); + + it('extracts an image nested inside a tool result', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + makeJsonResponse({ choices: [{ message: { content: 'ok' } }] }), + ); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + model: 'claude-sonnet-4.6', + max_tokens: 1024, + messages: [ + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'toolu_shot', + name: 'screenshot', + input: {}, + }, + ], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_shot', + content: [ + { type: 'text', text: 'Took a screenshot.' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'iVBORw0KGgo=', + }, + }, + ], + }, + ], + }, + ], + }, + ); + + const upstreamBody = JSON.parse( + String((fetchMock.mock.calls[0]?.[1] as RequestInit).body), + ) as { + messages: Array<{ + role: string; + content: unknown; + tool_call_id?: string; + }>; + }; + const toolMessage = upstreamBody.messages.find( + (m) => m.tool_call_id === 'toolu_shot', + ); + + expect(toolMessage?.content).toEqual([ + { type: 'text', text: 'Took a screenshot.' }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + }, + ]); + }); + + it('drops a nested image whose source is unusable', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { + type: 'tool_result', + tool_use_id: 'toolu_bad', + content: [ + { type: 'text', text: 'took one' }, + { type: 'image', source: {} }, + ], + }, + ], + ); + + const messages = upstreamBody.messages as Array>; + const toolMessage = messages.find((m) => m.role === 'tool'); + expect(toolMessage?.content).toBe('took one'); + }); + + it('keeps an image in an assistant message alongside tool calls', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + makeJsonResponse({ choices: [{ message: { content: 'ok' } }] }), + ); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + model: 'claude-sonnet-4.6', + max_tokens: 1024, + messages: [ + { + role: 'assistant', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'AAAA', + }, + cache_control: { type: 'ephemeral' }, + }, + { + type: 'tool_use', + id: 'toolu_mixed', + name: 'search', + input: { query: 'q' }, + }, + ], + }, + ], + }, + ); + + const upstreamBody = JSON.parse( + String((fetchMock.mock.calls[0]?.[1] as RequestInit).body), + ) as { messages: Array> }; + const assistant = upstreamBody.messages.find( + (m) => m.role === 'assistant', + ); + + expect(assistant?.content).toEqual([ + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + cache_control: { type: 'ephemeral' }, + }, + ]); + expect(assistant?.tool_calls).toEqual([ + expect.objectContaining({ id: 'toolu_mixed' }), + ]); + }); + + it('emits structured text blocks when an image has a cache_control sibling', async () => { + const upstreamBody = await captureUpstreamBody( + { responses_passthrough: false }, + [ + { type: 'text', text: 'look', cache_control: { type: 'ephemeral' } }, + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAA' }, + }, + ], + ); + + expect(getLastUserContent(upstreamBody)).toEqual([ + { type: 'text', text: 'look', cache_control: { type: 'ephemeral' } }, + { + type: 'image_url', + image_url: { url: 'data:image/png;base64,AAAA' }, + }, + ]); + }); + + it('leaves a tool result without images as plain text', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValueOnce( + makeJsonResponse({ choices: [{ message: { content: 'ok' } }] }), + ); + + await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + model: 'claude-sonnet-4.6', + max_tokens: 1024, + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'toolu_plain', + content: [{ type: 'text', text: 'just text' }], + }, + ], + }, + ], + }, + ); + + const upstreamBody = JSON.parse( + String((fetchMock.mock.calls[0]?.[1] as RequestInit).body), + ) as { messages: Array<{ role: string; content: unknown }> }; + + expect(upstreamBody.messages.some((m) => m.content === 'just text')).toBe( + true, + ); + }); + }); }); diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts new file mode 100644 index 0000000..dbe8523 --- /dev/null +++ b/tests/server/image-generation.test.ts @@ -0,0 +1,807 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { NextRequest } from 'next/server'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAccessKey } from '@/lib/server/domain/access-keys'; +import { + addCredential, + resetCredentialRuntimeState, +} from '@/lib/server/domain/credentials'; +import { + executeImageGeneration, + isImageGenerationToolCall, +} from '@/lib/server/proxy/image-generation'; +import type { ProxyContext } from '@/lib/server/proxy/codebuddy'; +import { + handleResponsesRequest, + resetResponseSessions, +} from '@/lib/server/proxy/responses'; + +const tempRootDir = path.join(process.cwd(), '.tmp-test-image-generation'); + +const cleanupTempState = (): void => { + fs.rmSync(tempRootDir, { force: true, maxRetries: 5, recursive: true }); +}; + +const makeRequest = (secret?: string): NextRequest => { + return new NextRequest('http://localhost/v1/responses', { + headers: secret ? { authorization: `Bearer ${secret}` } : {}, + method: 'POST', + }); +}; + +const makeChatResponse = (message: Record): Response => { + return new Response( + JSON.stringify({ choices: [{ finish_reason: 'stop', message }] }), + { headers: { 'Content-Type': 'application/json' } }, + ); +}; + +const makeImageResponse = (data: unknown): Response => { + return new Response(JSON.stringify({ data }), { + headers: { 'Content-Type': 'application/json' }, + }); +}; + +const requestBodies = (): Array> => { + return vi + .mocked(globalThis.fetch) + .mock.calls.map(([, init]) => + JSON.parse(String((init as RequestInit | undefined)?.body ?? '{}')), + ) as Array>; +}; + +const makeContext = (): ProxyContext => { + return { + accessKeyId: null, + accessKeyName: null, + auth: { + bearerToken: 'image-gen-token', + credentialData: {}, + type: 'bearer', + userId: 'image-gen@example.com', + }, + credentialFilename: null, + preferences: { + firstMessageRoleToSystem: false, + firstSystemMessageRoleToUser: false, + upstreamProtocol: 'chat', + }, + }; +}; + +const addCredentialWith = async ( + overrides: Record = {}, +): Promise => { + const credential = await addCredential({ + bearer_token: 'image-gen-token', + user_id: 'image-gen@example.com', + ...overrides, + }); + const accessKey = await createAccessKey({ + credentialFilenames: [credential.filename], + name: 'Image Gen Key', + }); + + return accessKey.secret; +}; + +describe('Responses image support', () => { + beforeEach(async () => { + cleanupTempState(); + resetCredentialRuntimeState(); + resetResponseSessions(); + vi.restoreAllMocks(); + vi.clearAllMocks(); + vi.spyOn(process, 'cwd').mockReturnValue(tempRootDir); + process.env.CODEBUDDY_AUTH_MODE = 'api_key'; + process.env.CODEBUDDY_API_KEY = 'image-gen-key'; + }); + + afterEach(() => { + cleanupTempState(); + }); + + describe('input_image on the chat path', () => { + it('preserves an image part instead of stringifying it', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'a cat' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { text: 'what is this', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies[bodies.length - 1]?.messages).toEqual([ + { + content: [ + 'what is this', + { + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + type: 'image_url', + }, + ], + role: 'user', + }, + ]); + }); + + it('keeps a plain text message as a string', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { content: [{ text: 'hello', type: 'input_text' }], role: 'user' }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies[bodies.length - 1]?.messages).toEqual([ + { content: 'hello', role: 'user' }, + ]); + }); + + it('preserves an image returned by a tool', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: [ + { text: 'screenshot taken', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + // The chat path maps the output to a tool message, then the Responses + // converter rebuilds it as `function_call_output` with the image intact. + const toolMessages = requestBodies() + .flatMap( + (candidate) => + (candidate.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toEqual([ + 'screenshot taken', + { + image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' }, + type: 'image_url', + }, + ]); + }); + + it('accepts an image with an object or bare-URL shape', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { image_url: { url: 'data:image/png;base64,AAAA' } }, + { image_url: 'https://example.com/b.png', type: 'input_image' }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies.at(-1)?.messages).toEqual([ + { + content: [ + { + image_url: { url: 'data:image/png;base64,AAAA' }, + type: 'image_url', + }, + { + image_url: { url: 'https://example.com/b.png' }, + type: 'image_url', + }, + ], + role: 'user', + }, + ]); + }); + + it('drops an image part whose url cannot be read', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { image_url: '', type: 'input_image' }, + { text: 'still here', type: 'input_text' }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + // The unusable part is dropped, leaving a single text part. + expect(bodies.at(-1)?.messages).toEqual([ + { content: ['still here'], role: 'user' }, + ]); + }); + + it('keeps a tool output without images as plain text', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: [{ text: 'plain result', type: 'input_text' }], + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toBe('plain result'); + }); + }); + + describe('image_generation tool', () => { + it('executes a generation and replays the result to the model', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + + chatCall += 1; + + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here is your cat.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + const payload = (await response.json()) as { + output: Array<{ content: Array<{ text: string }> }>; + }; + expect(payload.output[0]?.content[0]?.text).toBe('Here is your cat.'); + + const imageRequest = requestBodies().find((body) => 'prompt' in body); + expect(imageRequest).toEqual({ + prompt: 'a cat', + response_format: 'b64_json', + }); + + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages).toEqual([ + { + content: 'data:image/png;base64,QUJD', + role: 'tool', + tool_call_id: 'call_1', + }, + ]); + }); + + it('reports a failure as a tool result so the turn continues', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return new Response('upstream exploded', { status: 500 }); + } + + chatCall += 1; + + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Sorry, that failed.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toContain('Image generation failed'); + }); + + it('makes no extra upstream call when the model does not ask for an image', async () => { + const secret = await addCredentialWith(); + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(makeChatResponse({ content: 'Sure.' })); + + await handleResponsesRequest(makeRequest(secret), { + input: 'hello', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const imageCalls = fetchMock.mock.calls.filter(([url]) => + String(url).includes('/v2/images/generations'), + ); + expect(imageCalls).toHaveLength(0); + }); + + it('passes a streamed response through untouched', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('data: {}\n\ndata: [DONE]\n\n', { + headers: { 'Content-Type': 'text/event-stream' }, + }), + ); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + }); + + it('forwards the native declaration on the responses passthrough', async () => { + const secret = await addCredentialWith({ + upstream_protocol: 'responses', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ id: 'resp_1', output: [], output_text: 'ok' }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation', model: 'gpt-image-2' }], + } as never); + + const body = requestBodies()[0]; + expect(body).toBeDefined(); + expect(body?.tools).toEqual([ + { model: 'gpt-image-2', type: 'image_generation' }, + ]); + }); + + it('handles string, unusable and empty parts alongside an image', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + 'leading text', + 42, + { image_url: 'data:image/png;base64,AAAA', type: 'input_image' }, + { text: 'trailing text', type: 'input_text' }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const bodies = requestBodies().filter((body) => + Array.isArray(body.messages), + ); + expect(bodies.at(-1)?.messages).toEqual([ + { + content: [ + 'leading text', + { + image_url: { url: 'data:image/png;base64,AAAA' }, + type: 'image_url', + }, + 'trailing text', + ], + role: 'user', + }, + ]); + }); + + it('preserves a tool-returned image on the responses passthrough', async () => { + const secret = await addCredentialWith({ + upstream_protocol: 'responses', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ id: 'resp_1', output: [], output_text: 'ok' }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: [ + { text: 'screenshot', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + expect(requestBodies()[0]?.input).toEqual([ + { + call_id: 'call_1', + output: [ + { text: 'screenshot', type: 'input_text' }, + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + type: 'function_call_output', + }, + ]); + }); + + it('preserves input_image on the responses passthrough', async () => { + const secret = await addCredentialWith({ + upstream_protocol: 'responses', + }); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ id: 'resp_1', output: [], output_text: 'ok' }), + { headers: { 'Content-Type': 'application/json' } }, + ), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + content: [ + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + role: 'user', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + expect(requestBodies()[0]?.input).toEqual([ + { + content: [ + { + image_url: 'data:image/png;base64,iVBORw0KGgo=', + type: 'input_image', + }, + ], + role: 'user', + }, + ]); + }); + }); + + describe('executeImageGeneration', () => { + it('returns null without a prompt', async () => { + const result = await executeImageGeneration({ + arguments: '{}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toBeNull(); + }); + + it('prefers base64 over a url and tolerates malformed arguments', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue( + makeImageResponse([ + { b64_json: 'QUJD', url: 'https://example.com/a.png' }, + ]), + ); + + const result = await executeImageGeneration({ + arguments: 'not json at all', + context: makeContext(), + request: makeRequest(), + }); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(result).toBeNull(); + }); + + it('returns a hosted url when no inline data is present', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue( + makeImageResponse([{ url: 'https://example.com/a.png' }]), + ); + + const result = await executeImageGeneration({ + arguments: '{"prompt":"a cat","size":"512x512","quality":"high"}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toEqual({ url: 'https://example.com/a.png' }); + const body = JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)); + expect(body).toMatchObject({ + prompt: 'a cat', + quality: 'high', + size: '512x512', + }); + }); + + it('returns null for a malformed or empty upstream payload', async () => { + for (const data of [undefined, [], [null], [{}], [{}]]) { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeImageResponse(data), + ); + + const result = await executeImageGeneration({ + arguments: '{"prompt":"a cat"}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toBeNull(); + } + }); + + it('forwards model, size and quality when supplied', async () => { + const fetchMock = vi + .spyOn(globalThis, 'fetch') + .mockResolvedValue(makeImageResponse([{ b64_json: 'QUJD' }])); + + const result = await executeImageGeneration({ + arguments: + '{"prompt":"a cat","model":"gpt-image-2","size":"1024x1024","quality":"high"}', + context: makeContext(), + request: makeRequest(), + }); + + expect(result).toEqual({ b64Json: 'QUJD' }); + expect( + JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)), + ).toMatchObject({ + model: 'gpt-image-2', + quality: 'high', + size: '1024x1024', + }); + }); + + it('ignores blank optional fields and returns null on a network error', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeImageResponse([{ b64_json: 'QUJD' }]), + ); + await executeImageGeneration({ + arguments: '{"prompt":"a cat","size":" ","quality":"","model":" "}', + context: makeContext(), + request: makeRequest(), + }); + const body = JSON.parse( + String( + vi.mocked(globalThis.fetch).mock.calls.at(-1)?.[1]?.body as string, + ), + ); + expect(body).toEqual({ prompt: 'a cat', response_format: 'b64_json' }); + + vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('boom')); + await expect( + executeImageGeneration({ + arguments: '{"prompt":"a cat"}', + context: makeContext(), + request: makeRequest(), + }), + ).resolves.toBeNull(); + }); + }); + + describe('streaming image generation', () => { + it('passes an SSE response through without resuming it', async () => { + const secret = await addCredentialWith(); + const upstream = new Response( + 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + vi.spyOn(globalThis, 'fetch').mockResolvedValue(upstream); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain( + 'text/event-stream', + ); + }); + }); + + describe('edge cases', () => { + it('handles a tool output that is not an array', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeChatResponse({ content: 'ok' }), + ); + + await handleResponsesRequest(makeRequest(secret), { + input: [ + { + call_id: 'call_1', + output: 'plain string result', + type: 'function_call_output', + }, + ], + model: 'claude-sonnet-4.6', + } as never); + + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.content).toBe('plain string result'); + }); + + it('tolerates a tool call missing its id and arguments', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [{ function: { name: 'image_generation' } }], + } + : { content: 'done' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + // The missing id/arguments fall back to empty values rather than + // aborting the turn. + const toolMessages = requestBodies() + .flatMap( + (body) => (body.messages ?? []) as Array>, + ) + .filter((message) => message.role === 'tool'); + expect(toolMessages[0]?.tool_call_id).toBe(''); + }); + }); + + describe('isImageGenerationToolCall', () => { + it('matches the rewritten function name loosely', () => { + expect( + isImageGenerationToolCall({ + function: { name: 'image_generation' }, + }), + ).toBe(true); + expect( + isImageGenerationToolCall({ function: { name: 'image-generation' } }), + ).toBe(true); + expect( + isImageGenerationToolCall({ function: { name: 'web_search' } }), + ).toBe(false); + expect(isImageGenerationToolCall(null)).toBe(false); + }); + }); +}); diff --git a/tests/server/units.test.ts b/tests/server/units.test.ts index 9243a4a..d4ec55d 100644 --- a/tests/server/units.test.ts +++ b/tests/server/units.test.ts @@ -5494,12 +5494,32 @@ describe('server units', () => { }); it('returns undefined when only unsupported tool types are provided', () => { + expect( + translateResponsesToolsToChat([{ type: 'file_search' }]), + ).toBeUndefined(); + }); + + it('rewrites an image_generation tool as a chat function', () => { expect( translateResponsesToolsToChat([ - { type: 'file_search' }, - { type: 'image_generation' }, + { type: 'image_generation', model: 'gpt-image-2' }, ]), - ).toBeUndefined(); + ).toEqual([ + { + // Marked as server-declared so the proxy knows it executes the call. + 'x-codebuddy2api-server-tool': true, + type: 'function', + function: expect.objectContaining({ + name: 'image_generation', + parameters: expect.objectContaining({ + properties: expect.objectContaining({ + prompt: expect.any(Object), + }), + required: ['prompt'], + }), + }), + }, + ]); }); it('maps responses tool_choice object variants to chat-completions shapes', async () => {