diff --git a/lib/server/proxy/image-generation.ts b/lib/server/proxy/image-generation.ts index 7ddd56f..afe1c23 100644 --- a/lib/server/proxy/image-generation.ts +++ b/lib/server/proxy/image-generation.ts @@ -22,6 +22,12 @@ import type { NextRequest } from 'next/server'; import { getCodeBuddyApiEndpoint } from '../domain/config'; import type { ProxyContext } from './codebuddy'; import { buildUpstreamHeaders } from './codebuddy'; +import { + withIntermediateTurns, + type ChatCompletionMessage, + type ChatCompletionPayload, + type ChatCompletionToolCall, +} from './web-search-loop'; export const IMAGE_GENERATION_TOOL_TYPE = 'image_generation'; @@ -81,6 +87,16 @@ const parseArguments = (raw: string): ImageGenerationArguments => { } }; +/** The prompt the model asked for, used as the `image_generation_call` label. */ +const extractPrompt = (raw: string): string => { + // Model-generated arguments are untrusted JSON, so a non-string prompt is + // possible. `executeImageGeneration` already rejects it, and throwing here + // would turn that handled failure into a 500. + const { prompt } = parseArguments(raw); + + return typeof prompt === 'string' ? prompt.trim() : ''; +}; + /** * Rewrites an `image_generation` tool declaration as a Chat function so a * chat-protocol model can invoke it. The schema is deliberately permissive: @@ -228,20 +244,6 @@ export const executeImageGeneration = async ({ /** Bounded because each generation is slow and one round of results suffices. */ const MAX_IMAGE_ITERATIONS = 3; -interface ChatToolCall { - id?: string; - function?: { arguments?: string; name?: string }; -} - -interface ChatCompletionMessage { - content?: unknown; - tool_calls?: ChatToolCall[]; -} - -interface ChatCompletionPayload { - choices?: Array<{ message?: ChatCompletionMessage }>; -} - /** * True when a model tool call targets the rewritten image-generation function. * Compared loosely because upstream providers may normalize the name. @@ -251,7 +253,7 @@ export const isImageGenerationToolCall = (toolCall: unknown): boolean => { return false; } - const name = (toolCall as ChatToolCall).function?.name; + const name = (toolCall as ChatCompletionToolCall).function?.name; return ( typeof name === 'string' && @@ -276,16 +278,79 @@ const buildImageToolResult = (result: ImageGenerationResult | null): string => { return 'Image generation failed: the upstream service returned no image.'; }; /** - * Runs image-generation tool calls a chat-protocol model made and returns the - * final upstream response with the images folded back into the transcript. + * Runs image-generation tool calls a chat-protocol model makes, replaying the + * request with each generated image folded back in. * - * Returns `null` when the model made no image call, so the caller keeps its - * ordinary upstream path. + * Always returns the final upstream response, including when the model made no + * image call — callers must not re-issue the request themselves, since that + * would bill the turn twice and could return a different answer from the one + * already inspected. `executions` is empty when no image was generated. * * The returned response is always freshly constructed: reading an intermediate * response to inspect its tool calls consumes the body, and the caller needs to * read the final one again. */ +export interface ImageGenerationExecution { + id: string; + /** The rewritten prompt some providers echo back. Absent when unavailable. */ + prompt: string; + /** + * Base64-encoded image, when the upstream returned inline data. This is what + * an OpenAI `image_generation_call` carries in its `result` field. + */ + result: string | null; + status: 'completed' | 'failed'; +} + +export interface ImageGenerationLoopResult { + /** One entry per image call the model made, in call order. */ + executions: ImageGenerationExecution[]; + response: Response; +} + +/** + * Rebuilds a response whose body was already read. The loop consumes each + * response to inspect its tool calls, so anything handed back has to be + * reconstructed from the text that was read. + */ +const rebuildResponse = ( + response: Response, + payload: ChatCompletionPayload, +): Response => { + return new Response(JSON.stringify(payload), { + headers: response.headers, + status: response.status, + }); +}; + +/** Prose a hop wrote, i.e. text the model produced before calling the tool. */ +const readMessageText = ( + message: ChatCompletionMessage | undefined, +): string => { + return typeof message?.content === 'string' ? message.content.trim() : ''; +}; + +const buildImageGenerationExecution = ({ + id, + prompt, + result, +}: { + id: string; + prompt: string; + result: ImageGenerationResult | null; +}): ImageGenerationExecution => { + if (result?.b64Json) { + return { id, prompt, result: result.b64Json, status: 'completed' }; + } + + return { + id, + prompt, + result: null, + status: result?.url ? 'completed' : 'failed', + }; +}; + export const executeImageGenerationLoop = async ({ body, callUpstream, @@ -296,8 +361,14 @@ export const executeImageGenerationLoop = async ({ callUpstream: (body: Record) => Promise; context: ProxyContext; request: NextRequest; -}): Promise => { +}): Promise => { let currentBody: Record = body; + const executions: ImageGenerationExecution[] = []; + const intermediateTexts: string[] = []; + // Carried across iterations so the cap can hand back the last response + // instead of discarding every image already generated. + let lastResponse: Response | null = null; + let lastPayload: ChatCompletionPayload | null = null; for (let iteration = 0; iteration < MAX_IMAGE_ITERATIONS; iteration += 1) { const response = await callUpstream(currentBody); @@ -310,7 +381,7 @@ export const executeImageGenerationLoop = async ({ ?.toLowerCase() .includes('text/event-stream') ) { - return response; + return { executions, response }; } const payloadText = await response.text(); @@ -320,37 +391,63 @@ export const executeImageGenerationLoop = async ({ payload = JSON.parse(payloadText) as ChatCompletionPayload; } catch { // Unparseable upstream output cannot be continued; return it verbatim. - return new Response(payloadText, { - headers: response.headers, - status: response.status, - }); + return { + executions, + response: new Response(payloadText, { + headers: response.headers, + status: response.status, + }), + }; } const message = payload.choices?.[0]?.message; - const imageCalls: ChatToolCall[] = (message?.tool_calls ?? []).filter( - isImageGenerationToolCall, - ); + const imageCalls: ChatCompletionToolCall[] = ( + message?.tool_calls ?? [] + ).filter(isImageGenerationToolCall); if (!imageCalls.length) { - // Nothing to execute. Rebuild the response so the caller can still read - // it, since `payloadText` was consumed above. - return iteration === 0 - ? null - : new Response(payloadText, { - headers: response.headers, - status: response.status, - }); + // Nothing to execute. Hand the response back rather than returning null: + // the caller must not re-issue the request, since that would bill the + // turn twice and could yield a different answer. Prose from earlier hops + // is folded in first, because it is part of the turn the client sees. + return { + executions, + response: rebuildResponse( + response, + withIntermediateTurns({ + executions: [], + payload, + reasonings: [], + texts: intermediateTexts, + }), + ), + }; + } + + const iterationText = readMessageText(message); + + if (iterationText) { + intermediateTexts.push(iterationText); } const results: unknown[] = []; for (const toolCall of imageCalls) { + const arguments_ = toolCall.function?.arguments ?? ''; const result = await executeImageGeneration({ - arguments: toolCall.function?.arguments ?? '', + arguments: arguments_, context, request, }); + executions.push( + buildImageGenerationExecution({ + id: toolCall.id ?? '', + prompt: extractPrompt(arguments_), + result, + }), + ); + results.push({ role: 'tool', content: buildImageToolResult(result), @@ -369,7 +466,49 @@ export const executeImageGenerationLoop = async ({ messages.push(...results); currentBody = { ...currentBody, messages }; + lastPayload = payload; + lastResponse = response; } - return null; + // The cap was reached with the model still asking for images. Every image + // generated so far is kept, and the last response is handed back so the + // caller does not re-issue the request and discard them. + return { + executions, + response: rebuildResponse( + lastResponse ?? new Response(null, { status: 502 }), + withIntermediateTurns({ + executions: [], + payload: lastPayload ?? {}, + reasonings: [], + texts: intermediateTexts, + }), + ), + }; +}; + +// --------------------------------------------------------------------------- +// Responses output item +// --------------------------------------------------------------------------- + +/** + * Builds the `image_generation_call` output item OpenAI's Responses API + * defines, so a client driving the chat upstream still sees the standard shape: + * the generated image in `result` as base64, and the prompt it came from. + * + * `result` is null when generation failed. The field is still emitted, with + * status `failed`, because dropping it would leave the client with no way to + * tell that an image was attempted. + */ +export const buildResponsesImageGenerationCallItem = ( + execution: ImageGenerationExecution, + id = `ig_${crypto.randomUUID().replaceAll('-', '')}`, +): Record => { + return { + id, + result: execution.result, + status: execution.status, + type: 'image_generation_call', + ...(execution.prompt ? { revised_prompt: execution.prompt } : {}), + }; }; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 41e3f92..230f556 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -19,9 +19,11 @@ import { } from '../search/tool'; import { buildImageGenerationChatTool, + buildResponsesImageGenerationCallItem, executeImageGenerationLoop, IMAGE_GENERATION_CHAT_TOOL_NAME, IMAGE_GENERATION_TOOL_TYPE, + type ImageGenerationExecution, } from './image-generation'; import { extractImageUrl, @@ -1354,6 +1356,7 @@ const mapChatResponseToResponsesPayload = async ( previousResponseId: string | null, upstreamPayload: Record, serverToolExecutions: ServerToolExecution[], + imageExecutions: ImageGenerationExecution[] = [], ): Promise> => { const responseId = createResponseId(); const choices = Array.isArray(upstreamPayload.choices) @@ -1367,9 +1370,17 @@ const mapChatResponseToResponsesPayload = async ( : []; const outputText = stringifyContent(firstChoice.message?.content); const createdAt = Math.floor(Date.now() / 1000); - const output: Array> = serverToolExecutions.map( - (execution) => buildResponsesWebSearchCallItem(execution, 'completed'), - ); + const output: Array> = [ + ...serverToolExecutions.map((execution) => + buildResponsesWebSearchCallItem(execution, 'completed'), + ), + // Image generation is executed locally, so the standard + // `image_generation_call` item has to be synthesized here — the chat + // upstream has no notion of it. + ...imageExecutions.map((execution) => + buildResponsesImageGenerationCallItem(execution), + ), + ]; const transcriptToolCalls = buildAssistantTranscriptToolCalls( toolCalls, defaults.tools, @@ -1455,6 +1466,81 @@ const buildResponsesWebSearchCallItem = ( }, }); +/** + * Emits an already-buffered chat payload as a Responses SSE stream. + * + * Used when a request had to be buffered to inspect it — image generation is + * executed locally, so the call cannot be forwarded before it is seen. The + * client still asked for `stream: true`, so the buffered result is replayed as + * the same event sequence a live stream would have produced. + */ +const mapChatResponseToResponsesStream = async ( + upstreamPayload: Record, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + proxyContext: ProxyContext, + imageExecutions: ImageGenerationExecution[], +): Promise => { + const payload = await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + defaults, + transcript, + model, + previousResponseId, + upstreamPayload, + [], + imageExecutions, + ); + // The mapper creates and persists the session id, so the stream has to reuse + // it: advertising a different one would leave a client unable to continue the + // turn, because nothing was stored under the id it was given. + const responseId = String(payload.id); + const output = payload.output as Array>; + + const frames = [ + { + response: { ...payload, output: [], status: 'in_progress' }, + type: 'response.created', + }, + { + response: { id: responseId, status: 'in_progress' }, + type: 'response.in_progress', + }, + ...output.map((item, output_index) => ({ + item, + output_index, + response_id: responseId, + type: 'response.output_item.added', + })), + ...output.map((item, output_index) => ({ + item, + output_index, + response_id: responseId, + type: 'response.output_item.done', + })), + { response: { ...payload, id: responseId }, type: 'response.completed' }, + ]; + + const body = [ + ...frames.map( + (frame) => `event: ${frame.type}\ndata: ${JSON.stringify(frame)}`, + ), + 'data: [DONE]', + '', + ].join('\n\n'); + + return new Response(body, { + headers: { + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream; charset=utf-8', + }, + }); +}; + const mapChatStreamToResponsesEventStream = ( upstreamResponse: Response, defaults: ResponseSessionDefaults, @@ -2093,25 +2179,70 @@ const createResponsesEventStream = async ( : false, ]); + const chatBody = { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }; + + // Image generation is executed locally, so a streaming request has to be + // buffered first to see whether the model asked for an image. Without this + // the call is forwarded as an ordinary function_call the client is expected + // to resolve — and nothing would ever generate the image. + // + // Handled before the server-tool branch below: a turn may declare both, and + // gating on search/fetch would silently skip generation whenever those were + // enabled. + if (hasImageGenerationTool(defaults.tools)) { + const { executions, response } = await executeImageGenerationLoop({ + body: chatBody, + // Buffered so the tool call can be inspected before any delta reaches + // the client; the ordinary path below stays live. + callUpstream: (loopBody) => + proxyChatCompletions( + request, + { ...loopBody, stream: false } as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); + + // Always consumed, even when nothing was generated: the loop has already + // sent the turn upstream, and re-issuing it would bill twice and could + // return a different answer than the one inspected. + if (!response.ok) { + return response; + } + + return mapChatResponseToResponsesStream( + (await response.json()) as Record, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + executions, + ); + } + if (!searchEnabled && !fetchEnabled) { const upstreamResponse = await proxyChatCompletions( request, - { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }, + chatBody as never, proxyContext, debugTrace, '/v1/responses', @@ -2463,43 +2594,45 @@ export const handleResponsesRequest = async ( // the tool was actually declared; otherwise the loop returns null and the // ordinary upstream call runs. if (hasImageGenerationTool(prepared.defaults.tools)) { - const imageResponse = await executeImageGenerationLoop({ - body: chatBody, - callUpstream: (loopBody) => - proxyChatCompletions( - request, - loopBody as never, - proxyContext, - debugTrace, - '/v1/responses', - ), - context: proxyContext, - request, - }); - - if (imageResponse) { - if (!imageResponse.ok) { - return imageResponse; - } + const { executions, response: imageResponse } = + await executeImageGenerationLoop({ + body: chatBody, + callUpstream: (loopBody) => + proxyChatCompletions( + request, + loopBody as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); - const imagePayload = (await imageResponse.json()) as Record< - string, - unknown - >; - - return Response.json( - await mapChatResponseToResponsesPayload( - proxyContext.accessKeyId, - proxyContext.credentialFilename, - prepared.defaults, - prepared.transcript, - prepared.model, - prepared.previousResponseId, - imagePayload, - getServerToolExecutions(imageResponse), - ), - ); + // Always consumed: the loop has already sent the turn upstream, and + // re-issuing it would bill twice and could return a different answer. + if (!imageResponse.ok) { + return imageResponse; } + + const imagePayload = (await imageResponse.json()) as Record< + string, + unknown + >; + + return Response.json( + await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + prepared.defaults, + prepared.transcript, + prepared.model, + prepared.previousResponseId, + imagePayload, + getServerToolExecutions(imageResponse), + executions, + ), + ); } const upstreamResponse = await proxyChatCompletions( diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index a9ed5de..d70b08a 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -54,7 +54,7 @@ const STREAM_TEXT_CHUNK_LENGTH = 1024; type JsonRecord = Record; -interface ChatCompletionToolCall { +export interface ChatCompletionToolCall { id?: string; index?: number; type?: string; @@ -64,7 +64,7 @@ interface ChatCompletionToolCall { }; } -interface ChatCompletionMessage { +export interface ChatCompletionMessage { content?: string | null; reasoning?: string; reasoning_content?: string; @@ -572,7 +572,9 @@ const buildMixedTurnPayload = ({ }; }; -const readReasoning = (message: ChatCompletionMessage | undefined): string => { +export const readReasoning = ( + message: ChatCompletionMessage | undefined, +): string => { if (!message) { return ''; } @@ -624,8 +626,12 @@ const buildIntermediateTurns = ({ * * The same hops are also re-grouped into `turns`, because the folded strings * cannot express where one hop ends and the next begins. + * + * Shared with the image-generation loop, which has the same shape: a local + * tool call is replayed with its result appended, so only the final hop's + * message would otherwise survive. */ -const withIntermediateTurns = ({ +export const withIntermediateTurns = ({ executions, payload, reasonings, diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index dbe8523..dae2136 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { NextRequest } from 'next/server'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as config from '@/lib/server/domain/config'; import { createAccessKey } from '@/lib/server/domain/access-keys'; import { addCredential, @@ -336,9 +337,22 @@ describe('Responses image support', () => { expect(response.status).toBe(200); const payload = (await response.json()) as { - output: Array<{ content: Array<{ text: string }> }>; + output: Array>; }; - expect(payload.output[0]?.content[0]?.text).toBe('Here is your cat.'); + + // The generated image is returned as a standard image_generation_call + // output item, ahead of the assistant's text. + expect(payload.output[0]).toMatchObject({ + result: 'QUJD', + revised_prompt: 'a cat', + status: 'completed', + type: 'image_generation_call', + }); + + const message = payload.output[1] as { + content: Array<{ text: string }>; + }; + expect(message.content[0]?.text).toBe('Here is your cat.'); const imageRequest = requestBodies().find((body) => 'prompt' in body); expect(imageRequest).toEqual({ @@ -360,6 +374,317 @@ describe('Responses image support', () => { ]); }); + it('emits a failed image_generation_call when generation fails', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return new Response('upstream exploded', { status: 500 }); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Sorry, that failed.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output: Array>; + }; + // The item is still emitted so the client can tell an image was + // attempted, but carries no result. + expect(payload.output[0]).toMatchObject({ + result: null, + status: 'failed', + type: 'image_generation_call', + }); + }); + + it('omits revised_prompt when the model sent no prompt', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { arguments: '{}', name: 'image_generation' }, + id: 'c1', + }, + ], + } + : { content: 'done' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output: Array>; + }; + expect(payload.output[0]).toMatchObject({ + result: null, + status: 'failed', + type: 'image_generation_call', + }); + expect(payload.output[0]).not.toHaveProperty('revised_prompt'); + }); + + it('executes image calls when a server search tool is also enabled', async () => { + // Regression: the streaming image path used to sit inside the + // "no server search tools" branch, so a turn declaring both silently + // skipped generation and forwarded the call as an ordinary + // function_call for the client to resolve. + const spy = vi.spyOn(config, 'isWebSearchEnabled'); + spy.mockResolvedValue(true); + + const secret = await addCredentialWith(); + let chatCall = 0; + let imageCalls = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + imageCalls += 1; + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'done' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'search then draw', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }, { type: 'web_search_preview' }], + } as never); + + expect(imageCalls).toBe(1); + const text = await response.text(); + expect(text).toContain('image_generation_call'); + expect(text).not.toContain('"type":"function_call"'); + + spy.mockRestore(); + }); + + it('streams the image_generation_call as Responses SSE events', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here is your cat.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain( + 'text/event-stream', + ); + const text = await response.text(); + + // The client sees the standard item, not a function_call it must resolve. + expect(text).toContain('"type":"image_generation_call"'); + expect(text).toContain('"result":"QUJD"'); + expect(text).not.toContain('"type":"function_call"'); + expect(text).toContain('event: response.output_item.added'); + expect(text).toContain('event: response.completed'); + }); + + it('marks a URL-only result completed without inline data', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ url: 'https://example.com/a.png' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here it is.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output: Array>; + }; + // The upstream returned a hosted URL rather than inline bytes, so there + // is no base64 to hand back, but the call itself succeeded. + expect(payload.output[0]).toMatchObject({ + result: null, + status: 'completed', + type: 'image_generation_call', + }); + }); + + it('returns a failed upstream response from a streaming image call', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve(new Response('upstream down', { status: 502 })), + ); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(502); + }); + + it('keeps prose the model wrote before calling the tool', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + // A model commonly explains itself before calling a tool. + content: 'Sure, let me draw that for you.', + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here is your cat.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output_text: string; + output: Array>; + }; + + // The loop replays the request with the image appended, so only the final + // hop's message would survive without folding the earlier prose in. + expect(payload.output_text).toBe( + 'Sure, let me draw that for you.\n\nHere is your cat.', + ); + expect(payload.output[0]).toMatchObject({ + result: 'QUJD', + status: 'completed', + type: 'image_generation_call', + }); + }); + + it('returns a failed upstream response from a non-streaming image call', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve(new Response('upstream down', { status: 502 })), + ); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(502); + }); + it('reports a failure as a tool result so the turn continues', async () => { const secret = await addCredentialWith(); let chatCall = 0; @@ -424,10 +749,15 @@ describe('Responses image support', () => { it('passes a streamed response through untouched', async () => { const secret = await addCredentialWith(); - vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response('data: {}\n\ndata: [DONE]\n\n', { - headers: { 'Content-Type': 'text/event-stream' }, - }), + // A fresh Response per call: the loop issues more than one upstream + // request, and a reused body cannot be read twice. + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve( + new Response( + 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ), + ), ); const response = await handleResponsesRequest(makeRequest(secret), { @@ -706,11 +1036,15 @@ describe('Responses image support', () => { describe('streaming image generation', () => { it('passes an SSE response through without resuming it', async () => { const secret = await addCredentialWith(); - const upstream = new Response( - 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', - { headers: { 'Content-Type': 'text/event-stream' } }, + // A fresh Response per call: a reused body cannot be read twice. + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve( + new Response( + 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ), + ), ); - vi.spyOn(globalThis, 'fetch').mockResolvedValue(upstream); const response = await handleResponsesRequest(makeRequest(secret), { input: 'draw a cat',