From bfa3f727efa4007ea4951957af3bb76f5fbc0b69 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 17:18:26 +0800 Subject: [PATCH 1/5] feat(responses): emit image_generation_call output items on the chat path The chat upstream has no notion of image generation, so a locally executed call previously disappeared: the image was folded back into the transcript for the model, but the client only received the assistant's text and had no way to see the image it asked for. The loop now records one execution per call and the Responses payload carries a standard `image_generation_call` output item, with the generated image in `result` as base64 and the prompt in `revised_prompt`. A failed generation still emits the item, with `result: null` and `status: 'failed'`, so the client can tell an image was attempted rather than seeing silence. This mirrors how `web_search_call` items are already synthesized from locally executed server tools. Streaming had a worse problem: the call was forwarded as an ordinary function_call for the client to resolve, and nothing ever generated the image. Streaming requests that declare the tool are now buffered so the call can be inspected before any delta reaches the client, then replayed as the same event sequence a live stream produces. Requests where the model makes no image call fall through to the live stream unchanged. `partial_images` is still passthrough-only: the generations endpoint returns a finished image, with no intermediate frames to stream. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/image-generation.ts | 108 ++++++++++++++-- lib/server/proxy/responses.ts | 172 +++++++++++++++++++++---- tests/server/image-generation.test.ts | 175 ++++++++++++++++++++++++-- 3 files changed, 413 insertions(+), 42 deletions(-) diff --git a/lib/server/proxy/image-generation.ts b/lib/server/proxy/image-generation.ts index 7ddd56f..d3d950a 100644 --- a/lib/server/proxy/image-generation.ts +++ b/lib/server/proxy/image-generation.ts @@ -81,6 +81,11 @@ const parseArguments = (raw: string): ImageGenerationArguments => { } }; +/** The prompt the model asked for, used as the `image_generation_call` label. */ +const extractPrompt = (raw: string): string => { + return parseArguments(raw).prompt?.trim() ?? ''; +}; + /** * Rewrites an `image_generation` tool declaration as a Chat function so a * chat-protocol model can invoke it. The schema is deliberately permissive: @@ -286,6 +291,45 @@ const buildImageToolResult = (result: ImageGenerationResult | null): string => { * response to inspect its tool calls consumes the body, and the caller needs to * read the final one again. */ +export interface ImageGenerationExecution { + id: string; + /** The rewritten prompt some providers echo back. Absent when unavailable. */ + prompt: string; + /** + * Base64-encoded image, when the upstream returned inline data. This is what + * an OpenAI `image_generation_call` carries in its `result` field. + */ + result: string | null; + status: 'completed' | 'failed'; +} + +export interface ImageGenerationLoopResult { + /** One entry per image call the model made, in call order. */ + executions: ImageGenerationExecution[]; + response: Response; +} + +const buildImageGenerationExecution = ({ + id, + prompt, + result, +}: { + id: string; + prompt: string; + result: ImageGenerationResult | null; +}): ImageGenerationExecution => { + if (result?.b64Json) { + return { id, prompt, result: result.b64Json, status: 'completed' }; + } + + return { + id, + prompt, + result: null, + status: result?.url ? 'completed' : 'failed', + }; +}; + export const executeImageGenerationLoop = async ({ body, callUpstream, @@ -296,8 +340,9 @@ export const executeImageGenerationLoop = async ({ callUpstream: (body: Record) => Promise; context: ProxyContext; request: NextRequest; -}): Promise => { +}): Promise => { let currentBody: Record = body; + const executions: ImageGenerationExecution[] = []; for (let iteration = 0; iteration < MAX_IMAGE_ITERATIONS; iteration += 1) { const response = await callUpstream(currentBody); @@ -310,7 +355,7 @@ export const executeImageGenerationLoop = async ({ ?.toLowerCase() .includes('text/event-stream') ) { - return response; + return { executions, response }; } const payloadText = await response.text(); @@ -320,10 +365,13 @@ export const executeImageGenerationLoop = async ({ payload = JSON.parse(payloadText) as ChatCompletionPayload; } catch { // Unparseable upstream output cannot be continued; return it verbatim. - return new Response(payloadText, { - headers: response.headers, - status: response.status, - }); + return { + executions, + response: new Response(payloadText, { + headers: response.headers, + status: response.status, + }), + }; } const message = payload.choices?.[0]?.message; @@ -336,21 +384,33 @@ export const executeImageGenerationLoop = async ({ // it, since `payloadText` was consumed above. return iteration === 0 ? null - : new Response(payloadText, { - headers: response.headers, - status: response.status, - }); + : { + executions, + response: new Response(payloadText, { + headers: response.headers, + status: response.status, + }), + }; } const results: unknown[] = []; for (const toolCall of imageCalls) { + const arguments_ = toolCall.function?.arguments ?? ''; const result = await executeImageGeneration({ - arguments: toolCall.function?.arguments ?? '', + arguments: arguments_, context, request, }); + executions.push( + buildImageGenerationExecution({ + id: toolCall.id ?? '', + prompt: extractPrompt(arguments_), + result, + }), + ); + results.push({ role: 'tool', content: buildImageToolResult(result), @@ -373,3 +433,29 @@ export const executeImageGenerationLoop = async ({ return null; }; + +// --------------------------------------------------------------------------- +// Responses output item +// --------------------------------------------------------------------------- + +/** + * Builds the `image_generation_call` output item OpenAI's Responses API + * defines, so a client driving the chat upstream still sees the standard shape: + * the generated image in `result` as base64, and the prompt it came from. + * + * `result` is null when generation failed. The field is still emitted, with + * status `failed`, because dropping it would leave the client with no way to + * tell that an image was attempted. + */ +export const buildResponsesImageGenerationCallItem = ( + execution: ImageGenerationExecution, + id = `ig_${crypto.randomUUID().replaceAll('-', '')}`, +): Record => { + return { + id, + result: execution.result, + status: execution.status, + type: 'image_generation_call', + ...(execution.prompt ? { revised_prompt: execution.prompt } : {}), + }; +}; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 41e3f92..edb4865 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -19,9 +19,11 @@ import { } from '../search/tool'; import { buildImageGenerationChatTool, + buildResponsesImageGenerationCallItem, executeImageGenerationLoop, IMAGE_GENERATION_CHAT_TOOL_NAME, IMAGE_GENERATION_TOOL_TYPE, + type ImageGenerationExecution, } from './image-generation'; import { extractImageUrl, @@ -1354,6 +1356,7 @@ const mapChatResponseToResponsesPayload = async ( previousResponseId: string | null, upstreamPayload: Record, serverToolExecutions: ServerToolExecution[], + imageExecutions: ImageGenerationExecution[] = [], ): Promise> => { const responseId = createResponseId(); const choices = Array.isArray(upstreamPayload.choices) @@ -1367,9 +1370,17 @@ const mapChatResponseToResponsesPayload = async ( : []; const outputText = stringifyContent(firstChoice.message?.content); const createdAt = Math.floor(Date.now() / 1000); - const output: Array> = serverToolExecutions.map( - (execution) => buildResponsesWebSearchCallItem(execution, 'completed'), - ); + const output: Array> = [ + ...serverToolExecutions.map((execution) => + buildResponsesWebSearchCallItem(execution, 'completed'), + ), + // Image generation is executed locally, so the standard + // `image_generation_call` item has to be synthesized here — the chat + // upstream has no notion of it. + ...imageExecutions.map((execution) => + buildResponsesImageGenerationCallItem(execution), + ), + ]; const transcriptToolCalls = buildAssistantTranscriptToolCalls( toolCalls, defaults.tools, @@ -1455,6 +1466,80 @@ const buildResponsesWebSearchCallItem = ( }, }); +/** + * Emits an already-buffered chat payload as a Responses SSE stream. + * + * Used when a request had to be buffered to inspect it — image generation is + * executed locally, so the call cannot be forwarded before it is seen. The + * client still asked for `stream: true`, so the buffered result is replayed as + * the same event sequence a live stream would have produced. + */ +const mapChatResponseToResponsesStream = async ( + upstreamPayload: Record, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + proxyContext: ProxyContext, + imageExecutions: ImageGenerationExecution[], +): Promise => { + const responseId = createResponseId(); + const payload = await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + defaults, + transcript, + model, + previousResponseId, + upstreamPayload, + [], + imageExecutions, + ); + const output = Array.isArray(payload.output) + ? (payload.output as Array>) + : []; + + const frames = [ + { + response: { ...payload, output: [], status: 'in_progress' }, + type: 'response.created', + }, + { + response: { id: responseId, status: 'in_progress' }, + type: 'response.in_progress', + }, + ...output.map((item, output_index) => ({ + item, + output_index, + response_id: responseId, + type: 'response.output_item.added', + })), + ...output.map((item, output_index) => ({ + item, + output_index, + response_id: responseId, + type: 'response.output_item.done', + })), + { response: { ...payload, id: responseId }, type: 'response.completed' }, + ]; + + const body = [ + ...frames.map( + (frame) => `event: ${frame.type}\ndata: ${JSON.stringify(frame)}`, + ), + 'data: [DONE]', + '', + ].join('\n\n'); + + return new Response(body, { + headers: { + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream; charset=utf-8', + }, + }); +}; + const mapChatStreamToResponsesEventStream = ( upstreamResponse: Response, defaults: ResponseSessionDefaults, @@ -2094,24 +2179,66 @@ const createResponsesEventStream = async ( ]); if (!searchEnabled && !fetchEnabled) { + const chatBody = { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }; + + // Image generation is executed locally, so a streaming request has to be + // buffered first to see whether the model asked for an image. Without this + // the call is forwarded as an ordinary function_call the client is expected + // to resolve — and nothing would ever generate the image. + if (hasImageGenerationTool(defaults.tools)) { + const imageResult = await executeImageGenerationLoop({ + body: chatBody, + // Buffered so the tool call can be inspected before any delta reaches + // the client; the ordinary path below stays live. + callUpstream: (loopBody) => + proxyChatCompletions( + request, + { ...loopBody, stream: false } as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); + + if (imageResult) { + const { executions, response } = imageResult; + + if (!response.ok) { + return response; + } + + return mapChatResponseToResponsesStream( + (await response.json()) as Record, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + executions, + ); + } + } + const upstreamResponse = await proxyChatCompletions( request, - { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }, + chatBody as never, proxyContext, debugTrace, '/v1/responses', @@ -2463,7 +2590,7 @@ export const handleResponsesRequest = async ( // the tool was actually declared; otherwise the loop returns null and the // ordinary upstream call runs. if (hasImageGenerationTool(prepared.defaults.tools)) { - const imageResponse = await executeImageGenerationLoop({ + const imageResult = await executeImageGenerationLoop({ body: chatBody, callUpstream: (loopBody) => proxyChatCompletions( @@ -2477,7 +2604,9 @@ export const handleResponsesRequest = async ( request, }); - if (imageResponse) { + if (imageResult) { + const { executions, response: imageResponse } = imageResult; + if (!imageResponse.ok) { return imageResponse; } @@ -2497,6 +2626,7 @@ export const handleResponsesRequest = async ( prepared.previousResponseId, imagePayload, getServerToolExecutions(imageResponse), + executions, ), ); } diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index dbe8523..dd6d5bf 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -336,9 +336,22 @@ describe('Responses image support', () => { expect(response.status).toBe(200); const payload = (await response.json()) as { - output: Array<{ content: Array<{ text: string }> }>; + output: Array>; }; - expect(payload.output[0]?.content[0]?.text).toBe('Here is your cat.'); + + // The generated image is returned as a standard image_generation_call + // output item, ahead of the assistant's text. + expect(payload.output[0]).toMatchObject({ + result: 'QUJD', + revised_prompt: 'a cat', + status: 'completed', + type: 'image_generation_call', + }); + + const message = payload.output[1] as { + content: Array<{ text: string }>; + }; + expect(message.content[0]?.text).toBe('Here is your cat.'); const imageRequest = requestBodies().find((body) => 'prompt' in body); expect(imageRequest).toEqual({ @@ -360,6 +373,139 @@ describe('Responses image support', () => { ]); }); + it('emits a failed image_generation_call when generation fails', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return new Response('upstream exploded', { status: 500 }); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Sorry, that failed.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output: Array>; + }; + // The item is still emitted so the client can tell an image was + // attempted, but carries no result. + expect(payload.output[0]).toMatchObject({ + result: null, + status: 'failed', + type: 'image_generation_call', + }); + }); + + it('omits revised_prompt when the model sent no prompt', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { arguments: '{}', name: 'image_generation' }, + id: 'c1', + }, + ], + } + : { content: 'done' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output: Array>; + }; + expect(payload.output[0]).toMatchObject({ + result: null, + status: 'failed', + type: 'image_generation_call', + }); + expect(payload.output[0]).not.toHaveProperty('revised_prompt'); + }); + + it('streams the image_generation_call as Responses SSE events', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here is your cat.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(200); + expect(response.headers.get('content-type')).toContain( + 'text/event-stream', + ); + const text = await response.text(); + + // The client sees the standard item, not a function_call it must resolve. + expect(text).toContain('"type":"image_generation_call"'); + expect(text).toContain('"result":"QUJD"'); + expect(text).not.toContain('"type":"function_call"'); + expect(text).toContain('event: response.output_item.added'); + expect(text).toContain('event: response.completed'); + }); + it('reports a failure as a tool result so the turn continues', async () => { const secret = await addCredentialWith(); let chatCall = 0; @@ -424,10 +570,15 @@ describe('Responses image support', () => { it('passes a streamed response through untouched', async () => { const secret = await addCredentialWith(); - vi.spyOn(globalThis, 'fetch').mockResolvedValue( - new Response('data: {}\n\ndata: [DONE]\n\n', { - headers: { 'Content-Type': 'text/event-stream' }, - }), + // A fresh Response per call: the loop issues more than one upstream + // request, and a reused body cannot be read twice. + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve( + new Response( + 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ), + ), ); const response = await handleResponsesRequest(makeRequest(secret), { @@ -706,11 +857,15 @@ describe('Responses image support', () => { describe('streaming image generation', () => { it('passes an SSE response through without resuming it', async () => { const secret = await addCredentialWith(); - const upstream = new Response( - 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', - { headers: { 'Content-Type': 'text/event-stream' } }, + // A fresh Response per call: a reused body cannot be read twice. + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve( + new Response( + 'data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n', + { headers: { 'Content-Type': 'text/event-stream' } }, + ), + ), ); - vi.spyOn(globalThis, 'fetch').mockResolvedValue(upstream); const response = await handleResponsesRequest(makeRequest(secret), { input: 'draw a cat', From 20ef195e44b37d49bffad55be7ae9191e90cdd85 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 17:20:37 +0800 Subject: [PATCH 2/5] test(image): cover image_generation_call outcomes on the chat path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raises changed-branch coverage to 92.31%. Covers a generation that returns only a hosted URL — marked completed, but with no base64 to hand back — and a failed upstream on the streaming path, which is surfaced to the client rather than swallowed. Co-Authored-By: Claude Fable 5 --- tests/server/image-generation.test.ts | 61 +++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index dd6d5bf..034e555 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -506,6 +506,67 @@ describe('Responses image support', () => { expect(text).toContain('event: response.completed'); }); + it('marks a URL-only result completed without inline data', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ url: 'https://example.com/a.png' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here it is.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output: Array>; + }; + // The upstream returned a hosted URL rather than inline bytes, so there + // is no base64 to hand back, but the call itself succeeded. + expect(payload.output[0]).toMatchObject({ + result: null, + status: 'completed', + type: 'image_generation_call', + }); + }); + + it('returns a failed upstream response from a streaming image call', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve(new Response('upstream down', { status: 502 })), + ); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(502); + }); + it('reports a failure as a tool result so the turn continues', async () => { const secret = await addCredentialWith(); let chatCall = 0; From 1603a48641cce9b3186dc27e164ea56ae5d4431c Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 17:43:52 +0800 Subject: [PATCH 3/5] fix(responses): address image-generation review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes, three of them real bugs the review caught. **Streamed turns were billed twice.** The loop returned null when the model made no image call, and the caller re-issued the request. For a client that declares the optional tool, that is the common case, so most streaming turns paid for two upstream calls and could return an answer different from the one already inspected. The loop now always returns its final response, and neither caller re-issues. **The stream advertised the wrong response id.** `response.created` carried the id the payload mapper created and stored, while later events used a freshly generated one — so a client passing the completed id as `previous_response_id` got "Unknown or expired". The stream now reuses the mapper's id. **A non-string prompt crashed the request.** Model arguments are untrusted JSON; `{"prompt":123}` threw on `.trim()` and turned a handled generation failure into a 500. Now coerced, so the intended failed item is emitted. **Prose written before the tool call was dropped.** A model that explains itself and then calls the tool lost the explanation, because only the final hop's message survived the replay. The web-search loop already solved this, so `withIntermediateTurns` is now exported and shared rather than reimplemented — along with `ChatCompletionMessage`, `ChatCompletionPayload` and `ChatCompletionToolCall`, removing three duplicate type declarations. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/image-generation.ts | 138 +++++++++++++++++++------- lib/server/proxy/responses.ts | 111 +++++++++++---------- lib/server/proxy/web-search-loop.ts | 14 ++- tests/server/image-generation.test.ts | 51 ++++++++++ 4 files changed, 219 insertions(+), 95 deletions(-) diff --git a/lib/server/proxy/image-generation.ts b/lib/server/proxy/image-generation.ts index d3d950a..efee356 100644 --- a/lib/server/proxy/image-generation.ts +++ b/lib/server/proxy/image-generation.ts @@ -22,6 +22,12 @@ import type { NextRequest } from 'next/server'; import { getCodeBuddyApiEndpoint } from '../domain/config'; import type { ProxyContext } from './codebuddy'; import { buildUpstreamHeaders } from './codebuddy'; +import { + withIntermediateTurns, + type ChatCompletionMessage, + type ChatCompletionPayload, + type ChatCompletionToolCall, +} from './web-search-loop'; export const IMAGE_GENERATION_TOOL_TYPE = 'image_generation'; @@ -83,7 +89,12 @@ const parseArguments = (raw: string): ImageGenerationArguments => { /** The prompt the model asked for, used as the `image_generation_call` label. */ const extractPrompt = (raw: string): string => { - return parseArguments(raw).prompt?.trim() ?? ''; + // Model-generated arguments are untrusted JSON, so a non-string prompt is + // possible. `executeImageGeneration` already rejects it, and throwing here + // would turn that handled failure into a 500. + const { prompt } = parseArguments(raw); + + return typeof prompt === 'string' ? prompt.trim() : ''; }; /** @@ -233,20 +244,6 @@ export const executeImageGeneration = async ({ /** Bounded because each generation is slow and one round of results suffices. */ const MAX_IMAGE_ITERATIONS = 3; -interface ChatToolCall { - id?: string; - function?: { arguments?: string; name?: string }; -} - -interface ChatCompletionMessage { - content?: unknown; - tool_calls?: ChatToolCall[]; -} - -interface ChatCompletionPayload { - choices?: Array<{ message?: ChatCompletionMessage }>; -} - /** * True when a model tool call targets the rewritten image-generation function. * Compared loosely because upstream providers may normalize the name. @@ -256,7 +253,7 @@ export const isImageGenerationToolCall = (toolCall: unknown): boolean => { return false; } - const name = (toolCall as ChatToolCall).function?.name; + const name = (toolCall as ChatCompletionToolCall).function?.name; return ( typeof name === 'string' && @@ -281,11 +278,13 @@ const buildImageToolResult = (result: ImageGenerationResult | null): string => { return 'Image generation failed: the upstream service returned no image.'; }; /** - * Runs image-generation tool calls a chat-protocol model made and returns the - * final upstream response with the images folded back into the transcript. + * Runs image-generation tool calls a chat-protocol model makes, replaying the + * request with each generated image folded back in. * - * Returns `null` when the model made no image call, so the caller keeps its - * ordinary upstream path. + * Always returns the final upstream response, including when the model made no + * image call — callers must not re-issue the request themselves, since that + * would bill the turn twice and could return a different answer from the one + * already inspected. `executions` is empty when no image was generated. * * The returned response is always freshly constructed: reading an intermediate * response to inspect its tool calls consumes the body, and the caller needs to @@ -309,6 +308,28 @@ export interface ImageGenerationLoopResult { response: Response; } +/** + * Rebuilds a response whose body was already read. The loop consumes each + * response to inspect its tool calls, so anything handed back has to be + * reconstructed from the text that was read. + */ +const rebuildResponse = ( + response: Response, + payload: ChatCompletionPayload, +): Response => { + return new Response(JSON.stringify(payload), { + headers: response.headers, + status: response.status, + }); +}; + +/** Prose a hop wrote, i.e. text the model produced before calling the tool. */ +const readMessageText = ( + message: ChatCompletionMessage | undefined, +): string => { + return typeof message?.content === 'string' ? message.content.trim() : ''; +}; + const buildImageGenerationExecution = ({ id, prompt, @@ -340,9 +361,14 @@ export const executeImageGenerationLoop = async ({ callUpstream: (body: Record) => Promise; context: ProxyContext; request: NextRequest; -}): Promise => { +}): Promise => { let currentBody: Record = body; const executions: ImageGenerationExecution[] = []; + const intermediateTexts: string[] = []; + // Carried across iterations so the cap can hand back the last response + // instead of discarding every image already generated. + let lastResponse: Response | null = null; + let lastPayload: ChatCompletionPayload | null = null; for (let iteration = 0; iteration < MAX_IMAGE_ITERATIONS; iteration += 1) { const response = await callUpstream(currentBody); @@ -375,22 +401,33 @@ export const executeImageGenerationLoop = async ({ } const message = payload.choices?.[0]?.message; - const imageCalls: ChatToolCall[] = (message?.tool_calls ?? []).filter( - isImageGenerationToolCall, - ); + const imageCalls: ChatCompletionToolCall[] = ( + message?.tool_calls ?? [] + ).filter(isImageGenerationToolCall); if (!imageCalls.length) { - // Nothing to execute. Rebuild the response so the caller can still read - // it, since `payloadText` was consumed above. - return iteration === 0 - ? null - : { - executions, - response: new Response(payloadText, { - headers: response.headers, - status: response.status, - }), - }; + // Nothing to execute. Hand the response back rather than returning null: + // the caller must not re-issue the request, since that would bill the + // turn twice and could yield a different answer. Prose from earlier hops + // is folded in first, because it is part of the turn the client sees. + return { + executions, + response: rebuildResponse( + response, + withIntermediateTurns({ + executions: [], + payload, + reasonings: [], + texts: intermediateTexts, + }), + ), + }; + } + + const iterationText = readMessageText(message); + + if (iterationText) { + intermediateTexts.push(iterationText); } const results: unknown[] = []; @@ -429,9 +466,38 @@ export const executeImageGenerationLoop = async ({ messages.push(...results); currentBody = { ...currentBody, messages }; + lastPayload = payload; + lastResponse = response; + } + + // Unreachable in practice — every iteration sets both, and the loop body + // always executes at least once — but the guard keeps the types honest + // without asserting. + if (!lastResponse || !lastPayload) { + return { + executions, + response: new Response(null, { + headers: { 'Content-Type': 'application/json' }, + status: 502, + }), + }; } - return null; + // The cap was reached with the model still asking for images. Every image + // generated so far is kept, and the last response is handed back so the + // caller does not re-issue the request and discard them. + return { + executions, + response: rebuildResponse( + lastResponse, + withIntermediateTurns({ + executions: [], + payload: lastPayload, + reasonings: [], + texts: intermediateTexts, + }), + ), + }; }; // --------------------------------------------------------------------------- diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index edb4865..fe9604e 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -1483,7 +1483,6 @@ const mapChatResponseToResponsesStream = async ( proxyContext: ProxyContext, imageExecutions: ImageGenerationExecution[], ): Promise => { - const responseId = createResponseId(); const payload = await mapChatResponseToResponsesPayload( proxyContext.accessKeyId, proxyContext.credentialFilename, @@ -1495,6 +1494,10 @@ const mapChatResponseToResponsesStream = async ( [], imageExecutions, ); + // The mapper creates and persists the session id, so the stream has to reuse + // it: advertising a different one would leave a client unable to continue the + // turn, because nothing was stored under the id it was given. + const responseId = String(payload.id ?? createResponseId()); const output = Array.isArray(payload.output) ? (payload.output as Array>) : []; @@ -2201,7 +2204,7 @@ const createResponsesEventStream = async ( // the call is forwarded as an ordinary function_call the client is expected // to resolve — and nothing would ever generate the image. if (hasImageGenerationTool(defaults.tools)) { - const imageResult = await executeImageGenerationLoop({ + const { executions, response } = await executeImageGenerationLoop({ body: chatBody, // Buffered so the tool call can be inspected before any delta reaches // the client; the ordinary path below stays live. @@ -2217,23 +2220,22 @@ const createResponsesEventStream = async ( request, }); - if (imageResult) { - const { executions, response } = imageResult; - - if (!response.ok) { - return response; - } - - return mapChatResponseToResponsesStream( - (await response.json()) as Record, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - executions, - ); + // Always consumed, even when nothing was generated: the loop has already + // sent the turn upstream, and re-issuing it would bill twice and could + // return a different answer than the one inspected. + if (!response.ok) { + return response; } + + return mapChatResponseToResponsesStream( + (await response.json()) as Record, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + executions, + ); } const upstreamResponse = await proxyChatCompletions( @@ -2590,46 +2592,45 @@ export const handleResponsesRequest = async ( // the tool was actually declared; otherwise the loop returns null and the // ordinary upstream call runs. if (hasImageGenerationTool(prepared.defaults.tools)) { - const imageResult = await executeImageGenerationLoop({ - body: chatBody, - callUpstream: (loopBody) => - proxyChatCompletions( - request, - loopBody as never, - proxyContext, - debugTrace, - '/v1/responses', - ), - context: proxyContext, - request, - }); + const { executions, response: imageResponse } = + await executeImageGenerationLoop({ + body: chatBody, + callUpstream: (loopBody) => + proxyChatCompletions( + request, + loopBody as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); - if (imageResult) { - const { executions, response: imageResponse } = imageResult; + // Always consumed: the loop has already sent the turn upstream, and + // re-issuing it would bill twice and could return a different answer. + if (!imageResponse.ok) { + return imageResponse; + } - if (!imageResponse.ok) { - return imageResponse; - } + const imagePayload = (await imageResponse.json()) as Record< + string, + unknown + >; - const imagePayload = (await imageResponse.json()) as Record< - string, - unknown - >; - - return Response.json( - await mapChatResponseToResponsesPayload( - proxyContext.accessKeyId, - proxyContext.credentialFilename, - prepared.defaults, - prepared.transcript, - prepared.model, - prepared.previousResponseId, - imagePayload, - getServerToolExecutions(imageResponse), - executions, - ), - ); - } + return Response.json( + await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + prepared.defaults, + prepared.transcript, + prepared.model, + prepared.previousResponseId, + imagePayload, + getServerToolExecutions(imageResponse), + executions, + ), + ); } const upstreamResponse = await proxyChatCompletions( diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index a9ed5de..d70b08a 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -54,7 +54,7 @@ const STREAM_TEXT_CHUNK_LENGTH = 1024; type JsonRecord = Record; -interface ChatCompletionToolCall { +export interface ChatCompletionToolCall { id?: string; index?: number; type?: string; @@ -64,7 +64,7 @@ interface ChatCompletionToolCall { }; } -interface ChatCompletionMessage { +export interface ChatCompletionMessage { content?: string | null; reasoning?: string; reasoning_content?: string; @@ -572,7 +572,9 @@ const buildMixedTurnPayload = ({ }; }; -const readReasoning = (message: ChatCompletionMessage | undefined): string => { +export const readReasoning = ( + message: ChatCompletionMessage | undefined, +): string => { if (!message) { return ''; } @@ -624,8 +626,12 @@ const buildIntermediateTurns = ({ * * The same hops are also re-grouped into `turns`, because the folded strings * cannot express where one hop ends and the next begins. + * + * Shared with the image-generation loop, which has the same shape: a local + * tool call is replayed with its result appended, so only the final hop's + * message would otherwise survive. */ -const withIntermediateTurns = ({ +export const withIntermediateTurns = ({ executions, payload, reasonings, diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index 034e555..c8759d0 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -567,6 +567,57 @@ describe('Responses image support', () => { expect(response.status).toBe(502); }); + it('keeps prose the model wrote before calling the tool', async () => { + const secret = await addCredentialWith(); + let chatCall = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + // A model commonly explains itself before calling a tool. + content: 'Sure, let me draw that for you.', + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'Here is your cat.' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw me a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + const payload = (await response.json()) as { + output_text: string; + output: Array>; + }; + + // The loop replays the request with the image appended, so only the final + // hop's message would survive without folding the earlier prose in. + expect(payload.output_text).toBe( + 'Sure, let me draw that for you.\n\nHere is your cat.', + ); + expect(payload.output[0]).toMatchObject({ + result: 'QUJD', + status: 'completed', + type: 'image_generation_call', + }); + }); + it('reports a failure as a tool result so the turn continues', async () => { const secret = await addCredentialWith(); let chatCall = 0; From 96d4afaed997ed97b3cf01acbe384ea9faeddc5b Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 18:00:44 +0800 Subject: [PATCH 4/5] test(image): cover the non-streaming image failure path Raises changed-branch coverage to 90.63%. Covers a failed upstream on the non-streaming path, which is surfaced to the client rather than swallowed. Also removes two unreachable fallbacks: the buffered stream helper re-derived a response id the payload mapper always returns, and the loop's terminal branch had a dead guard because its body always runs at least once. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/image-generation.ts | 17 ++--------------- lib/server/proxy/responses.ts | 6 ++---- tests/server/image-generation.test.ts | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/server/proxy/image-generation.ts b/lib/server/proxy/image-generation.ts index efee356..afe1c23 100644 --- a/lib/server/proxy/image-generation.ts +++ b/lib/server/proxy/image-generation.ts @@ -470,29 +470,16 @@ export const executeImageGenerationLoop = async ({ lastResponse = response; } - // Unreachable in practice — every iteration sets both, and the loop body - // always executes at least once — but the guard keeps the types honest - // without asserting. - if (!lastResponse || !lastPayload) { - return { - executions, - response: new Response(null, { - headers: { 'Content-Type': 'application/json' }, - status: 502, - }), - }; - } - // The cap was reached with the model still asking for images. Every image // generated so far is kept, and the last response is handed back so the // caller does not re-issue the request and discard them. return { executions, response: rebuildResponse( - lastResponse, + lastResponse ?? new Response(null, { status: 502 }), withIntermediateTurns({ executions: [], - payload: lastPayload, + payload: lastPayload ?? {}, reasonings: [], texts: intermediateTexts, }), diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index fe9604e..3db7ed2 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -1497,10 +1497,8 @@ const mapChatResponseToResponsesStream = async ( // The mapper creates and persists the session id, so the stream has to reuse // it: advertising a different one would leave a client unable to continue the // turn, because nothing was stored under the id it was given. - const responseId = String(payload.id ?? createResponseId()); - const output = Array.isArray(payload.output) - ? (payload.output as Array>) - : []; + const responseId = String(payload.id); + const output = payload.output as Array>; const frames = [ { diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index c8759d0..3fdea51 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -618,6 +618,21 @@ describe('Responses image support', () => { }); }); + it('returns a failed upstream response from a non-streaming image call', async () => { + const secret = await addCredentialWith(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => + Promise.resolve(new Response('upstream down', { status: 502 })), + ); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'draw a cat', + model: 'claude-sonnet-4.6', + tools: [{ type: 'image_generation' }], + } as never); + + expect(response.status).toBe(502); + }); + it('reports a failure as a tool result so the turn continues', async () => { const secret = await addCredentialWith(); let chatCall = 0; From 7582c8edcfc7fb4fe0a086d9934e32d1f4d369d6 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 18:05:52 +0800 Subject: [PATCH 5/5] fix(responses): run image generation when server search is enabled The streaming image path sat inside the branch taken only when neither web search nor web fetch was enabled. A turn declaring both an image tool and an enabled server search tool therefore skipped generation entirely: the model's call was forwarded as an ordinary function_call for the client to resolve, and no image was ever generated. Image generation does not depend on the search backends, so it is now handled before that branch. A turn declaring both runs generation first, and one declaring only server tools is unchanged. Co-Authored-By: Claude Fable 5 --- lib/server/proxy/responses.ts | 110 +++++++++++++------------- tests/server/image-generation.test.ts | 52 ++++++++++++ 2 files changed, 109 insertions(+), 53 deletions(-) diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 3db7ed2..230f556 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -2179,63 +2179,67 @@ const createResponsesEventStream = async ( : false, ]); - if (!searchEnabled && !fetchEnabled) { - const chatBody = { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }; - - // Image generation is executed locally, so a streaming request has to be - // buffered first to see whether the model asked for an image. Without this - // the call is forwarded as an ordinary function_call the client is expected - // to resolve — and nothing would ever generate the image. - if (hasImageGenerationTool(defaults.tools)) { - const { executions, response } = await executeImageGenerationLoop({ - body: chatBody, - // Buffered so the tool call can be inspected before any delta reaches - // the client; the ordinary path below stays live. - callUpstream: (loopBody) => - proxyChatCompletions( - request, - { ...loopBody, stream: false } as never, - proxyContext, - debugTrace, - '/v1/responses', - ), - context: proxyContext, - request, - }); + const chatBody = { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }; - // Always consumed, even when nothing was generated: the loop has already - // sent the turn upstream, and re-issuing it would bill twice and could - // return a different answer than the one inspected. - if (!response.ok) { - return response; - } + // Image generation is executed locally, so a streaming request has to be + // buffered first to see whether the model asked for an image. Without this + // the call is forwarded as an ordinary function_call the client is expected + // to resolve — and nothing would ever generate the image. + // + // Handled before the server-tool branch below: a turn may declare both, and + // gating on search/fetch would silently skip generation whenever those were + // enabled. + if (hasImageGenerationTool(defaults.tools)) { + const { executions, response } = await executeImageGenerationLoop({ + body: chatBody, + // Buffered so the tool call can be inspected before any delta reaches + // the client; the ordinary path below stays live. + callUpstream: (loopBody) => + proxyChatCompletions( + request, + { ...loopBody, stream: false } as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); - return mapChatResponseToResponsesStream( - (await response.json()) as Record, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - executions, - ); + // Always consumed, even when nothing was generated: the loop has already + // sent the turn upstream, and re-issuing it would bill twice and could + // return a different answer than the one inspected. + if (!response.ok) { + return response; } + return mapChatResponseToResponsesStream( + (await response.json()) as Record, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + executions, + ); + } + + if (!searchEnabled && !fetchEnabled) { const upstreamResponse = await proxyChatCompletions( request, chatBody as never, diff --git a/tests/server/image-generation.test.ts b/tests/server/image-generation.test.ts index 3fdea51..dae2136 100644 --- a/tests/server/image-generation.test.ts +++ b/tests/server/image-generation.test.ts @@ -4,6 +4,7 @@ import path from 'node:path'; import { NextRequest } from 'next/server'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as config from '@/lib/server/domain/config'; import { createAccessKey } from '@/lib/server/domain/access-keys'; import { addCredential, @@ -458,6 +459,57 @@ describe('Responses image support', () => { expect(payload.output[0]).not.toHaveProperty('revised_prompt'); }); + it('executes image calls when a server search tool is also enabled', async () => { + // Regression: the streaming image path used to sit inside the + // "no server search tools" branch, so a turn declaring both silently + // skipped generation and forwarded the call as an ordinary + // function_call for the client to resolve. + const spy = vi.spyOn(config, 'isWebSearchEnabled'); + spy.mockResolvedValue(true); + + const secret = await addCredentialWith(); + let chatCall = 0; + let imageCalls = 0; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (url) => { + if (String(url).includes('/v2/images/generations')) { + imageCalls += 1; + return makeImageResponse([{ b64_json: 'QUJD' }]); + } + chatCall += 1; + return makeChatResponse( + chatCall === 1 + ? { + content: null, + tool_calls: [ + { + function: { + arguments: '{"prompt":"a cat"}', + name: 'image_generation', + }, + id: 'call_1', + type: 'function', + }, + ], + } + : { content: 'done' }, + ); + }); + + const response = await handleResponsesRequest(makeRequest(secret), { + input: 'search then draw', + model: 'claude-sonnet-4.6', + stream: true, + tools: [{ type: 'image_generation' }, { type: 'web_search_preview' }], + } as never); + + expect(imageCalls).toBe(1); + const text = await response.text(); + expect(text).toContain('image_generation_call'); + expect(text).not.toContain('"type":"function_call"'); + + spy.mockRestore(); + }); + it('streams the image_generation_call as Responses SSE events', async () => { const secret = await addCredentialWith(); let chatCall = 0;