feat(responses): emit image_generation_call output items on the chat path

lib/server/proxy/image-generation.ts

@@ -22,6 +22,12 @@ import type { NextRequest } from 'next/server';
 import { getCodeBuddyApiEndpoint } from '../domain/config';
 import type { ProxyContext } from './codebuddy';
 import { buildUpstreamHeaders } from './codebuddy';
+import {
+  withIntermediateTurns,
+  type ChatCompletionMessage,
+  type ChatCompletionPayload,
+  type ChatCompletionToolCall,
+} from './web-search-loop';
 
 export const IMAGE_GENERATION_TOOL_TYPE = 'image_generation';
 
@@ -81,6 +87,16 @@ const parseArguments = (raw: string): ImageGenerationArguments => {
   }
 };
 
+/** The prompt the model asked for, used as the `image_generation_call` label. */
+const extractPrompt = (raw: string): string => {
+  // Model-generated arguments are untrusted JSON, so a non-string prompt is
+  // possible. `executeImageGeneration` already rejects it, and throwing here
+  // would turn that handled failure into a 500.
+  const { prompt } = parseArguments(raw);
+
+  return typeof prompt === 'string' ? prompt.trim() : '';
+};
+
 /**
  * Rewrites an `image_generation` tool declaration as a Chat function so a
  * chat-protocol model can invoke it. The schema is deliberately permissive:
@@ -228,20 +244,6 @@ export const executeImageGeneration = async ({
 /** Bounded because each generation is slow and one round of results suffices. */
 const MAX_IMAGE_ITERATIONS = 3;
 
-interface ChatToolCall {
-  id?: string;
-  function?: { arguments?: string; name?: string };
-}
-
-interface ChatCompletionMessage {
-  content?: unknown;
-  tool_calls?: ChatToolCall[];
-}
-
-interface ChatCompletionPayload {
-  choices?: Array<{ message?: ChatCompletionMessage }>;
-}
-
 /**
  * True when a model tool call targets the rewritten image-generation function.
  * Compared loosely because upstream providers may normalize the name.
@@ -251,7 +253,7 @@ export const isImageGenerationToolCall = (toolCall: unknown): boolean => {
     return false;
   }
 
-  const name = (toolCall as ChatToolCall).function?.name;
+  const name = (toolCall as ChatCompletionToolCall).function?.name;
 
   return (
     typeof name === 'string' &&
@@ -276,16 +278,79 @@ const buildImageToolResult = (result: ImageGenerationResult | null): string => {
   return 'Image generation failed: the upstream service returned no image.';
 };
 /**
- * Runs image-generation tool calls a chat-protocol model made and returns the
- * final upstream response with the images folded back into the transcript.
+ * Runs image-generation tool calls a chat-protocol model makes, replaying the
+ * request with each generated image folded back in.
  *
- * Returns `null` when the model made no image call, so the caller keeps its
- * ordinary upstream path.
+ * Always returns the final upstream response, including when the model made no
+ * image call — callers must not re-issue the request themselves, since that
+ * would bill the turn twice and could return a different answer from the one
+ * already inspected. `executions` is empty when no image was generated.
  *
  * The returned response is always freshly constructed: reading an intermediate
  * response to inspect its tool calls consumes the body, and the caller needs to
  * read the final one again.
  */
+export interface ImageGenerationExecution {
+  id: string;
+  /** The rewritten prompt some providers echo back. Absent when unavailable. */
+  prompt: string;
+  /**
+   * Base64-encoded image, when the upstream returned inline data. This is what
+   * an OpenAI `image_generation_call` carries in its `result` field.
+   */
+  result: string | null;
+  status: 'completed' | 'failed';
+}
+
+export interface ImageGenerationLoopResult {
+  /** One entry per image call the model made, in call order. */
+  executions: ImageGenerationExecution[];
+  response: Response;
+}
+
+/**
+ * Rebuilds a response whose body was already read. The loop consumes each
+ * response to inspect its tool calls, so anything handed back has to be
+ * reconstructed from the text that was read.
+ */
+const rebuildResponse = (
+  response: Response,
+  payload: ChatCompletionPayload,
+): Response => {
+  return new Response(JSON.stringify(payload), {
+    headers: response.headers,
+    status: response.status,
+  });
+};
+
+/** Prose a hop wrote, i.e. text the model produced before calling the tool. */
+const readMessageText = (
+  message: ChatCompletionMessage | undefined,
+): string => {
+  return typeof message?.content === 'string' ? message.content.trim() : '';
+};
+
+const buildImageGenerationExecution = ({
+  id,
+  prompt,
+  result,
+}: {
+  id: string;
+  prompt: string;
+  result: ImageGenerationResult | null;
+}): ImageGenerationExecution => {
+  if (result?.b64Json) {
+    return { id, prompt, result: result.b64Json, status: 'completed' };
+  }
+
+  return {
+    id,
+    prompt,
+    result: null,
+    status: result?.url ? 'completed' : 'failed',
+  };
+};
+
 export const executeImageGenerationLoop = async ({
   body,
   callUpstream,
@@ -296,8 +361,14 @@ export const executeImageGenerationLoop = async ({
   callUpstream: (body: Record<string, unknown>) => Promise<Response>;
   context: ProxyContext;
   request: NextRequest;
-}): Promise<Response | null> => {
+}): Promise<ImageGenerationLoopResult> => {
   let currentBody: Record<string, unknown> = body;
+  const executions: ImageGenerationExecution[] = [];
+  const intermediateTexts: string[] = [];
+  // Carried across iterations so the cap can hand back the last response
+  // instead of discarding every image already generated.
+  let lastResponse: Response | null = null;
+  let lastPayload: ChatCompletionPayload | null = null;
 
   for (let iteration = 0; iteration < MAX_IMAGE_ITERATIONS; iteration += 1) {
     const response = await callUpstream(currentBody);
@@ -310,7 +381,7 @@ export const executeImageGenerationLoop = async ({
         ?.toLowerCase()
         .includes('text/event-stream')
     ) {
-      return response;
+      return { executions, response };
     }
 
     const payloadText = await response.text();
@@ -320,37 +391,63 @@ export const executeImageGenerationLoop = async ({
       payload = JSON.parse(payloadText) as ChatCompletionPayload;
     } catch {
       // Unparseable upstream output cannot be continued; return it verbatim.
-      return new Response(payloadText, {
-        headers: response.headers,
-        status: response.status,
-      });
+      return {
+        executions,
+        response: new Response(payloadText, {
+          headers: response.headers,
+          status: response.status,
+        }),
+      };
     }
 
     const message = payload.choices?.[0]?.message;
-    const imageCalls: ChatToolCall[] = (message?.tool_calls ?? []).filter(
-      isImageGenerationToolCall,
-    );
+    const imageCalls: ChatCompletionToolCall[] = (
+      message?.tool_calls ?? []
+    ).filter(isImageGenerationToolCall);
 
     if (!imageCalls.length) {
-      // Nothing to execute. Rebuild the response so the caller can still read
-      // it, since `payloadText` was consumed above.
-      return iteration === 0
-        ? null
-        : new Response(payloadText, {
-            headers: response.headers,
-            status: response.status,
-          });
+      // Nothing to execute. Hand the response back rather than returning null:
+      // the caller must not re-issue the request, since that would bill the
+      // turn twice and could yield a different answer. Prose from earlier hops
+      // is folded in first, because it is part of the turn the client sees.
+      return {
+        executions,
+        response: rebuildResponse(
+          response,
+          withIntermediateTurns({
+            executions: [],
+            payload,
+            reasonings: [],
+            texts: intermediateTexts,
+          }),
+        ),
+      };
+    }
+
+    const iterationText = readMessageText(message);
+
+    if (iterationText) {
+      intermediateTexts.push(iterationText);
     }
 
     const results: unknown[] = [];
 
     for (const toolCall of imageCalls) {
+      const arguments_ = toolCall.function?.arguments ?? '';
       const result = await executeImageGeneration({
-        arguments: toolCall.function?.arguments ?? '',
+        arguments: arguments_,
         context,
         request,
       });
 
+      executions.push(
+        buildImageGenerationExecution({
+          id: toolCall.id ?? '',
+          prompt: extractPrompt(arguments_),
+          result,
+        }),
+      );
+
       results.push({
         role: 'tool',
         content: buildImageToolResult(result),
@@ -369,7 +466,49 @@ export const executeImageGenerationLoop = async ({
     messages.push(...results);
 
     currentBody = { ...currentBody, messages };
+    lastPayload = payload;
+    lastResponse = response;
   }
 
-  return null;
+  // The cap was reached with the model still asking for images. Every image
+  // generated so far is kept, and the last response is handed back so the
+  // caller does not re-issue the request and discard them.
+  return {
+    executions,
+    response: rebuildResponse(
+      lastResponse ?? new Response(null, { status: 502 }),
+      withIntermediateTurns({
+        executions: [],
+        payload: lastPayload ?? {},
+        reasonings: [],
+        texts: intermediateTexts,
+      }),
+    ),
+  };
+};
+
+// ---------------------------------------------------------------------------
+// Responses output item
+// ---------------------------------------------------------------------------
+
+/**
+ * Builds the `image_generation_call` output item OpenAI's Responses API
+ * defines, so a client driving the chat upstream still sees the standard shape:
+ * the generated image in `result` as base64, and the prompt it came from.
+ *
+ * `result` is null when generation failed. The field is still emitted, with
+ * status `failed`, because dropping it would leave the client with no way to
+ * tell that an image was attempted.
+ */
+export const buildResponsesImageGenerationCallItem = (
+  execution: ImageGenerationExecution,
+  id = `ig_${crypto.randomUUID().replaceAll('-', '')}`,
+): Record<string, unknown> => {
+  return {
+    id,
+    result: execution.result,
+    status: execution.status,
+    type: 'image_generation_call',
+    ...(execution.prompt ? { revised_prompt: execution.prompt } : {}),
+  };
 };