feat(anthropic): pass image content blocks through to the upstream model

lib/server/proxy/anthropic.ts

@@ -33,6 +33,13 @@ const MAX_STREAM_FRAME_LENGTH = 1_000_000;
 // Anthropic Messages API types
 // ---------------------------------------------------------------------------
 
+interface AnthropicImageSource {
+  type?: string;
+  media_type?: string;
+  data?: string;
+  url?: string;
+}
+
 interface AnthropicContentBlock {
   type: string;
   text?: string;
@@ -43,6 +50,7 @@ interface AnthropicContentBlock {
   thinking?: string;
   tool_use_id?: string;
   content?: unknown;
+  source?: AnthropicImageSource;
 }
 
 interface AnthropicMessage {
@@ -144,6 +152,26 @@ interface ChatTextBlock {
   type: 'text';
 }
 
+/**
+ * An image part in the OpenAI Chat shape. Emitted in this shape rather than a
+ * native Anthropic one because the request is translated to Chat before it
+ * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the
+ * `responses` upstream converts it to `input_image`.
+ */
+interface ChatImageBlock {
+  cache_control?: { type?: string };
+  image_url: { url: string };
+  type: 'image_url';
+}
+
+type ChatContentPart = string | ChatTextBlock | ChatImageBlock;
+
+type ChatContent = string | Array<ChatTextBlock | ChatImageBlock>;
+
+/**
+ * Text-only content, used where images are not representable — the system
+ * prompt and the intermediate text-part buffer.
+ */
 type ChatTextContent = string | ChatTextBlock[];
 
 // ---------------------------------------------------------------------------
@@ -200,6 +228,87 @@ const mapTextPartsToChatContent = (
   ]);
 };
 
+/**
+ * Builds the `image_url` value for an Anthropic image block. Base64 sources
+ * become a data URI because the upstream Chat/Responses APIs expect a URL;
+ * `url` sources pass through untouched. Returns undefined for an unusable
+ * source so the caller can fall back to a text placeholder rather than
+ * emitting a block the upstream would reject.
+ */
+const buildChatImageUrl = (
+  source: AnthropicImageSource | undefined,
+): string | undefined => {
+  if (!source || typeof source !== 'object') {
+    return undefined;
+  }
+
+  if (source.type === 'url' || (!source.data && source.url)) {
+    return typeof source.url === 'string' && source.url
+      ? source.url
+      : undefined;
+  }
+
+  if (typeof source.data !== 'string' || !source.data) {
+    return undefined;
+  }
+
+  const mediaType =
+    typeof source.media_type === 'string' && source.media_type
+      ? source.media_type
+      : 'image/png';
+
+  return `data:${mediaType};base64,${source.data}`;
+};
+
+/**
+ * Like `mapTextPartsToChatContent`, but keeps image parts as real image
+ * blocks instead of collapsing them into text. Falls back to the text-only
+ * result when nothing resolved to an image.
+ */
+const mapContentPartsToChat = (parts: ChatContentPart[]): ChatContent => {
+  const hasImage = parts.some(
+    (part) => typeof part === 'object' && part.type === 'image_url',
+  );
+
+  if (!hasImage) {
+    return mapTextPartsToChatContent(
+      parts.filter(
+        (part): part is string | ChatTextBlock =>
+          typeof part === 'string' || part.type === 'text',
+      ),
+    );
+  }
+
+  const blocks: Array<ChatTextBlock | ChatImageBlock> = [];
+  let pendingText: Array<string | ChatTextBlock> = [];
+
+  const flushText = (): void => {
+    if (!pendingText.length) {
+      return;
+    }
+    const textContent = mapTextPartsToChatContent(pendingText);
+    if (typeof textContent === 'string') {
+      blocks.push({ type: 'text', text: textContent });
+    } else {
+      blocks.push(...textContent);
+    }
+    pendingText = [];
+  };
+
+  for (const part of parts) {
+    if (typeof part === 'object' && part.type === 'image_url') {
+      flushText();
+      blocks.push(part);
+      continue;
+    }
+    pendingText.push(part);
+  }
+
+  flushText();
+
+  return blocks;
+};
+
 const extractSystemText = (
   system: string | AnthropicContentBlock[] | undefined,
 ): ChatTextContent => {
@@ -232,7 +341,7 @@ const extractSystemText = (
 
 interface ChatMessage {
   role: string;
-  content: ChatTextContent | null;
+  content: ChatContent | null;
   tool_calls?: Array<{
     id: string;
     type: string;
@@ -311,11 +420,58 @@ const formatAnthropicServerToolResult = (
     return [url, text].filter(Boolean).join('\n\n');
   }
 
+  // Nested images are emitted as real image parts by
+  // `collectAnthropicNestedImages`, so they are excluded here to keep their
+  // base64 payload out of the text.
+  if (Array.isArray(block.content)) {
+    return stringifyContent(
+      block.content.filter((value) => {
+        return !(
+          value &&
+          typeof value === 'object' &&
+          (value as AnthropicContentBlock).type === 'image'
+        );
+      }),
+    );
+  }
+
   return typeof block.content === 'string'
     ? block.content
     : stringifyContent(block.content);
 };
 
+/**
+ * Images nested inside a `tool_result` content array, e.g. a screenshot a tool
+ * returned. The outer block is handled by the `tool_result` branch, whose
+ * formatter stringifies nested content — so without extracting them here the
+ * model would receive the base64 payload as text.
+ */
+const collectAnthropicNestedImages = (
+  block: AnthropicContentBlock,
+): ChatImageBlock[] => {
+  if (!Array.isArray(block.content)) {
+    return [];
+  }
+
+  return block.content.flatMap((value): ChatImageBlock[] => {
+    if (!value || typeof value !== 'object') {
+      return [];
+    }
+
+    const nested = value as AnthropicContentBlock;
+
+    if (nested.type !== 'image') {
+      return [];
+    }
+
+    const imageUrl = buildChatImageUrl(nested.source);
+
+    return imageUrl
+      ? [{ type: 'image_url', image_url: { url: imageUrl } }]
+      : [];
+  });
+};
+
 const mapAnthropicContentToChat = (
   content: string | AnthropicContentBlock[],
   role: 'user' | 'assistant',
@@ -324,7 +480,7 @@ const mapAnthropicContentToChat = (
     return [{ role, content }];
   }
 
-  const parts: Array<string | ChatTextBlock> = [];
+  const parts: ChatContentPart[] = [];
   const toolCalls: Array<{
     id: string;
     type: string;
@@ -336,15 +492,16 @@ const mapAnthropicContentToChat = (
   const toolResults: ChatMessage[] = [];
   const messages: ChatMessage[] = [];
   const flushAssistantMessage = (): void => {
-    const textContent = mapTextPartsToChatContent(parts);
+    const content = mapContentPartsToChat(parts);
+    const hasContent = typeof content === 'string' ? content.length > 0 : true;
 
-    if (!toolCalls.length && !textContent.length) {
+    if (!toolCalls.length && !hasContent) {
       return;
     }
 
     messages.push({
       role: 'assistant',
-      content: textContent.length ? textContent : null,
+      content: hasContent ? content : null,
       ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}),
     });
     parts.length = 0;
@@ -374,9 +531,16 @@ const mapAnthropicContentToChat = (
       block.type === 'web_search_tool_result' ||
       block.type === 'web_fetch_tool_result'
     ) {
+      const nestedImages = collectAnthropicNestedImages(block);
+
       const resultMessage: ChatMessage = {
         role: 'tool',
-        content: formatAnthropicServerToolResult(block),
+        content: nestedImages.length
+          ? mapContentPartsToChat([
+              formatAnthropicServerToolResult(block),
+              ...nestedImages,
+            ])
+          : formatAnthropicServerToolResult(block),
         tool_call_id: block.tool_use_id ?? '',
       };
 
@@ -388,16 +552,41 @@ const mapAnthropicContentToChat = (
       }
     } else if (block.type === 'thinking') {
       // Skip thinking blocks in conversation history for OpenAI compat.
+    } else if (block.type === 'image' && block.source) {
+      // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url',
+      // media_type, data | url } }`. Emit a real image block so the upstream
+      // model sees the image; without this branch the block fell through to
+      // `stringifyContent` and the model received a JSON dump of the base64
+      // payload as text. An `image` block with no `source` is not a real
+      // Anthropic image, so it keeps the generic stringified handling.
+      const imageUrl = buildChatImageUrl(block.source);
+
+      parts.push(
+        imageUrl
+          ? {
+              type: 'image_url',
+              image_url: { url: imageUrl },
+              // Preserve an explicit cache breakpoint, matching how text
+              // blocks carry `cache_control` through. Without this the
+              // requested breakpoint is dropped and `applyPromptCacheControl`
+              // falls back to its own automatic placement.
+              ...(block.cache_control
+                ? { cache_control: block.cache_control }
+                : {}),
+            }
+          : stringifyContent(block),
+      );
     } else {
       parts.push(stringifyContent(block));
     }
   }
 
   if (role === 'user') {
     messages.push(...toolResults);
-    const textContent = mapTextPartsToChatContent(parts);
-    if (textContent.length) {
-      messages.push({ role: 'user', content: textContent });
+    const content = mapContentPartsToChat(parts);
+    const hasContent = typeof content === 'string' ? content.length > 0 : true;
+    if (hasContent) {
+      messages.push({ role: 'user', content });
     }
   } else {
     flushAssistantMessage();

lib/server/proxy/codebuddy.ts

@@ -759,7 +759,7 @@ const getCredentialValue = (
   return null;
 };
 
-const buildUpstreamHeaders = async (
+export const buildUpstreamHeaders = async (
   request: NextRequest,
   auth: ResolvedAuth,
 ): Promise<HeadersInit> => {
@@ -885,6 +885,58 @@ const buildUpstreamBody = async (
   };
 };
 
+export const isImageContentPart = (part: unknown): boolean => {
+  if (!part || typeof part !== 'object') {
+    return false;
+  }
+
+  const value = part as { image_url?: unknown; type?: unknown };
+
+  if (value.type === 'image_url' || value.type === 'input_image') {
+    return true;
+  }
+
+  // Accept the shapes an OpenAI-compatible client may send even when `type`
+  // is absent or unexpected: any part carrying an image URL is an image.
+  return (
+    typeof value.image_url === 'string' ||
+    Boolean(
+      value.image_url &&
+      typeof value.image_url === 'object' &&
+      typeof (value.image_url as { url?: unknown }).url === 'string',
+    )
+  );
+};
+
+/**
+ * Reads the image URL out of a Responses `input_image` / `image_url` part.
+ * Returns undefined when the part carries no usable URL, so callers can drop it
+ * rather than forwarding a block the upstream would reject.
+ */
+export const extractImageUrl = (part: unknown): string | undefined => {
+  if (!part || typeof part !== 'object') {
+    return undefined;
+  }
+
+  const { image_url: imageUrl } = part as { image_url?: unknown };
+
+  // `input_image` carries a bare URL string; the OpenAI Chat-style
+  // `image_url` part nests it under `url`.
+  if (typeof imageUrl === 'string') {
+    return imageUrl || undefined;
+  }
+
+  if (
+    imageUrl &&
+    typeof imageUrl === 'object' &&
+    typeof (imageUrl as { url?: unknown }).url === 'string'
+  ) {
+    return (imageUrl as { url: string }).url || undefined;
+  }
+
+  return undefined;
+};
+
 const stringifyResponsesInputContent = (content: unknown): string => {
   if (typeof content === 'string') return content;
   if (content === null || content === undefined) return '';
@@ -1149,9 +1201,24 @@ const buildResponsesBodyFromChat = async (
       )
       .map((message) => {
         if (message.role === 'tool') {
+          // A tool may return an image, e.g. a screenshot. The upstream
+          // `function_call_output` carries `output` as structured content, so
+          // an image part is preserved there; stringifying it would hand the
+          // model a base64 dump instead of the image.
+          const toolOutput = Array.isArray(message.content)
+            ? message.content.filter(
+                (part) => part !== null && part !== undefined,
+              )
+            : message.content;
+          const hasImage = Array.isArray(toolOutput)
+            ? toolOutput.some(isImageContentPart)
+            : isImageContentPart(toolOutput);
+
           return {
             call_id: message.tool_call_id,
-            output: stringifyResponsesInputContent(message.content),
+            output: hasImage
+              ? mapChatContentToResponses(toolOutput)
+              : stringifyResponsesInputContent(toolOutput),
             type: 'function_call_output',
           };
         }